okstra 0.108.0 → 0.110.0

package/runtime/python/okstra_vendor/graphify/hooks.py

@@ -0,0 +1,220 @@
+# git hook integration - install/uninstall graphify post-commit and post-checkout hooks
+from __future__ import annotations
+import re
+import subprocess
+from pathlib import Path
+
+_HOOK_MARKER = "# graphify-hook-start"
+_HOOK_MARKER_END = "# graphify-hook-end"
+_CHECKOUT_MARKER = "# graphify-checkout-hook-start"
+_CHECKOUT_MARKER_END = "# graphify-checkout-hook-end"
+
+_PYTHON_DETECT = """\
+# Detect the correct Python interpreter (handles pipx, venv, system installs)
+GRAPHIFY_BIN=$(command -v graphify 2>/dev/null)
+if [ -n "$GRAPHIFY_BIN" ]; then
+    _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | sed 's/^#![[:space:]]*//')
+    case "$_SHEBANG" in
+        */env\\ *) GRAPHIFY_PYTHON="${_SHEBANG#*/env }" ;;
+        *)         GRAPHIFY_PYTHON="$_SHEBANG" ;;
+    esac
+    # Allowlist: only keep characters valid in a filesystem path to prevent
+    # injection if the shebang contains shell metacharacters
+    case "$GRAPHIFY_PYTHON" in
+        *[!a-zA-Z0-9/_.-]*) GRAPHIFY_PYTHON="" ;;
+    esac
+    if [ -n "$GRAPHIFY_PYTHON" ] && ! "$GRAPHIFY_PYTHON" -c "import graphify" 2>/dev/null; then
+        GRAPHIFY_PYTHON=""
+    fi
+fi
+# Fall back: try python3, then python (Windows has no python3 shim)
+if [ -z "$GRAPHIFY_PYTHON" ]; then
+    if command -v python3 >/dev/null 2>&1 && python3 -c "import graphify" 2>/dev/null; then
+        GRAPHIFY_PYTHON="python3"
+    elif command -v python >/dev/null 2>&1 && python -c "import graphify" 2>/dev/null; then
+        GRAPHIFY_PYTHON="python"
+    else
+        exit 0
+    fi
+fi
+"""
+
+_HOOK_SCRIPT = """\
+# graphify-hook-start
+# Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed).
+# Installed by: graphify hook install
+
+CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || git diff --name-only HEAD 2>/dev/null)
+if [ -z "$CHANGED" ]; then
+    exit 0
+fi
+
+""" + _PYTHON_DETECT + """
+export GRAPHIFY_CHANGED="$CHANGED"
+$GRAPHIFY_PYTHON -c "
+import os, sys
+from pathlib import Path
+
+changed_raw = os.environ.get('GRAPHIFY_CHANGED', '')
+changed = [Path(f.strip()) for f in changed_raw.strip().splitlines() if f.strip()]
+
+if not changed:
+    sys.exit(0)
+
+print(f'[graphify hook] {len(changed)} file(s) changed - rebuilding graph...')
+
+try:
+    from graphify.watch import _rebuild_code
+    _rebuild_code(Path('.'))
+except Exception as exc:
+    print(f'[graphify hook] Rebuild failed: {exc}')
+    sys.exit(1)
+"
+# graphify-hook-end
+"""
+
+
+_CHECKOUT_SCRIPT = """\
+# graphify-checkout-hook-start
+# Auto-rebuilds the knowledge graph (code only) when switching branches.
+# Installed by: graphify hook install
+
+PREV_HEAD=$1
+NEW_HEAD=$2
+BRANCH_SWITCH=$3
+
+# Only run on branch switches, not file checkouts
+if [ "$BRANCH_SWITCH" != "1" ]; then
+    exit 0
+fi
+
+# Only run if graphify-out/ exists (graph has been built before)
+if [ ! -d "graphify-out" ]; then
+    exit 0
+fi
+
+""" + _PYTHON_DETECT + """
+echo "[graphify] Branch switched - rebuilding knowledge graph (code files)..."
+$GRAPHIFY_PYTHON -c "
+from graphify.watch import _rebuild_code
+from pathlib import Path
+import sys
+try:
+    _rebuild_code(Path('.'))
+except Exception as exc:
+    print(f'[graphify] Rebuild failed: {exc}')
+    sys.exit(1)
+"
+# graphify-checkout-hook-end
+"""
+
+
+def _git_root(path: Path) -> Path | None:
+    """Walk up to find .git directory."""
+    current = path.resolve()
+    for parent in [current, *current.parents]:
+        if (parent / ".git").exists():
+            return parent
+    return None
+
+
+def _hooks_dir(root: Path) -> Path:
+    """Return the git hooks directory, respecting core.hooksPath if set (e.g. Husky)."""
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(root), "config", "core.hooksPath"],
+            capture_output=True, text=True,
+        )
+        if result.returncode == 0:
+            custom = result.stdout.strip()
+            if custom:
+                p = Path(custom)
+                if not p.is_absolute():
+                    p = root / p
+                p.mkdir(parents=True, exist_ok=True)
+                return p
+    except (OSError, FileNotFoundError):
+        pass
+    d = root / ".git" / "hooks"
+    d.mkdir(exist_ok=True)
+    return d
+
+
+def _install_hook(hooks_dir: Path, name: str, script: str, marker: str) -> str:
+    """Install a single git hook, appending if an existing hook is present."""
+    hook_path = hooks_dir / name
+    if hook_path.exists():
+        content = hook_path.read_text(encoding="utf-8")
+        if marker in content:
+            return f"already installed at {hook_path}"
+        hook_path.write_text(content.rstrip() + "\n\n" + script, encoding="utf-8", newline="\n")
+        return f"appended to existing {name} hook at {hook_path}"
+    hook_path.write_text("#!/bin/sh\n" + script, encoding="utf-8", newline="\n")
+    hook_path.chmod(0o755)
+    return f"installed at {hook_path}"
+
+
+def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> str:
+    """Remove graphify section from a git hook using start/end markers."""
+    hook_path = hooks_dir / name
+    if not hook_path.exists():
+        return f"no {name} hook found - nothing to remove."
+    content = hook_path.read_text(encoding="utf-8")
+    if marker not in content:
+        return f"graphify hook not found in {name} - nothing to remove."
+    new_content = re.sub(
+        rf"{re.escape(marker)}.*?{re.escape(marker_end)}\n?",
+        "",
+        content,
+        flags=re.DOTALL,
+    ).strip()
+    if not new_content or new_content in ("#!/bin/bash", "#!/bin/sh"):
+        hook_path.unlink()
+        return f"removed {name} hook at {hook_path}"
+    hook_path.write_text(new_content + "\n", encoding="utf-8", newline="\n")
+    return f"graphify removed from {name} at {hook_path} (other hook content preserved)"
+
+
+def install(path: Path = Path(".")) -> str:
+    """Install graphify post-commit and post-checkout hooks in the nearest git repo."""
+    root = _git_root(path)
+    if root is None:
+        raise RuntimeError(f"No git repository found at or above {path.resolve()}")
+
+    hooks_dir = _hooks_dir(root)
+
+    commit_msg = _install_hook(hooks_dir, "post-commit", _HOOK_SCRIPT, _HOOK_MARKER)
+    checkout_msg = _install_hook(hooks_dir, "post-checkout", _CHECKOUT_SCRIPT, _CHECKOUT_MARKER)
+
+    return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}"
+
+
+def uninstall(path: Path = Path(".")) -> str:
+    """Remove graphify post-commit and post-checkout hooks."""
+    root = _git_root(path)
+    if root is None:
+        raise RuntimeError(f"No git repository found at or above {path.resolve()}")
+
+    hooks_dir = _hooks_dir(root)
+    commit_msg = _uninstall_hook(hooks_dir, "post-commit", _HOOK_MARKER, _HOOK_MARKER_END)
+    checkout_msg = _uninstall_hook(hooks_dir, "post-checkout", _CHECKOUT_MARKER, _CHECKOUT_MARKER_END)
+
+    return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}"
+
+
+def status(path: Path = Path(".")) -> str:
+    """Check if graphify hooks are installed."""
+    root = _git_root(path)
+    if root is None:
+        return "Not in a git repository."
+    hooks_dir = _hooks_dir(root)
+
+    def _check(name: str, marker: str) -> str:
+        p = hooks_dir / name
+        if not p.exists():
+            return "not installed"
+        return "installed" if marker in p.read_text(encoding="utf-8") else "not installed (hook exists but graphify not found)"
+
+    commit = _check("post-commit", _HOOK_MARKER)
+    checkout = _check("post-checkout", _CHECKOUT_MARKER)
+    return f"post-commit: {commit}\npost-checkout: {checkout}"

package/runtime/python/okstra_vendor/graphify/ingest.py

@@ -0,0 +1,297 @@
+# fetch URLs (tweet/arxiv/pdf/web) and save as annotated markdown
+from __future__ import annotations
+import json
+import re
+import urllib.error
+import urllib.parse
+from datetime import datetime, timezone
+from pathlib import Path
+
+from graphify.security import safe_fetch, safe_fetch_text, validate_url
+
+
+def _yaml_str(s: str) -> str:
+    """Escape a string for embedding in a YAML double-quoted scalar."""
+    return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")
+
+
+def _safe_filename(url: str, suffix: str) -> str:
+    """Turn a URL into a safe filename."""
+    parsed = urllib.parse.urlparse(url)
+    name = parsed.netloc + parsed.path
+    name = re.sub(r"[^\w\-]", "_", name).strip("_")
+    name = re.sub(r"_+", "_", name)[:80]
+    return name + suffix
+
+
+def _detect_url_type(url: str) -> str:
+    """Classify the URL for targeted extraction."""
+    lower = url.lower()
+    if "twitter.com" in lower or "x.com" in lower:
+        return "tweet"
+    if "arxiv.org" in lower:
+        return "arxiv"
+    if "github.com" in lower:
+        return "github"
+    if "youtube.com" in lower or "youtu.be" in lower:
+        return "youtube"
+    parsed = urllib.parse.urlparse(url)
+    path = parsed.path.lower()
+    if path.endswith(".pdf"):
+        return "pdf"
+    if any(path.endswith(ext) for ext in (".png", ".jpg", ".jpeg", ".webp", ".gif")):
+        return "image"
+    return "webpage"
+
+
+def _fetch_html(url: str) -> str:
+    return safe_fetch_text(url)
+
+
+def _html_to_markdown(html: str, url: str) -> str:
+    """Convert HTML to clean markdown. Uses html2text if available, else basic strip."""
+    try:
+        import html2text
+        h = html2text.HTML2Text()
+        h.ignore_links = False
+        h.ignore_images = True
+        h.body_width = 0
+        return h.handle(html)
+    except ImportError:
+        # Fallback: strip tags
+        text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = re.sub(r"\s+", " ", text).strip()
+        return text[:8000]
+
+
+def _fetch_tweet(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
+    """Fetch a tweet URL. Returns (content, filename)."""
+    # Normalize to twitter.com for oEmbed
+    oembed_url = url.replace("x.com", "twitter.com")
+    oembed_api = f"https://publish.twitter.com/oembed?url={urllib.parse.quote(oembed_url)}&omit_script=true"
+    try:
+        data = json.loads(safe_fetch_text(oembed_api))
+        tweet_text = re.sub(r"<[^>]+>", "", data.get("html", "")).strip()
+        tweet_author = data.get("author_name", "unknown")
+    except Exception:
+        # oEmbed failed - save URL stub
+        tweet_text = f"Tweet at {url} (could not fetch content)"
+        tweet_author = "unknown"
+
+    now = datetime.now(timezone.utc).isoformat()
+    content = f"""---
+source_url: "{_yaml_str(url)}"
+type: tweet
+author: "{_yaml_str(tweet_author)}"
+captured_at: {now}
+contributor: "{_yaml_str(contributor or author or 'unknown')}"
+---
+
+# Tweet by @{tweet_author}
+
+{tweet_text}
+
+Source: {url}
+"""
+    filename = _safe_filename(url, ".md")
+    return content, filename
+
+
+def _fetch_webpage(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
+    """Fetch a generic webpage and convert to markdown."""
+    html = _fetch_html(url)
+    # Extract title
+    title_match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+    title = re.sub(r"\s+", " ", title_match.group(1)).strip() if title_match else url
+
+    markdown = _html_to_markdown(html, url)
+    now = datetime.now(timezone.utc).isoformat()
+    content = f"""---
+source_url: "{_yaml_str(url)}"
+type: webpage
+title: "{_yaml_str(title)}"
+captured_at: {now}
+contributor: "{_yaml_str(contributor or author or 'unknown')}"
+---
+
+# {title}
+
+Source: {url}
+
+---
+
+{markdown[:12000]}
+"""
+    filename = _safe_filename(url, ".md")
+    return content, filename
+
+
+def _fetch_arxiv(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
+    """Fetch arXiv abstract page."""
+    # Convert /abs/ or /pdf/ to abs for the API
+    arxiv_id = re.search(r"(\d{4}\.\d{4,5})", url)
+    if arxiv_id:
+        api_url = f"https://export.arxiv.org/abs/{arxiv_id.group(1)}"
+        try:
+            html = _fetch_html(api_url)
+            abstract_match = re.search(r'class="abstract[^"]*"[^>]*>(.*?)</blockquote>', html, re.DOTALL | re.IGNORECASE)
+            abstract = re.sub(r"<[^>]+>", "", abstract_match.group(1)).strip() if abstract_match else ""
+            title_match = re.search(r'class="title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL | re.IGNORECASE)
+            title = re.sub(r"<[^>]+>", " ", title_match.group(1)).strip() if title_match else arxiv_id.group(1)
+            authors_match = re.search(r'class="authors"[^>]*>(.*?)</div>', html, re.DOTALL | re.IGNORECASE)
+            paper_authors = re.sub(r"<[^>]+>", "", authors_match.group(1)).strip() if authors_match else ""
+        except Exception:
+            title, abstract, paper_authors = arxiv_id.group(1), "", ""
+    else:
+        return _fetch_webpage(url, author, contributor)
+
+    now = datetime.now(timezone.utc).isoformat()
+    content = f"""---
+source_url: "{_yaml_str(url)}"
+arxiv_id: "{_yaml_str(arxiv_id.group(1) if arxiv_id else '')}"
+type: paper
+title: "{_yaml_str(title)}"
+paper_authors: "{_yaml_str(paper_authors)}"
+captured_at: {now}
+contributor: "{_yaml_str(contributor or author or 'unknown')}"
+---
+
+# {title}
+
+**Authors:** {paper_authors}
+**arXiv:** {arxiv_id.group(1) if arxiv_id else url}
+
+## Abstract
+
+{abstract}
+
+Source: {url}
+"""
+    filename = f"arxiv_{arxiv_id.group(1).replace('.', '_')}.md" if arxiv_id else _safe_filename(url, ".md")
+    return content, filename
+
+
+def _download_binary(url: str, suffix: str, target_dir: Path) -> Path:
+    """Download a binary file (PDF, image) directly."""
+    filename = _safe_filename(url, suffix)
+    out_path = target_dir / filename
+    out_path.write_bytes(safe_fetch(url))
+    return out_path
+
+
+def ingest(url: str, target_dir: Path, author: str | None = None, contributor: str | None = None) -> Path:
+    """
+    Fetch a URL and save it into target_dir as a graphify-ready file.
+
+    Returns the path of the saved file.
+    """
+    target_dir.mkdir(parents=True, exist_ok=True)
+    url_type = _detect_url_type(url)
+
+    try:
+        validate_url(url)
+    except ValueError as exc:
+        raise ValueError(f"ingest: {exc}") from exc
+
+    try:
+        if url_type == "pdf":
+            out = _download_binary(url, ".pdf", target_dir)
+            print(f"Downloaded PDF: {out.name}")
+            return out
+
+        if url_type == "image":
+            suffix = Path(urllib.parse.urlparse(url).path).suffix or ".jpg"
+            out = _download_binary(url, suffix, target_dir)
+            print(f"Downloaded image: {out.name}")
+            return out
+
+        if url_type == "youtube":
+            from graphify.transcribe import download_audio
+            out = download_audio(url, target_dir)
+            print(f"Downloaded audio: {out.name}")
+            return out
+
+        if url_type == "tweet":
+            content, filename = _fetch_tweet(url, author, contributor)
+        elif url_type == "arxiv":
+            content, filename = _fetch_arxiv(url, author, contributor)
+        else:
+            content, filename = _fetch_webpage(url, author, contributor)
+    except (urllib.error.HTTPError, urllib.error.URLError, OSError) as exc:
+        raise RuntimeError(f"ingest: failed to fetch {url!r}: {exc}") from exc
+
+    out_path = target_dir / filename
+    # Avoid overwriting - append counter if needed
+    counter = 1
+    while out_path.exists() and counter < 1000:
+        stem = Path(filename).stem
+        out_path = target_dir / f"{stem}_{counter}.md"
+        counter += 1
+
+    out_path.write_text(content, encoding="utf-8")
+    print(f"Saved {url_type}: {out_path.name}")
+    return out_path
+
+
+def save_query_result(
+    question: str,
+    answer: str,
+    memory_dir: Path,
+    query_type: str = "query",
+    source_nodes: list[str] | None = None,
+) -> Path:
+    """Save a Q&A result as markdown so it gets extracted into the graph on next --update.
+
+    Files are stored in memory_dir (typically graphify-out/memory/) with YAML frontmatter
+    that graphify's extractor reads as node metadata. This closes the feedback loop:
+    the system grows smarter from both what you add AND what you ask.
+    """
+    memory_dir = Path(memory_dir)
+    memory_dir.mkdir(parents=True, exist_ok=True)
+
+    now = datetime.now(timezone.utc)
+    slug = re.sub(r"[^\w]", "_", question.lower())[:50].strip("_")
+    filename = f"query_{now.strftime('%Y%m%d_%H%M%S')}_{slug}.md"
+
+    frontmatter_lines = [
+        "---",
+        f'type: "{query_type}"',
+        f'date: "{now.isoformat()}"',
+        f'question: "{_yaml_str(question)}"',
+        'contributor: "graphify"',
+    ]
+    if source_nodes:
+        nodes_str = ", ".join(f'"{n}"' for n in source_nodes[:10])
+        frontmatter_lines.append(f"source_nodes: [{nodes_str}]")
+    frontmatter_lines.append("---")
+
+    body_lines = [
+        "",
+        f"# Q: {question}",
+        "",
+        "## Answer",
+        "",
+        answer,
+    ]
+    if source_nodes:
+        body_lines += ["", "## Source Nodes", ""]
+        body_lines += [f"- {n}" for n in source_nodes]
+
+    content = "\n".join(frontmatter_lines + body_lines)
+    out_path = memory_dir / filename
+    out_path.write_text(content, encoding="utf-8")
+    return out_path
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Fetch a URL into a graphify /raw folder")
+    parser.add_argument("url", help="URL to fetch")
+    parser.add_argument("target_dir", nargs="?", default="./raw", help="Target directory (default: ./raw)")
+    parser.add_argument("--author", help="Your name (stored as node metadata)")
+    parser.add_argument("--contributor", help="Contributor name for team graphs")
+    args = parser.parse_args()
+    out = ingest(args.url, Path(args.target_dir), author=args.author, contributor=args.contributor)
+    print(f"Ready for graphify: {out}")

package/runtime/python/okstra_vendor/graphify/manifest.py

@@ -0,0 +1,4 @@
+# re-export manifest helpers from detect for backwards compatibility
+from graphify.detect import save_manifest, load_manifest, detect_incremental
+
+__all__ = ["save_manifest", "load_manifest", "detect_incremental"]