okstra 0.130.3 → 0.131.0

package/runtime/python/okstra_vendor/graphify/hooks.py

@@ -1,220 +0,0 @@
-# git hook integration - install/uninstall graphify post-commit and post-checkout hooks
-from __future__ import annotations
-import re
-import subprocess
-from pathlib import Path
-
-_HOOK_MARKER = "# graphify-hook-start"
-_HOOK_MARKER_END = "# graphify-hook-end"
-_CHECKOUT_MARKER = "# graphify-checkout-hook-start"
-_CHECKOUT_MARKER_END = "# graphify-checkout-hook-end"
-
-_PYTHON_DETECT = """\
-# Detect the correct Python interpreter (handles pipx, venv, system installs)
-GRAPHIFY_BIN=$(command -v graphify 2>/dev/null)
-if [ -n "$GRAPHIFY_BIN" ]; then
-    _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | sed 's/^#![[:space:]]*//')
-    case "$_SHEBANG" in
-        */env\\ *) GRAPHIFY_PYTHON="${_SHEBANG#*/env }" ;;
-        *)         GRAPHIFY_PYTHON="$_SHEBANG" ;;
-    esac
-    # Allowlist: only keep characters valid in a filesystem path to prevent
-    # injection if the shebang contains shell metacharacters
-    case "$GRAPHIFY_PYTHON" in
-        *[!a-zA-Z0-9/_.-]*) GRAPHIFY_PYTHON="" ;;
-    esac
-    if [ -n "$GRAPHIFY_PYTHON" ] && ! "$GRAPHIFY_PYTHON" -c "import graphify" 2>/dev/null; then
-        GRAPHIFY_PYTHON=""
-    fi
-fi
-# Fall back: try python3, then python (Windows has no python3 shim)
-if [ -z "$GRAPHIFY_PYTHON" ]; then
-    if command -v python3 >/dev/null 2>&1 && python3 -c "import graphify" 2>/dev/null; then
-        GRAPHIFY_PYTHON="python3"
-    elif command -v python >/dev/null 2>&1 && python -c "import graphify" 2>/dev/null; then
-        GRAPHIFY_PYTHON="python"
-    else
-        exit 0
-    fi
-fi
-"""
-
-_HOOK_SCRIPT = """\
-# graphify-hook-start
-# Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed).
-# Installed by: graphify hook install
-
-CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || git diff --name-only HEAD 2>/dev/null)
-if [ -z "$CHANGED" ]; then
-    exit 0
-fi
-
-""" + _PYTHON_DETECT + """
-export GRAPHIFY_CHANGED="$CHANGED"
-$GRAPHIFY_PYTHON -c "
-import os, sys
-from pathlib import Path
-
-changed_raw = os.environ.get('GRAPHIFY_CHANGED', '')
-changed = [Path(f.strip()) for f in changed_raw.strip().splitlines() if f.strip()]
-
-if not changed:
-    sys.exit(0)
-
-print(f'[graphify hook] {len(changed)} file(s) changed - rebuilding graph...')
-
-try:
-    from graphify.watch import _rebuild_code
-    _rebuild_code(Path('.'))
-except Exception as exc:
-    print(f'[graphify hook] Rebuild failed: {exc}')
-    sys.exit(1)
-"
-# graphify-hook-end
-"""
-
-
-_CHECKOUT_SCRIPT = """\
-# graphify-checkout-hook-start
-# Auto-rebuilds the knowledge graph (code only) when switching branches.
-# Installed by: graphify hook install
-
-PREV_HEAD=$1
-NEW_HEAD=$2
-BRANCH_SWITCH=$3
-
-# Only run on branch switches, not file checkouts
-if [ "$BRANCH_SWITCH" != "1" ]; then
-    exit 0
-fi
-
-# Only run if graphify-out/ exists (graph has been built before)
-if [ ! -d "graphify-out" ]; then
-    exit 0
-fi
-
-""" + _PYTHON_DETECT + """
-echo "[graphify] Branch switched - rebuilding knowledge graph (code files)..."
-$GRAPHIFY_PYTHON -c "
-from graphify.watch import _rebuild_code
-from pathlib import Path
-import sys
-try:
-    _rebuild_code(Path('.'))
-except Exception as exc:
-    print(f'[graphify] Rebuild failed: {exc}')
-    sys.exit(1)
-"
-# graphify-checkout-hook-end
-"""
-
-
-def _git_root(path: Path) -> Path | None:
-    """Walk up to find .git directory."""
-    current = path.resolve()
-    for parent in [current, *current.parents]:
-        if (parent / ".git").exists():
-            return parent
-    return None
-
-
-def _hooks_dir(root: Path) -> Path:
-    """Return the git hooks directory, respecting core.hooksPath if set (e.g. Husky)."""
-    try:
-        result = subprocess.run(
-            ["git", "-C", str(root), "config", "core.hooksPath"],
-            capture_output=True, text=True,
-        )
-        if result.returncode == 0:
-            custom = result.stdout.strip()
-            if custom:
-                p = Path(custom)
-                if not p.is_absolute():
-                    p = root / p
-                p.mkdir(parents=True, exist_ok=True)
-                return p
-    except (OSError, FileNotFoundError):
-        pass
-    d = root / ".git" / "hooks"
-    d.mkdir(exist_ok=True)
-    return d
-
-
-def _install_hook(hooks_dir: Path, name: str, script: str, marker: str) -> str:
-    """Install a single git hook, appending if an existing hook is present."""
-    hook_path = hooks_dir / name
-    if hook_path.exists():
-        content = hook_path.read_text(encoding="utf-8")
-        if marker in content:
-            return f"already installed at {hook_path}"
-        hook_path.write_text(content.rstrip() + "\n\n" + script, encoding="utf-8", newline="\n")
-        return f"appended to existing {name} hook at {hook_path}"
-    hook_path.write_text("#!/bin/sh\n" + script, encoding="utf-8", newline="\n")
-    hook_path.chmod(0o755)
-    return f"installed at {hook_path}"
-
-
-def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> str:
-    """Remove graphify section from a git hook using start/end markers."""
-    hook_path = hooks_dir / name
-    if not hook_path.exists():
-        return f"no {name} hook found - nothing to remove."
-    content = hook_path.read_text(encoding="utf-8")
-    if marker not in content:
-        return f"graphify hook not found in {name} - nothing to remove."
-    new_content = re.sub(
-        rf"{re.escape(marker)}.*?{re.escape(marker_end)}\n?",
-        "",
-        content,
-        flags=re.DOTALL,
-    ).strip()
-    if not new_content or new_content in ("#!/bin/bash", "#!/bin/sh"):
-        hook_path.unlink()
-        return f"removed {name} hook at {hook_path}"
-    hook_path.write_text(new_content + "\n", encoding="utf-8", newline="\n")
-    return f"graphify removed from {name} at {hook_path} (other hook content preserved)"
-
-
-def install(path: Path = Path(".")) -> str:
-    """Install graphify post-commit and post-checkout hooks in the nearest git repo."""
-    root = _git_root(path)
-    if root is None:
-        raise RuntimeError(f"No git repository found at or above {path.resolve()}")
-
-    hooks_dir = _hooks_dir(root)
-
-    commit_msg = _install_hook(hooks_dir, "post-commit", _HOOK_SCRIPT, _HOOK_MARKER)
-    checkout_msg = _install_hook(hooks_dir, "post-checkout", _CHECKOUT_SCRIPT, _CHECKOUT_MARKER)
-
-    return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}"
-
-
-def uninstall(path: Path = Path(".")) -> str:
-    """Remove graphify post-commit and post-checkout hooks."""
-    root = _git_root(path)
-    if root is None:
-        raise RuntimeError(f"No git repository found at or above {path.resolve()}")
-
-    hooks_dir = _hooks_dir(root)
-    commit_msg = _uninstall_hook(hooks_dir, "post-commit", _HOOK_MARKER, _HOOK_MARKER_END)
-    checkout_msg = _uninstall_hook(hooks_dir, "post-checkout", _CHECKOUT_MARKER, _CHECKOUT_MARKER_END)
-
-    return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}"
-
-
-def status(path: Path = Path(".")) -> str:
-    """Check if graphify hooks are installed."""
-    root = _git_root(path)
-    if root is None:
-        return "Not in a git repository."
-    hooks_dir = _hooks_dir(root)
-
-    def _check(name: str, marker: str) -> str:
-        p = hooks_dir / name
-        if not p.exists():
-            return "not installed"
-        return "installed" if marker in p.read_text(encoding="utf-8") else "not installed (hook exists but graphify not found)"
-
-    commit = _check("post-commit", _HOOK_MARKER)
-    checkout = _check("post-checkout", _CHECKOUT_MARKER)
-    return f"post-commit: {commit}\npost-checkout: {checkout}"

package/runtime/python/okstra_vendor/graphify/ingest.py

@@ -1,297 +0,0 @@
-# fetch URLs (tweet/arxiv/pdf/web) and save as annotated markdown
-from __future__ import annotations
-import json
-import re
-import urllib.error
-import urllib.parse
-from datetime import datetime, timezone
-from pathlib import Path
-
-from graphify.security import safe_fetch, safe_fetch_text, validate_url
-
-
-def _yaml_str(s: str) -> str:
-    """Escape a string for embedding in a YAML double-quoted scalar."""
-    return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")
-
-
-def _safe_filename(url: str, suffix: str) -> str:
-    """Turn a URL into a safe filename."""
-    parsed = urllib.parse.urlparse(url)
-    name = parsed.netloc + parsed.path
-    name = re.sub(r"[^\w\-]", "_", name).strip("_")
-    name = re.sub(r"_+", "_", name)[:80]
-    return name + suffix
-
-
-def _detect_url_type(url: str) -> str:
-    """Classify the URL for targeted extraction."""
-    lower = url.lower()
-    if "twitter.com" in lower or "x.com" in lower:
-        return "tweet"
-    if "arxiv.org" in lower:
-        return "arxiv"
-    if "github.com" in lower:
-        return "github"
-    if "youtube.com" in lower or "youtu.be" in lower:
-        return "youtube"
-    parsed = urllib.parse.urlparse(url)
-    path = parsed.path.lower()
-    if path.endswith(".pdf"):
-        return "pdf"
-    if any(path.endswith(ext) for ext in (".png", ".jpg", ".jpeg", ".webp", ".gif")):
-        return "image"
-    return "webpage"
-
-
-def _fetch_html(url: str) -> str:
-    return safe_fetch_text(url)
-
-
-def _html_to_markdown(html: str, url: str) -> str:
-    """Convert HTML to clean markdown. Uses html2text if available, else basic strip."""
-    try:
-        import html2text
-        h = html2text.HTML2Text()
-        h.ignore_links = False
-        h.ignore_images = True
-        h.body_width = 0
-        return h.handle(html)
-    except ImportError:
-        # Fallback: strip tags
-        text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
-        text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
-        text = re.sub(r"<[^>]+>", " ", text)
-        text = re.sub(r"\s+", " ", text).strip()
-        return text[:8000]
-
-
-def _fetch_tweet(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
-    """Fetch a tweet URL. Returns (content, filename)."""
-    # Normalize to twitter.com for oEmbed
-    oembed_url = url.replace("x.com", "twitter.com")
-    oembed_api = f"https://publish.twitter.com/oembed?url={urllib.parse.quote(oembed_url)}&omit_script=true"
-    try:
-        data = json.loads(safe_fetch_text(oembed_api))
-        tweet_text = re.sub(r"<[^>]+>", "", data.get("html", "")).strip()
-        tweet_author = data.get("author_name", "unknown")
-    except Exception:
-        # oEmbed failed - save URL stub
-        tweet_text = f"Tweet at {url} (could not fetch content)"
-        tweet_author = "unknown"
-
-    now = datetime.now(timezone.utc).isoformat()
-    content = f"""---
-source_url: "{_yaml_str(url)}"
-type: tweet
-author: "{_yaml_str(tweet_author)}"
-captured_at: {now}
-contributor: "{_yaml_str(contributor or author or 'unknown')}"
----
-
-# Tweet by @{tweet_author}
-
-{tweet_text}
-
-Source: {url}
-"""
-    filename = _safe_filename(url, ".md")
-    return content, filename
-
-
-def _fetch_webpage(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
-    """Fetch a generic webpage and convert to markdown."""
-    html = _fetch_html(url)
-    # Extract title
-    title_match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
-    title = re.sub(r"\s+", " ", title_match.group(1)).strip() if title_match else url
-
-    markdown = _html_to_markdown(html, url)
-    now = datetime.now(timezone.utc).isoformat()
-    content = f"""---
-source_url: "{_yaml_str(url)}"
-type: webpage
-title: "{_yaml_str(title)}"
-captured_at: {now}
-contributor: "{_yaml_str(contributor or author or 'unknown')}"
----
-
-# {title}
-
-Source: {url}
-
----
-
-{markdown[:12000]}
-"""
-    filename = _safe_filename(url, ".md")
-    return content, filename
-
-
-def _fetch_arxiv(url: str, author: str | None, contributor: str | None) -> tuple[str, str]:
-    """Fetch arXiv abstract page."""
-    # Convert /abs/ or /pdf/ to abs for the API
-    arxiv_id = re.search(r"(\d{4}\.\d{4,5})", url)
-    if arxiv_id:
-        api_url = f"https://export.arxiv.org/abs/{arxiv_id.group(1)}"
-        try:
-            html = _fetch_html(api_url)
-            abstract_match = re.search(r'class="abstract[^"]*"[^>]*>(.*?)</blockquote>', html, re.DOTALL | re.IGNORECASE)
-            abstract = re.sub(r"<[^>]+>", "", abstract_match.group(1)).strip() if abstract_match else ""
-            title_match = re.search(r'class="title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL | re.IGNORECASE)
-            title = re.sub(r"<[^>]+>", " ", title_match.group(1)).strip() if title_match else arxiv_id.group(1)
-            authors_match = re.search(r'class="authors"[^>]*>(.*?)</div>', html, re.DOTALL | re.IGNORECASE)
-            paper_authors = re.sub(r"<[^>]+>", "", authors_match.group(1)).strip() if authors_match else ""
-        except Exception:
-            title, abstract, paper_authors = arxiv_id.group(1), "", ""
-    else:
-        return _fetch_webpage(url, author, contributor)
-
-    now = datetime.now(timezone.utc).isoformat()
-    content = f"""---
-source_url: "{_yaml_str(url)}"
-arxiv_id: "{_yaml_str(arxiv_id.group(1) if arxiv_id else '')}"
-type: paper
-title: "{_yaml_str(title)}"
-paper_authors: "{_yaml_str(paper_authors)}"
-captured_at: {now}
-contributor: "{_yaml_str(contributor or author or 'unknown')}"
----
-
-# {title}
-
-**Authors:** {paper_authors}
-**arXiv:** {arxiv_id.group(1) if arxiv_id else url}
-
-## Abstract
-
-{abstract}
-
-Source: {url}
-"""
-    filename = f"arxiv_{arxiv_id.group(1).replace('.', '_')}.md" if arxiv_id else _safe_filename(url, ".md")
-    return content, filename
-
-
-def _download_binary(url: str, suffix: str, target_dir: Path) -> Path:
-    """Download a binary file (PDF, image) directly."""
-    filename = _safe_filename(url, suffix)
-    out_path = target_dir / filename
-    out_path.write_bytes(safe_fetch(url))
-    return out_path
-
-
-def ingest(url: str, target_dir: Path, author: str | None = None, contributor: str | None = None) -> Path:
-    """
-    Fetch a URL and save it into target_dir as a graphify-ready file.
-
-    Returns the path of the saved file.
-    """
-    target_dir.mkdir(parents=True, exist_ok=True)
-    url_type = _detect_url_type(url)
-
-    try:
-        validate_url(url)
-    except ValueError as exc:
-        raise ValueError(f"ingest: {exc}") from exc
-
-    try:
-        if url_type == "pdf":
-            out = _download_binary(url, ".pdf", target_dir)
-            print(f"Downloaded PDF: {out.name}")
-            return out
-
-        if url_type == "image":
-            suffix = Path(urllib.parse.urlparse(url).path).suffix or ".jpg"
-            out = _download_binary(url, suffix, target_dir)
-            print(f"Downloaded image: {out.name}")
-            return out
-
-        if url_type == "youtube":
-            from graphify.transcribe import download_audio
-            out = download_audio(url, target_dir)
-            print(f"Downloaded audio: {out.name}")
-            return out
-
-        if url_type == "tweet":
-            content, filename = _fetch_tweet(url, author, contributor)
-        elif url_type == "arxiv":
-            content, filename = _fetch_arxiv(url, author, contributor)
-        else:
-            content, filename = _fetch_webpage(url, author, contributor)
-    except (urllib.error.HTTPError, urllib.error.URLError, OSError) as exc:
-        raise RuntimeError(f"ingest: failed to fetch {url!r}: {exc}") from exc
-
-    out_path = target_dir / filename
-    # Avoid overwriting - append counter if needed
-    counter = 1
-    while out_path.exists() and counter < 1000:
-        stem = Path(filename).stem
-        out_path = target_dir / f"{stem}_{counter}.md"
-        counter += 1
-
-    out_path.write_text(content, encoding="utf-8")
-    print(f"Saved {url_type}: {out_path.name}")
-    return out_path
-
-
-def save_query_result(
-    question: str,
-    answer: str,
-    memory_dir: Path,
-    query_type: str = "query",
-    source_nodes: list[str] | None = None,
-) -> Path:
-    """Save a Q&A result as markdown so it gets extracted into the graph on next --update.
-
-    Files are stored in memory_dir (typically graphify-out/memory/) with YAML frontmatter
-    that graphify's extractor reads as node metadata. This closes the feedback loop:
-    the system grows smarter from both what you add AND what you ask.
-    """
-    memory_dir = Path(memory_dir)
-    memory_dir.mkdir(parents=True, exist_ok=True)
-
-    now = datetime.now(timezone.utc)
-    slug = re.sub(r"[^\w]", "_", question.lower())[:50].strip("_")
-    filename = f"query_{now.strftime('%Y%m%d_%H%M%S')}_{slug}.md"
-
-    frontmatter_lines = [
-        "---",
-        f'type: "{query_type}"',
-        f'date: "{now.isoformat()}"',
-        f'question: "{_yaml_str(question)}"',
-        'contributor: "graphify"',
-    ]
-    if source_nodes:
-        nodes_str = ", ".join(f'"{n}"' for n in source_nodes[:10])
-        frontmatter_lines.append(f"source_nodes: [{nodes_str}]")
-    frontmatter_lines.append("---")
-
-    body_lines = [
-        "",
-        f"# Q: {question}",
-        "",
-        "## Answer",
-        "",
-        answer,
-    ]
-    if source_nodes:
-        body_lines += ["", "## Source Nodes", ""]
-        body_lines += [f"- {n}" for n in source_nodes]
-
-    content = "\n".join(frontmatter_lines + body_lines)
-    out_path = memory_dir / filename
-    out_path.write_text(content, encoding="utf-8")
-    return out_path
-
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser(description="Fetch a URL into a graphify /raw folder")
-    parser.add_argument("url", help="URL to fetch")
-    parser.add_argument("target_dir", nargs="?", default="./raw", help="Target directory (default: ./raw)")
-    parser.add_argument("--author", help="Your name (stored as node metadata)")
-    parser.add_argument("--contributor", help="Contributor name for team graphs")
-    args = parser.parse_args()
-    out = ingest(args.url, Path(args.target_dir), author=args.author, contributor=args.contributor)
-    print(f"Ready for graphify: {out}")

package/runtime/python/okstra_vendor/graphify/manifest.py

@@ -1,4 +0,0 @@
-# re-export manifest helpers from detect for backwards compatibility
-from graphify.detect import save_manifest, load_manifest, detect_incremental
-
-__all__ = ["save_manifest", "load_manifest", "detect_incremental"]