okstra 0.130.4 → 0.131.0

package/runtime/python/okstra_vendor/graphify/detect.py

@@ -1,510 +0,0 @@
-# file discovery, type classification, and corpus health checks
-from __future__ import annotations
-import fnmatch
-import json
-import os
-import re
-from enum import Enum
-from pathlib import Path
-
-
-class FileType(str, Enum):
-    CODE = "code"
-    DOCUMENT = "document"
-    PAPER = "paper"
-    IMAGE = "image"
-    VIDEO = "video"
-
-
-_MANIFEST_PATH = "graphify-out/manifest.json"
-
-CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv'}
-DOC_EXTENSIONS = {'.md', '.mdx', '.txt', '.rst', '.html'}
-PAPER_EXTENSIONS = {'.pdf'}
-IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
-OFFICE_EXTENSIONS = {'.docx', '.xlsx'}
-VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'}
-
-CORPUS_WARN_THRESHOLD = 50_000    # words - below this, warn "you may not need a graph"
-CORPUS_UPPER_THRESHOLD = 500_000  # words - above this, warn about token cost
-FILE_COUNT_UPPER = 200             # files - above this, warn about token cost
-
-# Files that may contain secrets - skip silently
-_SENSITIVE_PATTERNS = [
-    re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE),
-    re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE),
-    re.compile(r'(credential|secret|passwd|password|token|private_key)', re.IGNORECASE),
-    re.compile(r'(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$'),
-    re.compile(r'(\.netrc|\.pgpass|\.htpasswd)$', re.IGNORECASE),
-    re.compile(r'(aws_credentials|gcloud_credentials|service.account)', re.IGNORECASE),
-]
-
-# Signals that a .md/.txt file is actually a converted academic paper
-_PAPER_SIGNALS = [
-    re.compile(r'\barxiv\b', re.IGNORECASE),
-    re.compile(r'\bdoi\s*:', re.IGNORECASE),
-    re.compile(r'\babstract\b', re.IGNORECASE),
-    re.compile(r'\bproceedings\b', re.IGNORECASE),
-    re.compile(r'\bjournal\b', re.IGNORECASE),
-    re.compile(r'\bpreprint\b', re.IGNORECASE),
-    re.compile(r'\\cite\{'),          # LaTeX citation
-    re.compile(r'\[\d+\]'),           # Numbered citation [1], [23] (inline)
-    re.compile(r'\[\n\d+\n\]'),       # Numbered citation spread across lines (markdown conversion)
-    re.compile(r'eq\.\s*\d+|equation\s+\d+', re.IGNORECASE),
-    re.compile(r'\d{4}\.\d{4,5}'),   # arXiv ID like 1706.03762
-    re.compile(r'\bwe propose\b', re.IGNORECASE),   # common academic phrasing
-    re.compile(r'\bliterature\b', re.IGNORECASE),   # "from the literature"
-]
-_PAPER_SIGNAL_THRESHOLD = 3  # need at least this many signals to call it a paper
-
-
-def _is_sensitive(path: Path) -> bool:
-    """Return True if this file likely contains secrets and should be skipped."""
-    name = path.name
-    full = str(path)
-    return any(p.search(name) or p.search(full) for p in _SENSITIVE_PATTERNS)
-
-
-def _looks_like_paper(path: Path) -> bool:
-    """Heuristic: does this text file read like an academic paper?"""
-    try:
-        # Only scan first 3000 chars for speed
-        text = path.read_text(encoding="utf-8", errors="ignore")[:3000]
-        hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text))
-        return hits >= _PAPER_SIGNAL_THRESHOLD
-    except Exception:
-        return False
-
-
-_ASSET_DIR_MARKERS = {".imageset", ".xcassets", ".appiconset", ".colorset", ".launchimage"}
-
-
-def classify_file(path: Path) -> FileType | None:
-    # Compound extensions must be checked before simple suffix lookup
-    if path.name.lower().endswith(".blade.php"):
-        return FileType.CODE
-    ext = path.suffix.lower()
-    if ext in CODE_EXTENSIONS:
-        return FileType.CODE
-    if ext in PAPER_EXTENSIONS:
-        # PDFs inside Xcode asset catalogs are vector icons, not papers
-        if any(part.endswith(tuple(_ASSET_DIR_MARKERS)) for part in path.parts):
-            return None
-        return FileType.PAPER
-    if ext in IMAGE_EXTENSIONS:
-        return FileType.IMAGE
-    if ext in DOC_EXTENSIONS:
-        # Check if it's a converted paper
-        if _looks_like_paper(path):
-            return FileType.PAPER
-        return FileType.DOCUMENT
-    if ext in OFFICE_EXTENSIONS:
-        return FileType.DOCUMENT
-    if ext in VIDEO_EXTENSIONS:
-        return FileType.VIDEO
-    return None
-
-
-def extract_pdf_text(path: Path) -> str:
-    """Extract plain text from a PDF file using pypdf."""
-    try:
-        from pypdf import PdfReader
-        reader = PdfReader(str(path))
-        pages = []
-        for page in reader.pages:
-            text = page.extract_text()
-            if text:
-                pages.append(text)
-        return "\n".join(pages)
-    except Exception:
-        return ""
-
-
-def docx_to_markdown(path: Path) -> str:
-    """Convert a .docx file to markdown text using python-docx."""
-    try:
-        from docx import Document
-        from docx.oxml.ns import qn
-        doc = Document(str(path))
-        lines = []
-        for para in doc.paragraphs:
-            style = para.style.name if para.style else ""
-            text = para.text.strip()
-            if not text:
-                lines.append("")
-                continue
-            if style.startswith("Heading 1"):
-                lines.append(f"# {text}")
-            elif style.startswith("Heading 2"):
-                lines.append(f"## {text}")
-            elif style.startswith("Heading 3"):
-                lines.append(f"### {text}")
-            elif style.startswith("List"):
-                lines.append(f"- {text}")
-            else:
-                lines.append(text)
-        # Tables
-        for table in doc.tables:
-            rows = [[cell.text.strip() for cell in row.cells] for row in table.rows]
-            if not rows:
-                continue
-            header = "| " + " | ".join(rows[0]) + " |"
-            sep = "| " + " | ".join("---" for _ in rows[0]) + " |"
-            lines.extend([header, sep])
-            for row in rows[1:]:
-                lines.append("| " + " | ".join(row) + " |")
-        return "\n".join(lines)
-    except ImportError:
-        return ""
-    except Exception:
-        return ""
-
-
-def xlsx_to_markdown(path: Path) -> str:
-    """Convert an .xlsx file to markdown text using openpyxl."""
-    try:
-        import openpyxl
-        wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True)
-        sections = []
-        for sheet_name in wb.sheetnames:
-            ws = wb[sheet_name]
-            rows = []
-            for row in ws.iter_rows(values_only=True):
-                # Skip entirely empty rows
-                if all(cell is None for cell in row):
-                    continue
-                rows.append([str(cell) if cell is not None else "" for cell in row])
-            if not rows:
-                continue
-            sections.append(f"## Sheet: {sheet_name}")
-            if len(rows) >= 1:
-                header = "| " + " | ".join(rows[0]) + " |"
-                sep = "| " + " | ".join("---" for _ in rows[0]) + " |"
-                sections.extend([header, sep])
-                for row in rows[1:]:
-                    sections.append("| " + " | ".join(row) + " |")
-        wb.close()
-        return "\n".join(sections)
-    except ImportError:
-        return ""
-    except Exception:
-        return ""
-
-
-def convert_office_file(path: Path, out_dir: Path) -> Path | None:
-    """Convert a .docx or .xlsx to a markdown sidecar in out_dir.
-
-    Returns the path of the converted .md file, or None if conversion failed
-    or the required library is not installed.
-    """
-    ext = path.suffix.lower()
-    if ext == ".docx":
-        text = docx_to_markdown(path)
-    elif ext == ".xlsx":
-        text = xlsx_to_markdown(path)
-    else:
-        return None
-
-    if not text.strip():
-        return None
-
-    out_dir.mkdir(parents=True, exist_ok=True)
-    # Use a stable name derived from the original path to avoid collisions
-    import hashlib
-    name_hash = hashlib.sha256(str(path.resolve()).encode()).hexdigest()[:8]
-    out_path = out_dir / f"{path.stem}_{name_hash}.md"
-    out_path.write_text(
-        f"<!-- converted from {path.name} -->\n\n{text}",
-        encoding="utf-8",
-    )
-    return out_path
-
-
-def count_words(path: Path) -> int:
-    try:
-        ext = path.suffix.lower()
-        if ext == ".pdf":
-            return len(extract_pdf_text(path).split())
-        if ext == ".docx":
-            return len(docx_to_markdown(path).split())
-        if ext == ".xlsx":
-            return len(xlsx_to_markdown(path).split())
-        return len(path.read_text(encoding="utf-8", errors="ignore").split())
-    except Exception:
-        return 0
-
-
-# Directory names to always skip - venvs, caches, build artifacts, deps
-_SKIP_DIRS = {
-    "venv", ".venv", "env", ".env",
-    "node_modules", "__pycache__", ".git",
-    "dist", "build", "target", "out",
-    "site-packages", "lib64",
-    ".pytest_cache", ".mypy_cache", ".ruff_cache",
-    ".tox", ".eggs", "*.egg-info",
-}
-
-# Large generated files that are never useful to extract
-_SKIP_FILES = {
-    "package-lock.json", "yarn.lock", "pnpm-lock.yaml",
-    "Cargo.lock", "poetry.lock", "Gemfile.lock",
-    "composer.lock", "go.sum", "go.work.sum",
-}
-
-def _is_noise_dir(part: str) -> bool:
-    """Return True if this directory name looks like a venv, cache, or dep dir."""
-    if part in _SKIP_DIRS:
-        return True
-    # Catch *_venv, *_repo/site-packages patterns
-    if part.endswith("_venv") or part.endswith("_env"):
-        return True
-    if part.endswith(".egg-info"):
-        return True
-    return False
-
-
-def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]:
-    """Read .graphifyignore from root **and ancestor directories**.
-
-    Returns a list of (anchor_dir, pattern) pairs. Each pattern is matched
-    against paths relative to both the scan root and the anchor_dir where
-    the .graphifyignore file was found — so patterns written relative to a
-    parent directory still work when graphify is run on a subfolder.
-
-    Walks upward from *root* towards the filesystem root, stopping at a
-    ``.git`` boundary. Lines starting with # are comments; blank lines ignored.
-    """
-    patterns: list[tuple[Path, str]] = []
-    current = root.resolve()
-    while True:
-        ignore_file = current / ".graphifyignore"
-        if ignore_file.exists():
-            for line in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    patterns.append((current, line))
-        # Stop climbing once we've processed the git repo root
-        if (current / ".git").exists():
-            break
-        parent = current.parent
-        if parent == current:
-            break  # filesystem root
-        current = parent
-    return patterns
-
-
-def _is_ignored(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool:
-    """Return True if path matches any .graphifyignore pattern."""
-    if not patterns:
-        return False
-
-    def _matches(rel: str, p: str) -> bool:
-        parts = rel.split("/")
-        if fnmatch.fnmatch(rel, p):
-            return True
-        if fnmatch.fnmatch(path.name, p):
-            return True
-        for i, part in enumerate(parts):
-            if fnmatch.fnmatch(part, p):
-                return True
-            if fnmatch.fnmatch("/".join(parts[:i + 1]), p):
-                return True
-        return False
-
-    for anchor, pattern in patterns:
-        p = pattern.strip("/")
-        if not p:
-            continue
-        # Try path relative to the scan root
-        try:
-            rel = str(path.relative_to(root)).replace(os.sep, "/")
-            if _matches(rel, p):
-                return True
-        except ValueError:
-            pass
-        # Also try relative to the anchor dir (the .graphifyignore's location),
-        # so patterns written at a parent level still fire when running on a subfolder
-        if anchor != root:
-            try:
-                rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/")
-                if _matches(rel_anchor, p):
-                    return True
-            except ValueError:
-                pass
-    return False
-
-
-def detect(root: Path, *, follow_symlinks: bool = False) -> dict:
-    files: dict[FileType, list[str]] = {
-        FileType.CODE: [],
-        FileType.DOCUMENT: [],
-        FileType.PAPER: [],
-        FileType.IMAGE: [],
-        FileType.VIDEO: [],
-    }
-    total_words = 0
-
-    skipped_sensitive: list[str] = []
-    ignore_patterns = _load_graphifyignore(root)
-
-    # Always include graphify-out/memory/ - query results filed back into the graph
-    memory_dir = root / "graphify-out" / "memory"
-    scan_paths = [root]
-    if memory_dir.exists():
-        scan_paths.append(memory_dir)
-
-    seen: set[Path] = set()
-    all_files: list[Path] = []
-
-    for scan_root in scan_paths:
-        in_memory_tree = memory_dir.exists() and str(scan_root).startswith(str(memory_dir))
-        for dirpath, dirnames, filenames in os.walk(scan_root, followlinks=follow_symlinks):
-            dp = Path(dirpath)
-            if follow_symlinks and os.path.islink(dirpath):
-                real = os.path.realpath(dirpath)
-                parent_real = os.path.realpath(os.path.dirname(dirpath))
-                if parent_real == real or parent_real.startswith(real + os.sep):
-                    dirnames.clear()
-                    continue
-            if not in_memory_tree:
-                # Prune noise dirs in-place so os.walk never descends into them
-                dirnames[:] = [
-                    d for d in dirnames
-                    if not d.startswith(".")
-                    and not _is_noise_dir(d)
-                    and not _is_ignored(dp / d, root, ignore_patterns)
-                ]
-            for fname in filenames:
-                if fname in _SKIP_FILES:
-                    continue
-                p = dp / fname
-                if p not in seen:
-                    seen.add(p)
-                    all_files.append(p)
-
-    converted_dir = root / "graphify-out" / "converted"
-
-    for p in all_files:
-        # For memory dir files, skip hidden/noise filtering
-        in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir))
-        if not in_memory:
-            # Hidden files are already excluded via dir pruning above,
-            # but catch hidden files at the root level
-            if p.name.startswith("."):
-                continue
-            # Skip files inside our own converted/ dir (avoid re-processing sidecars)
-            if str(p).startswith(str(converted_dir)):
-                continue
-        if _is_ignored(p, root, ignore_patterns):
-            continue
-        if _is_sensitive(p):
-            skipped_sensitive.append(str(p))
-            continue
-        ftype = classify_file(p)
-        if ftype:
-            # Office files: convert to markdown sidecar so subagents can read them
-            if p.suffix.lower() in OFFICE_EXTENSIONS:
-                md_path = convert_office_file(p, converted_dir)
-                if md_path:
-                    files[ftype].append(str(md_path))
-                    total_words += count_words(md_path)
-                else:
-                    # Conversion failed (library not installed) - skip with note
-                    skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]")
-                continue
-            files[ftype].append(str(p))
-            if ftype != FileType.VIDEO:
-                total_words += count_words(p)
-
-    total_files = sum(len(v) for v in files.values())
-    needs_graph = total_words >= CORPUS_WARN_THRESHOLD
-
-    # Determine warning - lower bound, upper bound, or sensitive files skipped
-    warning: str | None = None
-    if not needs_graph:
-        warning = (
-            f"Corpus is ~{total_words:,} words - fits in a single context window. "
-            f"You may not need a graph."
-        )
-    elif total_words >= CORPUS_UPPER_THRESHOLD or total_files >= FILE_COUNT_UPPER:
-        warning = (
-            f"Large corpus: {total_files} files · ~{total_words:,} words. "
-            f"Semantic extraction will be expensive (many Claude tokens). "
-            f"Consider running on a subfolder, or use --no-semantic to run AST-only."
-        )
-
-    return {
-        "files": {k.value: v for k, v in files.items()},
-        "total_files": total_files,
-        "total_words": total_words,
-        "needs_graph": needs_graph,
-        "warning": warning,
-        "skipped_sensitive": skipped_sensitive,
-        "graphifyignore_patterns": len(ignore_patterns),
-    }
-
-
-def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]:
-    """Load the file modification time manifest from a previous run."""
-    try:
-        return json.loads(Path(manifest_path).read_text(encoding="utf-8"))
-    except Exception:
-        return {}
-
-
-def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PATH) -> None:
-    """Save current file mtimes so the next --update run can diff against them."""
-    manifest: dict[str, float] = {}
-    for file_list in files.values():
-        for f in file_list:
-            try:
-                manifest[f] = Path(f).stat().st_mtime
-            except OSError:
-                pass  # file deleted between detect() and manifest write - skip it
-    Path(manifest_path).parent.mkdir(parents=True, exist_ok=True)
-    Path(manifest_path).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
-
-
-def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict:
-    """Like detect(), but returns only new or modified files since the last run.
-
-    Compares current file mtimes against the stored manifest.
-    Use for --update mode: re-extract only what changed, merge into existing graph.
-    """
-    full = detect(root)
-    manifest = load_manifest(manifest_path)
-
-    if not manifest:
-        # No previous run - treat everything as new
-        full["incremental"] = True
-        full["new_files"] = full["files"]
-        full["unchanged_files"] = {k: [] for k in full["files"]}
-        full["new_total"] = full["total_files"]
-        return full
-
-    new_files: dict[str, list[str]] = {k: [] for k in full["files"]}
-    unchanged_files: dict[str, list[str]] = {k: [] for k in full["files"]}
-
-    for ftype, file_list in full["files"].items():
-        for f in file_list:
-            stored_mtime = manifest.get(f)
-            try:
-                current_mtime = Path(f).stat().st_mtime
-            except Exception:
-                current_mtime = 0
-            if stored_mtime is None or current_mtime > stored_mtime:
-                new_files[ftype].append(f)
-            else:
-                unchanged_files[ftype].append(f)
-
-    # Files in manifest that no longer exist - their cached nodes are now ghost nodes
-    current_files = {f for flist in full["files"].values() for f in flist}
-    deleted_files = [f for f in manifest if f not in current_files]
-
-    new_total = sum(len(v) for v in new_files.values())
-    full["incremental"] = True
-    full["new_files"] = new_files
-    full["unchanged_files"] = unchanged_files
-    full["new_total"] = new_total
-    full["deleted_files"] = deleted_files
-    return full