okstra 0.130.3 → 0.131.0

package/runtime/python/okstra_vendor/graphify/build.py

@@ -1,107 +0,0 @@
-# assemble node+edge dicts into a NetworkX graph, preserving edge direction
-#
-# Node deduplication — three layers:
-#
-# 1. Within a file (AST): each extractor tracks a `seen_ids` set. A node ID is
-#    emitted at most once per file, so duplicate class/function definitions in
-#    the same source file are collapsed to the first occurrence.
-#
-# 2. Between files (build): NetworkX G.add_node() is idempotent — calling it
-#    twice with the same ID overwrites the attributes with the second call's
-#    values. Nodes are added in extraction order (AST first, then semantic),
-#    so if the same entity is extracted by both passes the semantic node
-#    silently overwrites the AST node. This is intentional: semantic nodes
-#    carry richer labels and cross-file context, while AST nodes have precise
-#    source_location. If you need to change the priority, reorder extractions
-#    passed to build().
-#
-# 3. Semantic merge (skill): before calling build(), the skill merges cached
-#    and new semantic results using an explicit `seen` set keyed on node["id"],
-#    so duplicates across cache hits and new extractions are resolved there
-#    before any graph construction happens.
-#
-from __future__ import annotations
-import re
-import sys
-import networkx as nx
-from .validate import validate_extraction
-
-
-def _normalize_id(s: str) -> str:
-    """Normalize an ID string the same way extract._make_id does.
-
-    Used to reconcile edge endpoints when the LLM generates IDs with slightly
-    different punctuation or casing than the AST extractor.
-    """
-    cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", s)
-    return cleaned.strip("_").lower()
-
-
-def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
-    """Build a NetworkX graph from an extraction dict.
-
-    directed=True produces a DiGraph that preserves edge direction (source→target).
-    directed=False (default) produces an undirected Graph for backward compatibility.
-    """
-    # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility.
-    if "edges" not in extraction and "links" in extraction:
-        extraction = dict(extraction, edges=extraction["links"])
-    errors = validate_extraction(extraction)
-    # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.
-    real_errors = [e for e in errors if "does not match any node id" not in e]
-    if real_errors:
-        print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr)
-    G: nx.Graph = nx.DiGraph() if directed else nx.Graph()
-    for node in extraction.get("nodes", []):
-        G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
-    node_set = set(G.nodes())
-    # Normalized ID map: lets edges survive when the LLM generates IDs with
-    # slightly different casing or punctuation than the AST extractor.
-    # e.g. "Session_ValidateToken" maps to "session_validatetoken".
-    norm_to_id: dict[str, str] = {_normalize_id(nid): nid for nid in node_set}
-    for edge in extraction.get("edges", []):
-        if "source" not in edge and "from" in edge:
-            edge["source"] = edge["from"]
-        if "target" not in edge and "to" in edge:
-            edge["target"] = edge["to"]
-        if "source" not in edge or "target" not in edge:
-            continue
-        src, tgt = edge["source"], edge["target"]
-        # Remap mismatched IDs via normalization before dropping the edge.
-        if src not in node_set:
-            src = norm_to_id.get(_normalize_id(src), src)
-        if tgt not in node_set:
-            tgt = norm_to_id.get(_normalize_id(tgt), tgt)
-        if src not in node_set or tgt not in node_set:
-            continue  # skip edges to external/stdlib nodes - expected, not an error
-        attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
-        # Preserve original edge direction - undirected graphs lose it otherwise,
-        # causing display functions to show edges backwards.
-        attrs["_src"] = src
-        attrs["_tgt"] = tgt
-        G.add_edge(src, tgt, **attrs)
-    hyperedges = extraction.get("hyperedges", [])
-    if hyperedges:
-        G.graph["hyperedges"] = hyperedges
-    return G
-
-
-def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph:
-    """Merge multiple extraction results into one graph.
-
-    directed=True produces a DiGraph that preserves edge direction (source→target).
-    directed=False (default) produces an undirected Graph for backward compatibility.
-
-    Extractions are merged in order. For nodes with the same ID, the last
-    extraction's attributes win (NetworkX add_node overwrites). Pass AST
-    results before semantic results so semantic labels take precedence, or
-    reverse the order if you prefer AST source_location precision to win.
-    """
-    combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0}
-    for ext in extractions:
-        combined["nodes"].extend(ext.get("nodes", []))
-        combined["edges"].extend(ext.get("edges", []))
-        combined["hyperedges"].extend(ext.get("hyperedges", []))
-        combined["input_tokens"] += ext.get("input_tokens", 0)
-        combined["output_tokens"] += ext.get("output_tokens", 0)
-    return build_from_json(combined, directed=directed)

package/runtime/python/okstra_vendor/graphify/cache.py

@@ -1,169 +0,0 @@
-# per-file extraction cache - skip unchanged files on re-run
-from __future__ import annotations
-
-import hashlib
-import json
-import os
-from pathlib import Path
-
-
-def _body_content(content: bytes) -> bytes:
-    """Strip YAML frontmatter from Markdown content, returning only the body."""
-    text = content.decode(errors="replace")
-    if text.startswith("---"):
-        end = text.find("\n---", 3)
-        if end != -1:
-            return text[end + 4:].encode()
-    return content
-
-
-def file_hash(path: Path, root: Path = Path(".")) -> str:
-    """SHA256 of file contents + path relative to root.
-
-    Using a relative path (not absolute) makes cache entries portable across
-    machines and checkout directories, so shared caches and CI work correctly.
-    Falls back to the resolved absolute path if the file is outside root.
-
-    For Markdown files (.md), only the body below the YAML frontmatter is hashed,
-    so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache.
-    """
-    p = Path(path)
-    raw = p.read_bytes()
-    content = _body_content(raw) if p.suffix.lower() == ".md" else raw
-    h = hashlib.sha256()
-    h.update(content)
-    h.update(b"\x00")
-    try:
-        rel = p.resolve().relative_to(Path(root).resolve())
-        h.update(str(rel).encode())
-    except ValueError:
-        h.update(str(p.resolve()).encode())
-    return h.hexdigest()
-
-
-def cache_dir(root: Path = Path(".")) -> Path:
-    """Returns graphify-out/cache/ - creates it if needed."""
-    d = Path(root).resolve() / "graphify-out" / "cache"
-    d.mkdir(parents=True, exist_ok=True)
-    return d
-
-
-def load_cached(path: Path, root: Path = Path(".")) -> dict | None:
-    """Return cached extraction for this file if hash matches, else None.
-
-    Cache key: SHA256 of file contents.
-    Cache value: stored as graphify-out/cache/{hash}.json
-    Returns None if no cache entry or file has changed.
-    """
-    try:
-        h = file_hash(path, root)
-    except OSError:
-        return None
-    entry = cache_dir(root) / f"{h}.json"
-    if not entry.exists():
-        return None
-    try:
-        return json.loads(entry.read_text(encoding="utf-8"))
-    except (json.JSONDecodeError, OSError):
-        return None
-
-
-def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None:
-    """Save extraction result for this file.
-
-    Stores as graphify-out/cache/{hash}.json where hash = SHA256 of current file contents.
-    result should be a dict with 'nodes' and 'edges' lists.
-    """
-    h = file_hash(path, root)
-    entry = cache_dir(root) / f"{h}.json"
-    tmp = entry.with_suffix(".tmp")
-    try:
-        tmp.write_text(json.dumps(result), encoding="utf-8")
-        try:
-            os.replace(tmp, entry)
-        except PermissionError:
-            # Windows: os.replace can fail with WinError 5 if the target is
-            # briefly locked. Fall back to copy-then-delete.
-            import shutil
-            shutil.copy2(tmp, entry)
-            tmp.unlink(missing_ok=True)
-    except Exception:
-        tmp.unlink(missing_ok=True)
-        raise
-
-
-def cached_files(root: Path = Path(".")) -> set[str]:
-    """Return set of file paths that have a valid cache entry (hash still matches)."""
-    d = cache_dir(root)
-    return {p.stem for p in d.glob("*.json")}
-
-
-def clear_cache(root: Path = Path(".")) -> None:
-    """Delete all graphify-out/cache/*.json files."""
-    d = cache_dir(root)
-    for f in d.glob("*.json"):
-        f.unlink()
-
-
-def check_semantic_cache(
-    files: list[str],
-    root: Path = Path("."),
-) -> tuple[list[dict], list[dict], list[dict], list[str]]:
-    """Check semantic extraction cache for a list of absolute file paths.
-
-    Returns (cached_nodes, cached_edges, cached_hyperedges, uncached_files).
-    Uncached files need Claude extraction; cached files are merged directly.
-    """
-    cached_nodes: list[dict] = []
-    cached_edges: list[dict] = []
-    cached_hyperedges: list[dict] = []
-    uncached: list[str] = []
-
-    for fpath in files:
-        result = load_cached(Path(fpath), root)
-        if result is not None:
-            cached_nodes.extend(result.get("nodes", []))
-            cached_edges.extend(result.get("edges", []))
-            cached_hyperedges.extend(result.get("hyperedges", []))
-        else:
-            uncached.append(fpath)
-
-    return cached_nodes, cached_edges, cached_hyperedges, uncached
-
-
-def save_semantic_cache(
-    nodes: list[dict],
-    edges: list[dict],
-    hyperedges: list[dict] | None = None,
-    root: Path = Path("."),
-) -> int:
-    """Save semantic extraction results to cache, keyed by source_file.
-
-    Groups nodes and edges by source_file, then saves one cache entry per file.
-    Returns the number of files cached.
-    """
-    from collections import defaultdict
-
-    by_file: dict[str, dict] = defaultdict(lambda: {"nodes": [], "edges": [], "hyperedges": []})
-    for n in nodes:
-        src = n.get("source_file", "")
-        if src:
-            by_file[src]["nodes"].append(n)
-    for e in edges:
-        src = e.get("source_file", "")
-        if src:
-            by_file[src]["edges"].append(e)
-    for h in (hyperedges or []):
-        src = h.get("source_file", "")
-        if src:
-            by_file[src]["hyperedges"].append(h)
-
-    saved = 0
-    for fpath, result in by_file.items():
-        p = Path(fpath)
-        if not p.is_absolute():
-            p = Path(root) / p
-        if p.exists():
-            save_cached(p, result, root)
-            saved += 1
-    return saved

package/runtime/python/okstra_vendor/graphify/cluster.py

@@ -1,137 +0,0 @@
-"""Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores."""
-from __future__ import annotations
-import contextlib
-import inspect
-import io
-import sys
-import networkx as nx
-
-
-def _suppress_output():
-    """Context manager to suppress stdout/stderr during library calls.
-
-    graspologic's leiden() emits ANSI escape sequences (progress bars,
-    colored warnings) that corrupt PowerShell 5.1's scroll buffer on
-    Windows (see issue #19). Redirecting stdout/stderr to devnull during
-    the call prevents this without losing any graphify output.
-    """
-    return contextlib.redirect_stdout(io.StringIO())
-
-
-def _partition(G: nx.Graph) -> dict[str, int]:
-    """Run community detection. Returns {node_id: community_id}.
-
-    Tries Leiden (graspologic) first — best quality.
-    Falls back to Louvain (built into networkx) if graspologic is not installed.
-
-    Output from graspologic is suppressed to prevent ANSI escape codes
-    from corrupting terminal scroll buffers on Windows PowerShell 5.1.
-    """
-    try:
-        from graspologic.partition import leiden
-        # Suppress graspologic output to prevent ANSI escape codes from
-        # corrupting PowerShell 5.1 scroll buffer (issue #19)
-        old_stderr = sys.stderr
-        try:
-            sys.stderr = io.StringIO()
-            with _suppress_output():
-                result = leiden(G)
-        finally:
-            sys.stderr = old_stderr
-        return result
-    except ImportError:
-        pass
-
-    # Fallback: networkx louvain (available since networkx 2.7).
-    # Inspect kwargs to stay compatible across NetworkX versions — max_level
-    # was added in a later release and prevents hangs on large sparse graphs.
-    kwargs: dict = {"seed": 42, "threshold": 1e-4}
-    if "max_level" in inspect.signature(nx.community.louvain_communities).parameters:
-        kwargs["max_level"] = 10
-    communities = nx.community.louvain_communities(G, **kwargs)
-    return {node: cid for cid, nodes in enumerate(communities) for node in nodes}
-
-
-_MAX_COMMUNITY_FRACTION = 0.25   # communities larger than 25% of graph get split
-_MIN_SPLIT_SIZE = 10             # only split if community has at least this many nodes
-
-
-def cluster(G: nx.Graph) -> dict[int, list[str]]:
-    """Run Leiden community detection. Returns {community_id: [node_ids]}.
-
-    Community IDs are stable across runs: 0 = largest community after splitting.
-    Oversized communities (> 25% of graph nodes, min 10) are split by running
-    a second Leiden pass on the subgraph.
-
-    Accepts directed or undirected graphs. DiGraphs are converted to undirected
-    internally since Louvain/Leiden require undirected input.
-    """
-    if G.number_of_nodes() == 0:
-        return {}
-    if G.is_directed():
-        G = G.to_undirected()
-    if G.number_of_edges() == 0:
-        return {i: [n] for i, n in enumerate(sorted(G.nodes))}
-
-    # Leiden warns and drops isolates - handle them separately
-    isolates = [n for n in G.nodes() if G.degree(n) == 0]
-    connected_nodes = [n for n in G.nodes() if G.degree(n) > 0]
-    connected = G.subgraph(connected_nodes)
-
-    raw: dict[int, list[str]] = {}
-    if connected.number_of_nodes() > 0:
-        partition = _partition(connected)
-        for node, cid in partition.items():
-            raw.setdefault(cid, []).append(node)
-
-    # Each isolate becomes its own single-node community
-    next_cid = max(raw.keys(), default=-1) + 1
-    for node in isolates:
-        raw[next_cid] = [node]
-        next_cid += 1
-
-    # Split oversized communities
-    max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))
-    final_communities: list[list[str]] = []
-    for nodes in raw.values():
-        if len(nodes) > max_size:
-            final_communities.extend(_split_community(G, nodes))
-        else:
-            final_communities.append(nodes)
-
-    # Re-index by size descending for deterministic ordering
-    final_communities.sort(key=len, reverse=True)
-    return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}
-
-
-def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:
-    """Run a second Leiden pass on a community subgraph to split it further."""
-    subgraph = G.subgraph(nodes)
-    if subgraph.number_of_edges() == 0:
-        # No edges - split into individual nodes
-        return [[n] for n in sorted(nodes)]
-    try:
-        sub_partition = _partition(subgraph)
-        sub_communities: dict[int, list[str]] = {}
-        for node, cid in sub_partition.items():
-            sub_communities.setdefault(cid, []).append(node)
-        if len(sub_communities) <= 1:
-            return [sorted(nodes)]
-        return [sorted(v) for v in sub_communities.values()]
-    except Exception:
-        return [sorted(nodes)]
-
-
-def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:
-    """Ratio of actual intra-community edges to maximum possible."""
-    n = len(community_nodes)
-    if n <= 1:
-        return 1.0
-    subgraph = G.subgraph(community_nodes)
-    actual = subgraph.number_of_edges()
-    possible = n * (n - 1) / 2
-    return round(actual / possible, 2) if possible > 0 else 0.0
-
-
-def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:
-    return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}