okstra 0.108.0 → 0.110.0

package/runtime/python/okstra_vendor/graphify/build.py

@@ -0,0 +1,107 @@
+# assemble node+edge dicts into a NetworkX graph, preserving edge direction
+#
+# Node deduplication — three layers:
+#
+# 1. Within a file (AST): each extractor tracks a `seen_ids` set. A node ID is
+#    emitted at most once per file, so duplicate class/function definitions in
+#    the same source file are collapsed to the first occurrence.
+#
+# 2. Between files (build): NetworkX G.add_node() is idempotent — calling it
+#    twice with the same ID overwrites the attributes with the second call's
+#    values. Nodes are added in extraction order (AST first, then semantic),
+#    so if the same entity is extracted by both passes the semantic node
+#    silently overwrites the AST node. This is intentional: semantic nodes
+#    carry richer labels and cross-file context, while AST nodes have precise
+#    source_location. If you need to change the priority, reorder extractions
+#    passed to build().
+#
+# 3. Semantic merge (skill): before calling build(), the skill merges cached
+#    and new semantic results using an explicit `seen` set keyed on node["id"],
+#    so duplicates across cache hits and new extractions are resolved there
+#    before any graph construction happens.
+#
+from __future__ import annotations
+import re
+import sys
+import networkx as nx
+from .validate import validate_extraction
+
+
+def _normalize_id(s: str) -> str:
+    """Normalize an ID string the same way extract._make_id does.
+
+    Used to reconcile edge endpoints when the LLM generates IDs with slightly
+    different punctuation or casing than the AST extractor.
+    """
+    cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", s)
+    return cleaned.strip("_").lower()
+
+
+def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
+    """Build a NetworkX graph from an extraction dict.
+
+    directed=True produces a DiGraph that preserves edge direction (source→target).
+    directed=False (default) produces an undirected Graph for backward compatibility.
+    """
+    # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility.
+    if "edges" not in extraction and "links" in extraction:
+        extraction = dict(extraction, edges=extraction["links"])
+    errors = validate_extraction(extraction)
+    # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.
+    real_errors = [e for e in errors if "does not match any node id" not in e]
+    if real_errors:
+        print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr)
+    G: nx.Graph = nx.DiGraph() if directed else nx.Graph()
+    for node in extraction.get("nodes", []):
+        G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
+    node_set = set(G.nodes())
+    # Normalized ID map: lets edges survive when the LLM generates IDs with
+    # slightly different casing or punctuation than the AST extractor.
+    # e.g. "Session_ValidateToken" maps to "session_validatetoken".
+    norm_to_id: dict[str, str] = {_normalize_id(nid): nid for nid in node_set}
+    for edge in extraction.get("edges", []):
+        if "source" not in edge and "from" in edge:
+            edge["source"] = edge["from"]
+        if "target" not in edge and "to" in edge:
+            edge["target"] = edge["to"]
+        if "source" not in edge or "target" not in edge:
+            continue
+        src, tgt = edge["source"], edge["target"]
+        # Remap mismatched IDs via normalization before dropping the edge.
+        if src not in node_set:
+            src = norm_to_id.get(_normalize_id(src), src)
+        if tgt not in node_set:
+            tgt = norm_to_id.get(_normalize_id(tgt), tgt)
+        if src not in node_set or tgt not in node_set:
+            continue  # skip edges to external/stdlib nodes - expected, not an error
+        attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
+        # Preserve original edge direction - undirected graphs lose it otherwise,
+        # causing display functions to show edges backwards.
+        attrs["_src"] = src
+        attrs["_tgt"] = tgt
+        G.add_edge(src, tgt, **attrs)
+    hyperedges = extraction.get("hyperedges", [])
+    if hyperedges:
+        G.graph["hyperedges"] = hyperedges
+    return G
+
+
+def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph:
+    """Merge multiple extraction results into one graph.
+
+    directed=True produces a DiGraph that preserves edge direction (source→target).
+    directed=False (default) produces an undirected Graph for backward compatibility.
+
+    Extractions are merged in order. For nodes with the same ID, the last
+    extraction's attributes win (NetworkX add_node overwrites). Pass AST
+    results before semantic results so semantic labels take precedence, or
+    reverse the order if you prefer AST source_location precision to win.
+    """
+    combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0}
+    for ext in extractions:
+        combined["nodes"].extend(ext.get("nodes", []))
+        combined["edges"].extend(ext.get("edges", []))
+        combined["hyperedges"].extend(ext.get("hyperedges", []))
+        combined["input_tokens"] += ext.get("input_tokens", 0)
+        combined["output_tokens"] += ext.get("output_tokens", 0)
+    return build_from_json(combined, directed=directed)

package/runtime/python/okstra_vendor/graphify/cache.py

@@ -0,0 +1,169 @@
+# per-file extraction cache - skip unchanged files on re-run
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+
+
+def _body_content(content: bytes) -> bytes:
+    """Strip YAML frontmatter from Markdown content, returning only the body."""
+    text = content.decode(errors="replace")
+    if text.startswith("---"):
+        end = text.find("\n---", 3)
+        if end != -1:
+            return text[end + 4:].encode()
+    return content
+
+
+def file_hash(path: Path, root: Path = Path(".")) -> str:
+    """SHA256 of file contents + path relative to root.
+
+    Using a relative path (not absolute) makes cache entries portable across
+    machines and checkout directories, so shared caches and CI work correctly.
+    Falls back to the resolved absolute path if the file is outside root.
+
+    For Markdown files (.md), only the body below the YAML frontmatter is hashed,
+    so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache.
+    """
+    p = Path(path)
+    raw = p.read_bytes()
+    content = _body_content(raw) if p.suffix.lower() == ".md" else raw
+    h = hashlib.sha256()
+    h.update(content)
+    h.update(b"\x00")
+    try:
+        rel = p.resolve().relative_to(Path(root).resolve())
+        h.update(str(rel).encode())
+    except ValueError:
+        h.update(str(p.resolve()).encode())
+    return h.hexdigest()
+
+
+def cache_dir(root: Path = Path(".")) -> Path:
+    """Returns graphify-out/cache/ - creates it if needed."""
+    d = Path(root).resolve() / "graphify-out" / "cache"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def load_cached(path: Path, root: Path = Path(".")) -> dict | None:
+    """Return cached extraction for this file if hash matches, else None.
+
+    Cache key: SHA256 of file contents.
+    Cache value: stored as graphify-out/cache/{hash}.json
+    Returns None if no cache entry or file has changed.
+    """
+    try:
+        h = file_hash(path, root)
+    except OSError:
+        return None
+    entry = cache_dir(root) / f"{h}.json"
+    if not entry.exists():
+        return None
+    try:
+        return json.loads(entry.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None:
+    """Save extraction result for this file.
+
+    Stores as graphify-out/cache/{hash}.json where hash = SHA256 of current file contents.
+    result should be a dict with 'nodes' and 'edges' lists.
+    """
+    h = file_hash(path, root)
+    entry = cache_dir(root) / f"{h}.json"
+    tmp = entry.with_suffix(".tmp")
+    try:
+        tmp.write_text(json.dumps(result), encoding="utf-8")
+        try:
+            os.replace(tmp, entry)
+        except PermissionError:
+            # Windows: os.replace can fail with WinError 5 if the target is
+            # briefly locked. Fall back to copy-then-delete.
+            import shutil
+            shutil.copy2(tmp, entry)
+            tmp.unlink(missing_ok=True)
+    except Exception:
+        tmp.unlink(missing_ok=True)
+        raise
+
+
+def cached_files(root: Path = Path(".")) -> set[str]:
+    """Return set of file paths that have a valid cache entry (hash still matches)."""
+    d = cache_dir(root)
+    return {p.stem for p in d.glob("*.json")}
+
+
+def clear_cache(root: Path = Path(".")) -> None:
+    """Delete all graphify-out/cache/*.json files."""
+    d = cache_dir(root)
+    for f in d.glob("*.json"):
+        f.unlink()
+
+
+def check_semantic_cache(
+    files: list[str],
+    root: Path = Path("."),
+) -> tuple[list[dict], list[dict], list[dict], list[str]]:
+    """Check semantic extraction cache for a list of absolute file paths.
+
+    Returns (cached_nodes, cached_edges, cached_hyperedges, uncached_files).
+    Uncached files need Claude extraction; cached files are merged directly.
+    """
+    cached_nodes: list[dict] = []
+    cached_edges: list[dict] = []
+    cached_hyperedges: list[dict] = []
+    uncached: list[str] = []
+
+    for fpath in files:
+        result = load_cached(Path(fpath), root)
+        if result is not None:
+            cached_nodes.extend(result.get("nodes", []))
+            cached_edges.extend(result.get("edges", []))
+            cached_hyperedges.extend(result.get("hyperedges", []))
+        else:
+            uncached.append(fpath)
+
+    return cached_nodes, cached_edges, cached_hyperedges, uncached
+
+
+def save_semantic_cache(
+    nodes: list[dict],
+    edges: list[dict],
+    hyperedges: list[dict] | None = None,
+    root: Path = Path("."),
+) -> int:
+    """Save semantic extraction results to cache, keyed by source_file.
+
+    Groups nodes and edges by source_file, then saves one cache entry per file.
+    Returns the number of files cached.
+    """
+    from collections import defaultdict
+
+    by_file: dict[str, dict] = defaultdict(lambda: {"nodes": [], "edges": [], "hyperedges": []})
+    for n in nodes:
+        src = n.get("source_file", "")
+        if src:
+            by_file[src]["nodes"].append(n)
+    for e in edges:
+        src = e.get("source_file", "")
+        if src:
+            by_file[src]["edges"].append(e)
+    for h in (hyperedges or []):
+        src = h.get("source_file", "")
+        if src:
+            by_file[src]["hyperedges"].append(h)
+
+    saved = 0
+    for fpath, result in by_file.items():
+        p = Path(fpath)
+        if not p.is_absolute():
+            p = Path(root) / p
+        if p.exists():
+            save_cached(p, result, root)
+            saved += 1
+    return saved

package/runtime/python/okstra_vendor/graphify/cluster.py

@@ -0,0 +1,137 @@
+"""Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores."""
+from __future__ import annotations
+import contextlib
+import inspect
+import io
+import sys
+import networkx as nx
+
+
+def _suppress_output():
+    """Context manager to suppress stdout/stderr during library calls.
+
+    graspologic's leiden() emits ANSI escape sequences (progress bars,
+    colored warnings) that corrupt PowerShell 5.1's scroll buffer on
+    Windows (see issue #19). Redirecting stdout/stderr to devnull during
+    the call prevents this without losing any graphify output.
+    """
+    return contextlib.redirect_stdout(io.StringIO())
+
+
+def _partition(G: nx.Graph) -> dict[str, int]:
+    """Run community detection. Returns {node_id: community_id}.
+
+    Tries Leiden (graspologic) first — best quality.
+    Falls back to Louvain (built into networkx) if graspologic is not installed.
+
+    Output from graspologic is suppressed to prevent ANSI escape codes
+    from corrupting terminal scroll buffers on Windows PowerShell 5.1.
+    """
+    try:
+        from graspologic.partition import leiden
+        # Suppress graspologic output to prevent ANSI escape codes from
+        # corrupting PowerShell 5.1 scroll buffer (issue #19)
+        old_stderr = sys.stderr
+        try:
+            sys.stderr = io.StringIO()
+            with _suppress_output():
+                result = leiden(G)
+        finally:
+            sys.stderr = old_stderr
+        return result
+    except ImportError:
+        pass
+
+    # Fallback: networkx louvain (available since networkx 2.7).
+    # Inspect kwargs to stay compatible across NetworkX versions — max_level
+    # was added in a later release and prevents hangs on large sparse graphs.
+    kwargs: dict = {"seed": 42, "threshold": 1e-4}
+    if "max_level" in inspect.signature(nx.community.louvain_communities).parameters:
+        kwargs["max_level"] = 10
+    communities = nx.community.louvain_communities(G, **kwargs)
+    return {node: cid for cid, nodes in enumerate(communities) for node in nodes}
+
+
+_MAX_COMMUNITY_FRACTION = 0.25   # communities larger than 25% of graph get split
+_MIN_SPLIT_SIZE = 10             # only split if community has at least this many nodes
+
+
+def cluster(G: nx.Graph) -> dict[int, list[str]]:
+    """Run Leiden community detection. Returns {community_id: [node_ids]}.
+
+    Community IDs are stable across runs: 0 = largest community after splitting.
+    Oversized communities (> 25% of graph nodes, min 10) are split by running
+    a second Leiden pass on the subgraph.
+
+    Accepts directed or undirected graphs. DiGraphs are converted to undirected
+    internally since Louvain/Leiden require undirected input.
+    """
+    if G.number_of_nodes() == 0:
+        return {}
+    if G.is_directed():
+        G = G.to_undirected()
+    if G.number_of_edges() == 0:
+        return {i: [n] for i, n in enumerate(sorted(G.nodes))}
+
+    # Leiden warns and drops isolates - handle them separately
+    isolates = [n for n in G.nodes() if G.degree(n) == 0]
+    connected_nodes = [n for n in G.nodes() if G.degree(n) > 0]
+    connected = G.subgraph(connected_nodes)
+
+    raw: dict[int, list[str]] = {}
+    if connected.number_of_nodes() > 0:
+        partition = _partition(connected)
+        for node, cid in partition.items():
+            raw.setdefault(cid, []).append(node)
+
+    # Each isolate becomes its own single-node community
+    next_cid = max(raw.keys(), default=-1) + 1
+    for node in isolates:
+        raw[next_cid] = [node]
+        next_cid += 1
+
+    # Split oversized communities
+    max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))
+    final_communities: list[list[str]] = []
+    for nodes in raw.values():
+        if len(nodes) > max_size:
+            final_communities.extend(_split_community(G, nodes))
+        else:
+            final_communities.append(nodes)
+
+    # Re-index by size descending for deterministic ordering
+    final_communities.sort(key=len, reverse=True)
+    return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}
+
+
+def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:
+    """Run a second Leiden pass on a community subgraph to split it further."""
+    subgraph = G.subgraph(nodes)
+    if subgraph.number_of_edges() == 0:
+        # No edges - split into individual nodes
+        return [[n] for n in sorted(nodes)]
+    try:
+        sub_partition = _partition(subgraph)
+        sub_communities: dict[int, list[str]] = {}
+        for node, cid in sub_partition.items():
+            sub_communities.setdefault(cid, []).append(node)
+        if len(sub_communities) <= 1:
+            return [sorted(nodes)]
+        return [sorted(v) for v in sub_communities.values()]
+    except Exception:
+        return [sorted(nodes)]
+
+
+def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:
+    """Ratio of actual intra-community edges to maximum possible."""
+    n = len(community_nodes)
+    if n <= 1:
+        return 1.0
+    subgraph = G.subgraph(community_nodes)
+    actual = subgraph.number_of_edges()
+    possible = n * (n - 1) / 2
+    return round(actual / possible, 2) if possible > 0 else 0.0
+
+
+def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:
+    return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}