imhcode 1.0.0

package/skills/graphify/benchmark.py

@@ -0,0 +1,152 @@
+"""Token-reduction benchmark - measures how much context graphify saves vs naive full-corpus approach."""
+from __future__ import annotations
+import json
+import sys
+from pathlib import Path
+import networkx as nx
+from networkx.readwrite import json_graph
+
+from graphify.build import edge_data
+
+
+_CHARS_PER_TOKEN = 4  # standard approximation
+
+
+def _safe(unicode_char: str, ascii_fallback: str) -> str:
+    """Return unicode_char if stdout can encode it, else ascii_fallback.
+
+    Windows consoles often default to cp1252 which cannot encode box-drawing
+    or arrow glyphs; printing them raises UnicodeEncodeError mid-output.
+    """
+    encoding = getattr(sys.stdout, "encoding", None) or ""
+    try:
+        unicode_char.encode(encoding)
+        return unicode_char
+    except (UnicodeEncodeError, LookupError):
+        return ascii_fallback
+
+
+def _hr(width: int = 50) -> str:
+    """Horizontal rule that survives non-UTF-8 stdout (e.g. Windows cp1252 console)."""
+    return _safe("─", "-") * width
+
+
+def _estimate_tokens(text: str) -> int:
+    return max(1, len(text) // _CHARS_PER_TOKEN)
+
+
+def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int:
+    """Run BFS from best-matching nodes and return estimated tokens in the subgraph context."""
+    terms = [t.lower() for t in question.split() if len(t) > 2]
+    scored = []
+    for nid, data in G.nodes(data=True):
+        label = data.get("label", "").lower()
+        score = sum(1 for t in terms if t in label)
+        if score > 0:
+            scored.append((score, nid))
+    scored.sort(reverse=True)
+    start_nodes = [nid for _, nid in scored[:3]]
+    if not start_nodes:
+        return 0
+
+    visited: set[str] = set(start_nodes)
+    frontier = set(start_nodes)
+    edges_seen: list[tuple] = []
+    for _ in range(depth):
+        next_frontier: set[str] = set()
+        for n in frontier:
+            for neighbor in G.neighbors(n):
+                if neighbor not in visited:
+                    next_frontier.add(neighbor)
+                    edges_seen.append((n, neighbor))
+        visited.update(next_frontier)
+        frontier = next_frontier
+
+    lines = []
+    for nid in visited:
+        d = G.nodes[nid]
+        lines.append(f"NODE {d.get('label', nid)} src={d.get('source_file', '')} loc={d.get('source_location', '')}")
+    for u, v in edges_seen:
+        if u in visited and v in visited:
+            d = edge_data(G, u, v)
+            lines.append(f"EDGE {G.nodes[u].get('label', u)} --{d.get('relation', '')}--> {G.nodes[v].get('label', v)}")
+
+    return _estimate_tokens("\n".join(lines))
+
+
+_SAMPLE_QUESTIONS = [
+    "how does authentication work",
+    "what is the main entry point",
+    "how are errors handled",
+    "what connects the data layer to the api",
+    "what are the core abstractions",
+]
+
+
+def run_benchmark(
+    graph_path: str = "graphify-out/graph.json",
+    corpus_words: int | None = None,
+    questions: list[str] | None = None,
+) -> dict:
+    """Measure token reduction: corpus tokens vs graphify query tokens.
+
+    Args:
+        graph_path: path to the built graph
+        corpus_words: total word count from detect() output; if None, estimated from graph
+        questions: list of questions to benchmark; defaults to _SAMPLE_QUESTIONS
+
+    Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question
+    """
+    data = json.loads(Path(graph_path).read_text(encoding="utf-8"))
+    try:
+        G = json_graph.node_link_graph(data, edges="links")
+    except TypeError:
+        G = json_graph.node_link_graph(data)
+
+    if corpus_words is None:
+        # Rough estimate: each node label is ~3 words, plus source context
+        corpus_words = G.number_of_nodes() * 50
+
+    corpus_tokens = corpus_words * 100 // 75  # words → tokens (100 words ≈ 133 tokens)
+
+    qs = questions or _SAMPLE_QUESTIONS
+    per_question = []
+    for q in qs:
+        qt = _query_subgraph_tokens(G, q)
+        if qt > 0:
+            per_question.append({"question": q, "query_tokens": qt, "reduction": round(corpus_tokens / qt, 1)})
+
+    if not per_question:
+        return {"error": "No matching nodes found for sample questions. Build the graph first."}
+
+    avg_query_tokens = sum(p["query_tokens"] for p in per_question) // len(per_question)
+    reduction_ratio = round(corpus_tokens / avg_query_tokens, 1) if avg_query_tokens > 0 else 0
+
+    return {
+        "corpus_tokens": corpus_tokens,
+        "corpus_words": corpus_words,
+        "nodes": G.number_of_nodes(),
+        "edges": G.number_of_edges(),
+        "avg_query_tokens": avg_query_tokens,
+        "reduction_ratio": reduction_ratio,
+        "per_question": per_question,
+    }
+
+
+def print_benchmark(result: dict) -> None:
+    """Print a human-readable benchmark report."""
+    if "error" in result:
+        print(f"Benchmark error: {result['error']}")
+        return
+
+    print(f"\ngraphify token reduction benchmark")
+    print(_hr(50))
+    arrow = _safe("→", "->")
+    print(f"  Corpus:          {result['corpus_words']:,} words {arrow} ~{result['corpus_tokens']:,} tokens (naive)")
+    print(f"  Graph:           {result['nodes']:,} nodes, {result['edges']:,} edges")
+    print(f"  Avg query cost:  ~{result['avg_query_tokens']:,} tokens")
+    print(f"  Reduction:       {result['reduction_ratio']}x fewer tokens per query")
+    print(f"\n  Per question:")
+    for p in result["per_question"]:
+        print(f"    [{p['reduction']}x] {p['question'][:55]}")
+    print()

package/skills/graphify/build.py

@@ -0,0 +1,366 @@
+# assemble node+edge dicts into a NetworkX graph, preserving edge direction
+#
+# Node deduplication — three layers:
+#
+# 1. Within a file (AST): each extractor tracks a `seen_ids` set. A node ID is
+#    emitted at most once per file, so duplicate class/function definitions in
+#    the same source file are collapsed to the first occurrence.
+#
+# 2. Between files (build): NetworkX G.add_node() is idempotent — calling it
+#    twice with the same ID overwrites the attributes with the second call's
+#    values. Nodes are added in extraction order (AST first, then semantic),
+#    so if the same entity is extracted by both passes the semantic node
+#    silently overwrites the AST node. This is intentional: semantic nodes
+#    carry richer labels and cross-file context, while AST nodes have precise
+#    source_location. If you need to change the priority, reorder extractions
+#    passed to build().
+#
+# 3. Semantic merge (skill): before calling build(), the skill merges cached
+#    and new semantic results using an explicit `seen` set keyed on node["id"],
+#    so duplicates across cache hits and new extractions are resolved there
+#    before any graph construction happens.
+#
+from __future__ import annotations
+import json
+import re
+import sys
+import unicodedata
+from pathlib import Path
+import networkx as nx
+from .validate import validate_extraction
+
+
+# Synonym mapper for known invalid file_type values that LLM subagents commonly
+# emit. Keeps semantic intent close (markdown→document, tool→code) and falls
+# back to "concept" for any other invalid value (see #840).
+_FILE_TYPE_SYNONYMS = {
+    "markdown": "document",
+    "text": "document",
+    "tool": "code",
+    "library": "code",
+    "pattern": "concept",
+    "principle": "concept",
+    "constraint": "concept",
+    "tech": "concept",
+    "technology": "concept",
+    "data-source": "concept",
+    "data_source": "concept",
+    "gotcha": "concept",
+    "framework": "concept",
+}
+
+
+def _normalize_id(s: str) -> str:
+    r"""Normalize an ID string the same way extract._make_id does.
+
+    Used to reconcile edge endpoints when the LLM generates IDs with slightly
+    different punctuation or casing than the AST extractor. Must stay in sync
+    with extract._make_id — NFKC normalization, \w with re.UNICODE, underscore
+    collapse, and casefold must all match (#811).
+    """
+    s = unicodedata.normalize("NFKC", s)
+    cleaned = re.sub(r"[^\w]+", "_", s, flags=re.UNICODE)
+    cleaned = re.sub(r"_+", "_", cleaned)
+    return cleaned.strip("_").casefold()
+
+
+def _norm_source_file(p: str | None) -> str | None:
+    """Normalize path separators to forward slashes so Windows backslash paths
+    and POSIX paths from semantic subagents resolve to the same node identity."""
+    return p.replace("\\", "/") if p else p
+
+
+def edge_data(G: nx.Graph, u: str, v: str) -> dict:
+    """Return one edge attribute dict for (u, v), tolerating MultiGraph.
+
+    For MultiGraph/MultiDiGraph there can be multiple parallel edges;
+    this returns the first one (sufficient for callers that only need
+    relation/confidence for rendering). Fixes #796.
+    """
+    raw = G[u][v]
+    if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)):
+        return next(iter(raw.values()), {})
+    return raw
+
+
+def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]:
+    """Return every edge attribute dict for (u, v); always a list."""
+    raw = G[u][v]
+    if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)):
+        return list(raw.values())
+    return [raw]
+
+
+def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
+    """Build a NetworkX graph from an extraction dict.
+
+    directed=True produces a DiGraph that preserves edge direction (source→target).
+    directed=False (default) produces an undirected Graph for backward compatibility.
+    """
+    # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility.
+    if "edges" not in extraction and "links" in extraction:
+        extraction = dict(extraction, edges=extraction["links"])
+
+    # Canonicalize legacy node/edge schema before validation.
+    for node in extraction.get("nodes", []):
+        if not isinstance(node, dict):
+            continue
+        if "source" in node and "source_file" not in node:
+            # Count edges that reference this node so the warning is actionable (#479)
+            node_id = node.get("id", "?")
+            affected_edges = sum(
+                1 for e in extraction.get("edges", [])
+                if e.get("source") == node_id or e.get("target") == node_id
+            )
+            print(
+                f"[graphify] WARNING: node '{node_id}' uses field 'source' instead of "
+                f"'source_file' — {affected_edges} edge(s) may be misrouted. "
+                f"Rename the field to 'source_file' to silence this warning.",
+                file=sys.stderr,
+            )
+            node["source_file"] = node.pop("source")
+        # Default missing/None file_type to "concept" so legacy graph.json
+        # entries (and stub nodes preserved by `_rebuild_code` from older
+        # graphify versions that didn't always populate file_type) don't
+        # trigger spurious "invalid file_type 'None'" validator warnings (#660).
+        if node.get("file_type") in (None, ""):
+            node["file_type"] = "concept"
+        ft = node.get("file_type", "")
+        if ft and ft not in {"code", "document", "paper", "image", "rationale", "concept"}:
+            node["file_type"] = _FILE_TYPE_SYNONYMS.get(ft, "concept")
+
+    errors = validate_extraction(extraction)
+    # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.
+    real_errors = [e for e in errors if "does not match any node id" not in e]
+    if real_errors:
+        print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr)
+    G: nx.Graph = nx.DiGraph() if directed else nx.Graph()
+    for node in extraction.get("nodes", []):
+        if "source_file" in node:
+            node["source_file"] = _norm_source_file(node["source_file"])
+        G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
+    node_set = set(G.nodes())
+    # Normalized ID map: lets edges survive when the LLM generates IDs with
+    # slightly different casing or punctuation than the AST extractor.
+    # e.g. "Session_ValidateToken" maps to "session_validatetoken".
+    norm_to_id: dict[str, str] = {_normalize_id(nid): nid for nid in node_set}
+    for edge in extraction.get("edges", []):
+        if "source" not in edge and "from" in edge:
+            edge["source"] = edge["from"]
+        if "target" not in edge and "to" in edge:
+            edge["target"] = edge["to"]
+        if "source" not in edge or "target" not in edge:
+            continue
+        src, tgt = edge["source"], edge["target"]
+        # Remap mismatched IDs via normalization before dropping the edge.
+        if src not in node_set:
+            src = norm_to_id.get(_normalize_id(src), src)
+        if tgt not in node_set:
+            tgt = norm_to_id.get(_normalize_id(tgt), tgt)
+        if src not in node_set or tgt not in node_set:
+            continue  # skip edges to external/stdlib nodes - expected, not an error
+        attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
+        if "source_file" in attrs:
+            attrs["source_file"] = _norm_source_file(attrs["source_file"])
+        # Preserve original edge direction - undirected graphs lose it otherwise,
+        # causing display functions to show edges backwards.
+        attrs["_src"] = src
+        attrs["_tgt"] = tgt
+        G.add_edge(src, tgt, **attrs)
+    hyperedges = extraction.get("hyperedges", [])
+    if hyperedges:
+        G.graph["hyperedges"] = hyperedges
+    return G
+
+
+def build(
+    extractions: list[dict],
+    *,
+    directed: bool = False,
+    dedup: bool = True,
+    dedup_llm_backend: str | None = None,
+) -> nx.Graph:
+    """Merge multiple extraction results into one graph.
+
+    directed=True produces a DiGraph that preserves edge direction (source→target).
+    directed=False (default) produces an undirected Graph for backward compatibility.
+    dedup=True (default) runs entity deduplication before building the graph.
+    dedup_llm_backend: if set (e.g. "gemini", "claude", or "kimi"), uses LLM to resolve
+        ambiguous pairs in the 75–92 Jaro-Winkler score zone.
+
+    Extractions are merged in order. For nodes with the same ID, the last
+    extraction's attributes win (NetworkX add_node overwrites). Pass AST
+    results before semantic results so semantic labels take precedence, or
+    reverse the order if you prefer AST source_location precision to win.
+    """
+    from graphify.dedup import deduplicate_entities
+    combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0}
+    for ext in extractions:
+        combined["nodes"].extend(ext.get("nodes", []))
+        combined["edges"].extend(ext.get("edges", []))
+        combined["hyperedges"].extend(ext.get("hyperedges", []))
+        combined["input_tokens"] += ext.get("input_tokens", 0)
+        combined["output_tokens"] += ext.get("output_tokens", 0)
+    if dedup and combined["nodes"]:
+        combined["nodes"], combined["edges"] = deduplicate_entities(
+            combined["nodes"], combined["edges"], communities={},
+            dedup_llm_backend=dedup_llm_backend,
+        )
+    return build_from_json(combined, directed=directed)
+
+
+def _norm_label(label: str) -> str:
+    """Canonical dedup key — lowercase, alphanumeric only."""
+    return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip()
+
+
+def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]:
+    """Merge nodes that share a normalised label, rewriting edge references.
+
+    Prefers IDs without chunk suffixes (_c\\d+) and shorter IDs when tied.
+    Drops self-loops created by the merge. Called in build() automatically.
+    """
+    _CHUNK_SUFFIX = re.compile(r"_c\d+$")
+    canonical: dict[str, dict] = {}  # norm_label -> surviving node
+    remap: dict[str, str] = {}       # old_id -> surviving_id
+
+    for node in nodes:
+        key = _norm_label(node.get("label", node.get("id", "")))
+        if not key:
+            continue
+        existing = canonical.get(key)
+        if existing is None:
+            canonical[key] = node
+        else:
+            has_suffix = bool(_CHUNK_SUFFIX.search(node["id"]))
+            existing_has_suffix = bool(_CHUNK_SUFFIX.search(existing["id"]))
+            if has_suffix and not existing_has_suffix:
+                remap[node["id"]] = existing["id"]
+            elif existing_has_suffix and not has_suffix:
+                remap[existing["id"]] = node["id"]
+                canonical[key] = node
+            elif len(node["id"]) < len(existing["id"]):
+                remap[existing["id"]] = node["id"]
+                canonical[key] = node
+            else:
+                remap[node["id"]] = existing["id"]
+
+    if not remap:
+        return nodes, edges
+
+    print(f"[graphify] Deduplicated {len(remap)} duplicate node(s) by label.", file=sys.stderr)
+    deduped_nodes = list(canonical.values())
+    deduped_edges = []
+    for edge in edges:
+        e = dict(edge)
+        e["source"] = remap.get(e["source"], e["source"])
+        e["target"] = remap.get(e["target"], e["target"])
+        if e["source"] != e["target"]:
+            deduped_edges.append(e)
+    return deduped_nodes, deduped_edges
+
+
+def build_merge(
+    new_chunks: list[dict],
+    graph_path: str | Path = "graphify-out/graph.json",
+    prune_sources: list[str] | None = None,
+    *,
+    directed: bool = False,
+    dedup: bool = True,
+    dedup_llm_backend: str | None = None,
+) -> nx.Graph:
+    """Load existing graph.json, merge new chunks into it, and save back.
+
+    Never replaces - only grows (or prunes deleted-file nodes via prune_sources).
+    Safe to call repeatedly: existing nodes and edges are preserved.
+    """
+    graph_path = Path(graph_path)
+    if graph_path.exists():
+        # Read JSON directly instead of going through node_link_graph().
+        # The latter rebuilds an undirected nx.Graph and then enumerating
+        # edges() yields endpoints based on node insertion order, which
+        # silently flips directional edges (e.g. `calls`) when the callee
+        # was inserted before the caller. The _src/_tgt direction-preserving
+        # attrs are popped before saving in export.py, so going through the
+        # NetworkX round-trip loses direction permanently (#760).
+        data = json.loads(graph_path.read_text(encoding="utf-8"))
+        links_key = "links" if "links" in data else "edges"
+        existing_nodes = list(data.get("nodes", []))
+        existing_edges = list(data.get(links_key, []))
+        base = [{"nodes": existing_nodes, "edges": existing_edges}]
+    else:
+        existing_nodes = []
+        base = []
+
+    all_chunks = base + list(new_chunks)
+    G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend)
+
+    # Prune nodes and edges from deleted source files
+    if prune_sources:
+        prune_set = set(prune_sources)
+        to_remove = [
+            n for n, d in G.nodes(data=True)
+            if d.get("source_file") in prune_set
+        ]
+        G.remove_nodes_from(to_remove)
+        n_files = len(prune_sources)
+        n_nodes = len(to_remove)
+        if n_nodes:
+            print(
+                f"[graphify] Pruned {n_nodes} node(s) from {n_files} deleted source file(s).",
+                file=sys.stderr,
+            )
+
+        edges_to_remove = [
+            (u, v) for u, v, d in G.edges(data=True)
+            if d.get("source_file") in prune_set
+        ]
+        if edges_to_remove:
+            G.remove_edges_from(edges_to_remove)
+            print(
+                f"[graphify] Pruned {len(edges_to_remove)} edge(s) from deleted source file(s).",
+                file=sys.stderr,
+            )
+
+        if not n_nodes and not edges_to_remove:
+            print(
+                f"[graphify] {n_files} source file(s) deleted since last run — "
+                f"no matching nodes or edges in graph, already clean.",
+                file=sys.stderr,
+            )
+
+    # Safety check: refuse to shrink the graph silently (#479)
+    # Skip when dedup or prune_sources is active — shrinkage is intentional there.
+    if graph_path.exists() and not dedup and not prune_sources:
+        existing_n = len(existing_nodes)
+        new_n = G.number_of_nodes()
+        if new_n < existing_n:
+            raise ValueError(
+                f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. "
+                f"Pass prune_sources explicitly if you intend to remove nodes."
+            )
+
+    return G
+
+
+def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph:
+    """Return a copy of G with all node IDs prefixed with repo_tag::.
+
+    Labels are preserved unchanged (for display). A 'local_id' attribute
+    is added to each node so the original ID can be recovered. Edges are
+    rewritten to match the new prefixed IDs. The 'repo' attribute is set
+    on every node.
+    """
+    relabel = {n: f"{repo_tag}::{n}" for n in G.nodes}
+    H = nx.relabel_nodes(G, relabel, copy=True)
+    for node, data in H.nodes(data=True):
+        data["repo"] = repo_tag
+        data.setdefault("local_id", node.split("::", 1)[1])
+    return H
+
+
+def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int:
+    """Remove all nodes tagged with repo_tag from G in-place. Returns count removed."""
+    to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag]
+    G.remove_nodes_from(to_remove)
+    return len(to_remove)