deepdebug-local-agent 1.0.23 → 1.1.0

package/tools/graphify-validation/lib/pip_audit_summary.py

@@ -0,0 +1,18 @@
+"""Condense pip-audit JSON into results/pip_audit.json (exit code kept: 0 clean, 1 vulnerabilities found)."""
+import json
+import sys
+from pathlib import Path
+
+src, rc, dest = sys.argv[1], int(sys.argv[2]), sys.argv[3]
+vulns = []
+try:
+    data = json.loads(Path(src).read_text())
+    for dep in data.get("dependencies", []):
+        for v in dep.get("vulns", []):
+            vulns.append({"package": dep["name"], "version": dep.get("version"), "id": v["id"], "fix": v.get("fix_versions"), "aliases": v.get("aliases", [])})
+    summary = "%d vulnerabilidades conhecidas em %d pacotes (exit %d)" % (len(vulns), len(data.get("dependencies", [])), rc)
+except Exception as exc:  # no JSON means pip-audit itself failed (network, resolver)
+    summary = "pip-audit nao produziu resultado (exit %d): %s" % (rc, exc)
+Path(dest).parent.mkdir(parents=True, exist_ok=True)
+Path(dest).write_text(json.dumps({"exit_code": rc, "vulnerabilities": vulns, "summary": summary}, indent=2))
+print(summary)

package/tools/graphify-validation/lib/recall.py

@@ -0,0 +1,115 @@
+"""recall@8 of retrieval backends against a hand-written reference question set (G5).
+
+Backends:
+  graph    `graphify query "<question>" --budget N` -> distinct files in output order
+  lexical_auto     ripgrep keyword ranking (the D14 fallback) with keywords taken from the question text
+  lexical_curated  same ranking with human-picked keywords (optimistic ceiling: the author knew the answer)
+  vector   optional: results exported from pgvector, passed with --vector-results
+Test files and non-code files are removed from every ranking (loop context rule).
+"""
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from collections import Counter
+from pathlib import Path
+
+from gv_common import files_with_word, is_test_path, CODE_EXTS
+
+SRC = re.compile(r"\[src=([^\s\]]*)")
+
+
+def rank_graph(graphify, workdir, question, env, budget=8000, context=None):
+    args = [graphify, "query", question, "--budget", str(budget)]
+    if context:
+        args += ["--context", context]
+    res = subprocess.run(args, cwd=str(workdir), env=env, capture_output=True, text=True, timeout=300)
+    seen, order = set(), []
+    for m in SRC.finditer(res.stdout):
+        f = m.group(1)
+        if f and f not in seen:
+            seen.add(f)
+            order.append(f)
+    return order, res.returncode
+
+
+STOP = set("where which what does that this with from into when then than have been being their there about "
+           "after before using used uses over under also each only some such code class classes file files "
+           "module modules component components function functions method methods before".split())
+
+
+def auto_keywords(question):
+    """Fair keyword extraction from the question text only: no knowledge of the answer."""
+    toks = [t.lower() for t in re.findall(r"[A-Za-z][A-Za-z0-9]+", question)]
+    out = []
+    for t in toks:
+        if len(t) < 4 or t in STOP:
+            continue
+        stem = t[:5] if len(t) > 5 else t
+        if stem not in out:
+            out.append(stem)
+    return out
+
+
+def rank_lexical(workdir, keywords, whole_word=True, ignore_case=False):
+    distinct = Counter()
+    for kw in keywords:
+        for f in files_with_word(workdir, kw, include_tests=False, whole_word=whole_word, ignore_case=ignore_case):
+            distinct[f] += 1
+    # also reward keyword appearance in the file name (cheap and typical for rg + fd)
+    for f in list(distinct):
+        base = Path(f).stem.lower()
+        for kw in keywords:
+            if kw.lower() in base:
+                distinct[f] += 1
+    return [f for f, _ in sorted(distinct.items(), key=lambda kv: (-kv[1], kv[0]))]
+
+
+def clean(ranking):
+    return [f for f in ranking if Path(f).suffix in CODE_EXTS and not is_test_path(f)]
+
+
+def recall_at(ranking, truth, k=8):
+    top = clean(ranking)[:k]
+    hit = [t for t in truth if t in top]
+    return len(hit) / len(truth), hit
+
+
+def evaluate(graphify, workdir, repo, questions, env, vector=None, k=8):
+    rows = []
+    for q in questions:
+        truth = q["truth"]
+        missing = [t for t in truth if not (Path(workdir) / t).exists()]
+        if missing:
+            raise SystemExit("question %s: truth files not in snapshot: %s" % (q["id"], missing))
+        entry = {"id": q["id"], "kind": q["kind"], "question": q["question"], "truth": truth}
+        rank, rc = rank_graph(graphify, workdir, q["question"], env)
+        entry["graph"] = _score(rank, truth, k)
+        entry["graph"]["exit_code"] = rc
+        rank_c, _ = rank_graph(graphify, workdir, q["question"], env, context="call")
+        entry["graph_call_only"] = _score(rank_c, truth, k)
+        # lexical_curated: keywords picked by a human who knew the answer (optimistic ceiling, NOT a fair baseline)
+        entry["lexical_curated"] = _score(rank_lexical(workdir, q["keywords"], whole_word=False, ignore_case=True), truth, k)
+        # lexical_auto: keywords extracted from the question text only (fair baseline for the D14 ripgrep fallback)
+        entry["lexical_auto"] = _score(rank_lexical(workdir, auto_keywords(q["question"]), whole_word=False, ignore_case=True), truth, k)
+        if vector is not None:
+            entry["vector"] = _score(vector.get(q["id"], []), truth, k)
+        rows.append(entry)
+    backends = ["graph", "graph_call_only", "lexical_auto", "lexical_curated"] + (["vector"] if vector is not None else [])
+    agg = {}
+    for b in backends:
+        for kind in ("all", "symbolic", "conceptual"):
+            sel = [r for r in rows if kind == "all" or r["kind"] == kind]
+            if sel:
+                agg.setdefault(b, {})[kind] = {
+                    "n": len(sel),
+                    "recall_at_%d_pct" % k: round(100.0 * sum(r[b]["recall"] for r in sel) / len(sel), 1),
+                    "hit_at_%d_pct" % k: round(100.0 * sum(1 for r in sel if r[b]["recall"] > 0) / len(sel), 1),
+                }
+    return {"k": k, "questions": rows, "aggregate": agg, "vector_measured": vector is not None}
+
+
+def _score(ranking, truth, k):
+    r, hit = recall_at(ranking, truth, k)
+    return {"recall": round(r, 3), "hits": hit, "top": clean(ranking)[:k]}

package/tools/graphify-validation/lib/report.py

@@ -0,0 +1,226 @@
+"""Render out/REPORT.md from out/results/*.json. Thresholds are the proposed G1-G10 limits (Local Agent GA plan)."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+PASS, FAIL, PART, NM = "PASSA", "FALHA", "PARCIAL", "NAO MEDIDO"
+T = {"g1_cov": 95.0, "g2_inf": 15.0, "g2_prec": 85.0, "g4_cold_s": 60.0, "g4_rss_mb": 1024.0, "g4_inc_s": 30.0,
+     "g5_calls": 90.0, "max_files_perf": 2000}
+
+
+def load(out):
+    res = {}
+    for p in sorted((out / "results").glob("*.json")):
+        res[p.stem] = json.loads(p.read_text())
+    return res
+
+
+def fmt(v, suffix=""):
+    return "n/d" if v is None else ("%s%s" % (v, suffix))
+
+
+def repos(res):
+    return [k for k in res if k not in ("environment", "spring", "licenses", "pip_audit")]
+
+
+def section_env(res):
+    e = res["environment"]
+    lines = ["## 0. Ambiente e versoes", "",
+             "| Item | Valor |", "|---|---|",
+             "| Data (UTC) | %s |" % e["date_utc"], "| Graphify | graphifyy==%s (fixado por hash) |" % e["graphify_version"],
+             "| Python | %s |" % e["python"], "| SO / CPU / RAM | %s / %s CPUs / %s GB |" % (e["os"], e["cpu_count"], e["ram_gb"]),
+             "| ripgrep | %s |" % e["ripgrep"],
+             "| Bloqueio de rede | %s; eficaz: %s |" % (e["network_block"], e.get("network_block_effective")),
+             "| Ambiente do processo | so %s (sem chaves LLM nem credenciais) |" % ", ".join(e["env_keys_passed"]), ""]
+    lines += ["| Repo | Ref | SHA | Ficheiros de codigo |", "|---|---|---|---|"]
+    for r in repos(res):
+        s = res[r].get("snapshot", {})
+        lines.append("| %s | %s | %s | %s |" % (r, s.get("ref"), (s.get("sha") or "")[:10], s.get("code_files")))
+    lines.append("")
+    return lines
+
+
+def verdicts(res):
+    v = {}
+    rs = repos(res)
+    # G1
+    covs = {r: res[r]["cold"]["g1"]["coverage_pct"] for r in rs if "cold" in res[r]}
+    v["G1"] = (PASS if covs and all(c >= T["g1_cov"] for c in covs.values()) else (FAIL if covs else NM),
+               "; ".join("%s %s%% (%d sem simbolos)" % (r, covs[r], res[r]["cold"]["g1"]["files_without_symbols_total"]) for r in covs))
+    # G2
+    parts, ok_inf, ok_prec, have = [], True, True, False
+    for r in rs:
+        if "cold" not in res[r]:
+            continue
+        have = True
+        inf = res[r]["cold"]["g2"]["inferred_pct"]
+        ed = res[r].get("edges", {})
+        prec = ed.get("oracle_precision_pct")
+        ok_inf &= inf <= T["g2_inf"]
+        if prec is not None:
+            ok_prec &= prec >= T["g2_prec"]
+        parts.append("%s INFERRED %s%%, precisao oraculo %s%% (n=%s, decididas %s)" % (r, inf, fmt(prec), ed.get("sample_size", "n/d"), ed.get("oracle_decided", "n/d")))
+    if not have:
+        v["G2"] = (NM, "")
+    elif not (ok_inf and ok_prec):
+        v["G2"] = (FAIL, "; ".join(parts))
+    else:
+        v["G2"] = (PART, "; ".join(parts) + "; falta revisao manual por 2 pessoas")
+    # G3
+    sp = res.get("spring")
+    v["G3"] = ((PASS if sp["pass"] else FAIL), "interface->impl %s%% (limiar %s%%)" % (sp["headline_pct"], sp["threshold_pct"])) if sp else (NM, "")
+    # G4: FALHA only on wall time or on the largest single process; a tree sum above the limit is
+    # reported as PARCIAL because summed RSS double-counts pages shared between workers.
+    parts, fail, soft = [], False, False
+    for r in rs:
+        c = res[r].get("cold")
+        i = res[r].get("incremental")
+        if not c:
+            continue
+        run = c["run"]
+        small = res[r]["snapshot"]["code_files"] <= T["max_files_perf"]
+        s = "%s frio %ss, arvore %s MB, maior filho %s MB" % (r, run["wall_s"], run["peak_tree_rss_mb"], run["max_child_rss_mb"])
+        if small:
+            fail |= run["wall_s"] > T["g4_cold_s"] or run["max_child_rss_mb"] > T["g4_rss_mb"]
+            soft |= run["peak_tree_rss_mb"] > T["g4_rss_mb"]
+        if i:
+            ch = i["changed"]
+            s += "; incremental %ss, arvore %s MB, maior filho %s MB" % (ch["wall_s"], ch["peak_tree_rss_mb"], ch["max_child_rss_mb"])
+            fail |= ch["wall_s"] > T["g4_inc_s"] or ch["max_child_rss_mb"] > T["g4_rss_mb"]
+            soft |= ch["peak_tree_rss_mb"] > T["g4_rss_mb"]
+        parts.append(s)
+    note = "; maquina de desenvolvimento (nao 2 vCPU/4 GB)" + ("; soma de RSS da arvore acima de 1 GB em pelo menos uma corrida (pode contar paginas partilhadas)" if soft else "")
+    v["G4"] = (FAIL if fail else (PART if parts else NM), "; ".join(parts) + note)
+    # G5
+    parts, calls_ok = [], True
+    for r in rs:
+        a = res[r].get("affected")
+        rc = res[r].get("recall")
+        if a:
+            x = a["aggregate_main"]
+            calls_ok &= (x["recall_vs_rg_pct"] or 0) >= T["g5_calls"]
+            parts.append("%s affected recall vs rg %s%% (precisao %s%%)" % (r, fmt(x["recall_vs_rg_pct"]), fmt(x["precision_vs_rg_pct"])))
+        if rc:
+            g = rc["aggregate"].get("graph", {}).get("all", {})
+            lx = rc["aggregate"].get("lexical_auto", {}).get("all", {})
+            parts.append("%s recall@8 grafo %s%% vs lexical (palavras da pergunta) %s%%" % (r, fmt(g.get("recall_at_8_pct")), fmt(lx.get("recall_at_8_pct"))))
+    vec = any(res[r].get("recall", {}).get("vector_measured") for r in rs)
+    v["G5"] = ((PASS if (calls_ok and vec) else (FAIL if not calls_ok else PART)) if parts else NM,
+               "; ".join(parts) + ("" if vec else "; pgvector NAO medido (sem BD de QA)"))
+    v["G6"] = (NM, "exige agent2/LA com LLM e incidentes reais; fora do harness")
+    # G7
+    socks, pa = [], res.get("pip_audit")
+    for r in rs:
+        n = res[r].get("network")
+        if n and n.get("status") == "measured":
+            socks.append((r, n["run"]["audit"]["socket_events"], n["block_effective"], n["run"]["exit_code"]))
+    if socks:
+        zero = all(s[1] == 0 and s[3] == 0 for s in socks)
+        detail = "; ".join("%s: %d eventos socket, exit %s" % (s[0], s[1], s[3]) for s in socks)
+        pa_txt = ("pip-audit: %s" % pa["summary"]) if pa else "pip-audit nao corrido"
+        v["G7"] = ((PART if zero else FAIL), detail + "; " + pa_txt + "; comandos de rede desativados = requisito de empacotamento, nao testado aqui")
+    else:
+        v["G7"] = (NM, "")
+    # G8
+    det = [res[r]["determinism"]["same_as_first"] for r in rs if "determinism" in res[r]]
+    tr = {r: res[r]["snapshot"]["tracked_graphify_out_files"] for r in rs if "snapshot" in res[r]}
+    parts = ["determinismo %s" % ("igual em %d/%d repos" % (sum(det), len(det)) if det else "n/d")] + ["%s: %d ficheiros graphify-out versionados" % (r, n) for r, n in tr.items()]
+    v["G8"] = ((PASS if det and all(det) and not any(tr.values()) else FAIL) if det else NM, "; ".join(parts))
+    v["G9"] = (NM, "sem Docker local; ver docs/graphify-packaging-plan.md")
+    v["G10"] = (NM, "Java e JS cobertos por repos VH; Python/SQL/Go/Terraform e repo do piloto fora deste conjunto")
+    return v
+
+
+def render(res):
+    L = ["# Relatorio de validacao do Graphify (harness reprodutivel)", "",
+         "Gerado por `tools/graphify-validation/run.sh`. Marcacao: PASSA/FALHA contra o limiar proposto; PARCIAL = medido mas falta um passo humano ou de ambiente; NAO MEDIDO = nao deu para medir aqui.", ""]
+    L += section_env(res)
+    L += ["## 1. Resumo por criterio", "", "| ID | Veredito | Medido |", "|---|---|---|"]
+    vs = verdicts(res)
+    for k in ["G%d" % i for i in range(1, 11)]:
+        L.append("| %s | %s | %s |" % (k, vs[k][0], vs[k][1].replace("|", "/")))
+    L.append("")
+    L += ["## 2. Detalhe por repo", ""]
+    for r in repos(res):
+        d = res[r]
+        L += ["### %s" % r, ""]
+        if "cold" in d:
+            c = d["cold"]
+            L += ["- Corrida a frio: %ss, pico de RSS da arvore %s MB (estimativa; maior filho %s MB), nos %s, arestas %s, graphify-out %s MB." % (
+                c["run"]["wall_s"], c["run"]["peak_tree_rss_mb"], c["run"]["max_child_rss_mb"], c["g2"]["nodes"], c["g2"]["edges"], c["graphify_out_mb"]),
+                "- G1: %s/%s ficheiros de codigo com simbolos (%s%%); %s sem simbolos, dos quais %s em testes. Lista: %s" % (c["g1"]["files_with_symbols"], c["g1"]["code_files"], c["g1"]["coverage_pct"],
+                                                                                      c["g1"]["files_without_symbols_total"], c["g1"]["of_which_in_tests"], ", ".join(c["g1"]["files_without_symbols"][:8]) or "nenhum"),
+                "- G2: INFERRED %s%% (%s%% so em fontes main); INFERRED por relacao %s." % (c["g2"]["inferred_pct"], c["g2"]["inferred_pct_main_sources"], c["g2"]["inferred_by_relation"])]
+        if "determinism" in d:
+            L.append("- G8 determinismo: assinatura %s vs %s, igual: %s." % (d["cold"]["signature"]["sha256"], d["determinism"]["signature"]["sha256"], d["determinism"]["same_as_first"]))
+        if "incremental" in d:
+            i = d["incremental"]
+            L.append("- Incremental (1 ficheiro alterado: %s): %ss, %s MB; simbolo novo no grafo: %s; sem alteracoes: %ss." % (
+                i["probe_file"], i["changed"]["wall_s"], i["changed"]["peak_tree_rss_mb"], i["new_symbol_in_graph"], i["unchanged"]["wall_s"]))
+        if "network" in d and d["network"].get("status") == "measured":
+            n = d["network"]
+            a = n["run"]["audit"]
+            L.append("- G7 rede bloqueada (%s): exit %s, %d eventos socket, %d subprocess.Popen (%s)." % (n["mechanism"], n["run"]["exit_code"], a["socket_events"], a["subprocess_popen"], "; ".join(s[:70] for s in a["subprocess_samples"][:1])))
+        if "edges" in d:
+            e = d["edges"]
+            L.append("- G2 amostra (seed %s, n=%s): oraculo ok/wrong/unknown %s; precisao nas decididas %s%%; estrato ambiguo %s; estrato outros %s. Revisao manual: %s linhas." % (
+                e["seed"], e["sample_size"], e["oracle_counts"], e["oracle_precision_pct"], e["by_stratum"]["ambiguous"], e["by_stratum"]["other"], e["manual_reviewed"]))
+        if "affected" in d:
+            x = d["affected"]
+            L += ["", "`affected` (profundidade 1) contra `rg -w`, so fontes main:", "", "| Simbolo | grafo | rg | ambos | so grafo | so rg | recall vs rg | precisao vs rg |", "|---|---|---|---|---|---|---|---|"]
+            for s in x["symbols"]:
+                m = s["main"]
+                L.append("| %s (%s) | %d | %d | %d | %d | %d | %s | %s |" % (s["symbol"], s["graph_status"], m["graph"], m["rg"], m["both"], m["graph_only"], m["rg_only"], fmt(m["recall_vs_rg_pct"], "%"), fmt(m["precision_vs_rg_pct"], "%")))
+            L.append("")
+            L.append("Agregado (profundidade 1): recall %s%%, precisao %s%% (%s simbolos); profundidade 2: recall %s%%." % (fmt(x["aggregate_main"]["recall_vs_rg_pct"]), fmt(x["aggregate_main"]["precision_vs_rg_pct"]), x["aggregate_main"]["symbols_compared"], fmt(x["aggregate_main"]["depth2_recall_vs_rg_pct"])))
+        if "recall" in d:
+            rc = d["recall"]
+            L += ["", "recall@%d no conjunto de referencia (%d perguntas):" % (rc["k"], len(rc["questions"])), "", "| Backend | Tipo | n | recall@8 | acerto@8 |", "|---|---|---|---|---|"]
+            for b, kinds in rc["aggregate"].items():
+                for kind, m in kinds.items():
+                    L.append("| %s | %s | %d | %s%% | %s%% |" % (b, kind, m["n"], m["recall_at_8_pct"], m["hit_at_8_pct"]))
+            L.append("")
+            L.append("`lexical_auto` usa so o texto da pergunta (baseline justo para o fallback ripgrep); `lexical_curated` usa palavras escolhidas por quem conhecia a resposta (teto otimista, nao comparar).")
+            if not rc["vector_measured"]:
+                L.append("| vector (pgvector) | - | - | NAO MEDIDO | NAO MEDIDO |")
+        L.append("")
+    sp = res.get("spring")
+    if sp:
+        L += ["## 3. Spring/DI (fixture sintetica)", "", "| Grupo | Encontrado | Esperado | % |", "|---|---|---|---|"]
+        for g, m in sp["groups"].items():
+            L.append("| %s | %d | %d | %s%% |" % (g, m["found"], m["expected"], m["pct"]))
+        L += ["", "Itens em falta:", ""]
+        for g, m in sp["groups"].items():
+            for it in m["items"]:
+                if not it["found"]:
+                    L.append("- %s / %s: %s" % (g, it["id"], it["detail"]))
+        L.append("")
+    lic = res.get("licenses")
+    if lic:
+        s = lic["summary"]
+        L += ["## 4. Licencas das dependencias instaladas", "", "%d distribuicoes; permissivas %d; a rever %s; desconhecidas %s; sem ficheiro de licenca %s." % (s["total"], s["permissive"], s["review"] or "nenhuma", s["unknown"] or "nenhuma", s["without_license_file"] or "nenhuma"), ""]
+    pa = res.get("pip_audit")
+    if pa:
+        L += ["## 5. pip-audit", "", "%s" % pa["summary"], ""]
+    L += ["## 6. O que este relatorio nao mediu", "",
+          "- recall@8 contra pgvector: precisa do dataset do RagEvalController e da BD de QA (passar `--vector-results`).",
+          "- G6 (incidentes ponta a ponta sem Voyage), G9 (imagem local-agent-full) e G10 (repo do piloto, Python/SQL/Go/Terraform reais).",
+          "- Revisao manual por 2 pessoas das arestas INFERRED (o CSV em `review/` esta pronto para isso).",
+          "- Desempenho em 2 vCPU/4 GB e no hardware do piloto.", ""]
+    return "\n".join(L)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", required=True)
+    a = ap.parse_args()
+    out = Path(a.out)
+    res = load(out)
+    (out / "REPORT.md").write_text(render(res), encoding="utf-8")
+    print(out / "REPORT.md")
+
+
+if __name__ == "__main__":
+    main()

package/tools/graphify-validation/lib/run_all.py

@@ -0,0 +1,237 @@
+"""Orchestrator: snapshot -> graphify runs -> metrics -> JSON results (then report.py renders REPORT.md)."""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import affected_vs_rg  # noqa: E402
+import graph_metrics as gm  # noqa: E402
+import recall as recall_mod  # noqa: E402
+import run_graphify as rgf  # noqa: E402
+import sample_edges as se  # noqa: E402
+import spring_fixture as sf  # noqa: E402
+from gv_common import HARNESS_DIR, clean_env, dump_json, is_test_path, iter_code_files, load_json, log, rg_version  # noqa: E402
+
+ALL_STEPS = ["snapshot", "cold", "determinism", "network", "incremental", "edges", "affected", "recall", "spring", "licenses"]
+
+
+def sh(args, cwd=None, check=True):
+    res = subprocess.run(args, cwd=cwd, capture_output=True, text=True)
+    if check and res.returncode != 0:
+        raise SystemExit("command failed: %s\n%s" % (" ".join(map(str, args)), res.stderr.strip()))
+    return res.stdout.strip()
+
+
+def check_origin(clone, allowed):
+    url = sh(["git", "-C", str(clone), "remote", "get-url", "origin"])
+    owner = re.search(r"[:/]([^/:]+)/[^/]+?(?:\.git)?$", url)
+    if not owner or owner.group(1) not in allowed:
+        raise SystemExit("refusing %s: origin owner is not in allowed_origin_owners %s" % (clone, allowed))
+    return url
+
+
+def snapshot(clone, ref, dest):
+    if dest.exists():
+        shutil.rmtree(dest)
+    dest.mkdir(parents=True)
+    archive = subprocess.Popen(["git", "-C", str(clone), "archive", ref], stdout=subprocess.PIPE)
+    subprocess.run(["tar", "-x", "-C", str(dest)], stdin=archive.stdout, check=True)
+    archive.wait()
+    if archive.returncode != 0:
+        raise SystemExit("git archive failed for %s %s" % (clone, ref))
+    shutil.rmtree(dest / "graphify-out", ignore_errors=True)
+
+
+def venv_paths(venv):
+    return str(Path(venv) / "bin"), str(Path(venv) / "bin" / "graphify"), str(Path(venv) / "bin" / "python")
+
+
+def graphify_version(venv_python):
+    out = sh([venv_python, "-c", "import importlib.metadata as m;print(m.version('graphifyy'))"])
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--repos-dir", required=True, help="directory holding the git clones named as in config/repos.json")
+    ap.add_argument("--out", required=True, help="output directory (work copies, results, review sheets)")
+    ap.add_argument("--venv", required=True, help="venv created from requirements.lock (run.sh does this)")
+    ap.add_argument("--only", default=",".join(ALL_STEPS), help="comma separated subset of: " + ",".join(ALL_STEPS))
+    ap.add_argument("--repo", action="append", help="limit to these repo names")
+    ap.add_argument("--fetch", action="store_true", help="run `git fetch origin` in each clone first (only network use against repos)")
+    ap.add_argument("--vector-results", help="optional JSON {question_id: [files...]} exported from pgvector")
+    ap.add_argument("--manual-dir", help="directory with filled review CSVs named <repo>.csv")
+    a = ap.parse_args()
+
+    steps = [s for s in a.only.split(",") if s]
+    out = Path(a.out).resolve()
+    (out / "results").mkdir(parents=True, exist_ok=True)
+    cfg = load_json(HARNESS_DIR / "config" / "repos.json")
+    symbols = load_json(HARNESS_DIR / "config" / "symbols.json")
+    questions = load_json(HARNESS_DIR / "config" / "questions.json")
+    vbin, graphify, vpy = venv_paths(a.venv)
+    home = out / "home"
+    home.mkdir(exist_ok=True)
+    env = clean_env(home, vbin)
+    vector = load_json(a.vector_results) if a.vector_results else None
+    wrapper, mech = rgf.network_wrapper()
+
+    meta = {
+        "date_utc": dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "graphify_version": graphify_version(vpy),
+        "python": sh([vpy, "--version"]),
+        "os": "%s %s %s" % (platform.system(), platform.release(), platform.machine()),
+        "cpu_count": os.cpu_count(),
+        "ram_gb": _ram_gb(),
+        "ripgrep": rg_version(),
+        "network_block": mech,
+        "env_keys_passed": sorted(env.keys()),
+    }
+    if wrapper:
+        ok, evidence = rgf.verify_block(wrapper, vpy)
+        meta["network_block_effective"] = ok
+        meta["network_block_probe"] = evidence
+    dump_json(out / "results" / "environment.json", meta)
+    log("graphify %s on %s" % (meta["graphify_version"], meta["os"]))
+
+    for repo in cfg["repos"]:
+        name = repo["name"]
+        if a.repo and name not in a.repo:
+            continue
+        clone = Path(a.repos_dir) / name
+        res_path = out / "results" / ("%s.json" % name)
+        res = load_json(res_path) if res_path.exists() else {"repo": name}
+        origin = check_origin(clone, cfg["allowed_origin_owners"])
+        if a.fetch:
+            sh(["git", "-C", str(clone), "fetch", "origin", "--quiet"])
+        ref = repo["ref"]
+        sha = sh(["git", "-C", str(clone), "rev-parse", ref])
+        work = out / "work" / name
+
+        if "snapshot" in steps:
+            log("%s: snapshot %s@%s" % (name, ref, sha[:8]))
+            snapshot(clone, ref, work)
+            tracked = [p for p in sh(["git", "-C", str(clone), "ls-tree", "-r", "--name-only", ref]).splitlines()]
+            symlinks = sh(["find", str(work), "-type", "l"], check=False).splitlines()
+            res["snapshot"] = {
+                "ref": ref, "sha": sha, "origin_owner_ok": True,
+                "code_files": len(iter_code_files(work)),
+                "tracked_graphify_out_files": sum(1 for p in tracked if p.startswith("graphify-out/")),
+                "symlinks_in_snapshot": len(symlinks),
+            }
+        if not work.exists():
+            raise SystemExit("no snapshot for %s; run the snapshot step first" % name)
+
+        if "cold" in steps:
+            log("%s: cold run" % name)
+            shutil.rmtree(work / "graphify-out", ignore_errors=True)
+            run = rgf.measure([graphify, "update", "."], work, env)
+            g = gm.load_graph(work)
+            res["cold"] = {"run": run, "g1": gm.g1_extraction(g, work), "g2": gm.g2_edges(g), "signature": gm.signature(g),
+                           "graphify_out_mb": round(sum(f.stat().st_size for f in (work / "graphify-out").rglob("*") if f.is_file()) / 1e6, 1)}
+        if "determinism" in steps:
+            log("%s: second cold run (determinism)" % name)
+            work_b = out / "work" / (name + "-b")
+            snapshot(clone, ref, work_b)
+            run = rgf.measure([graphify, "update", "."], work_b, env)
+            sig = gm.signature(gm.load_graph(work_b))
+            res["determinism"] = {"run": run, "signature": sig, "same_as_first": sig == res.get("cold", {}).get("signature")}
+            shutil.rmtree(work_b, ignore_errors=True)
+        if "network" in steps:
+            log("%s: cold run with network blocked + audit hook" % name)
+            work_n = out / "work" / (name + "-net")
+            snapshot(clone, ref, work_n)
+            if wrapper is None:
+                res["network"] = {"status": "not_measured", "reason": mech}
+            else:
+                run = rgf.measure([graphify, "update", "."], work_n, env, wrapper=wrapper, audit_file=out / ("audit-%s.tsv" % name))
+                res["network"] = {"status": "measured", "mechanism": mech, "block_effective": meta.get("network_block_effective"), "run": run}
+            shutil.rmtree(work_n, ignore_errors=True)
+        if "incremental" in steps:
+            log("%s: incremental update" % name)
+            probe = _append_probe(work)
+            changed = rgf.measure([graphify, "update", "."], work, env)
+            found = any(n["label"] in ("GvProbe", ".gvProbeMethod()", "gvProbeFunction()") for n in gm.load_graph(work)["nodes"])
+            unchanged = rgf.measure([graphify, "update", "."], work, env)
+            res["incremental"] = {"probe_file": probe, "changed": changed, "new_symbol_in_graph": found, "unchanged": unchanged}
+            # restore the probe so later steps see the original sources
+            (work / probe).write_text(_strip_probe((work / probe).read_text()), encoding="utf-8")
+        if "edges" in steps:
+            log("%s: edge sample" % name)
+            g = gm.load_graph(work)
+            rows, smeta = se.build_sample(g, work)
+            rows = se.merge_manual(rows, (Path(a.manual_dir) / ("%s.csv" % name)) if a.manual_dir else None)
+            se.write_csv(rows, out / "review" / ("%s.csv" % name))
+            res["edges"] = se.summarize(rows, smeta)
+        if "affected" in steps or "recall" in steps:
+            # incremental step modified the graph; rebuild a pristine graph for query-based steps
+            shutil.rmtree(work / "graphify-out", ignore_errors=True)
+            rgf.measure([graphify, "update", "."], work, env)
+            g = gm.load_graph(work)
+        if "affected" in steps:
+            log("%s: affected vs ripgrep" % name)
+            res["affected"] = affected_vs_rg.compare(graphify, work, g, symbols[name], env)
+        if "recall" in steps:
+            log("%s: recall@8" % name)
+            res["recall"] = recall_mod.evaluate(graphify, work, name, questions[name], env, vector=vector)
+        dump_json(res_path, res)
+
+    if "spring" in steps:
+        log("spring fixture")
+        fx = out / "work" / "spring-fixture"
+        if fx.exists():
+            shutil.rmtree(fx)
+        shutil.copytree(HARNESS_DIR / "fixtures" / "spring", fx, ignore=shutil.ignore_patterns("expected.json"))
+        run = rgf.measure([graphify, "update", "."], fx, env)
+        g = gm.load_graph(fx)
+        result = sf.check(g, sf.load_expected(HARNESS_DIR / "fixtures" / "spring" / "expected.json"))
+        result["run"] = run
+        dump_json(out / "results" / "spring.json", result)
+    if "licenses" in steps:
+        log("licenses")
+        raw = sh([vpy, "-I", str(HARNESS_DIR / "lib" / "licenses.py")])
+        (out / "results" / "licenses.json").write_text(raw, encoding="utf-8")
+    log("done; now run: python3 lib/report.py --out %s" % out)
+
+
+def _ram_gb():
+    try:
+        if platform.system() == "Darwin":
+            return round(int(sh(["sysctl", "-n", "hw.memsize"])) / 2**30, 1)
+        with open("/proc/meminfo") as fh:
+            return round(int(fh.readline().split()[1]) / 2**20, 1)
+    except Exception:
+        return None
+
+
+def _append_probe(work):
+    """Append a new symbol to a deterministic main source file; return its repo-relative path."""
+    exts = (".java", ".js", ".mjs", ".ts")
+    cands = [f for f in iter_code_files(work) if f.endswith(exts) and not is_test_path(f) and (work / f).stat().st_size > 1500]
+    probe = sorted(cands)[0]
+    path = work / probe
+    text = path.read_text(encoding="utf-8")
+    if probe.endswith(".java"):
+        text += "\n// GV-PROBE-BEGIN\nclass GvProbe {\n    int gvProbeMethod() { return 42; }\n}\n// GV-PROBE-END\n"
+    else:
+        text += "\n// GV-PROBE-BEGIN\nexport function gvProbeFunction() { return 42; }\n// GV-PROBE-END\n"
+    path.write_text(text, encoding="utf-8")
+    return probe
+
+
+def _strip_probe(text):
+    return re.sub(r"\n// GV-PROBE-BEGIN.*?// GV-PROBE-END\n", "", text, flags=re.S)
+
+
+if __name__ == "__main__":
+    main()