@vizalice/codegraph 1.5.0

package/scripts/agent-eval/audit.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# One-shot CodeGraph quality audit:
+#   set version -> ensure corpus repo -> wipe+reindex with that version ->
+#   run with/without A/B -> restore the local dev link.
+#
+# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
+#   <version>    "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
+#   <repo-name>  dir name under the corpus dir
+#   <repo-url>   git URL (cloned --depth 1 when the repo dir is missing)
+#   [mode]       headless (default) | all (also the interactive tmux arms)
+# Env: CORPUS  corpus dir (default: /tmp/codegraph-corpus)
+set -uo pipefail
+
+VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
+NAME="${2:?repo-name required}"
+URL="${3:?repo-url required}"
+Q="${4:?question required}"
+MODE="${5:-headless}"
+
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$HARNESS/../.." && pwd)"     # codegraph repo root
+CORPUS="${CORPUS:-/tmp/codegraph-corpus}"
+REPO="$CORPUS/$NAME"
+PKG="@colbymchenry/codegraph"
+
+echo "==================== CodeGraph audit ===================="
+echo "version=$VERSION  repo=$NAME  mode=$MODE  corpus=$CORPUS"
+echo
+
+# 1. Set the codegraph version under test (mutates the global install).
+if [ "$VERSION" = local ]; then
+  echo "→ [1/4] building + linking local dev build (local-install.sh)"
+  ( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
+else
+  echo "→ [1/4] installing $PKG@$VERSION globally"
+  npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
+fi
+ACTUAL="$(codegraph --version 2>/dev/null || echo '?')"
+echo "  codegraph on PATH: $(command -v codegraph) -> $ACTUAL"
+
+# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
+mkdir -p "$CORPUS"
+if [ -d "$REPO/.git" ]; then
+  echo "→ [2/4] reusing existing checkout: $REPO"
+else
+  echo "→ [2/4] cloning $URL"
+  git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
+fi
+
+# 3. Wipe + re-index with THIS version (the index must be built by the same
+#    binary that serves it — different versions extract differently).
+echo "→ [3/4] wiping .codegraph and re-indexing with $ACTUAL"
+rm -rf "$REPO/.codegraph"
+( cd "$REPO" && codegraph init -i ) || { echo "indexing failed"; exit 1; }
+
+# 4. Run the with/without A/B.
+echo "→ [4/4] running A/B harness (mode=$MODE)"
+bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
+
+# Restore the dev link (the normal working state in this repo).
+echo
+echo "→ restoring local dev link (local-install.sh)"
+if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
+  echo "  global codegraph restored to dev build"
+else
+  echo "  WARN: restore failed — run ./scripts/local-install.sh manually"
+fi
+echo "==================== audit complete ===================="

package/scripts/agent-eval/bench-readme.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Re-run the README "Benchmark Results" A/B (with vs without codegraph) on the
+# current build: the 7 README repos, same queries, RUNS per arm (default 4).
+# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}[.tN].jsonl
+# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
+# $CORPUS (default /tmp/codegraph-corpus) by the build under test.
+#
+# Each row is a THREE-TURN session: the README question, then two follow-ups
+# that stay inside the same flow. Turns 2-3 are where residual context occupancy
+# is actually charged — the first answer's tool output is still in the window,
+# so the arms diverge on how much headroom each left behind. CG_TURNS=1 runs the
+# README question alone (the original single-question A/B).
+set -uo pipefail
+H="$(cd "$(dirname "$0")" && pwd)"
+C="${CORPUS:-/tmp/codegraph-corpus}"
+RUNS="${RUNS:-4}"
+RUN_FROM="${RUN_FROM:-1}"   # extend an existing pass: RUN_FROM=3 RUNS=3 adds run3 only
+TURNS="${CG_TURNS:-3}"
+ROWS=(
+"vscode|How does the extension host communicate with the main process?|Where in that path would a message be dropped if the extension host crashes?|What would I need to change to add a new message type to that protocol?"
+"excalidraw|How does Excalidraw render and update canvas elements?|Which part of that path decides whether a full re-render happens or an incremental one?|If I added a new element type, what in that render path would need to change?"
+"django|How does Django's ORM build and execute a query from a QuerySet?|Where in that path is the SQL actually compiled into a string?|What would I change to add a new lookup type to that pipeline?"
+"tokio|How does tokio schedule and run async tasks on its runtime?|Where does a task move between the local and the global queue in that path?|What in that path would I touch to add a per-task instrumentation hook?"
+"okhttp|How does OkHttp process a request through its interceptor chain?|Where in that chain is the connection actually acquired?|What would I change to add a new interceptor stage before the cache?"
+"gin|How does gin route requests through its middleware chain?|Where is the 404 / no-route case handled in that same chain?|What would I change to add a per-route middleware that runs before the global ones?"
+"alamofire|How does Alamofire build, send, and validate a request?|Where does retry / interceptor logic hook into that path?|What would I change to add a new validation step to it?"
+)
+echo "### README A/B START $(date) RUNS=$RUN_FROM..$RUNS TURNS=$TURNS"
+for row in "${ROWS[@]}"; do
+  repo="${row%%|*}"; rest="${row#*|}"
+  # Take the first $TURNS questions and join them with "||" for run-all.sh.
+  q=""; n=0
+  while [ "$n" -lt "$TURNS" ] && [ -n "$rest" ]; do
+    part="${rest%%|*}"
+    if [ "$rest" = "$part" ]; then rest=""; else rest="${rest#*|}"; fi
+    [ -n "$q" ] && q="$q||"
+    q="$q$part"; n=$((n + 1))
+  done
+  echo "===== $repo ($n turns) ====="
+  for run in $(seq "$RUN_FROM" "$RUNS"); do
+    out="/tmp/ab-readme/$repo/run$run"
+    mkdir -p "$out"
+    AGENT_EVAL_OUT="$out" bash "$H/run-all.sh" "$C/$repo" "$q" headless > "$out/console.log" 2>&1
+    grep -E "^exit [0-9]" "$out/console.log" | sed 's/^/  /' || echo "  run$run: (no exit line)"
+    grep -E "codegraph +[0-9,]+ tok|→ file-access" "$out/console.log" | sed 's/^/  /' || true
+  done
+done
+echo "### README A/B DONE $(date)"

package/scripts/agent-eval/bench-why-repo.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# One README repo, WITH-codegraph only, N runs. Each run appends a why-Read
+# diagnostic so the agent explains any Read/Grep. (The WITHOUT baseline is
+# codegraph-independent and already in the README — no point re-running it.)
+# Output -> /tmp/ab-why/<repo>/with<n>.jsonl
+# Usage: bench-why-repo.sh <repo-path> "<query>" [N]
+set -uo pipefail
+REPO="$1"; Q="$2"; N="${3:-4}"
+NAME="$(basename "$REPO")"
+CG="/Users/colby/Development/Personal/codegraph/dist/bin/codegraph.js"
+OUT="/tmp/ab-why/$NAME"; mkdir -p "$OUT"
+WHY=$'\n\nIMPORTANT — diagnostic: if you use the Read or Grep tool at ANY point, for EACH such call explain why codegraph_explore / codegraph_node did not already give you what you needed. End your entire answer with a section titled exactly "## Why I read" listing every Read and Grep you made and the precise reason codegraph fell short for it. If you used neither, write "## Why I read" then "none — codegraph was sufficient."'
+printf '{"mcpServers":{"codegraph":{"command":"%s","args":["serve","--mcp","--path","%s"]}}}' "$CG" "$REPO" > "$OUT/cg.json"
+
+for i in $(seq 1 "$N"); do
+  pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.codegraph/daemon.sock"
+  ( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
+      --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
+      --strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
+  echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
+done
+echo "DONE $NAME"

package/scripts/agent-eval/block-read-hook.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# PreToolUse hook (experiment): deny Read of codegraph-indexed source files and
+# steer the agent to codegraph_explore/codegraph_node instead. Tests whether
+# codegraph can FULLY replace Read for code-understanding once the escape hatch
+# is removed. Non-source reads (config, .env, markdown, new files) pass through.
+#
+# Wire via:  claude ... --settings scripts/agent-eval/hook-settings.json
+set -uo pipefail
+input="$(cat)"
+fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
+
+case "$fp" in
+  *.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
+    msg="Read is disabled for source files in this session — codegraph already has this file indexed (with line numbers, kept in sync on every change). Use codegraph_explore (several related symbols at once) or codegraph_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER codegraph_explore with its exact name instead of reading the file."
+    jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
+    exit 0
+    ;;
+esac
+exit 0

package/scripts/agent-eval/compare-arms.mjs

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+// One side-by-side table for the three feedback metrics, across the arms of a
+// single A/B output directory. This is the "did it move?" view — the per-run
+// blocks parse-run.mjs prints are the "why did it move?" view, and both are
+// printed by the harnesses (ab-new-vs-baseline.sh, run-all.sh).
+//
+//   residual context occupancy  how much window the arm's retrieval still holds
+//   explore sufficiency         whether a response was ENOUGH (agent's next act)
+//   allocation efficiency       what share of returned bytes the answer used
+//
+// Usage: compare-arms.mjs <out-dir> <label> [<label> ...]
+//   e.g.  compare-arms.mjs /tmp/ab-new-vs-baseline new baseline
+//         compare-arms.mjs /tmp/agent-eval headless-with headless-without
+//
+// Run discovery handles both shapes the harnesses write, per label:
+//   run-<label>-<i>.jsonl        N independent runs   (ab-new-vs-baseline, RUNS=N)
+//   run-<label>.jsonl + .tN      ONE session, N turns (run-all.sh multi-turn)
+// A `.tN` file is always a resumed SEGMENT of the run it hangs off, never a run
+// of its own — mixing those up would report a three-turn session as three runs
+// and average away the residual the later turns exist to charge.
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { parseSession, SUFFICIENCY } from './parse-run.mjs';
+
+/** Segment files of one run, in turn order: run-X.jsonl, run-X.t2.jsonl, … */
+function segmentsOf(dir, stem) {
+  const first = join(dir, `${stem}.jsonl`);
+  if (!existsSync(first)) return null;
+  const rest = readdirSync(dir)
+    .map((f) => [f, new RegExp(`^${stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.t(\\d+)\\.jsonl$`).exec(f)])
+    .filter(([, m]) => m)
+    .sort((a, b) => Number(a[1][1]) - Number(b[1][1]))
+    .map(([f]) => join(dir, f));
+  return [first, ...rest];
+}
+
+/** Every run of one arm, newest-numbering-first-run order. */
+export function discoverRuns(dir, label) {
+  const session = segmentsOf(dir, `run-${label}`);
+  if (session) return [{ name: label, files: session }];
+  const indexed = readdirSync(dir)
+    .map((f) => new RegExp(`^run-${label.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}-(\\d+)\\.jsonl$`).exec(f))
+    .filter(Boolean)
+    .map((m) => Number(m[1]))
+    .sort((a, b) => a - b);
+  return indexed.map((i) => ({ name: `${label}-${i}`, files: segmentsOf(dir, `run-${label}-${i}`) }));
+}
+
+const median = (xs) => {
+  if (!xs.length) return null;
+  const a = [...xs].sort((x, y) => x - y);
+  const m = a.length >> 1;
+  return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
+};
+
+/** The numbers one arm's runs contribute to the table. */
+function measure(run) {
+  const s = parseSession(run.files);
+  const o = s.occupancy;
+  const a = s.allocation;
+  return {
+    name: run.name,
+    ok: s.ok,
+    raced: s.raced,
+    turns: s.turns,
+    dur: s.dur,
+    tools: s.tools,
+    reads: s.reads,
+    grep: s.grep,
+    bash: s.counts.Bash || 0,
+    cg: s.cg,
+    cliCalls: s.cliCalls,
+    cliContaminated: s.cliContaminated,
+    ctx: o.ctxFinal,
+    occCg: o.residual.codegraph,
+    occFile: o.residualFileAccess,
+    // The arm's OWN retrieval residual: codegraph in a with-arm, Read/Grep/Bash
+    // in a without-arm. Comparing these two is the apples-to-apples pair.
+    occSelf: o.residual.codegraph + o.residualFileAccess,
+    occShare: o.ctxFinal > 0 ? ((o.residual.codegraph + o.residualFileAccess) / o.ctxFinal) * 100 : 0,
+    suffAnswered: s.sufficiency.answered,
+    suffCounts: s.sufficiency.counts,
+    suffErrors: s.sufficiency.errors,
+    // Allocation is byte-weighted, so a run with no explore contributes nothing
+    // rather than a zero — a zero would drag the pooled number toward "wasteful"
+    // for a run that never spent a byte.
+    allocUsed: a.envelope ? a.used : null,
+    allocEnvelope: a.envelope || null,
+    allocCalls: a.calls.length,
+  };
+}
+
+/** median [min–max] over runs; the range is the point — never quote one run. */
+function span(runs, pick, fmt = (x) => String(Math.round(x))) {
+  const xs = runs.map(pick).filter((x) => x !== null && x !== undefined && Number.isFinite(x));
+  if (!xs.length) return '—';
+  const m = fmt(median(xs));
+  if (xs.length === 1) return m;
+  const lo = fmt(Math.min(...xs)); const hi = fmt(Math.max(...xs));
+  return lo === hi ? m : `${m} [${lo}–${hi}]`;
+}
+
+const int = (x) => Math.round(x).toLocaleString('en-US');
+const pct1 = (x) => `${x.toFixed(1)}%`;
+
+export function formatComparison(arms) {
+  const W = 36; const C = 24;
+  const out = [];
+  // The leading space is a separator, not padding: a `median [min–max]` cell can
+  // fill its column, and two of those with only padStart between them run
+  // together into one unreadable number.
+  const row = (label, cells) => out.push('  ' + label.padEnd(W) + cells.map((c) => ' ' + String(c).padStart(C - 1)).join(''));
+  const rule = (title) => out.push(`  ${title}`);
+
+  row('', arms.map((a) => a.label));
+  row('runs', arms.map((a) => a.runs.length));
+  const anyFailed = arms.some((a) => a.runs.some((r) => !r.ok));
+  if (anyFailed) row('  of which non-success', arms.map((a) => a.runs.filter((r) => !r.ok).length));
+  if (arms.some((a) => a.runs.some((r) => r.raced))) {
+    row('  MCP cold-start race', arms.map((a) => a.runs.filter((r) => r.raced).length));
+  }
+  out.push('');
+
+  rule('behavior');
+  row('  duration (s)', arms.map((a) => span(a.runs, (r) => r.dur)));
+  row('  tool calls', arms.map((a) => span(a.runs, (r) => r.tools)));
+  row('  Read', arms.map((a) => span(a.runs, (r) => r.reads)));
+  row('  Grep/Glob', arms.map((a) => span(a.runs, (r) => r.grep)));
+  row('  Bash', arms.map((a) => span(a.runs, (r) => r.bash)));
+  row('  codegraph calls', arms.map((a) => span(a.runs, (r) => r.cg)));
+  out.push('');
+
+  rule('residual context occupancy (CG-7) — tokens still resident at end of run');
+  row('  final context (tok)', arms.map((a) => span(a.runs, (r) => r.ctx, int)));
+  row('  codegraph residual (tok)', arms.map((a) => span(a.runs, (r) => r.occCg, int)));
+  row('  file-access residual (tok)', arms.map((a) => span(a.runs, (r) => r.occFile, int)));
+  row('  → retrieval residual (tok)', arms.map((a) => span(a.runs, (r) => r.occSelf, int)));
+  row('  → share of final context', arms.map((a) => span(a.runs, (r) => r.occShare, pct1)));
+  out.push('');
+
+  rule('explore sufficiency (CG-8) — pooled over every answered explore call');
+  row('  answered explore calls', arms.map((a) => a.runs.reduce((s, r) => s + r.suffAnswered, 0)));
+  for (const [key, label] of SUFFICIENCY) {
+    row(`  ${label}`, arms.map((a) => {
+      const n = a.runs.reduce((s, r) => s + r.suffCounts[key], 0);
+      const tot = a.runs.reduce((s, r) => s + r.suffAnswered, 0);
+      return tot ? `${n}  ${((n / tot) * 100).toFixed(0)}%` : '—';
+    }));
+  }
+  if (arms.some((a) => a.runs.some((r) => r.suffErrors))) {
+    row('  errored/unanswered (not bucketed)', arms.map((a) => a.runs.reduce((s, r) => s + r.suffErrors, 0)));
+  }
+  out.push('');
+
+  rule('explore allocation efficiency (CG-9) — share of returned bytes the answer cited');
+  row('  explore calls with source', arms.map((a) => a.runs.reduce((s, r) => s + r.allocCalls, 0)));
+  row('  pooled efficiency', arms.map((a) => {
+    const env = a.runs.reduce((s, r) => s + (r.allocEnvelope || 0), 0);
+    const used = a.runs.reduce((s, r) => s + (r.allocUsed || 0), 0);
+    return env ? pct1((used / env) * 100) : '—';
+  }));
+  row('  per-run efficiency', arms.map((a) =>
+    span(a.runs, (r) => (r.allocEnvelope ? (r.allocUsed / r.allocEnvelope) * 100 : null), pct1)));
+  row('  envelope (chars)', arms.map((a) => int(a.runs.reduce((s, r) => s + (r.allocEnvelope || 0), 0))));
+  out.push('');
+
+  rule('contamination — the CLI must never be how codegraph is reached');
+  row('  CLI calls that RETURNED output', arms.map((a) => a.runs.reduce((s, r) => s + r.cliContaminated, 0)));
+  row('  CLI attempts blocked', arms.map((a) => a.runs.reduce((s, r) => s + r.cliCalls, 0)));
+  const contaminated = arms.filter((a) => a.runs.some((r) => r.cliContaminated));
+  if (contaminated.length) {
+    out.push(`  !! ${contaminated.map((a) => a.label).join(', ')} reached codegraph through Bash — those runs are CONTAMINATED`);
+    out.push('     (a without-arm was not without codegraph; a with-arm has bytes attributed to Bash, not codegraph)');
+  }
+  out.push('');
+
+  out.push('  how to read this');
+  out.push('    occupancy  compare each arm\'s RETRIEVAL residual (codegraph in a with-arm,');
+  out.push('               file-access in a without-arm). Shares are Claude Code on a 200k');
+  out.push('               window and do NOT transfer to another host; the ratio does.');
+  out.push('    sufficiency  pooled across runs because it is per-CALL. "explore again" is');
+  out.push('               ambiguous by construction; "Read a file we returned" is an');
+  out.push('               allocation miss, the two recall rows are recall misses.');
+  out.push('    allocation  RELATIVE, not absolute — attribution is by citation, and an agent');
+  out.push('               can use a file without naming it. Compare builds on the SAME');
+  out.push('               question; never quote it as "codegraph wastes N% of what it returns."');
+  out.push('    all three  small-n. Runs make 1–5 explore calls, so read the range, not the');
+  out.push('               median of one run. RUNS>=2, and the 7-repo campaign for a verdict.');
+  return out.join('\n');
+}
+
+const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) {
+  const [dir, ...labels] = process.argv.slice(2);
+  if (!dir || !labels.length) {
+    console.error('usage: compare-arms.mjs <out-dir> <label> [<label> ...]');
+    process.exit(1);
+  }
+  const arms = labels.map((label) => ({ label, runs: discoverRuns(dir, label).map(measure) }));
+  const empty = arms.filter((a) => !a.runs.length);
+  if (empty.length === arms.length) {
+    console.error(`no run logs for ${labels.join('/')} in ${dir}`);
+    process.exit(1);
+  }
+  for (const a of empty) console.error(`  WARN: no run logs for arm '${a.label}' in ${dir}`);
+  console.log(`\n====== ARM COMPARISON — ${dir} ======`);
+  console.log(formatComparison(arms.filter((a) => a.runs.length)));
+}

package/scripts/agent-eval/diff-index-drift.mjs

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+/**
+ * Diff two CodeGraph indexes of the SAME tree — typically a live,
+ * incrementally-synced `.codegraph/codegraph.db` against a clean full rebuild
+ * of the identical working tree (CG-33).
+ *
+ * Non-destructive: it only reads. Rebuilding is the caller's job, so the live
+ * index is never clobbered by the tool measuring it — the mistake that cost the
+ * original CG-33 artifact.
+ *
+ *   # snapshot the live index BEFORE touching it
+ *   cp .codegraph/codegraph.db /tmp/live.db
+ *   node dist/bin/codegraph.js index .
+ *   node scripts/agent-eval/diff-index-drift.mjs /tmp/live.db .codegraph/codegraph.db
+ *
+ * Edges are compared as distinct `(source, target, kind)` triples. Raw row
+ * counts are NOT a drift signal: a bidirectional divergence nets out. On the
+ * codegraph repo the raw counts differed by +0.7% while 4.3% of distinct edges
+ * were actually wrong.
+ */
+import { DatabaseSync } from 'node:sqlite';
+import { existsSync } from 'node:fs';
+
+const [livePath, rebuiltPath] = process.argv.slice(2);
+if (!livePath || !rebuiltPath) {
+  console.error('usage: diff-index-drift.mjs <live.db> <rebuilt.db>');
+  process.exit(2);
+}
+for (const p of [livePath, rebuiltPath]) {
+  if (!existsSync(p)) {
+    // node:sqlite CREATES a missing file rather than failing, which silently
+    // yields an empty schema and a confident, wrong conclusion. Refuse first.
+    console.error(`not found: ${p}`);
+    process.exit(2);
+  }
+}
+
+const open = (p) => new DatabaseSync(p, { readOnly: true });
+const live = open(livePath);
+const rebuilt = open(rebuiltPath);
+
+const scalar = (db, q) => db.prepare(q).get().n;
+const edgeKey = (r) => `${r.source}\u0000${r.target}\u0000${r.kind}`;
+
+const liveEdges = live.prepare('select source, target, kind from edges').all();
+const rebuiltEdges = rebuilt.prepare('select source, target, kind from edges').all();
+const liveSet = new Set(liveEdges.map(edgeKey));
+const rebuiltSet = new Set(rebuiltEdges.map(edgeKey));
+
+const missing = rebuiltEdges.filter((r) => !liveSet.has(edgeKey(r)));   // should exist, doesn't
+const stale = liveEdges.filter((r) => !rebuiltSet.has(edgeKey(r)));      // exists, shouldn't
+const divergent = missing.length + stale.length;
+
+const byKind = (rows) => {
+  const m = new Map();
+  for (const r of rows) m.set(r.kind, (m.get(r.kind) ?? 0) + 1);
+  return [...m].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(', ') || '(none)';
+};
+
+const pct = (n, d) => (d ? ((n / d) * 100).toFixed(1) : '0.0');
+
+console.log(`live    ${livePath}`);
+console.log(`rebuilt ${rebuiltPath}`);
+console.log('');
+console.log('counts                 live      rebuilt');
+for (const [label, q] of [
+  ['files', 'select count(*) n from files'],
+  ['nodes', 'select count(*) n from nodes'],
+  ['edges (rows)', 'select count(*) n from edges'],
+  // Grouped rather than `count(distinct a || b || c)`: bare concatenation has no
+  // separator, so `(ab, c)` and `(a, bc)` would collapse into one.
+  ['edges (distinct)', 'select count(*) n from (select distinct source, target, kind from edges)'],
+  ['heuristic edges', "select count(*) n from edges where provenance='heuristic'"],
+]) {
+  console.log(`  ${label.padEnd(20)} ${String(scalar(live, q)).padEnd(9)} ${scalar(rebuilt, q)}`);
+}
+
+console.log('');
+console.log('edge divergence (distinct triples)');
+console.log(`  missing from live: ${missing.length}  — ${byKind(missing)}`);
+console.log(`  stale in live:     ${stale.length}  — ${byKind(stale)}`);
+console.log(`  TOTAL divergent:   ${divergent}  (${pct(divergent, rebuiltSet.size)}% of ${rebuiltSet.size})`);
+
+// Integrity checks — these separate "resolution went stale" (edges wrong, nodes
+// identical) from "residue accumulated" (duplicate/orphan rows). CG-33 is the
+// former: on the codegraph repo every check below was 0 on BOTH indexes.
+console.log('');
+console.log('integrity                        live   rebuilt');
+for (const [label, q] of [
+  ['duplicate nodes', 'select count(*) n from (select file_path,name,kind,start_line from nodes group by 1,2,3,4 having count(*)>1)'],
+  ['orphan edges', 'select count(*) n from edges e where not exists(select 1 from nodes where id=e.source) or not exists(select 1 from nodes where id=e.target)'],
+  ['nodes w/ missing file row', 'select count(*) n from nodes nd where not exists(select 1 from files f where f.path=nd.file_path)'],
+]) {
+  console.log(`  ${label.padEnd(30)} ${String(scalar(live, q)).padEnd(6)} ${scalar(rebuilt, q)}`);
+}
+
+console.log('');
+console.log(divergent === 0
+  ? 'CONVERGED — the synced index matches a full rebuild.'
+  : `DRIFTED — ${divergent} edges differ. Rebuild-vs-rebuild is 0 (the indexer is deterministic), so this is sync divergence, not noise.`);
+
+process.exitCode = divergent === 0 ? 0 : 1;

package/scripts/agent-eval/hook-settings.json

@@ -0,0 +1,15 @@
+{
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Read",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash /Users/colby/Development/Personal/codegraph/scripts/agent-eval/block-read-hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}

package/scripts/agent-eval/itrun.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
+# agent to finish, then print the tool-call breakdown from the session logs.
+#
+# Why interactive (not `claude -p`): headless print-mode picks the
+# general-purpose subagent, while real interactive sessions delegate to the
+# Explore subagent (or drive codegraph from the main thread). Only the
+# interactive TUI reproduces the behavior users actually see. (Idle-detection
+# technique borrowed from devpit's WaitForIdle.)
+#
+# Usage: itrun.sh <repo-path> <label> "<prompt>"
+# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
+# Requires: tmux 3.0+, a logged-in `claude` CLI, codegraph MCP configured.
+set -uo pipefail
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+SESSION="cgt_${LABEL}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/itrun-${LABEL}.txt"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
+
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Wide pane so the TUI doesn't hard-wrap tool lines.
+tmux new-session -d -s "$SESSION" -x 230 -y 60
+tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
+
+# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
+# welcome screen seconds before the input actually accepts keystrokes, so this is
+# necessary but NOT sufficient — the type-and-verify loop below is what proves
+# the input is live.
+ready=0
+for _ in $(seq 1 120); do
+  cap | grep -q "❯" && { ready=1; break; }
+  sleep 0.5
+done
+[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
+# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
+# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
+# cleared before the type-and-verify loop or keystrokes land on the menu.
+for _ in $(seq 1 20); do
+  cap | grep -q "trust this folder" || break
+  tmux send-keys -t "$SESSION" Enter
+  sleep 1
+done
+
+# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
+# landed in the input box, retry if it didn't (handles the early-❯ race where
+# the welcome screen shows the prompt glyph but MCP init is still eating keys).
+needle="${PROMPT:0:24}"
+typed=0
+for _ in $(seq 1 30); do
+  tmux send-keys -l -t "$SESSION" "$PROMPT"
+  sleep 1
+  if cap | grep -Fq "$needle"; then typed=1; break; fi
+  # Clear whatever partial text may have landed, then retry.
+  tmux send-keys -t "$SESSION" C-u
+  sleep 1
+done
+[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+sleep 0.5
+tmux send-keys -t "$SESSION" Enter
+
+# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
+# EVERY working state shows — both the pre-stream thinking phase
+# "(8s · thinking with max effort)" and the streaming phase
+# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
+# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
+# (some TUI versions/states show one but not the others).
+BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
+
+# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
+# fail loudly rather than silently reporting an empty run.
+started=0
+for _ in $(seq 1 120); do
+  cap | grep -qE "$BUSY_RE" && { started=1; break; }
+  sleep 0.5
+done
+[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Poll for idle. CRITICAL: Opus 4.8 (extended thinking) renders NO spinner /
+# "esc to interrupt" / timer while it STREAMS its final answer — those appear
+# only during the thinking + tool-use phases ("✻ Marinating… (32s · ↓ 1.3k
+# tokens · thinking with max effort)"). So BUSY_RE reads "not busy" for the whole
+# 10-30s answer stream, and any short not-busy threshold kills the run mid-answer
+# (the truncation bug). We therefore detect "done" by CONTENT STABILITY, not by a
+# spinner string: while the agent streams, the captured pane changes every poll,
+# so stability never accrues; it accrues only once the agent has finished and the
+# static "✻ Brewed for 1m 9s" summary is all that is left. BUSY_RE still hard-
+# resets stability (covers thinking/tool-use/live-timer, where text can briefly
+# sit still). Need STABLE_NEEDED polls (~8s) of zero pane change + ❯ present.
+# Content-stability is model-agnostic — it survives future spinner re-wordings.
+STABLE_NEEDED=16
+prev=""; stable=0
+for _ in $(seq 1 2400); do            # up to ~20 min
+  pane="$(cap)"
+  sig="$(printf '%s' "$pane" | tr -s '[:space:]' ' ')"
+  if printf '%s' "$pane" | grep -qE "$BUSY_RE"; then
+    stable=0                          # thinking / tool use / live timer → busy
+  elif [ -n "$sig" ] && [ "$sig" = "$prev" ] && printf '%s' "$pane" | grep -q "❯"; then
+    stable=$((stable+1)); [ "$stable" -ge "$STABLE_NEEDED" ] && break
+  else
+    stable=0                          # answer still streaming → pane changing
+  fi
+  prev="$sig"
+  sleep 0.5
+done
+sleep 1
+
+tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
+echo "captured $(wc -l < "$OUT") lines -> $OUT"
+grep -oE "Done \([^)]*\)|[A-Z][a-z]+ for ([0-9]+m )?[0-9]+s" "$OUT" | tail -1
+grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Clean tool breakdown from the session logs (main + subagents).
+node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true