@saluzi/codegraph 0.1.0 → 0.2.0

package/scripts/agent-eval/offload-eval-metrics.mjs

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+// Extract one eval run's metrics from its Claude stream-json transcript + the
+// offload usage sidecar log, emit ONE merged JSON line.
+//
+// Usage: extract-metrics.mjs --run <run.jsonl> --usage <usage.jsonl|-> \
+//          --arm <a> --rep <n> --repo <r> --tier <t> --q <question>
+import { readFileSync, existsSync } from 'fs';
+
+const args = {};
+for (let i = 2; i < process.argv.length; i += 2) args[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
+
+const runFile = args.run;
+const lines = existsSync(runFile) ? readFileSync(runFile, 'utf8').split('\n').filter(Boolean) : [];
+
+const toolCounts = {};
+let result = null;
+const tok = { gen: 0, fresh: 0, cached: 0 };
+const offloadAnswers = [];
+let exploreResults = 0; // tool_results from explore (offload or raw)
+let lastAssistantText = '';
+
+for (const line of lines) {
+  let ev; try { ev = JSON.parse(line); } catch { continue; }
+
+  // per-turn token usage (authoritative token measure; result.usage is last-turn only)
+  const u = ev.message?.usage;
+  if (u) {
+    tok.gen += u.output_tokens || 0;
+    tok.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+    tok.cached += u.cache_read_input_tokens || 0;
+  }
+
+  if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
+    for (const b of ev.message.content) {
+      if (b.type === 'tool_use') toolCounts[b.name] = (toolCounts[b.name] || 0) + 1;
+      if (b.type === 'text' && b.text?.trim()) lastAssistantText = b.text.trim();
+    }
+  }
+  // tool_results arrive in user messages
+  if (ev.type === 'user' && Array.isArray(ev.message?.content)) {
+    for (const b of ev.message.content) {
+      if (b.type !== 'tool_result') continue;
+      const text = Array.isArray(b.content)
+        ? b.content.map(c => (typeof c === 'string' ? c : c.text || '')).join('')
+        : (typeof b.content === 'string' ? b.content : '');
+      // An offload answer is either the 'plain'/'report' synthesis (carries the
+      // "Synthesized by CodeGraph" footer) or a 'refs' answer (carries the re-expanded
+      // "### Referenced source — verbatim" appendix). A refs call that cited nothing
+      // valid falls back to RAW source, which is correctly counted as a raw explore below.
+      if (/Synthesized by CodeGraph|### Referenced source — verbatim/.test(text)) { offloadAnswers.push(text); exploreResults++; }
+      else if (/Found \d+ symbols? across|\*\*Exploration:/.test(text)) exploreResults++;
+    }
+  }
+  if (ev.type === 'result') result = ev;
+}
+
+// offload usage sidecar (CodeGraph AI tokens + cost) — one JSON line per offload call
+const ai = { calls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, credits: 0, costUsd: 0, ms: 0 };
+if (args.usage && args.usage !== '-' && existsSync(args.usage)) {
+  for (const line of readFileSync(args.usage, 'utf8').split('\n').filter(Boolean)) {
+    let e; try { e = JSON.parse(line); } catch { continue; }
+    ai.calls++;
+    ai.promptTokens += e.promptTokens || 0;
+    ai.completionTokens += e.completionTokens || 0;
+    ai.totalTokens += e.totalTokens || 0;
+    ai.credits += e.creditsCharged || 0;
+    ai.costUsd += e.costUsd || 0;
+    ai.ms += e.ms || 0;
+  }
+}
+
+// front-load hook fired iff its injected header appears in the transcript
+const frontload = lines.some(l => l.includes('auto-retrieved for this question'));
+const get = (n) => toolCounts[n] || 0;
+const read = get('Read');
+const grep = get('Grep') + get('Bash') + get('Glob');
+const explore = get('mcp__codegraph__codegraph_explore');
+const cgAny = Object.keys(toolCounts).filter(k => /mcp__codegraph__/.test(k)).reduce((s, k) => s + toolCounts[k], 0);
+
+const out = {
+  repo: args.repo, tier: args.tier, arm: args.arm, rep: Number(args.rep), question: args.q,
+  ok: result?.subtype === 'success',
+  durationSec: result ? +(result.duration_ms / 1000).toFixed(1) : null,
+  numTurns: result?.num_turns ?? null,
+  costUsdMain: result ? +(result.total_cost_usd || 0).toFixed(4) : null,
+  tokGen: tok.gen, tokFresh: tok.fresh, tokCached: tok.cached, tokBillable: tok.gen + tok.fresh,
+  read, grep, explore, cgAny, frontload,
+  offloadFired: offloadAnswers.length,
+  ai,
+  // text payloads for the accuracy judge (kept separate; large)
+  finalAnswer: (result?.result || lastAssistantText || '').slice(0, 8000),
+  offloadAnswers: offloadAnswers.map(a => a.slice(0, 6000)),
+};
+process.stdout.write(JSON.stringify(out) + '\n');

package/scripts/agent-eval/offload-eval-refs1.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# ONE offload run on ONE indexed repo at a given offload STYLE (plain|refs), so we can
+# watch a single agent transcript at a time (the user's one-run-at-a-time methodology).
+# The OFFLOAD reasoning runs in the prewarmed DAEMON process, so the style env must be
+# set on BOTH the daemon and the client MCP config. Writes one metrics line to RESULTS
+# and leaves the raw stream-json at $RUNS/<repo>-<style>-<n>.jsonl for inspection.
+#
+# Usage: offload-eval-refs1.sh <indexed-repo> <style> <n> "<question>"
+set -uo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"; ENGINE="$(cd "$HERE/../.." && pwd)"; BIN="$ENGINE/dist/bin/codegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"; RUNS="$OUT/runs"; EXTRACT="$HERE/offload-eval-metrics.mjs"
+TARGET="${1:?repo}"; STYLE="${2:?style}"; N="${3:?run-tag}"; Q="${4:?question}"
+RESULTS="${RESULTS:-$OUT/results-refs.jsonl}"; REPO=$(basename "$TARGET"); TARGET=$(cd "$TARGET" && pwd -P)
+mkdir -p "$RUNS"; command -v claude >/dev/null || { echo "no claude"; exit 1; }
+USAGE="$RUNS/$REPO-$STYLE-usage.jsonl"; : > "$USAGE"
+CFG="$RUNS/mcp-$REPO-$STYLE.json"
+# `raw` is a pseudo-style: codegraph attached but the offload DISABLED (the ceiling —
+# verbatim source, no reasoning model). Any other value is an offload style (plain|refs).
+if [ "$STYLE" = "raw" ]; then
+  DAEMON_ENV="CODEGRAPH_OFFLOAD_DISABLE=1"
+  printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' \
+    "$BIN" "$TARGET" > "$CFG"
+  USAGE="-"
+else
+  DAEMON_ENV="CODEGRAPH_OFFLOAD_STYLE=$STYLE CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
+  printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_STYLE=%s","CODEGRAPH_OFFLOAD_USAGE_LOG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \
+    "$STYLE" "$USAGE" "$BIN" "$TARGET" > "$CFG"
+fi
+
+# Prewarm a persistent daemon carrying the SAME offload config (it does the reasoning).
+pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
+env $DAEMON_ENV CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 \
+  node "$BIN" serve --mcp --path "$TARGET" </dev/null >/dev/null 2>&1 &
+node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$TARGET" \
+  && echo "daemon warm ($STYLE)" || echo "WARN daemon never bound"
+
+tag="$REPO-$STYLE-$N"
+echo "== run $tag =="
+# DISALLOW (optional): block tools that confound the offload-sufficiency signal —
+# chiefly "Agent" (sub-agent delegation: the spawned Explore subagent has low MCP
+# salience, ignores codegraph, and thrashes via Bash+Read, making the A/B noise).
+( cd "$TARGET" && claude -p "$Q" --output-format stream-json --verbose --permission-mode bypassPermissions \
+    --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
+    ${DISALLOW:+--disallowedTools "$DISALLOW"} \
+    --strict-mcp-config --mcp-config "$CFG" </dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
+node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$USAGE" --arm "offload-$STYLE" --rep "$N" \
+    --repo "$REPO" --tier "complex" --q "$Q" >> "$RESULTS"
+node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(`  [${o.arm} #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable} tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}call/${o.ai.totalTokens}tok/$${o.ai.costUsd.toFixed(4)} | ok=${o.ok}`)' "$RESULTS"
+pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null
+echo "raw transcript: $RUNS/$tag.jsonl"

package/scripts/agent-eval/offload-eval-setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Clone + index the 4 "not-trained-on" eval repos into $AGENT_EVAL_OUT/repos. These were
+# selected via a no-tools memory-probe gate (Sonnet cannot answer their flow questions from
+# memory — so the no-codegraph baseline is honest). Env: AGENT_EVAL_OUT=<scratch dir>
+set -uo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ENGINE="$(cd "$HERE/../.." && pwd)"
+BIN="$ENGINE/dist/bin/codegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
+ROOT="$OUT/repos"; mkdir -p "$ROOT"
+export CODEGRAPH_TELEMETRY=0 DO_NOT_TRACK=1
+[ -f "$BIN" ] || { echo "engine not built: run 'npm run build' in $ENGINE first"; exit 1; }
+
+clone_index() { # url name
+  echo "=== $2: clone ==="; rm -rf "$ROOT/$2"
+  git clone --quiet --depth 1 "$1" "$ROOT/$2" || { echo "  clone FAILED"; return 1; }
+  echo "=== $2: index ==="
+  node "$BIN" init "$ROOT/$2" 2>&1 | grep -iE 'indexed|nodes|edges|error' | tail -2
+}
+clone_index https://github.com/MTKruto/MTKruto.git mtkruto          # small  (~322 TS)
+clone_index https://github.com/mvdicarlo/postybirb-plus.git postybirb  # medium (~608 TS)
+clone_index https://github.com/shapeshift/web.git shapeshift        # complex (~3.2k TS, 35-pkg monorepo)
+clone_index https://github.com/trezor/trezor-suite.git trezor       # large  (~8k TS monorepo)
+echo "###### SETUP DONE -> $ROOT"

package/scripts/agent-eval/offload-eval-styles.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Offload reasoning-OUTPUT-STYLE A/B — all codegraph-on, isolating the Worker's
+# output shape's effect on main-session tokens / latency / accuracy:
+#   raw  : CODEGRAPH_OFFLOAD_DISABLE=1            (verbatim explore source, the floor)
+#   refs : managed offload, default              (Cerebras map re-expanded to verbatim, ~24K)
+#   map  : managed offload, STYLE=map            (compact reasoned map + file:line anchors, ~1-3K)
+#   src  : managed offload, STYLE=src            (map + cited line ranges only, ~1-5K)
+# Delegation BLOCKED by default (DISALLOW=Agent) so we measure the offload payload's
+# effect on the main Sonnet agent, not whether it spawns a Haiku Explore subagent.
+#
+# Usage: offload-eval-styles.sh <indexed-repo> <reps> "<question>"
+# Env:   RESULTS=<file>  AGENT_EVAL_OUT=<dir>  REP_START=1  DISALLOW=Agent  MODEL/EFFORT
+set -uo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ENGINE="$(cd "$HERE/../.." && pwd)"
+BIN="$ENGINE/dist/bin/codegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
+TARGET="${1:?usage: offload-eval-styles.sh <indexed-repo> <reps> \"<question>\"}"
+REPS="${2:?reps}"; Q="${3:?question}"
+RUNS="$OUT/runs"; EXTRACT="$HERE/offload-eval-metrics.mjs"
+RESULTS="${RESULTS:-$OUT/results-styles.jsonl}"
+REPO=$(basename "$TARGET")
+DISALLOW="${DISALLOW-Agent}"   # default: block delegation. `DISALLOW= ` to allow.
+START="${REP_START:-1}"; END=$((START + REPS - 1))
+mkdir -p "$RUNS"
+command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
+[ -d "$TARGET/.codegraph" ] || { echo "not indexed: $TARGET"; exit 1; }
+TARGET=$(cd "$TARGET" && pwd -P)
+
+prewarm() { # path  extra-env
+  pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
+  env ${2:-} CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
+    && echo "  daemon warm" || echo "  WARN daemon never bound"
+}
+kill_daemon() { pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 1; }
+
+run() { # arm rep mcp-config usage-log-or-dash
+  local arm="$1" rep="$2" cfg="$3" usage="$4" tag="$REPO-$1-$2"
+  [ "$usage" != "-" ] && : > "$usage"
+  ( cd "$TARGET" && claude -p "$Q" \
+      --output-format stream-json --verbose --permission-mode bypassPermissions \
+      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
+      ${DISALLOW:+--disallowedTools "$DISALLOW"} \
+      --strict-mcp-config --mcp-config "$cfg" \
+      </dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
+  node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$usage" --arm "$arm" --rep "$rep" \
+      --repo "$REPO" --tier styles --q "$Q" >> "$RESULTS"
+  node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(`  [${o.arm} #${o.rep}] ${o.durationSec}s | ${o.tokBillable} billable tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}c/${o.ai.totalTokens}t | ok=${o.ok}`)' "$RESULTS"
+}
+
+# MCP configs: env baked into the daemon-spawn command claude uses.
+USAGE="$RUNS/$REPO-usage.jsonl"
+mkcfg() { # file extra-env-pairs(JSON array entries, comma-led or empty)
+  printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1"%s,"node","%s","serve","--mcp","--path","%s"]}}}' "$1" "$BIN" "$TARGET"
+}
+CFG_RAW="$RUNS/mcp-sty-raw-$REPO.json";   mkcfg ',"CODEGRAPH_OFFLOAD_DISABLE=1"' > "$CFG_RAW"
+CFG_REFS="$RUNS/mcp-sty-refs-$REPO.json"; mkcfg ",\"CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE\"" > "$CFG_REFS"
+CFG_MAP="$RUNS/mcp-sty-map-$REPO.json";   mkcfg ",\"CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE\",\"CODEGRAPH_OFFLOAD_STYLE=map\"" > "$CFG_MAP"
+CFG_SRC="$RUNS/mcp-sty-src-$REPO.json";   mkcfg ",\"CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE\",\"CODEGRAPH_OFFLOAD_STYLE=src\"" > "$CFG_SRC"
+
+echo "###### repo=$REPO reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high} disallow=${DISALLOW:-<none>}"
+echo "###### Q=$Q"
+echo "== ARM raw ==";  prewarm "$TARGET" "CODEGRAPH_OFFLOAD_DISABLE=1"
+for r in $(seq "$START" "$END"); do run raw  "$r" "$CFG_RAW"  "-"; done; kill_daemon
+echo "== ARM refs =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
+for r in $(seq "$START" "$END"); do run refs "$r" "$CFG_REFS" "$USAGE"; done; kill_daemon
+echo "== ARM map ==";  prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE CODEGRAPH_OFFLOAD_STYLE=map"
+for r in $(seq "$START" "$END"); do run map  "$r" "$CFG_MAP"  "$USAGE"; done; kill_daemon
+echo "== ARM src ==";  prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE CODEGRAPH_OFFLOAD_STYLE=src"
+for r in $(seq "$START" "$END"); do run src  "$r" "$CFG_SRC"  "$USAGE"; done; kill_daemon
+echo "###### DONE $REPO — judge: node $HERE/offload-eval-judge.mjs --results $RESULTS --truth $HERE/offload-eval-ground-truth.json --out $OUT/judged-styles.jsonl"

package/scripts/agent-eval/offload-eval-summarize.mjs

@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+// Aggregate judged.jsonl (or results.jsonl) into a per-repo, per-arm report:
+// time, main tokens/cost, AI tokens/cost, total cost, tool mix, accuracy.
+// Usage: summarize.mjs <judged-or-results.jsonl>
+import { readFileSync } from 'fs';
+const rows = readFileSync(process.argv[2], 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
+
+const med = (xs) => { const a = xs.filter(x => x != null).sort((p, q) => p - q); if (!a.length) return null; const m = Math.floor(a.length / 2); return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2; };
+const rng = (xs) => { const a = xs.filter(x => x != null); return a.length ? `${Math.min(...a)}–${Math.max(...a)}` : '—'; };
+const d2 = (x) => x == null ? '—' : (+x).toFixed(2);
+const d3 = (x) => x == null ? '—' : (+x).toFixed(3);
+const d4 = (x) => x == null ? '—' : (+x).toFixed(4);
+
+const ARM_ORDER = ['frontload', 'offload', 'raw', 'nocg'];
+const byRepo = {};
+for (const r of rows) (byRepo[r.repo] ??= {});
+for (const r of rows) ((byRepo[r.repo][r.arm] ??= []).push(r));
+
+const verdictTally = (rs, field) => {
+  const t = { pass: 0, partial: 0, fail: 0, error: 0 };
+  for (const r of rs) { const v = r[field]?.verdict; if (v in t) t[v]++; }
+  return t;
+};
+
+for (const repo of Object.keys(byRepo)) {
+  const tier = byRepo[repo][Object.keys(byRepo[repo])[0]][0].tier;
+  console.log(`\n${'='.repeat(78)}\n${repo}  [${tier}]\n${'='.repeat(78)}`);
+  console.log(`${'arm'.padEnd(9)} n  ${'time(s)'.padStart(9)} ${'mainCost'.padStart(9)} ${'aiCost'.padStart(8)} ${'totCost'.padStart(8)} ${'mainTok'.padStart(8)} ${'aiTok'.padStart(7)} ${'rd'.padStart(3)} ${'gr'.padStart(3)} ${'exp'.padStart(3)} ${'off'.padStart(3)}  e2e(P/p/F)  fidScore`);
+  for (const arm of ARM_ORDER) {
+    const rs = byRepo[repo][arm]; if (!rs) continue;
+    const n = rs.length;
+    const mainCost = med(rs.map(r => r.costUsdMain));
+    const aiCost = med(rs.map(r => r.ai?.costUsd ?? 0));
+    const totCost = (mainCost ?? 0) + (aiCost ?? 0);
+    const e2e = verdictTally(rs, 'e2e');
+    const fidScores = arm === 'offload' ? rs.flatMap(r => r.fidelity?.scores ?? []) : [];
+    const fid = fidScores.length ? med(fidScores) : null;
+    const fab = arm === 'offload' && rs.some(r => r.fidelity?.anyFabrication);
+    const e2eScore = med(rs.map(r => r.e2e?.score).filter(x => x != null));
+    console.log(
+      `${arm.padEnd(9)} ${String(n).padStart(1)}  ${String(med(rs.map(r => r.durationSec))).padStart(9)} ` +
+      `${('$' + d3(mainCost)).padStart(9)} ${('$' + d3(aiCost)).padStart(8)} ${('$' + d3(totCost)).padStart(8)} ` +
+      `${String(Math.round(med(rs.map(r => r.tokBillable)) / 1000) + 'k').padStart(8)} ${String(Math.round(med(rs.map(r => r.ai?.totalTokens ?? 0)) / 1000) + 'k').padStart(7)} ` +
+      `${String(med(rs.map(r => r.read))).padStart(3)} ${String(med(rs.map(r => r.grep))).padStart(3)} ${String(med(rs.map(r => r.explore))).padStart(3)} ${String(med(rs.map(r => r.offloadFired))).padStart(3)}  ` +
+      `${(e2e.pass + '/' + e2e.partial + '/' + e2e.fail).padStart(9)}  ${e2eScore != null ? 'e2e=' + e2eScore : ''} ${fid != null ? 'fid=' + fid + (fab ? ' FAB!' : '') : ''}`
+    );
+  }
+  // ranges line for the two key metrics (variance matters)
+  for (const arm of ARM_ORDER) {
+    const rs = byRepo[repo][arm]; if (!rs) continue;
+    console.log(`   ${arm} ranges: time ${rng(rs.map(r => r.durationSec))}s · mainCost $${rng(rs.map(r => r.costUsdMain))} · read ${rng(rs.map(r => r.read))} · explore ${rng(rs.map(r => r.explore))} · offloadFired ${rng(rs.map(r => r.offloadFired))}`);
+  }
+}
+
+// Cross-repo roll-up: offload vs raw vs nocg deltas
+console.log(`\n${'='.repeat(78)}\nCROSS-REPO SUMMARY (medians per repo, then averaged)\n${'='.repeat(78)}`);
+console.log(`${'repo'.padEnd(12)} ${'arm'.padEnd(8)} ${'time'.padStart(7)} ${'totCost'.padStart(8)} ${'read'.padStart(5)} ${'e2e pass%'.padStart(9)} ${'fid'.padStart(5)}`);
+for (const repo of Object.keys(byRepo)) {
+  for (const arm of ARM_ORDER) {
+    const rs = byRepo[repo][arm]; if (!rs) continue;
+    const e2e = verdictTally(rs, 'e2e');
+    const passPct = Math.round(100 * e2e.pass / rs.length);
+    const totCost = (med(rs.map(r => r.costUsdMain)) ?? 0) + (med(rs.map(r => r.ai?.costUsd ?? 0)) ?? 0);
+    const fid = arm === 'offload' ? med(rs.flatMap(r => r.fidelity?.scores ?? [])) : null;
+    console.log(`${repo.padEnd(12)} ${arm.padEnd(8)} ${(med(rs.map(r => r.durationSec)) + 's').padStart(7)} ${('$' + d3(totCost)).padStart(8)} ${String(med(rs.map(r => r.read))).padStart(5)} ${(passPct + '%').padStart(9)} ${String(fid ?? '—').padStart(5)}`);
+  }
+}
+console.log('');

package/scripts/agent-eval/offload-eval.md

@@ -0,0 +1,76 @@
+# CodeGraph AI offload — accuracy & adoption eval harness
+
+Measures the managed **offload** (`codegraph_explore` → reasoning model synthesis) and the
+**front-load hook** (approach 1) against plain codegraph and no-codegraph, across repo sizes,
+on **time · main-session tokens/cost · CodeGraph-AI tokens/cost · accuracy**.
+
+All agent arms run `claude -p --model sonnet --effort high` (the deliberate floor model — an
+affordance that lands on Sonnet generalizes up). Everything writes to a scratch dir
+(`AGENT_EVAL_OUT`, default `/tmp/cg-offload-eval`); nothing here is shipped to users.
+
+## Repos (selected via a memory-probe gate — NOT trained on)
+
+Famous repos (express, excalidraw, n8n, …) are useless for *accuracy* evals: Sonnet answers their
+flow questions from memory, so the no-codegraph baseline is dishonest. These four passed a no-tools
+probe (Sonnet could not name their real flow internals) and are cloned fresh by `offload-eval-setup.sh`:
+
+| tier | repo | ~src files | canonical flow |
+|---|---|---|---|
+| small | MTKruto/MTKruto | 322 TS | `sendMessage` → invoke → TL serialize → transport |
+| medium | mvdicarlo/postybirb-plus | 608 TS | submission → queue → per-website `.post()` |
+| complex | shapeshift/web | 3.2k TS (35-pkg monorepo) | swap → swapper registry → concrete swapper |
+| large | trezor/trezor-suite | 8k TS monorepo | send-form → sign thunk → `@trezor/connect` |
+
+Verified ground-truth flows (the judge's reference) live in `offload-eval-ground-truth.json`.
+
+## Arms
+
+- **offload** — codegraph + managed offload ON (requires `codegraph login`); records AI tokens/credits via `CODEGRAPH_OFFLOAD_USAGE_LOG`.
+- **raw** — codegraph, `CODEGRAPH_OFFLOAD_DISABLE=1` (returns raw source).
+- **nocg** — empty MCP config; Read/Grep baseline.
+- **frontload** — codegraph (offload-disabled) + a `UserPromptSubmit` hook (`offload-eval-hook.mjs`) that runs raw explore on the prompt and injects the result into context (approach 1).
+
+## Run it
+
+```bash
+npm run build                       # the harness shells out to dist/
+codegraph login                     # only needed for the offload arm
+export AGENT_EVAL_OUT=/tmp/cg-offload-eval
+
+bash scripts/agent-eval/offload-eval-setup.sh            # clone + index the 4 repos
+bash scripts/agent-eval/offload-eval-matrix.sh           # 3 arms × 4 tiers × REPS (default 3)
+node scripts/agent-eval/offload-eval-judge.mjs \
+     --results $AGENT_EVAL_OUT/results.jsonl \
+     --truth  scripts/agent-eval/offload-eval-ground-truth.json \
+     --out    $AGENT_EVAL_OUT/judged.jsonl
+node scripts/agent-eval/offload-eval-summarize.mjs $AGENT_EVAL_OUT/judged.jsonl
+
+bash scripts/agent-eval/offload-eval-frontload-matrix.sh # frontload arm + judge + merged summary
+```
+
+Single repo: `offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"` (or `-frontload.sh`).
+
+## Files
+
+- `offload-eval-setup.sh` — clone + index the 4 repos.
+- `offload-eval-3arm.sh` / `-frontload.sh` — one repo, the arms.
+- `offload-eval-matrix.sh` / `-frontload-matrix.sh` — drive all 4 tiers.
+- `offload-eval-hook.mjs` — the front-load `UserPromptSubmit` hook (resolves its own engine; `CG_FRONTLOAD_DEBUG=<path>` to log injections; `CG_FRONTLOAD_BUDGET` to cap injected chars).
+- `offload-eval-metrics.mjs` — one run's stream-json + usage log → one JSON metrics line.
+- `offload-eval-judge.mjs` — Sonnet judge: end-to-end (agent final vs ground truth) + per-answer offload fidelity.
+- `offload-eval-summarize.mjs` — per-tier, per-arm table + cross-repo roll-up.
+- `offload-eval-ground-truth.json` — source-verified canonical flows.
+
+## Findings (2026-06, n=3 — direction consistent, magnitudes noisy)
+
+- **Raw codegraph is the efficiency win** — ~nocg accuracy, fewer reads, faster, no AI cost.
+- **The offload is the least-accurate arm in all 4 tiers** — synthesized fidelity 12–27/100 with
+  fabrication in 3/4 (e.g. invented website services; traced `ClientPlain`/`SessionPlain` instead of
+  the real encrypted path). Its speed/cost win is narrow (medium-only) and inversely correlated with
+  accuracy. **Use raw until offload fidelity is fixed.**
+- **The front-load hook SOLVES adoption** — reads → 0–1 in every tier (incl. large, where the agent
+  otherwise read 12–24 files); fired 12/12, 0 errors. Wins on medium/complex (100% pass). But it
+  **regresses small/large to partial** — it suppresses the reads that compensate for explore's gaps at
+  **dynamic boundaries** (async queues, redux thunks, facade/factory indirection).
+- **Master lever for BOTH:** explore's dynamic-dispatch coverage. Fix it → front-load is complete
+  everywhere and the offload has the full flow to synthesize.

package/scripts/agent-eval/parse-arms.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+// Analyze the tool-surface ablation (/tmp/arms/<repo>/<arm>-r<n>.jsonl).
+// Compares arms A–E on trace adoption, Read/Grep fallback, codegraph payload,
+// round-trips, and duration — averaged across runs per arm.
+//
+// The decisive signal is READS: if removing a tool raises Reads on a question
+// class, that tool was load-bearing for it (not redundant). If removing it
+// changes nothing, it was redundant.
+//
+//   A control       all tools            no steering   (baseline)
+//   B steer         all tools            trace-first   (adoption)
+//   C no-explore    hide explore         trace-first   (is explore redundant?)
+//   D trace-centric hide explore+context trace-first   (is the survey pair redundant?)
+//   E control-probe hide explore+context trace-first   (NON-flow Q — should degrade)
+//
+// Usage: node scripts/agent-eval/parse-arms.mjs [/tmp/arms]
+import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
+import { join } from 'path';
+
+const ROOT = process.argv[2] || '/tmp/arms';
+const cgShort = (n) => n.replace('mcp__codegraph__codegraph_', '').replace('mcp__codegraph__', '');
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  const calls = []; let result = null, initCg = 0;
+  for (const l of lines) {
+    let ev; try { ev = JSON.parse(l); } catch { continue; }
+    if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /codegraph/.test(t)).length;
+    if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use')
+      calls.push({ id: b.id, name: b.name, out: 0 });
+    if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
+      const c = b.content;
+      const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
+      const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
+    }
+    if (ev.type === 'result') result = ev;
+  }
+  const cg = calls.filter(c => c.name.includes('codegraph'));
+  return {
+    initCg,
+    reads: calls.filter(c => c.name === 'Read').length,
+    greps: calls.filter(c => c.name === 'Grep').length + calls.filter(c => c.name === 'Glob').length,
+    cgCalls: cg.length,
+    cgSeq: cg.map(c => cgShort(c.name)),
+    cgOut: cg.reduce((s, c) => s + c.out, 0),
+    traceUsed: cg.some(c => c.name.includes('trace')),
+    turns: result?.num_turns ?? null,
+    dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
+    cost: result?.total_cost_usd || 0,
+    ok: result?.subtype === 'success',
+  };
+}
+
+// repo -> arm -> [runs]
+const data = {};
+if (!existsSync(ROOT)) { console.error(`no ${ROOT}`); process.exit(1); }
+for (const repo of readdirSync(ROOT)) {
+  const rdir = join(ROOT, repo);
+  if (!statSync(rdir).isDirectory()) continue;
+  for (const f of readdirSync(rdir)) {
+    const m = f.match(/^([A-I])-r(\d+)\.jsonl$/); if (!m) continue;
+    const p = parse(join(rdir, f)); if (!p || !p.ok) continue;
+    (((data[repo] ??= {})[m[1]]) ??= []).push(p);
+  }
+}
+
+const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
+const k = (n) => (n / 1000).toFixed(1);
+const pad = (s, n) => String(s).padEnd(n);
+const ARMS = ['A', 'H', 'I', 'B', 'F', 'G', 'C', 'D', 'E'];
+const LABEL = { A: 'A all/none(old)', H: 'H body-trace/none', I: 'I bodytrace+dest', B: 'B all/steer(thin)', F: 'F all/steer(body)', G: 'G ported(noprompt)', C: 'C no-explore', D: 'D trace-centric', E: 'E nonflow-probe' };
+
+// ---- per repo × arm ----
+console.log('\n=== PER REPO × ARM (avg over runs) ===');
+console.log(pad('repo', 22), pad('arm', 16), 'tools', 'trace', pad('reads', 6), pad('cgOutK', 7), pad('turns', 6), 'dur');
+for (const repo of Object.keys(data).sort()) {
+  for (const arm of ARMS) {
+    const runs = data[repo][arm]; if (!runs?.length) continue;
+    console.log(
+      pad(repo, 22), pad(LABEL[arm], 16),
+      pad(runs[0].initCg, 5),
+      pad(runs.filter(r => r.traceUsed).length + '/' + runs.length, 5),
+      pad(avg(runs, r => r.reads).toFixed(1), 6),
+      pad(k(avg(runs, r => r.cgOut)), 7),
+      pad(avg(runs, r => r.turns).toFixed(1), 6),
+      avg(runs, r => r.dur).toFixed(0) + 's',
+    );
+  }
+}
+
+// ---- aggregate per arm (flow arms A–D over the flow repos; E shown apart) ----
+console.log('\n=== AGGREGATE PER ARM (mean across repos) ===');
+console.log(pad('arm', 16), pad('adoption', 9), pad('reads', 7), pad('greps', 7), pad('cgOutK', 8), pad('turns', 7), pad('dur', 6), 'cost');
+for (const arm of ARMS) {
+  const all = [];
+  for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) all.push({ ...r, repo });
+  if (!all.length) continue;
+  const repos = new Set(all.map(r => r.repo)).size;
+  const adopt = all.filter(r => r.traceUsed).length;
+  console.log(
+    pad(LABEL[arm], 16),
+    pad(`${adopt}/${all.length}`, 9),
+    pad(avg(all, r => r.reads).toFixed(2), 7),
+    pad(avg(all, r => r.greps).toFixed(2), 7),
+    pad(k(avg(all, r => r.cgOut)), 8),
+    pad(avg(all, r => r.turns).toFixed(1), 7),
+    pad(avg(all, r => r.dur).toFixed(0) + 's', 6),
+    '$' + avg(all, r => r.cost).toFixed(3),
+    `  (${repos} repos)`,
+  );
+}
+
+console.log('\nRead the signal: B vs A = does steering alone fix adoption + cut payload.');
+console.log('C vs B = is explore redundant (reads should NOT jump). D vs C = is context redundant.');
+console.log('E = non-flow under trace-centric; reads SHOULD jump (proves survey tools are load-bearing).');

package/scripts/agent-eval/parse-bench-readme.mjs

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+// Aggregate the README A/B (bench-readme.sh output): per repo, median of N runs
+// per arm → time, tool calls, tokens, cost, and % saved. Plus an average row.
+//
+// Tokens = SUM of per-turn assistant `usage` (input + output + cache read +
+// cache creation) — the cumulative "total tokens processed". NOTE: `result.usage`
+// is last-turn-only in current Claude Code, so it under-counts badly; don't use it.
+// `total_cost_usd` and `duration_ms` are already cumulative.
+//
+// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+const ROOT = process.argv[2] || '/tmp/ab-readme';
+const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const L = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null, raced = false;
+  for (const l of L) { let e; try { e = JSON.parse(l); } catch { continue; }
+    if (e.type === 'assistant') {
+      const u = e.message?.usage;
+      if (u) tokens += (u.input_tokens || 0) + (u.output_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+      for (const b of (e.message?.content || [])) if (b.type === 'tool_use') {
+        const n = b.name;
+        if (n === 'ToolSearch') continue;
+        tools++;
+        if (n === 'Read') reads++;
+        else if (n === 'Grep' || n === 'Glob') grep++;
+        else if (/codegraph/.test(n)) cg++;
+      }
+    }
+    // MCP cold-start race: the headless agent fired before `codegraph serve --mcp`
+    // finished registering its tools, so early calls returned "No such tool
+    // available" and the agent floundered into grep/Read. That measures CodeGraph's
+    // startup latency, NOT its steady-state value — flag the run so the aggregate
+    // can exclude it (an artifact of headless first-turn timing, not the tool).
+    if (e.type === 'user') for (const b of (Array.isArray(e.message?.content) ? e.message.content : [])) {
+      if (b.type === 'tool_result') {
+        const t = Array.isArray(b.content) ? b.content.map(c => c.text || '').join('') : (b.content || '');
+        if (/No such tool available/.test(t)) raced = true;
+      }
+    }
+    if (e.type === 'result') r = e;
+  }
+  if (!r || r.subtype !== 'success') return null;
+  return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0, raced };
+}
+const median = (arr) => { const v = [...arr].sort((a, b) => a - b); const n = v.length; return n === 0 ? 0 : n % 2 ? v[(n - 1) / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; };
+const fmtTime = (s) => s >= 60 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${Math.round(s)}s`;
+const fmtTok = (t) => t >= 1e6 ? `${(t / 1e6).toFixed(1)}M` : `${Math.round(t / 1000)}k`;
+const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
+
+console.log('repo        n(w/wo)  time WITH→WITHOUT      tools W→WO   tokens W→WO (saved)     cost W→WO (saved)');
+const savings = { cost: [], tokens: [], time: [], tools: [] };
+for (const repo of REPOS) {
+  const dir = join(ROOT, repo);
+  const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)) : [];
+  // Exclude MCP-cold-start-raced WITH runs by default — they measure a startup
+  // race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw
+  // distribution). The WITHOUT arm has no MCP, so it's never raced.
+  const includeRaced = process.env.CG_INCLUDE_RACED === '1';
+  const W = [], WO = []; let racedExcluded = 0;
+  for (const rd of runDirs) {
+    const w = parse(join(dir, rd, 'run-headless-with.jsonl'));
+    if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); }
+    const wo = parse(join(dir, rd, 'run-headless-without.jsonl')); if (wo) WO.push(wo);
+  }
+  if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete: w=${W.length} wo=${WO.length})`); continue; }
+  const m = (arr, k) => median(arr.map(x => x[k]));
+  const wT = m(W, 'dur'), woT = m(WO, 'dur'), wTok = m(W, 'tokens'), woTok = m(WO, 'tokens');
+  const wC = m(W, 'cost'), woC = m(WO, 'cost'), wTl = m(W, 'tools'), woTl = m(WO, 'tools');
+  savings.time.push(pct(wT, woT)); savings.tokens.push(pct(wTok, woTok)); savings.cost.push(pct(wC, woC)); savings.tools.push(pct(wTl, woTl));
+  console.log(
+    `${repo.padEnd(11)} ${W.length}/${WO.length}      ` +
+    `${(fmtTime(wT) + '→' + fmtTime(woT)).padEnd(22)}` +
+    `${(Math.round(wTl) + '→' + Math.round(woTl)).padEnd(12)}` +
+    `${(fmtTok(wTok) + '→' + fmtTok(woTok) + ' (' + pct(wTok, woTok) + '%)').padEnd(24)}` +
+    `$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)` +
+    (racedExcluded ? `  [${racedExcluded} raced run${racedExcluded === 1 ? '' : 's'} excluded]` : '')
+  );
+}
+const avg = (a) => a.length ? Math.round(a.reduce((s, x) => s + x, 0) / a.length) : 0;
+console.log(`\nAVERAGE saved:  cost ${avg(savings.cost)}%  ·  tokens ${avg(savings.tokens)}%  ·  time ${avg(savings.time)}%  ·  tool calls ${avg(savings.tools)}%`);

package/scripts/agent-eval/parse-run.mjs

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
+import { readFileSync } from 'fs';
+const file = process.argv[2];
+const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+
+const toolCalls = [];
+let result = null;
+let initTools = null;
+
+for (const line of lines) {
+  let ev;
+  try { ev = JSON.parse(line); } catch { continue; }
+  if (ev.type === 'system' && ev.subtype === 'init') {
+    initTools = (ev.tools || []).filter(t => /codegraph/.test(t));
+  }
+  if (ev.type === 'assistant' && ev.message?.content) {
+    for (const block of ev.message.content) {
+      if (block.type === 'tool_use') {
+        let detail = '';
+        if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
+        else if (/codegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
+        else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
+        else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
+        toolCalls.push(`${block.name}${detail}`);
+      }
+    }
+  }
+  if (ev.type === 'result') result = ev;
+}
+
+console.log(`\n=== ${file.split('/').pop()} ===`);
+console.log(`codegraph tools exposed: ${initTools ? initTools.length : '?'}`);
+console.log(`\nTool calls (${toolCalls.length}):`);
+const counts = {};
+for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
+console.log('  by type:', JSON.stringify(counts));
+toolCalls.forEach((tc, i) => console.log(`  ${i+1}. ${tc}`));
+
+if (result) {
+  const u = result.usage || {};
+  const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
+  console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
+  console.log(`  tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
+}