homegraph 1.1.2 → 1.2.0

package/scripts/agent-eval/offload-eval-summarize.mjs

@@ -1,68 +1,68 @@
-#!/usr/bin/env node
-// Aggregate judged.jsonl (or results.jsonl) into a per-repo, per-arm report:
-// time, main tokens/cost, AI tokens/cost, total cost, tool mix, accuracy.
-// Usage: summarize.mjs <judged-or-results.jsonl>
-import { readFileSync } from 'fs';
-const rows = readFileSync(process.argv[2], 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
-
-const med = (xs) => { const a = xs.filter(x => x != null).sort((p, q) => p - q); if (!a.length) return null; const m = Math.floor(a.length / 2); return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2; };
-const rng = (xs) => { const a = xs.filter(x => x != null); return a.length ? `${Math.min(...a)}–${Math.max(...a)}` : '—'; };
-const d2 = (x) => x == null ? '—' : (+x).toFixed(2);
-const d3 = (x) => x == null ? '—' : (+x).toFixed(3);
-const d4 = (x) => x == null ? '—' : (+x).toFixed(4);
-
-const ARM_ORDER = ['frontload', 'offload', 'raw', 'nocg'];
-const byRepo = {};
-for (const r of rows) (byRepo[r.repo] ??= {});
-for (const r of rows) ((byRepo[r.repo][r.arm] ??= []).push(r));
-
-const verdictTally = (rs, field) => {
-  const t = { pass: 0, partial: 0, fail: 0, error: 0 };
-  for (const r of rs) { const v = r[field]?.verdict; if (v in t) t[v]++; }
-  return t;
-};
-
-for (const repo of Object.keys(byRepo)) {
-  const tier = byRepo[repo][Object.keys(byRepo[repo])[0]][0].tier;
-  console.log(`\n${'='.repeat(78)}\n${repo}  [${tier}]\n${'='.repeat(78)}`);
-  console.log(`${'arm'.padEnd(9)} n  ${'time(s)'.padStart(9)} ${'mainCost'.padStart(9)} ${'aiCost'.padStart(8)} ${'totCost'.padStart(8)} ${'mainTok'.padStart(8)} ${'aiTok'.padStart(7)} ${'rd'.padStart(3)} ${'gr'.padStart(3)} ${'exp'.padStart(3)} ${'off'.padStart(3)}  e2e(P/p/F)  fidScore`);
-  for (const arm of ARM_ORDER) {
-    const rs = byRepo[repo][arm]; if (!rs) continue;
-    const n = rs.length;
-    const mainCost = med(rs.map(r => r.costUsdMain));
-    const aiCost = med(rs.map(r => r.ai?.costUsd ?? 0));
-    const totCost = (mainCost ?? 0) + (aiCost ?? 0);
-    const e2e = verdictTally(rs, 'e2e');
-    const fidScores = arm === 'offload' ? rs.flatMap(r => r.fidelity?.scores ?? []) : [];
-    const fid = fidScores.length ? med(fidScores) : null;
-    const fab = arm === 'offload' && rs.some(r => r.fidelity?.anyFabrication);
-    const e2eScore = med(rs.map(r => r.e2e?.score).filter(x => x != null));
-    console.log(
-      `${arm.padEnd(9)} ${String(n).padStart(1)}  ${String(med(rs.map(r => r.durationSec))).padStart(9)} ` +
-      `${('$' + d3(mainCost)).padStart(9)} ${('$' + d3(aiCost)).padStart(8)} ${('$' + d3(totCost)).padStart(8)} ` +
-      `${String(Math.round(med(rs.map(r => r.tokBillable)) / 1000) + 'k').padStart(8)} ${String(Math.round(med(rs.map(r => r.ai?.totalTokens ?? 0)) / 1000) + 'k').padStart(7)} ` +
-      `${String(med(rs.map(r => r.read))).padStart(3)} ${String(med(rs.map(r => r.grep))).padStart(3)} ${String(med(rs.map(r => r.explore))).padStart(3)} ${String(med(rs.map(r => r.offloadFired))).padStart(3)}  ` +
-      `${(e2e.pass + '/' + e2e.partial + '/' + e2e.fail).padStart(9)}  ${e2eScore != null ? 'e2e=' + e2eScore : ''} ${fid != null ? 'fid=' + fid + (fab ? ' FAB!' : '') : ''}`
-    );
-  }
-  // ranges line for the two key metrics (variance matters)
-  for (const arm of ARM_ORDER) {
-    const rs = byRepo[repo][arm]; if (!rs) continue;
-    console.log(`   ${arm} ranges: time ${rng(rs.map(r => r.durationSec))}s · mainCost $${rng(rs.map(r => r.costUsdMain))} · read ${rng(rs.map(r => r.read))} · explore ${rng(rs.map(r => r.explore))} · offloadFired ${rng(rs.map(r => r.offloadFired))}`);
-  }
-}
-
-// Cross-repo roll-up: offload vs raw vs nocg deltas
-console.log(`\n${'='.repeat(78)}\nCROSS-REPO SUMMARY (medians per repo, then averaged)\n${'='.repeat(78)}`);
-console.log(`${'repo'.padEnd(12)} ${'arm'.padEnd(8)} ${'time'.padStart(7)} ${'totCost'.padStart(8)} ${'read'.padStart(5)} ${'e2e pass%'.padStart(9)} ${'fid'.padStart(5)}`);
-for (const repo of Object.keys(byRepo)) {
-  for (const arm of ARM_ORDER) {
-    const rs = byRepo[repo][arm]; if (!rs) continue;
-    const e2e = verdictTally(rs, 'e2e');
-    const passPct = Math.round(100 * e2e.pass / rs.length);
-    const totCost = (med(rs.map(r => r.costUsdMain)) ?? 0) + (med(rs.map(r => r.ai?.costUsd ?? 0)) ?? 0);
-    const fid = arm === 'offload' ? med(rs.flatMap(r => r.fidelity?.scores ?? [])) : null;
-    console.log(`${repo.padEnd(12)} ${arm.padEnd(8)} ${(med(rs.map(r => r.durationSec)) + 's').padStart(7)} ${('$' + d3(totCost)).padStart(8)} ${String(med(rs.map(r => r.read))).padStart(5)} ${(passPct + '%').padStart(9)} ${String(fid ?? '—').padStart(5)}`);
-  }
-}
-console.log('');
+#!/usr/bin/env node
+// Aggregate judged.jsonl (or results.jsonl) into a per-repo, per-arm report:
+// time, main tokens/cost, AI tokens/cost, total cost, tool mix, accuracy.
+// Usage: summarize.mjs <judged-or-results.jsonl>
+import { readFileSync } from 'fs';
+const rows = readFileSync(process.argv[2], 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
+
+const med = (xs) => { const a = xs.filter(x => x != null).sort((p, q) => p - q); if (!a.length) return null; const m = Math.floor(a.length / 2); return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2; };
+const rng = (xs) => { const a = xs.filter(x => x != null); return a.length ? `${Math.min(...a)}–${Math.max(...a)}` : '—'; };
+const d2 = (x) => x == null ? '—' : (+x).toFixed(2);
+const d3 = (x) => x == null ? '—' : (+x).toFixed(3);
+const d4 = (x) => x == null ? '—' : (+x).toFixed(4);
+
+const ARM_ORDER = ['frontload', 'offload', 'raw', 'nocg'];
+const byRepo = {};
+for (const r of rows) (byRepo[r.repo] ??= {});
+for (const r of rows) ((byRepo[r.repo][r.arm] ??= []).push(r));
+
+const verdictTally = (rs, field) => {
+  const t = { pass: 0, partial: 0, fail: 0, error: 0 };
+  for (const r of rs) { const v = r[field]?.verdict; if (v in t) t[v]++; }
+  return t;
+};
+
+for (const repo of Object.keys(byRepo)) {
+  const tier = byRepo[repo][Object.keys(byRepo[repo])[0]][0].tier;
+  console.log(`\n${'='.repeat(78)}\n${repo}  [${tier}]\n${'='.repeat(78)}`);
+  console.log(`${'arm'.padEnd(9)} n  ${'time(s)'.padStart(9)} ${'mainCost'.padStart(9)} ${'aiCost'.padStart(8)} ${'totCost'.padStart(8)} ${'mainTok'.padStart(8)} ${'aiTok'.padStart(7)} ${'rd'.padStart(3)} ${'gr'.padStart(3)} ${'exp'.padStart(3)} ${'off'.padStart(3)}  e2e(P/p/F)  fidScore`);
+  for (const arm of ARM_ORDER) {
+    const rs = byRepo[repo][arm]; if (!rs) continue;
+    const n = rs.length;
+    const mainCost = med(rs.map(r => r.costUsdMain));
+    const aiCost = med(rs.map(r => r.ai?.costUsd ?? 0));
+    const totCost = (mainCost ?? 0) + (aiCost ?? 0);
+    const e2e = verdictTally(rs, 'e2e');
+    const fidScores = arm === 'offload' ? rs.flatMap(r => r.fidelity?.scores ?? []) : [];
+    const fid = fidScores.length ? med(fidScores) : null;
+    const fab = arm === 'offload' && rs.some(r => r.fidelity?.anyFabrication);
+    const e2eScore = med(rs.map(r => r.e2e?.score).filter(x => x != null));
+    console.log(
+      `${arm.padEnd(9)} ${String(n).padStart(1)}  ${String(med(rs.map(r => r.durationSec))).padStart(9)} ` +
+      `${('$' + d3(mainCost)).padStart(9)} ${('$' + d3(aiCost)).padStart(8)} ${('$' + d3(totCost)).padStart(8)} ` +
+      `${String(Math.round(med(rs.map(r => r.tokBillable)) / 1000) + 'k').padStart(8)} ${String(Math.round(med(rs.map(r => r.ai?.totalTokens ?? 0)) / 1000) + 'k').padStart(7)} ` +
+      `${String(med(rs.map(r => r.read))).padStart(3)} ${String(med(rs.map(r => r.grep))).padStart(3)} ${String(med(rs.map(r => r.explore))).padStart(3)} ${String(med(rs.map(r => r.offloadFired))).padStart(3)}  ` +
+      `${(e2e.pass + '/' + e2e.partial + '/' + e2e.fail).padStart(9)}  ${e2eScore != null ? 'e2e=' + e2eScore : ''} ${fid != null ? 'fid=' + fid + (fab ? ' FAB!' : '') : ''}`
+    );
+  }
+  // ranges line for the two key metrics (variance matters)
+  for (const arm of ARM_ORDER) {
+    const rs = byRepo[repo][arm]; if (!rs) continue;
+    console.log(`   ${arm} ranges: time ${rng(rs.map(r => r.durationSec))}s · mainCost $${rng(rs.map(r => r.costUsdMain))} · read ${rng(rs.map(r => r.read))} · explore ${rng(rs.map(r => r.explore))} · offloadFired ${rng(rs.map(r => r.offloadFired))}`);
+  }
+}
+
+// Cross-repo roll-up: offload vs raw vs nocg deltas
+console.log(`\n${'='.repeat(78)}\nCROSS-REPO SUMMARY (medians per repo, then averaged)\n${'='.repeat(78)}`);
+console.log(`${'repo'.padEnd(12)} ${'arm'.padEnd(8)} ${'time'.padStart(7)} ${'totCost'.padStart(8)} ${'read'.padStart(5)} ${'e2e pass%'.padStart(9)} ${'fid'.padStart(5)}`);
+for (const repo of Object.keys(byRepo)) {
+  for (const arm of ARM_ORDER) {
+    const rs = byRepo[repo][arm]; if (!rs) continue;
+    const e2e = verdictTally(rs, 'e2e');
+    const passPct = Math.round(100 * e2e.pass / rs.length);
+    const totCost = (med(rs.map(r => r.costUsdMain)) ?? 0) + (med(rs.map(r => r.ai?.costUsd ?? 0)) ?? 0);
+    const fid = arm === 'offload' ? med(rs.flatMap(r => r.fidelity?.scores ?? [])) : null;
+    console.log(`${repo.padEnd(12)} ${arm.padEnd(8)} ${(med(rs.map(r => r.durationSec)) + 's').padStart(7)} ${('$' + d3(totCost)).padStart(8)} ${String(med(rs.map(r => r.read))).padStart(5)} ${(passPct + '%').padStart(9)} ${String(fid ?? '—').padStart(5)}`);
+  }
+}
+console.log('');

package/scripts/agent-eval/offload-eval.md

@@ -1,76 +1,76 @@
-# HomeGraph AI offload — accuracy & adoption eval harness
-
-Measures the managed **offload** (`homegraph_explore` → reasoning model synthesis) and the
-**front-load hook** (approach 1) against plain homegraph and no-homegraph, across repo sizes,
-on **time · main-session tokens/cost · HomeGraph-AI tokens/cost · accuracy**.
-
-All agent arms run `claude -p --model sonnet --effort high` (the deliberate floor model — an
-affordance that lands on Sonnet generalizes up). Everything writes to a scratch dir
-(`AGENT_EVAL_OUT`, default `/tmp/cg-offload-eval`); nothing here is shipped to users.
-
-## Repos (selected via a memory-probe gate — NOT trained on)
-
-Famous repos (express, excalidraw, n8n, …) are useless for *accuracy* evals: Sonnet answers their
-flow questions from memory, so the no-homegraph baseline is dishonest. These four passed a no-tools
-probe (Sonnet could not name their real flow internals) and are cloned fresh by `offload-eval-setup.sh`:
-
-| tier | repo | ~src files | canonical flow |
-|---|---|---|---|
-| small | MTKruto/MTKruto | 322 TS | `sendMessage` → invoke → TL serialize → transport |
-| medium | mvdicarlo/postybirb-plus | 608 TS | submission → queue → per-website `.post()` |
-| complex | shapeshift/web | 3.2k TS (35-pkg monorepo) | swap → swapper registry → concrete swapper |
-| large | trezor/trezor-suite | 8k TS monorepo | send-form → sign thunk → `@trezor/connect` |
-
-Verified ground-truth flows (the judge's reference) live in `offload-eval-ground-truth.json`.
-
-## Arms
-
-- **offload** — homegraph + managed offload ON (requires `homegraph login`); records AI tokens/credits via `HOMEGRAPH_OFFLOAD_USAGE_LOG`.
-- **raw** — homegraph, `HOMEGRAPH_OFFLOAD_DISABLE=1` (returns raw source).
-- **nocg** — empty MCP config; Read/Grep baseline.
-- **frontload** — homegraph (offload-disabled) + a `UserPromptSubmit` hook (`offload-eval-hook.mjs`) that runs raw explore on the prompt and injects the result into context (approach 1).
-
-## Run it
-
-```bash
-npm run build                       # the harness shells out to dist/
-homegraph login                     # only needed for the offload arm
-export AGENT_EVAL_OUT=/tmp/cg-offload-eval
-
-bash scripts/agent-eval/offload-eval-setup.sh            # clone + index the 4 repos
-bash scripts/agent-eval/offload-eval-matrix.sh           # 3 arms × 4 tiers × REPS (default 3)
-node scripts/agent-eval/offload-eval-judge.mjs \
-     --results $AGENT_EVAL_OUT/results.jsonl \
-     --truth  scripts/agent-eval/offload-eval-ground-truth.json \
-     --out    $AGENT_EVAL_OUT/judged.jsonl
-node scripts/agent-eval/offload-eval-summarize.mjs $AGENT_EVAL_OUT/judged.jsonl
-
-bash scripts/agent-eval/offload-eval-frontload-matrix.sh # frontload arm + judge + merged summary
-```
-
-Single repo: `offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"` (or `-frontload.sh`).
-
-## Files
-
-- `offload-eval-setup.sh` — clone + index the 4 repos.
-- `offload-eval-3arm.sh` / `-frontload.sh` — one repo, the arms.
-- `offload-eval-matrix.sh` / `-frontload-matrix.sh` — drive all 4 tiers.
-- `offload-eval-hook.mjs` — the front-load `UserPromptSubmit` hook (resolves its own engine; `CG_FRONTLOAD_DEBUG=<path>` to log injections; `CG_FRONTLOAD_BUDGET` to cap injected chars).
-- `offload-eval-metrics.mjs` — one run's stream-json + usage log → one JSON metrics line.
-- `offload-eval-judge.mjs` — Sonnet judge: end-to-end (agent final vs ground truth) + per-answer offload fidelity.
-- `offload-eval-summarize.mjs` — per-tier, per-arm table + cross-repo roll-up.
-- `offload-eval-ground-truth.json` — source-verified canonical flows.
-
-## Findings (2026-06, n=3 — direction consistent, magnitudes noisy)
-
-- **Raw homegraph is the efficiency win** — ~nocg accuracy, fewer reads, faster, no AI cost.
-- **The offload is the least-accurate arm in all 4 tiers** — synthesized fidelity 12–27/100 with
-  fabrication in 3/4 (e.g. invented website services; traced `ClientPlain`/`SessionPlain` instead of
-  the real encrypted path). Its speed/cost win is narrow (medium-only) and inversely correlated with
-  accuracy. **Use raw until offload fidelity is fixed.**
-- **The front-load hook SOLVES adoption** — reads → 0–1 in every tier (incl. large, where the agent
-  otherwise read 12–24 files); fired 12/12, 0 errors. Wins on medium/complex (100% pass). But it
-  **regresses small/large to partial** — it suppresses the reads that compensate for explore's gaps at
-  **dynamic boundaries** (async queues, redux thunks, facade/factory indirection).
-- **Master lever for BOTH:** explore's dynamic-dispatch coverage. Fix it → front-load is complete
-  everywhere and the offload has the full flow to synthesize.
+# HomeGraph AI offload — accuracy & adoption eval harness
+
+Measures the managed **offload** (`homegraph_explore` → reasoning model synthesis) and the
+**front-load hook** (approach 1) against plain homegraph and no-homegraph, across repo sizes,
+on **time · main-session tokens/cost · HomeGraph-AI tokens/cost · accuracy**.
+
+All agent arms run `claude -p --model sonnet --effort high` (the deliberate floor model — an
+affordance that lands on Sonnet generalizes up). Everything writes to a scratch dir
+(`AGENT_EVAL_OUT`, default `/tmp/cg-offload-eval`); nothing here is shipped to users.
+
+## Repos (selected via a memory-probe gate — NOT trained on)
+
+Famous repos (express, excalidraw, n8n, …) are useless for *accuracy* evals: Sonnet answers their
+flow questions from memory, so the no-homegraph baseline is dishonest. These four passed a no-tools
+probe (Sonnet could not name their real flow internals) and are cloned fresh by `offload-eval-setup.sh`:
+
+| tier | repo | ~src files | canonical flow |
+|---|---|---|---|
+| small | MTKruto/MTKruto | 322 TS | `sendMessage` → invoke → TL serialize → transport |
+| medium | mvdicarlo/postybirb-plus | 608 TS | submission → queue → per-website `.post()` |
+| complex | shapeshift/web | 3.2k TS (35-pkg monorepo) | swap → swapper registry → concrete swapper |
+| large | trezor/trezor-suite | 8k TS monorepo | send-form → sign thunk → `@trezor/connect` |
+
+Verified ground-truth flows (the judge's reference) live in `offload-eval-ground-truth.json`.
+
+## Arms
+
+- **offload** — homegraph + managed offload ON (requires `homegraph login`); records AI tokens/credits via `HOMEGRAPH_OFFLOAD_USAGE_LOG`.
+- **raw** — homegraph, `HOMEGRAPH_OFFLOAD_DISABLE=1` (returns raw source).
+- **nocg** — empty MCP config; Read/Grep baseline.
+- **frontload** — homegraph (offload-disabled) + a `UserPromptSubmit` hook (`offload-eval-hook.mjs`) that runs raw explore on the prompt and injects the result into context (approach 1).
+
+## Run it
+
+```bash
+npm run build                       # the harness shells out to dist/
+homegraph login                     # only needed for the offload arm
+export AGENT_EVAL_OUT=/tmp/cg-offload-eval
+
+bash scripts/agent-eval/offload-eval-setup.sh            # clone + index the 4 repos
+bash scripts/agent-eval/offload-eval-matrix.sh           # 3 arms × 4 tiers × REPS (default 3)
+node scripts/agent-eval/offload-eval-judge.mjs \
+     --results $AGENT_EVAL_OUT/results.jsonl \
+     --truth  scripts/agent-eval/offload-eval-ground-truth.json \
+     --out    $AGENT_EVAL_OUT/judged.jsonl
+node scripts/agent-eval/offload-eval-summarize.mjs $AGENT_EVAL_OUT/judged.jsonl
+
+bash scripts/agent-eval/offload-eval-frontload-matrix.sh # frontload arm + judge + merged summary
+```
+
+Single repo: `offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"` (or `-frontload.sh`).
+
+## Files
+
+- `offload-eval-setup.sh` — clone + index the 4 repos.
+- `offload-eval-3arm.sh` / `-frontload.sh` — one repo, the arms.
+- `offload-eval-matrix.sh` / `-frontload-matrix.sh` — drive all 4 tiers.
+- `offload-eval-hook.mjs` — the front-load `UserPromptSubmit` hook (resolves its own engine; `CG_FRONTLOAD_DEBUG=<path>` to log injections; `CG_FRONTLOAD_BUDGET` to cap injected chars).
+- `offload-eval-metrics.mjs` — one run's stream-json + usage log → one JSON metrics line.
+- `offload-eval-judge.mjs` — Sonnet judge: end-to-end (agent final vs ground truth) + per-answer offload fidelity.
+- `offload-eval-summarize.mjs` — per-tier, per-arm table + cross-repo roll-up.
+- `offload-eval-ground-truth.json` — source-verified canonical flows.
+
+## Findings (2026-06, n=3 — direction consistent, magnitudes noisy)
+
+- **Raw homegraph is the efficiency win** — ~nocg accuracy, fewer reads, faster, no AI cost.
+- **The offload is the least-accurate arm in all 4 tiers** — synthesized fidelity 12–27/100 with
+  fabrication in 3/4 (e.g. invented website services; traced `ClientPlain`/`SessionPlain` instead of
+  the real encrypted path). Its speed/cost win is narrow (medium-only) and inversely correlated with
+  accuracy. **Use raw until offload fidelity is fixed.**
+- **The front-load hook SOLVES adoption** — reads → 0–1 in every tier (incl. large, where the agent
+  otherwise read 12–24 files); fired 12/12, 0 errors. Wins on medium/complex (100% pass). But it
+  **regresses small/large to partial** — it suppresses the reads that compensate for explore's gaps at
+  **dynamic boundaries** (async queues, redux thunks, facade/factory indirection).
+- **Master lever for BOTH:** explore's dynamic-dispatch coverage. Fix it → front-load is complete
+  everywhere and the offload has the full flow to synthesize.

package/scripts/agent-eval/parse-arms.mjs

@@ -1,116 +1,116 @@
-#!/usr/bin/env node
-// Analyze the tool-surface ablation (/tmp/arms/<repo>/<arm>-r<n>.jsonl).
-// Compares arms A–E on trace adoption, Read/Grep fallback, homegraph payload,
-// round-trips, and duration — averaged across runs per arm.
-//
-// The decisive signal is READS: if removing a tool raises Reads on a question
-// class, that tool was load-bearing for it (not redundant). If removing it
-// changes nothing, it was redundant.
-//
-//   A control       all tools            no steering   (baseline)
-//   B steer         all tools            trace-first   (adoption)
-//   C no-explore    hide explore         trace-first   (is explore redundant?)
-//   D trace-centric hide explore+context trace-first   (is the survey pair redundant?)
-//   E control-probe hide explore+context trace-first   (NON-flow Q — should degrade)
-//
-// Usage: node scripts/agent-eval/parse-arms.mjs [/tmp/arms]
-import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
-import { join } from 'path';
-
-const ROOT = process.argv[2] || '/tmp/arms';
-const cgShort = (n) => n.replace('mcp__homegraph__homegraph_', '').replace('mcp__homegraph__', '');
-
-function parse(file) {
-  if (!existsSync(file)) return null;
-  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
-  const calls = []; let result = null, initCg = 0;
-  for (const l of lines) {
-    let ev; try { ev = JSON.parse(l); } catch { continue; }
-    if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /homegraph/.test(t)).length;
-    if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use')
-      calls.push({ id: b.id, name: b.name, out: 0 });
-    if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
-      const c = b.content;
-      const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
-      const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
-    }
-    if (ev.type === 'result') result = ev;
-  }
-  const cg = calls.filter(c => c.name.includes('homegraph'));
-  return {
-    initCg,
-    reads: calls.filter(c => c.name === 'Read').length,
-    greps: calls.filter(c => c.name === 'Grep').length + calls.filter(c => c.name === 'Glob').length,
-    cgCalls: cg.length,
-    cgSeq: cg.map(c => cgShort(c.name)),
-    cgOut: cg.reduce((s, c) => s + c.out, 0),
-    traceUsed: cg.some(c => c.name.includes('trace')),
-    turns: result?.num_turns ?? null,
-    dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
-    cost: result?.total_cost_usd || 0,
-    ok: result?.subtype === 'success',
-  };
-}
-
-// repo -> arm -> [runs]
-const data = {};
-if (!existsSync(ROOT)) { console.error(`no ${ROOT}`); process.exit(1); }
-for (const repo of readdirSync(ROOT)) {
-  const rdir = join(ROOT, repo);
-  if (!statSync(rdir).isDirectory()) continue;
-  for (const f of readdirSync(rdir)) {
-    const m = f.match(/^([A-I])-r(\d+)\.jsonl$/); if (!m) continue;
-    const p = parse(join(rdir, f)); if (!p || !p.ok) continue;
-    (((data[repo] ??= {})[m[1]]) ??= []).push(p);
-  }
-}
-
-const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
-const k = (n) => (n / 1000).toFixed(1);
-const pad = (s, n) => String(s).padEnd(n);
-const ARMS = ['A', 'H', 'I', 'B', 'F', 'G', 'C', 'D', 'E'];
-const LABEL = { A: 'A all/none(old)', H: 'H body-trace/none', I: 'I bodytrace+dest', B: 'B all/steer(thin)', F: 'F all/steer(body)', G: 'G ported(noprompt)', C: 'C no-explore', D: 'D trace-centric', E: 'E nonflow-probe' };
-
-// ---- per repo × arm ----
-console.log('\n=== PER REPO × ARM (avg over runs) ===');
-console.log(pad('repo', 22), pad('arm', 16), 'tools', 'trace', pad('reads', 6), pad('cgOutK', 7), pad('turns', 6), 'dur');
-for (const repo of Object.keys(data).sort()) {
-  for (const arm of ARMS) {
-    const runs = data[repo][arm]; if (!runs?.length) continue;
-    console.log(
-      pad(repo, 22), pad(LABEL[arm], 16),
-      pad(runs[0].initCg, 5),
-      pad(runs.filter(r => r.traceUsed).length + '/' + runs.length, 5),
-      pad(avg(runs, r => r.reads).toFixed(1), 6),
-      pad(k(avg(runs, r => r.cgOut)), 7),
-      pad(avg(runs, r => r.turns).toFixed(1), 6),
-      avg(runs, r => r.dur).toFixed(0) + 's',
-    );
-  }
-}
-
-// ---- aggregate per arm (flow arms A–D over the flow repos; E shown apart) ----
-console.log('\n=== AGGREGATE PER ARM (mean across repos) ===');
-console.log(pad('arm', 16), pad('adoption', 9), pad('reads', 7), pad('greps', 7), pad('cgOutK', 8), pad('turns', 7), pad('dur', 6), 'cost');
-for (const arm of ARMS) {
-  const all = [];
-  for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) all.push({ ...r, repo });
-  if (!all.length) continue;
-  const repos = new Set(all.map(r => r.repo)).size;
-  const adopt = all.filter(r => r.traceUsed).length;
-  console.log(
-    pad(LABEL[arm], 16),
-    pad(`${adopt}/${all.length}`, 9),
-    pad(avg(all, r => r.reads).toFixed(2), 7),
-    pad(avg(all, r => r.greps).toFixed(2), 7),
-    pad(k(avg(all, r => r.cgOut)), 8),
-    pad(avg(all, r => r.turns).toFixed(1), 7),
-    pad(avg(all, r => r.dur).toFixed(0) + 's', 6),
-    '$' + avg(all, r => r.cost).toFixed(3),
-    `  (${repos} repos)`,
-  );
-}
-
-console.log('\nRead the signal: B vs A = does steering alone fix adoption + cut payload.');
-console.log('C vs B = is explore redundant (reads should NOT jump). D vs C = is context redundant.');
-console.log('E = non-flow under trace-centric; reads SHOULD jump (proves survey tools are load-bearing).');
+#!/usr/bin/env node
+// Analyze the tool-surface ablation (/tmp/arms/<repo>/<arm>-r<n>.jsonl).
+// Compares arms A–E on trace adoption, Read/Grep fallback, homegraph payload,
+// round-trips, and duration — averaged across runs per arm.
+//
+// The decisive signal is READS: if removing a tool raises Reads on a question
+// class, that tool was load-bearing for it (not redundant). If removing it
+// changes nothing, it was redundant.
+//
+//   A control       all tools            no steering   (baseline)
+//   B steer         all tools            trace-first   (adoption)
+//   C no-explore    hide explore         trace-first   (is explore redundant?)
+//   D trace-centric hide explore+context trace-first   (is the survey pair redundant?)
+//   E control-probe hide explore+context trace-first   (NON-flow Q — should degrade)
+//
+// Usage: node scripts/agent-eval/parse-arms.mjs [/tmp/arms]
+import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
+import { join } from 'path';
+
+const ROOT = process.argv[2] || '/tmp/arms';
+const cgShort = (n) => n.replace('mcp__homegraph__homegraph_', '').replace('mcp__homegraph__', '');
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  const calls = []; let result = null, initCg = 0;
+  for (const l of lines) {
+    let ev; try { ev = JSON.parse(l); } catch { continue; }
+    if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /homegraph/.test(t)).length;
+    if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use')
+      calls.push({ id: b.id, name: b.name, out: 0 });
+    if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
+      const c = b.content;
+      const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
+      const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
+    }
+    if (ev.type === 'result') result = ev;
+  }
+  const cg = calls.filter(c => c.name.includes('homegraph'));
+  return {
+    initCg,
+    reads: calls.filter(c => c.name === 'Read').length,
+    greps: calls.filter(c => c.name === 'Grep').length + calls.filter(c => c.name === 'Glob').length,
+    cgCalls: cg.length,
+    cgSeq: cg.map(c => cgShort(c.name)),
+    cgOut: cg.reduce((s, c) => s + c.out, 0),
+    traceUsed: cg.some(c => c.name.includes('trace')),
+    turns: result?.num_turns ?? null,
+    dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
+    cost: result?.total_cost_usd || 0,
+    ok: result?.subtype === 'success',
+  };
+}
+
+// repo -> arm -> [runs]
+const data = {};
+if (!existsSync(ROOT)) { console.error(`no ${ROOT}`); process.exit(1); }
+for (const repo of readdirSync(ROOT)) {
+  const rdir = join(ROOT, repo);
+  if (!statSync(rdir).isDirectory()) continue;
+  for (const f of readdirSync(rdir)) {
+    const m = f.match(/^([A-I])-r(\d+)\.jsonl$/); if (!m) continue;
+    const p = parse(join(rdir, f)); if (!p || !p.ok) continue;
+    (((data[repo] ??= {})[m[1]]) ??= []).push(p);
+  }
+}
+
+const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
+const k = (n) => (n / 1000).toFixed(1);
+const pad = (s, n) => String(s).padEnd(n);
+const ARMS = ['A', 'H', 'I', 'B', 'F', 'G', 'C', 'D', 'E'];
+const LABEL = { A: 'A all/none(old)', H: 'H body-trace/none', I: 'I bodytrace+dest', B: 'B all/steer(thin)', F: 'F all/steer(body)', G: 'G ported(noprompt)', C: 'C no-explore', D: 'D trace-centric', E: 'E nonflow-probe' };
+
+// ---- per repo × arm ----
+console.log('\n=== PER REPO × ARM (avg over runs) ===');
+console.log(pad('repo', 22), pad('arm', 16), 'tools', 'trace', pad('reads', 6), pad('cgOutK', 7), pad('turns', 6), 'dur');
+for (const repo of Object.keys(data).sort()) {
+  for (const arm of ARMS) {
+    const runs = data[repo][arm]; if (!runs?.length) continue;
+    console.log(
+      pad(repo, 22), pad(LABEL[arm], 16),
+      pad(runs[0].initCg, 5),
+      pad(runs.filter(r => r.traceUsed).length + '/' + runs.length, 5),
+      pad(avg(runs, r => r.reads).toFixed(1), 6),
+      pad(k(avg(runs, r => r.cgOut)), 7),
+      pad(avg(runs, r => r.turns).toFixed(1), 6),
+      avg(runs, r => r.dur).toFixed(0) + 's',
+    );
+  }
+}
+
+// ---- aggregate per arm (flow arms A–D over the flow repos; E shown apart) ----
+console.log('\n=== AGGREGATE PER ARM (mean across repos) ===');
+console.log(pad('arm', 16), pad('adoption', 9), pad('reads', 7), pad('greps', 7), pad('cgOutK', 8), pad('turns', 7), pad('dur', 6), 'cost');
+for (const arm of ARMS) {
+  const all = [];
+  for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) all.push({ ...r, repo });
+  if (!all.length) continue;
+  const repos = new Set(all.map(r => r.repo)).size;
+  const adopt = all.filter(r => r.traceUsed).length;
+  console.log(
+    pad(LABEL[arm], 16),
+    pad(`${adopt}/${all.length}`, 9),
+    pad(avg(all, r => r.reads).toFixed(2), 7),
+    pad(avg(all, r => r.greps).toFixed(2), 7),
+    pad(k(avg(all, r => r.cgOut)), 8),
+    pad(avg(all, r => r.turns).toFixed(1), 7),
+    pad(avg(all, r => r.dur).toFixed(0) + 's', 6),
+    '$' + avg(all, r => r.cost).toFixed(3),
+    `  (${repos} repos)`,
+  );
+}
+
+console.log('\nRead the signal: B vs A = does steering alone fix adoption + cut payload.');
+console.log('C vs B = is explore redundant (reads should NOT jump). D vs C = is context redundant.');
+console.log('E = non-flow under trace-centric; reads SHOULD jump (proves survey tools are load-bearing).');