homegraph 1.1.2 → 1.2.0

package/scripts/agent-eval/offload-eval-cost.mjs

@@ -1,133 +1,133 @@
-#!/usr/bin/env node
-// Cost/token analysis for the 3-arm offload eval, with a MAIN-vs-SUBAGENT split.
-//
-// The explore-subagent question. With delegation ALLOWED, the nocg arm spawns a
-// Claude Code Explore subagent; the homegraph arms do all work in the main agent.
-// Two facts make naive accounting wrong:
-//   1. The Explore subagent runs on HAIKU 4.5; the main agent on SONNET 4.6.
-//      So per-token cost differs ~3x between them — you cannot price both the same.
-//   2. The subagent's consumption is ~95% cache-reads. At Haiku's $0.10/MTok
-//      cache-read rate, a huge TOKEN volume is a small DOLLAR cost.
-//
-// Rather than re-derive cost from raw token counts (and guess the cache TTL —
-// Claude Code uses 1-hour ephemeral cache here, 2x write, not 5-min), we read
-// Claude Code's OWN authoritative accounting from the `result` event:
-//   result.modelUsage[model].costUSD  — per-model cost CC itself billed
-//   result.total_cost_usd             — their sum (INCLUDES the Haiku subagent;
-//                                       the handoff's "excludes subagent" was wrong)
-// The model split IS the agent split here: sonnet => main, haiku => Explore subagent
-// (only nocg spawns one, and only nocg shows haiku usage). Token volume is still
-// summed per-model from modelUsage for the separate "tokens" story.
-//
-// Usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]
-//   e.g. offload-eval-cost.mjs /tmp/cg-offload-eval/runs trezor 3
-import { readFileSync, existsSync } from 'fs';
-
-const MAIN_TIER = /sonnet/;   // main agent
-const SUB_TIER  = /haiku/;    // Claude Code Explore subagent
-
-const [,, runsDir, repo, repsArg] = process.argv;
-if (!runsDir || !repo) { console.error('usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]   (env ARMS=nocg,raw,offload)'); process.exit(1); }
-const REPS = Number(repsArg || 3);
-// Arms to analyze (file stems `<repo>-<arm>-<rep>.jsonl`). Override for the style A/B:
-// ARMS=raw,refs,map,src. nocg's Haiku subagent is the only sub-tier; the rest are main-only.
-const ARMS = (process.env.ARMS || 'nocg,raw,offload').split(',').map((s) => s.trim()).filter(Boolean);
-
-const toks = (u) => (u.inputTokens||0)+(u.outputTokens||0)+(u.cacheReadInputTokens||0)+(u.cacheCreationInputTokens||0);
-
-function analyzeRun(file) {
-  let result = null, agentCalls = 0;
-  const tools = {}, subPids = new Set();
-  for (const line of readFileSync(file, 'utf8').split('\n')) {
-    if (!line) continue;
-    let e; try { e = JSON.parse(line); } catch { continue; }
-    if (e.parent_tool_use_id && e.message?.usage) subPids.add(e.parent_tool_use_id);
-    if (e.type === 'assistant' && Array.isArray(e.message?.content))
-      for (const b of e.message.content)
-        if (b.type === 'tool_use') { tools[b.name] = (tools[b.name]||0)+1; if (b.name === 'Agent') agentCalls++; }
-    if (e.type === 'result') result = e;
-  }
-  // Authoritative cost + tokens from Claude Code's per-model accounting.
-  const mu = result?.modelUsage || {};
-  const main = { cost: 0, tok: 0 }, sub = { cost: 0, tok: 0 };
-  for (const [model, u] of Object.entries(mu)) {
-    const bucket = SUB_TIER.test(model) ? sub : main; // sonnet/anything-else => main
-    bucket.cost += u.costUSD || 0;
-    bucket.tok  += toks(u);
-  }
-  return {
-    main, sub, subagents: subPids.size, agentCalls,
-    ccTotal: result?.total_cost_usd ?? null,
-    ok: result?.subtype === 'success',
-    durationSec: result?.duration_ms ? +(result.duration_ms/1000).toFixed(1) : null,
-    models: Object.keys(mu), tools,
-  };
-}
-
-const k = (n) => (n/1000).toFixed(0).padStart(5) + 'K';
-const d = (n) => '$' + n.toFixed(3);
-const cost = (b) => b.cost;
-const tot  = (b) => b.tok;
-
-const byArm = {};
-for (const arm of ARMS) {
-  const runs = [];
-  for (let r = 1; r <= REPS; r++) {
-    const f = `${runsDir}/${repo}-${arm}-${r}.jsonl`;
-    if (existsSync(f)) runs.push({ rep: r, ...analyzeRun(f) });
-  }
-  byArm[arm] = runs;
-}
-
-// Per-run detail. Cost is Claude Code's own modelUsage.costUSD (authoritative,
-// per-model pricing + correct cache TTL). MAIN=Sonnet, SUB=Haiku Explore subagent.
-// cc-check: main$+sub$ must equal result.total_cost_usd (delta should be ~0).
-console.log(`\n=== ${repo}: per-run main(Sonnet)/sub(Haiku) split — Claude Code's own cost accounting ===`);
-console.log('arm      rep | subAg | MAIN(sonnet) tok / $ | SUB(haiku) tok / $   | TOTAL tok / $   | cc_total Δ | dur  reads');
-for (const arm of ARMS) for (const r of byArm[arm]) {
-  const mC = cost(r.main), sC = cost(r.sub), mT = tot(r.main), sT = tot(r.sub);
-  const reads = r.tools['Read'] || 0, grep = (r.tools['Grep']||0)+(r.tools['Bash']||0)+(r.tools['Glob']||0);
-  const explore = r.tools['mcp__homegraph__homegraph_explore'] || 0;
-  const delta = (mC + sC) - (r.ccTotal || 0); // should be ~0
-  console.log(
-    `${arm.padEnd(8)} #${r.rep} | ${String(r.subagents).padStart(2)}    | ${k(mT)} ${d(mC).padStart(7)}     | ${k(sT)} ${d(sC).padStart(7)}     | ${k(mT+sT)} ${d(mC+sC).padStart(7)} | ${d(r.ccTotal||0).padStart(7)} ${(delta>=0?'+':'')+delta.toFixed(4)} | ${String(r.durationSec).padStart(5)} r=${reads} g=${grep} x=${explore}`
-  );
-}
-
-// Per-arm means
-const mean = (arr, f) => arr.length ? arr.reduce((s,x)=>s+f(x),0)/arr.length : 0;
-console.log(`\n=== ${repo}: per-arm MEANS (n per arm) ===`);
-console.log('arm      n | main $   sub $    TOTAL $  | main tok   sub tok    TOTAL tok | %$ in sub | %tok in sub');
-for (const arm of ARMS) {
-  const runs = byArm[arm]; if (!runs.length) continue;
-  const mC = mean(runs, r=>cost(r.main)), sC = mean(runs, r=>cost(r.sub));
-  const mT = mean(runs, r=>tot(r.main)),  sT = mean(runs, r=>tot(r.sub));
-  const pctSubC = (mC+sC) ? (100*sC/(mC+sC)) : 0;
-  const pctSubT = (mT+sT) ? (100*sT/(mT+sT)) : 0;
-  console.log(
-    `${arm.padEnd(8)} ${runs.length} | ${d(mC).padStart(7)} ${d(sC).padStart(7)} ${d(mC+sC).padStart(7)} | ${k(mT)} ${k(sT)} ${k(mT+sT)} | ${pctSubC.toFixed(0).padStart(3)}%      | ${pctSubT.toFixed(0).padStart(3)}%`
-  );
-}
-
-// Headline ladders — cost, tokens, duration, all vs a baseline (nocg if present, else first arm).
-console.log(`\n=== Ladders (mean, incl. subagent) ===`);
-const totals = ARMS.map(a => ({ a, c: mean(byArm[a], r=>cost(r.main)+cost(r.sub)), t: mean(byArm[a], r=>tot(r.main)+tot(r.sub)) })).filter(x=>byArm[x.a].length);
-const base = totals.find(x=>x.a==='nocg') ?? totals[0];
-const bn = base?.a ?? '?';
-console.log(`  COST (vs ${bn}):`);
-for (const x of totals) {
-  const vs = base && base.c ? ` (${((x.c/base.c-1)*100>=0?'+':'')}${((x.c/base.c-1)*100).toFixed(0)}%)` : '';
-  console.log(`    ${x.a.padEnd(8)} ${d(x.c)}${vs}`);
-}
-console.log(`  TOKENS (vs ${bn}):`);
-for (const x of totals) {
-  const vs = base && base.t ? ` (${((x.t/base.t-1)*100>=0?'+':'')}${((x.t/base.t-1)*100).toFixed(0)}%)` : '';
-  console.log(`    ${x.a.padEnd(8)} ${k(x.t)}${vs}`);
-}
-console.log(`  DURATION (wall-clock, vs ${bn}):`);
-const durs = ARMS.map(a => ({ a, s: mean(byArm[a].filter(r=>r.durationSec!=null), r=>r.durationSec) })).filter(x=>byArm[x.a].length);
-const dbase = durs.find(x=>x.a==='nocg') ?? durs[0];
-for (const x of durs) {
-  const vs = dbase && dbase.s ? ` (${((x.s/dbase.s-1)*100>=0?'+':'')}${((x.s/dbase.s-1)*100).toFixed(0)}%)` : '';
-  console.log(`    ${x.a.padEnd(8)} ${x.s.toFixed(0)}s${vs}`);
-}
+#!/usr/bin/env node
+// Cost/token analysis for the 3-arm offload eval, with a MAIN-vs-SUBAGENT split.
+//
+// The explore-subagent question. With delegation ALLOWED, the nocg arm spawns a
+// Claude Code Explore subagent; the homegraph arms do all work in the main agent.
+// Two facts make naive accounting wrong:
+//   1. The Explore subagent runs on HAIKU 4.5; the main agent on SONNET 4.6.
+//      So per-token cost differs ~3x between them — you cannot price both the same.
+//   2. The subagent's consumption is ~95% cache-reads. At Haiku's $0.10/MTok
+//      cache-read rate, a huge TOKEN volume is a small DOLLAR cost.
+//
+// Rather than re-derive cost from raw token counts (and guess the cache TTL —
+// Claude Code uses 1-hour ephemeral cache here, 2x write, not 5-min), we read
+// Claude Code's OWN authoritative accounting from the `result` event:
+//   result.modelUsage[model].costUSD  — per-model cost CC itself billed
+//   result.total_cost_usd             — their sum (INCLUDES the Haiku subagent;
+//                                       the handoff's "excludes subagent" was wrong)
+// The model split IS the agent split here: sonnet => main, haiku => Explore subagent
+// (only nocg spawns one, and only nocg shows haiku usage). Token volume is still
+// summed per-model from modelUsage for the separate "tokens" story.
+//
+// Usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]
+//   e.g. offload-eval-cost.mjs /tmp/cg-offload-eval/runs trezor 3
+import { readFileSync, existsSync } from 'fs';
+
+const MAIN_TIER = /sonnet/;   // main agent
+const SUB_TIER  = /haiku/;    // Claude Code Explore subagent
+
+const [,, runsDir, repo, repsArg] = process.argv;
+if (!runsDir || !repo) { console.error('usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]   (env ARMS=nocg,raw,offload)'); process.exit(1); }
+const REPS = Number(repsArg || 3);
+// Arms to analyze (file stems `<repo>-<arm>-<rep>.jsonl`). Override for the style A/B:
+// ARMS=raw,refs,map,src. nocg's Haiku subagent is the only sub-tier; the rest are main-only.
+const ARMS = (process.env.ARMS || 'nocg,raw,offload').split(',').map((s) => s.trim()).filter(Boolean);
+
+const toks = (u) => (u.inputTokens||0)+(u.outputTokens||0)+(u.cacheReadInputTokens||0)+(u.cacheCreationInputTokens||0);
+
+function analyzeRun(file) {
+  let result = null, agentCalls = 0;
+  const tools = {}, subPids = new Set();
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let e; try { e = JSON.parse(line); } catch { continue; }
+    if (e.parent_tool_use_id && e.message?.usage) subPids.add(e.parent_tool_use_id);
+    if (e.type === 'assistant' && Array.isArray(e.message?.content))
+      for (const b of e.message.content)
+        if (b.type === 'tool_use') { tools[b.name] = (tools[b.name]||0)+1; if (b.name === 'Agent') agentCalls++; }
+    if (e.type === 'result') result = e;
+  }
+  // Authoritative cost + tokens from Claude Code's per-model accounting.
+  const mu = result?.modelUsage || {};
+  const main = { cost: 0, tok: 0 }, sub = { cost: 0, tok: 0 };
+  for (const [model, u] of Object.entries(mu)) {
+    const bucket = SUB_TIER.test(model) ? sub : main; // sonnet/anything-else => main
+    bucket.cost += u.costUSD || 0;
+    bucket.tok  += toks(u);
+  }
+  return {
+    main, sub, subagents: subPids.size, agentCalls,
+    ccTotal: result?.total_cost_usd ?? null,
+    ok: result?.subtype === 'success',
+    durationSec: result?.duration_ms ? +(result.duration_ms/1000).toFixed(1) : null,
+    models: Object.keys(mu), tools,
+  };
+}
+
+const k = (n) => (n/1000).toFixed(0).padStart(5) + 'K';
+const d = (n) => '$' + n.toFixed(3);
+const cost = (b) => b.cost;
+const tot  = (b) => b.tok;
+
+const byArm = {};
+for (const arm of ARMS) {
+  const runs = [];
+  for (let r = 1; r <= REPS; r++) {
+    const f = `${runsDir}/${repo}-${arm}-${r}.jsonl`;
+    if (existsSync(f)) runs.push({ rep: r, ...analyzeRun(f) });
+  }
+  byArm[arm] = runs;
+}
+
+// Per-run detail. Cost is Claude Code's own modelUsage.costUSD (authoritative,
+// per-model pricing + correct cache TTL). MAIN=Sonnet, SUB=Haiku Explore subagent.
+// cc-check: main$+sub$ must equal result.total_cost_usd (delta should be ~0).
+console.log(`\n=== ${repo}: per-run main(Sonnet)/sub(Haiku) split — Claude Code's own cost accounting ===`);
+console.log('arm      rep | subAg | MAIN(sonnet) tok / $ | SUB(haiku) tok / $   | TOTAL tok / $   | cc_total Δ | dur  reads');
+for (const arm of ARMS) for (const r of byArm[arm]) {
+  const mC = cost(r.main), sC = cost(r.sub), mT = tot(r.main), sT = tot(r.sub);
+  const reads = r.tools['Read'] || 0, grep = (r.tools['Grep']||0)+(r.tools['Bash']||0)+(r.tools['Glob']||0);
+  const explore = r.tools['mcp__homegraph__homegraph_explore'] || 0;
+  const delta = (mC + sC) - (r.ccTotal || 0); // should be ~0
+  console.log(
+    `${arm.padEnd(8)} #${r.rep} | ${String(r.subagents).padStart(2)}    | ${k(mT)} ${d(mC).padStart(7)}     | ${k(sT)} ${d(sC).padStart(7)}     | ${k(mT+sT)} ${d(mC+sC).padStart(7)} | ${d(r.ccTotal||0).padStart(7)} ${(delta>=0?'+':'')+delta.toFixed(4)} | ${String(r.durationSec).padStart(5)} r=${reads} g=${grep} x=${explore}`
+  );
+}
+
+// Per-arm means
+const mean = (arr, f) => arr.length ? arr.reduce((s,x)=>s+f(x),0)/arr.length : 0;
+console.log(`\n=== ${repo}: per-arm MEANS (n per arm) ===`);
+console.log('arm      n | main $   sub $    TOTAL $  | main tok   sub tok    TOTAL tok | %$ in sub | %tok in sub');
+for (const arm of ARMS) {
+  const runs = byArm[arm]; if (!runs.length) continue;
+  const mC = mean(runs, r=>cost(r.main)), sC = mean(runs, r=>cost(r.sub));
+  const mT = mean(runs, r=>tot(r.main)),  sT = mean(runs, r=>tot(r.sub));
+  const pctSubC = (mC+sC) ? (100*sC/(mC+sC)) : 0;
+  const pctSubT = (mT+sT) ? (100*sT/(mT+sT)) : 0;
+  console.log(
+    `${arm.padEnd(8)} ${runs.length} | ${d(mC).padStart(7)} ${d(sC).padStart(7)} ${d(mC+sC).padStart(7)} | ${k(mT)} ${k(sT)} ${k(mT+sT)} | ${pctSubC.toFixed(0).padStart(3)}%      | ${pctSubT.toFixed(0).padStart(3)}%`
+  );
+}
+
+// Headline ladders — cost, tokens, duration, all vs a baseline (nocg if present, else first arm).
+console.log(`\n=== Ladders (mean, incl. subagent) ===`);
+const totals = ARMS.map(a => ({ a, c: mean(byArm[a], r=>cost(r.main)+cost(r.sub)), t: mean(byArm[a], r=>tot(r.main)+tot(r.sub)) })).filter(x=>byArm[x.a].length);
+const base = totals.find(x=>x.a==='nocg') ?? totals[0];
+const bn = base?.a ?? '?';
+console.log(`  COST (vs ${bn}):`);
+for (const x of totals) {
+  const vs = base && base.c ? ` (${((x.c/base.c-1)*100>=0?'+':'')}${((x.c/base.c-1)*100).toFixed(0)}%)` : '';
+  console.log(`    ${x.a.padEnd(8)} ${d(x.c)}${vs}`);
+}
+console.log(`  TOKENS (vs ${bn}):`);
+for (const x of totals) {
+  const vs = base && base.t ? ` (${((x.t/base.t-1)*100>=0?'+':'')}${((x.t/base.t-1)*100).toFixed(0)}%)` : '';
+  console.log(`    ${x.a.padEnd(8)} ${k(x.t)}${vs}`);
+}
+console.log(`  DURATION (wall-clock, vs ${bn}):`);
+const durs = ARMS.map(a => ({ a, s: mean(byArm[a].filter(r=>r.durationSec!=null), r=>r.durationSec) })).filter(x=>byArm[x.a].length);
+const dbase = durs.find(x=>x.a==='nocg') ?? durs[0];
+for (const x of durs) {
+  const vs = dbase && dbase.s ? ` (${((x.s/dbase.s-1)*100>=0?'+':'')}${((x.s/dbase.s-1)*100).toFixed(0)}%)` : '';
+  console.log(`    ${x.a.padEnd(8)} ${x.s.toFixed(0)}s${vs}`);
+}

package/scripts/agent-eval/offload-eval-effort.mjs

@@ -1,108 +1,108 @@
-#!/usr/bin/env node
-// Effort A/B — does HOMEGRAPH_OFFLOAD_EFFORT=high improve offload SYNTHESIS FIDELITY vs low?
-// Probe-based (no agent): for each repo × effort × rep, run homegraph_explore with the offload
-// ON on the canonical question, capture the synthesized answer + AI tokens/cost/latency, then
-// Sonnet-judge that answer's fidelity vs source-verified ground truth. Isolates the synthesis
-// from agent/adoption noise. Requires `homegraph login` (managed offload) + indexed repos.
-//
-// Env: REPS (default 3) · CG_ENGINE (engine repo) · AGENT_EVAL_OUT (repos under /repos) · CONC (judge concurrency)
-import { pathToFileURL, fileURLToPath } from 'node:url';
-import { resolve, dirname, join } from 'node:path';
-import { readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
-import { execFile } from 'node:child_process';
-import { tmpdir } from 'node:os';
-
-const HERE = dirname(fileURLToPath(import.meta.url));
-const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
-const OUT = process.env.AGENT_EVAL_OUT || '/tmp/cg-offload-eval';
-const REPOS = join(OUT, 'repos');
-const GT = JSON.parse(readFileSync(resolve(HERE, 'offload-eval-ground-truth.json'), 'utf8'));
-const REPS = Number(process.env.REPS || 3);
-const CONC = Number(process.env.CONC || 4);
-const EFFORTS = (process.env.EFFORTS_FILTER || 'low,high').split(',');
-const ONLY = process.env.REPOS_FILTER ? new Set(process.env.REPOS_FILTER.split(',')) : null;
-const TIER = { mtkruto: 'small', postybirb: 'medium', shapeshift: 'complex', trezor: 'large' };
-
-const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
-const idx = await load('dist/index.js');
-const toolsMod = await load('dist/mcp/tools.js');
-const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
-const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler;
-if (typeof HomeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
-  console.error('could not load engine from', ENGINE); process.exit(2);
-}
-
-const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. Do NOT use any tools.
-
-QUESTION: ${gt.question}
-
-VERIFIED GROUND TRUTH (the actual call path + files):
-${gt.truth}
-
-SYNTHESIZED ANSWER (to score):
-${ans || '(empty)'}
-
-Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was it honest? A confident WRONG trace is the worst outcome — penalize it harder than an honest partial.
-Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`;
-
-const askJudge = (prompt) => new Promise((res) => {
-  execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5',
-    '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
-    { cwd: OUT, maxBuffer: 1 << 24, timeout: 120000 }, (err, stdout) => {
-      const m = (stdout || '').match(/\{[\s\S]*\}/);
-      if (!m) return res({ verdict: 'error', score: null, note: (err ? err.message : 'no json').slice(0, 60) });
-      try { res(JSON.parse(m[0])); } catch { res({ verdict: 'error', score: null }); }
-    });
-});
-
-// ---- 1. Probe: collect synthesized answers at each effort -------------------
-const records = [];
-for (const repo of Object.keys(GT)) {
-  if (ONLY && !ONLY.has(repo)) continue;
-  const dir = join(REPOS, repo);
-  if (!existsSync(join(dir, '.homegraph'))) { console.error('skip (not indexed):', repo); continue; }
-  const cg = HomeGraph.openSync(dir);
-  const h = new ToolHandler(cg);
-  for (const effort of EFFORTS) {
-    for (let rep = 1; rep <= REPS; rep++) {
-      process.env.HOMEGRAPH_OFFLOAD_EFFORT = effort;
-      const usageLog = join(tmpdir(), `effort-${repo}-${effort}-${rep}.jsonl`);
-      try { rmSync(usageLog); } catch { /* none */ }
-      process.env.HOMEGRAPH_OFFLOAD_USAGE_LOG = usageLog;
-      let answer = '';
-      try { answer = (await h.execute('homegraph_explore', { query: GT[repo].question }))?.content?.[0]?.text ?? ''; }
-      catch (e) { console.error(`  ${repo}/${effort}#${rep} explore failed: ${e?.message}`); }
-      const fired = /Synthesized by HomeGraph/.test(answer);
-      const ai = { tokens: 0, cost: 0, ms: 0 };
-      if (existsSync(usageLog)) for (const e of readFileSync(usageLog, 'utf8').split('\n').filter(Boolean).map(JSON.parse)) {
-        ai.tokens += e.totalTokens || 0; ai.cost += e.costUsd || 0; ai.ms += e.ms || 0;
-      }
-      records.push({ repo, tier: TIER[repo], effort, rep, fired, ai, answer });
-      console.error(`  ${repo}/${effort}#${rep}: fired=${fired} ${ai.tokens}tok $${ai.cost.toFixed(4)} ${ai.ms}ms`);
-    }
-  }
-  try { cg.close?.(); } catch { /* none */ }
-}
-
-// ---- 2. Judge fidelity (concurrency) ---------------------------------------
-console.error(`\njudging ${records.length} answers (concurrency ${CONC})...`);
-let done = 0;
-const q = [...records];
-async function worker() { while (q.length) { const r = q.shift(); r.fid = await askJudge(fidPrompt(GT[r.repo], r.answer)); console.error(`  [${++done}/${records.length}] ${r.repo}/${r.effort}#${r.rep}: ${r.fid.verdict} ${r.fid.score ?? ''}`); } }
-await Promise.all(Array.from({ length: CONC }, worker));
-writeFileSync(join(OUT, 'effort-results.jsonl'), records.map((r) => JSON.stringify(r)).join('\n') + '\n');
-
-// ---- 3. Aggregate: low vs high per repo ------------------------------------
-const med = (a) => { a = a.filter((x) => x != null).sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
-console.log(`\n${'='.repeat(80)}\nEFFORT A/B — offload synthesis fidelity (probe, n=${REPS}/cell)\n${'='.repeat(80)}`);
-console.log(`${'repo'.padEnd(11)} ${'tier'.padEnd(8)} ${'effort'.padEnd(6)} fired  ${'fid(med)'.padStart(8)} ${'fab%'.padStart(5)} ${'AItok'.padStart(7)} ${'AIcost'.padStart(8)} ${'ms(med)'.padStart(8)}`);
-for (const repo of Object.keys(GT)) {
-  for (const effort of EFFORTS) {
-    const rs = records.filter((r) => r.repo === repo && r.effort === effort);
-    if (!rs.length) continue;
-    const fids = rs.map((r) => r.fid?.score).filter((x) => x != null);
-    const fab = rs.filter((r) => r.fid?.fabrication === true).length;
-    console.log(`${repo.padEnd(11)} ${TIER[repo].padEnd(8)} ${effort.padEnd(6)} ${rs.filter((r) => r.fired).length}/${rs.length}   ${String(med(fids) ?? '—').padStart(8)} ${String(Math.round(100 * fab / rs.length) + '%').padStart(5)} ${String(Math.round(med(rs.map((r) => r.ai.tokens)) / 1000) + 'k').padStart(7)} ${('$' + (med(rs.map((r) => r.ai.cost)) ?? 0).toFixed(4)).padStart(8)} ${String(med(rs.map((r) => r.ai.ms)) ?? '—').padStart(8)}`);
-  }
-}
-console.log('');
+#!/usr/bin/env node
+// Effort A/B — does HOMEGRAPH_OFFLOAD_EFFORT=high improve offload SYNTHESIS FIDELITY vs low?
+// Probe-based (no agent): for each repo × effort × rep, run homegraph_explore with the offload
+// ON on the canonical question, capture the synthesized answer + AI tokens/cost/latency, then
+// Sonnet-judge that answer's fidelity vs source-verified ground truth. Isolates the synthesis
+// from agent/adoption noise. Requires `homegraph login` (managed offload) + indexed repos.
+//
+// Env: REPS (default 3) · CG_ENGINE (engine repo) · AGENT_EVAL_OUT (repos under /repos) · CONC (judge concurrency)
+import { pathToFileURL, fileURLToPath } from 'node:url';
+import { resolve, dirname, join } from 'node:path';
+import { readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
+import { execFile } from 'node:child_process';
+import { tmpdir } from 'node:os';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
+const OUT = process.env.AGENT_EVAL_OUT || '/tmp/cg-offload-eval';
+const REPOS = join(OUT, 'repos');
+const GT = JSON.parse(readFileSync(resolve(HERE, 'offload-eval-ground-truth.json'), 'utf8'));
+const REPS = Number(process.env.REPS || 3);
+const CONC = Number(process.env.CONC || 4);
+const EFFORTS = (process.env.EFFORTS_FILTER || 'low,high').split(',');
+const ONLY = process.env.REPOS_FILTER ? new Set(process.env.REPOS_FILTER.split(',')) : null;
+const TIER = { mtkruto: 'small', postybirb: 'medium', shapeshift: 'complex', trezor: 'large' };
+
+const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
+const idx = await load('dist/index.js');
+const toolsMod = await load('dist/mcp/tools.js');
+const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
+const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler;
+if (typeof HomeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
+  console.error('could not load engine from', ENGINE); process.exit(2);
+}
+
+const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. Do NOT use any tools.
+
+QUESTION: ${gt.question}
+
+VERIFIED GROUND TRUTH (the actual call path + files):
+${gt.truth}
+
+SYNTHESIZED ANSWER (to score):
+${ans || '(empty)'}
+
+Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was it honest? A confident WRONG trace is the worst outcome — penalize it harder than an honest partial.
+Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`;
+
+const askJudge = (prompt) => new Promise((res) => {
+  execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5',
+    '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
+    { cwd: OUT, maxBuffer: 1 << 24, timeout: 120000 }, (err, stdout) => {
+      const m = (stdout || '').match(/\{[\s\S]*\}/);
+      if (!m) return res({ verdict: 'error', score: null, note: (err ? err.message : 'no json').slice(0, 60) });
+      try { res(JSON.parse(m[0])); } catch { res({ verdict: 'error', score: null }); }
+    });
+});
+
+// ---- 1. Probe: collect synthesized answers at each effort -------------------
+const records = [];
+for (const repo of Object.keys(GT)) {
+  if (ONLY && !ONLY.has(repo)) continue;
+  const dir = join(REPOS, repo);
+  if (!existsSync(join(dir, '.homegraph'))) { console.error('skip (not indexed):', repo); continue; }
+  const cg = HomeGraph.openSync(dir);
+  const h = new ToolHandler(cg);
+  for (const effort of EFFORTS) {
+    for (let rep = 1; rep <= REPS; rep++) {
+      process.env.HOMEGRAPH_OFFLOAD_EFFORT = effort;
+      const usageLog = join(tmpdir(), `effort-${repo}-${effort}-${rep}.jsonl`);
+      try { rmSync(usageLog); } catch { /* none */ }
+      process.env.HOMEGRAPH_OFFLOAD_USAGE_LOG = usageLog;
+      let answer = '';
+      try { answer = (await h.execute('homegraph_explore', { query: GT[repo].question }))?.content?.[0]?.text ?? ''; }
+      catch (e) { console.error(`  ${repo}/${effort}#${rep} explore failed: ${e?.message}`); }
+      const fired = /Synthesized by HomeGraph/.test(answer);
+      const ai = { tokens: 0, cost: 0, ms: 0 };
+      if (existsSync(usageLog)) for (const e of readFileSync(usageLog, 'utf8').split('\n').filter(Boolean).map(JSON.parse)) {
+        ai.tokens += e.totalTokens || 0; ai.cost += e.costUsd || 0; ai.ms += e.ms || 0;
+      }
+      records.push({ repo, tier: TIER[repo], effort, rep, fired, ai, answer });
+      console.error(`  ${repo}/${effort}#${rep}: fired=${fired} ${ai.tokens}tok $${ai.cost.toFixed(4)} ${ai.ms}ms`);
+    }
+  }
+  try { cg.close?.(); } catch { /* none */ }
+}
+
+// ---- 2. Judge fidelity (concurrency) ---------------------------------------
+console.error(`\njudging ${records.length} answers (concurrency ${CONC})...`);
+let done = 0;
+const q = [...records];
+async function worker() { while (q.length) { const r = q.shift(); r.fid = await askJudge(fidPrompt(GT[r.repo], r.answer)); console.error(`  [${++done}/${records.length}] ${r.repo}/${r.effort}#${r.rep}: ${r.fid.verdict} ${r.fid.score ?? ''}`); } }
+await Promise.all(Array.from({ length: CONC }, worker));
+writeFileSync(join(OUT, 'effort-results.jsonl'), records.map((r) => JSON.stringify(r)).join('\n') + '\n');
+
+// ---- 3. Aggregate: low vs high per repo ------------------------------------
+const med = (a) => { a = a.filter((x) => x != null).sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
+console.log(`\n${'='.repeat(80)}\nEFFORT A/B — offload synthesis fidelity (probe, n=${REPS}/cell)\n${'='.repeat(80)}`);
+console.log(`${'repo'.padEnd(11)} ${'tier'.padEnd(8)} ${'effort'.padEnd(6)} fired  ${'fid(med)'.padStart(8)} ${'fab%'.padStart(5)} ${'AItok'.padStart(7)} ${'AIcost'.padStart(8)} ${'ms(med)'.padStart(8)}`);
+for (const repo of Object.keys(GT)) {
+  for (const effort of EFFORTS) {
+    const rs = records.filter((r) => r.repo === repo && r.effort === effort);
+    if (!rs.length) continue;
+    const fids = rs.map((r) => r.fid?.score).filter((x) => x != null);
+    const fab = rs.filter((r) => r.fid?.fabrication === true).length;
+    console.log(`${repo.padEnd(11)} ${TIER[repo].padEnd(8)} ${effort.padEnd(6)} ${rs.filter((r) => r.fired).length}/${rs.length}   ${String(med(fids) ?? '—').padStart(8)} ${String(Math.round(100 * fab / rs.length) + '%').padStart(5)} ${String(Math.round(med(rs.map((r) => r.ai.tokens)) / 1000) + 'k').padStart(7)} ${('$' + (med(rs.map((r) => r.ai.cost)) ?? 0).toFixed(4)).padStart(8)} ${String(med(rs.map((r) => r.ai.ms)) ?? '—').padStart(8)}`);
+  }
+}
+console.log('');

package/scripts/agent-eval/offload-eval-frontload-matrix.sh

@@ -1,25 +1,25 @@
-#!/usr/bin/env bash
-# Run the FRONTLOAD arm across all 4 tiers (n reps), then judge + merge with the existing
-# matrix (offload/raw/nocg in $OUT/judged.jsonl, if present) + emit a combined summary.
-# Env: REPS (default 3)  AGENT_EVAL_OUT=<scratch dir>
-set -uo pipefail
-HERE="$(cd "$(dirname "$0")" && pwd)"
-OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
-GT="$HERE/offload-eval-ground-truth.json"
-REPS="${REPS:-3}"
-export RESULTS="$OUT/results-fl.jsonl"
-: > "$RESULTS"; rm -f "$OUT/runs/hook-debug.log"
-for repo in mtkruto postybirb shapeshift trezor; do
-  case "$repo" in mtkruto) tier=small;; postybirb) tier=medium;; shapeshift) tier=complex;; trezor) tier=large;; esac
-  Q=$(node -e "console.log(JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))[process.argv[2]].question)" "$GT" "$repo")
-  echo ""; echo "### $repo ($tier)  $(date +%H:%M:%S)"
-  bash "$HERE/offload-eval-frontload.sh" "$OUT/repos/$repo" "$tier" "$REPS" "$Q"
-done
-echo ""
-echo "frontload: $(wc -l < "$RESULTS") runs | hook injections: $(grep -c INJECTED "$OUT/runs/hook-debug.log" 2>/dev/null) | errors: $(grep -c ERROR "$OUT/runs/hook-debug.log" 2>/dev/null)"
-echo "=== JUDGE frontload ==="
-node "$HERE/offload-eval-judge.mjs" --results "$RESULTS" --truth "$GT" --out "$OUT/judged-fl.jsonl" --concurrency 4 2>&1 | tail -4
-if [ -f "$OUT/judged.jsonl" ]; then cat "$OUT/judged.jsonl" "$OUT/judged-fl.jsonl" > "$OUT/judged-all.jsonl"; else cp "$OUT/judged-fl.jsonl" "$OUT/judged-all.jsonl"; fi
-echo "=== COMBINED SUMMARY ==="
-node "$HERE/offload-eval-summarize.mjs" "$OUT/judged-all.jsonl"
-echo "###### FRONTLOAD MATRIX DONE"
+#!/usr/bin/env bash
+# Run the FRONTLOAD arm across all 4 tiers (n reps), then judge + merge with the existing
+# matrix (offload/raw/nocg in $OUT/judged.jsonl, if present) + emit a combined summary.
+# Env: REPS (default 3)  AGENT_EVAL_OUT=<scratch dir>
+set -uo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
+GT="$HERE/offload-eval-ground-truth.json"
+REPS="${REPS:-3}"
+export RESULTS="$OUT/results-fl.jsonl"
+: > "$RESULTS"; rm -f "$OUT/runs/hook-debug.log"
+for repo in mtkruto postybirb shapeshift trezor; do
+  case "$repo" in mtkruto) tier=small;; postybirb) tier=medium;; shapeshift) tier=complex;; trezor) tier=large;; esac
+  Q=$(node -e "console.log(JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))[process.argv[2]].question)" "$GT" "$repo")
+  echo ""; echo "### $repo ($tier)  $(date +%H:%M:%S)"
+  bash "$HERE/offload-eval-frontload.sh" "$OUT/repos/$repo" "$tier" "$REPS" "$Q"
+done
+echo ""
+echo "frontload: $(wc -l < "$RESULTS") runs | hook injections: $(grep -c INJECTED "$OUT/runs/hook-debug.log" 2>/dev/null) | errors: $(grep -c ERROR "$OUT/runs/hook-debug.log" 2>/dev/null)"
+echo "=== JUDGE frontload ==="
+node "$HERE/offload-eval-judge.mjs" --results "$RESULTS" --truth "$GT" --out "$OUT/judged-fl.jsonl" --concurrency 4 2>&1 | tail -4
+if [ -f "$OUT/judged.jsonl" ]; then cat "$OUT/judged.jsonl" "$OUT/judged-fl.jsonl" > "$OUT/judged-all.jsonl"; else cp "$OUT/judged-fl.jsonl" "$OUT/judged-all.jsonl"; fi
+echo "=== COMBINED SUMMARY ==="
+node "$HERE/offload-eval-summarize.mjs" "$OUT/judged-all.jsonl"
+echo "###### FRONTLOAD MATRIX DONE"