homegraph 1.1.2 → 1.1.3

package/scripts/agent-eval/offload-eval-hook.mjs

@@ -1,84 +1,84 @@
-#!/usr/bin/env node
-// UserPromptSubmit hook — APPROACH 1: additive context-injection.
-// Front-loads homegraph's structural answer for flow/impact/"how/where" prompts so the
-// agent's reflex grep/read has nothing left to find. Strictly additive (never blocks),
-// gated to structural prompts (no cost otherwise), and uses RAW explore (offload disabled)
-// so the injected context is accurate — never the (currently low-fidelity) synthesis.
-//
-// Reads {prompt, cwd} as JSON on stdin; prints the explore result to stdout (which Claude
-// Code injects into the agent's context). Any failure -> silent exit 0 (degradable).
-import { pathToFileURL, fileURLToPath } from 'node:url';
-import { resolve, join, dirname } from 'node:path';
-import { existsSync, readFileSync, appendFileSync } from 'node:fs';
-
-// Resolve the engine repo from this script's own location (scripts/agent-eval/ -> ../..),
-// overridable with CG_ENGINE. The hook ships inside the repo, so it finds its own dist.
-const HERE = dirname(fileURLToPath(import.meta.url));
-const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
-const BUDGET = Number(process.env.CG_FRONTLOAD_BUDGET || 16000);
-
-// Debug log only when CG_FRONTLOAD_DEBUG is set to a file path (the harness points it at a
-// log to count injections); off by default so the shipped hook writes nothing extra.
-const DBG = process.env.CG_FRONTLOAD_DEBUG;
-const dbg = (m) => { if (!DBG) return; try { appendFileSync(DBG, `[${new Date().toISOString()}] ${m}\n`); } catch { /* ignore */ } };
-
-let input = {};
-try { input = JSON.parse(readFileSync(0, 'utf8')); } catch (e) { dbg('stdin parse fail: ' + e.message); }
-const prompt = String(input.prompt || '');
-const cwd = String(input.cwd || process.cwd());
-dbg(`invoked: promptLen=${prompt.length} cwd=${cwd}`);
-
-// Gate: only structural / flow / impact / where-how questions. Cheap regex; silent no-op
-// otherwise so non-structural prompts ("fix this typo") cost nothing.
-const STRUCTURAL = /\b(how|where|trace|flow|path|reach(es|ed)?|call(s|ed|er|ers|ee)?|depend|impact|affect|wire[ds]?|connect|implement|architect|structure|breaks?|what calls|why does)\b/i;
-if (!prompt || !STRUCTURAL.test(prompt)) { dbg('gate: non-structural, no-op'); process.exit(0); }
-dbg('gate: structural PASS');
-
-// Find the index: cwd, then walk up a few levels.
-let root = cwd, found = null;
-for (let i = 0; i < 6 && root; i++) {
-  if (existsSync(join(root, '.homegraph'))) { found = root; break; }
-  const parent = resolve(root, '..'); if (parent === root) break; root = parent;
-}
-if (!found) { dbg(`no .homegraph found from cwd=${cwd}`); process.exit(0); }
-dbg(`found index at ${found}`);
-
-try {
-  process.env.HOMEGRAPH_OFFLOAD_DISABLE = '1'; // raw, accurate — never the unfixed offload
-  process.env.HOMEGRAPH_TELEMETRY = '0'; process.env.DO_NOT_TRACK = '1';
-  const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
-  const idx = await load('dist/index.js');
-  const tools = await load('dist/mcp/tools.js');
-  const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
-  const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
-  if (typeof HomeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') process.exit(0);
-
-  // Retry once on a transient busy/locked index (the hook's openSync can race a
-  // freshly-warming daemon on the first prompt of a session).
-  let text = '';
-  for (let attempt = 1; attempt <= 2; attempt++) {
-    try {
-      const cg = HomeGraph.openSync(found);
-      const h = new ToolHandler(cg);
-      const res = await h.execute('homegraph_explore', { query: prompt });
-      text = res?.content?.[0]?.text ?? '';
-      try { cg.close?.(); } catch { /* ignore */ }
-      dbg(`explore attempt ${attempt} returned ${text.length} chars`);
-      break;
-    } catch (e) {
-      dbg(`explore attempt ${attempt} failed: ${e?.message || e}`);
-      if (attempt === 2) throw e;
-      await new Promise((r) => setTimeout(r, 800));
-    }
-  }
-  if (!text.trim()) { dbg('empty explore result, no-op'); process.exit(0); }
-  if (text.length > BUDGET) text = text.slice(0, BUDGET) + '\n…[front-load truncated to budget]';
-
-  process.stdout.write(
-    `## HomeGraph structural context (auto-retrieved for this question)\n` +
-    `The code graph was queried for your question; the relevant symbols, source, and call flow are below. ` +
-    `Treat the quoted source as already read. If you need more, call homegraph_explore with specific symbol names rather than grepping or reading files.\n\n` +
-    text + '\n'
-  );
-  dbg(`INJECTED ${text.length} chars`);
-} catch (e) { dbg('ERROR: ' + (e?.stack || e?.message || e)); process.exit(0); } // degradable
+#!/usr/bin/env node
+// UserPromptSubmit hook — APPROACH 1: additive context-injection.
+// Front-loads homegraph's structural answer for flow/impact/"how/where" prompts so the
+// agent's reflex grep/read has nothing left to find. Strictly additive (never blocks),
+// gated to structural prompts (no cost otherwise), and uses RAW explore (offload disabled)
+// so the injected context is accurate — never the (currently low-fidelity) synthesis.
+//
+// Reads {prompt, cwd} as JSON on stdin; prints the explore result to stdout (which Claude
+// Code injects into the agent's context). Any failure -> silent exit 0 (degradable).
+import { pathToFileURL, fileURLToPath } from 'node:url';
+import { resolve, join, dirname } from 'node:path';
+import { existsSync, readFileSync, appendFileSync } from 'node:fs';
+
+// Resolve the engine repo from this script's own location (scripts/agent-eval/ -> ../..),
+// overridable with CG_ENGINE. The hook ships inside the repo, so it finds its own dist.
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
+const BUDGET = Number(process.env.CG_FRONTLOAD_BUDGET || 16000);
+
+// Debug log only when CG_FRONTLOAD_DEBUG is set to a file path (the harness points it at a
+// log to count injections); off by default so the shipped hook writes nothing extra.
+const DBG = process.env.CG_FRONTLOAD_DEBUG;
+const dbg = (m) => { if (!DBG) return; try { appendFileSync(DBG, `[${new Date().toISOString()}] ${m}\n`); } catch { /* ignore */ } };
+
+let input = {};
+try { input = JSON.parse(readFileSync(0, 'utf8')); } catch (e) { dbg('stdin parse fail: ' + e.message); }
+const prompt = String(input.prompt || '');
+const cwd = String(input.cwd || process.cwd());
+dbg(`invoked: promptLen=${prompt.length} cwd=${cwd}`);
+
+// Gate: only structural / flow / impact / where-how questions. Cheap regex; silent no-op
+// otherwise so non-structural prompts ("fix this typo") cost nothing.
+const STRUCTURAL = /\b(how|where|trace|flow|path|reach(es|ed)?|call(s|ed|er|ers|ee)?|depend|impact|affect|wire[ds]?|connect|implement|architect|structure|breaks?|what calls|why does)\b/i;
+if (!prompt || !STRUCTURAL.test(prompt)) { dbg('gate: non-structural, no-op'); process.exit(0); }
+dbg('gate: structural PASS');
+
+// Find the index: cwd, then walk up a few levels.
+let root = cwd, found = null;
+for (let i = 0; i < 6 && root; i++) {
+  if (existsSync(join(root, '.homegraph'))) { found = root; break; }
+  const parent = resolve(root, '..'); if (parent === root) break; root = parent;
+}
+if (!found) { dbg(`no .homegraph found from cwd=${cwd}`); process.exit(0); }
+dbg(`found index at ${found}`);
+
+try {
+  process.env.HOMEGRAPH_OFFLOAD_DISABLE = '1'; // raw, accurate — never the unfixed offload
+  process.env.HOMEGRAPH_TELEMETRY = '0'; process.env.DO_NOT_TRACK = '1';
+  const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
+  const idx = await load('dist/index.js');
+  const tools = await load('dist/mcp/tools.js');
+  const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
+  const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+  if (typeof HomeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') process.exit(0);
+
+  // Retry once on a transient busy/locked index (the hook's openSync can race a
+  // freshly-warming daemon on the first prompt of a session).
+  let text = '';
+  for (let attempt = 1; attempt <= 2; attempt++) {
+    try {
+      const cg = HomeGraph.openSync(found);
+      const h = new ToolHandler(cg);
+      const res = await h.execute('homegraph_explore', { query: prompt });
+      text = res?.content?.[0]?.text ?? '';
+      try { cg.close?.(); } catch { /* ignore */ }
+      dbg(`explore attempt ${attempt} returned ${text.length} chars`);
+      break;
+    } catch (e) {
+      dbg(`explore attempt ${attempt} failed: ${e?.message || e}`);
+      if (attempt === 2) throw e;
+      await new Promise((r) => setTimeout(r, 800));
+    }
+  }
+  if (!text.trim()) { dbg('empty explore result, no-op'); process.exit(0); }
+  if (text.length > BUDGET) text = text.slice(0, BUDGET) + '\n…[front-load truncated to budget]';
+
+  process.stdout.write(
+    `## HomeGraph structural context (auto-retrieved for this question)\n` +
+    `The code graph was queried for your question; the relevant symbols, source, and call flow are below. ` +
+    `Treat the quoted source as already read. If you need more, call homegraph_explore with specific symbol names rather than grepping or reading files.\n\n` +
+    text + '\n'
+  );
+  dbg(`INJECTED ${text.length} chars`);
+} catch (e) { dbg('ERROR: ' + (e?.stack || e?.message || e)); process.exit(0); } // degradable

package/scripts/agent-eval/offload-eval-judge.mjs

@@ -1,103 +1,103 @@
-#!/usr/bin/env node
-// Accuracy judge. For each run in results.jsonl:
-//   - end-to-end: agent finalAnswer vs verified ground truth (all arms)
-//   - fidelity:   offload synthesized answer vs ground truth (offload arm only)
-// Judge = claude -p sonnet --effort high, no tools, run from a neutral cwd,
-// JSON-only verdicts. Writes judged.jsonl (one line per run, verdicts merged).
-//
-// Usage: judge.mjs --results <f> --truth <f> --out <f> [--concurrency 4]
-import { readFileSync, writeFileSync, existsSync } from 'fs';
-import { execFile } from 'child_process';
-
-const A = {};
-for (let i = 2; i < process.argv.length; i += 2) A[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
-const results = readFileSync(A.results, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
-const truth = JSON.parse(readFileSync(A.truth, 'utf8'));
-const OUT = A.out || '/tmp/cg-offload-eval/judged.jsonl';
-const CONC = Number(A.concurrency || 4);
-
-function askJudge(prompt) {
-  return new Promise((resolve) => {
-    execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high',
-      '--max-budget-usd', '0.5', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
-      // Run from a neutral dir with no repo files so the judge can't "cheat" by reading source.
-      { cwd: process.env.AGENT_EVAL_OUT || '/tmp', maxBuffer: 1 << 24, timeout: 120000 },
-      (err, stdout) => {
-        const raw = (stdout || '').trim();
-        const m = raw.match(/\{[\s\S]*\}/);
-        if (!m) return resolve({ verdict: 'error', score: null, note: (err ? 'exec ' + err.message : 'no json').slice(0, 80) });
-        try { resolve(JSON.parse(m[0])); } catch { resolve({ verdict: 'error', score: null, note: 'parse fail' }); }
-      });
-  });
-}
-
-const e2ePrompt = (gt, ans) => `You are scoring whether an AI coding agent correctly answered a code-flow question about a repository. Judge ONLY against the verified ground truth. Do NOT use any tools.
-
-QUESTION: ${gt.question}
-
-VERIFIED GROUND TRUTH (the actual call path + files):
-${gt.truth}
-
-AGENT'S ANSWER:
-${ans || '(empty)'}
-
-Score how correct the agent's answer is vs the ground truth. A "pass" means it identifies the core mechanism and the major hops with the right files/symbols and makes no materially wrong claim. "partial" = right area but misses major hops or has notable errors. "fail" = wrong layer, fabricated, or misses the mechanism.
-Output ONLY minified JSON, no prose, no code fences:
-{"verdict":"pass|partial|fail","score":<0-100>,"missedHops":["..."],"wrongClaims":["..."],"note":"<=20 words"}`;
-
-const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. The synthesized answer claims to trace a flow and cite file:line locations. Do NOT use any tools.
-
-QUESTION: ${gt.question}
-
-VERIFIED GROUND TRUTH (the actual call path + files):
-${gt.truth}
-
-SYNTHESIZED ANSWER (to score):
-${ans || '(empty)'}
-
-Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was that verdict honest about what it actually covered? A confident WRONG trace is the worst outcome — penalize it harder than an honest "partial/not found".
-Output ONLY minified JSON, no prose, no code fences:
-{"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"missedHops":["..."],"note":"<=20 words"}`;
-
-// Build the job list
-const jobs = [];
-for (const r of results) {
-  const gt = truth[r.repo];
-  if (!gt) { r._nojudge = true; continue; }
-  jobs.push({ r, kind: 'e2e', prompt: e2ePrompt(gt, r.finalAnswer) });
-  if (r.arm === 'offload' && Array.isArray(r.offloadAnswers))
-    r.offloadAnswers.forEach((ans, i) => { if (ans && ans.trim()) jobs.push({ r, kind: 'fid', idx: i, prompt: fidPrompt(gt, ans) }); });
-}
-console.error(`judging ${jobs.length} verdicts across ${results.length} runs (concurrency ${CONC})...`);
-
-let done = 0;
-async function worker(queue) {
-  while (queue.length) {
-    const job = queue.shift();
-    const v = await askJudge(job.prompt);
-    if (job.kind === 'e2e') job.r.e2e = v; else (job.r._fid ??= []).push(v);
-    console.error(`  [${++done}/${jobs.length}] ${job.r.repo}/${job.r.arm}#${job.r.rep} ${job.kind}: ${v.verdict}${v.score != null ? ' ' + v.score : ''}`);
-  }
-}
-const q = [...jobs];
-await Promise.all(Array.from({ length: CONC }, () => worker(q)));
-
-// Aggregate per-answer fidelity verdicts into one fidelity object per offload run.
-const medOf = (a) => { a = [...a].sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
-for (const r of results) {
-  if (r._fid?.length) {
-    const scores = r._fid.map(v => v.score).filter(x => x != null);
-    r.fidelity = {
-      n: r._fid.length, scores,
-      max: scores.length ? Math.max(...scores) : null,
-      min: scores.length ? Math.min(...scores) : null,
-      median: medOf(scores),
-      anyFabrication: r._fid.some(v => v.fabrication === true),
-      allCoverageHonest: r._fid.every(v => v.coverageHonest !== false),
-      verdicts: r._fid.map(v => v.verdict),
-    };
-  }
-  delete r._fid;
-}
-writeFileSync(OUT, results.map(r => JSON.stringify(r)).join('\n') + '\n');
-console.error(`wrote ${OUT}`);
+#!/usr/bin/env node
+// Accuracy judge. For each run in results.jsonl:
+//   - end-to-end: agent finalAnswer vs verified ground truth (all arms)
+//   - fidelity:   offload synthesized answer vs ground truth (offload arm only)
+// Judge = claude -p sonnet --effort high, no tools, run from a neutral cwd,
+// JSON-only verdicts. Writes judged.jsonl (one line per run, verdicts merged).
+//
+// Usage: judge.mjs --results <f> --truth <f> --out <f> [--concurrency 4]
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { execFile } from 'child_process';
+
+const A = {};
+for (let i = 2; i < process.argv.length; i += 2) A[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
+const results = readFileSync(A.results, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
+const truth = JSON.parse(readFileSync(A.truth, 'utf8'));
+const OUT = A.out || '/tmp/cg-offload-eval/judged.jsonl';
+const CONC = Number(A.concurrency || 4);
+
+function askJudge(prompt) {
+  return new Promise((resolve) => {
+    execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high',
+      '--max-budget-usd', '0.5', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
+      // Run from a neutral dir with no repo files so the judge can't "cheat" by reading source.
+      { cwd: process.env.AGENT_EVAL_OUT || '/tmp', maxBuffer: 1 << 24, timeout: 120000 },
+      (err, stdout) => {
+        const raw = (stdout || '').trim();
+        const m = raw.match(/\{[\s\S]*\}/);
+        if (!m) return resolve({ verdict: 'error', score: null, note: (err ? 'exec ' + err.message : 'no json').slice(0, 80) });
+        try { resolve(JSON.parse(m[0])); } catch { resolve({ verdict: 'error', score: null, note: 'parse fail' }); }
+      });
+  });
+}
+
+const e2ePrompt = (gt, ans) => `You are scoring whether an AI coding agent correctly answered a code-flow question about a repository. Judge ONLY against the verified ground truth. Do NOT use any tools.
+
+QUESTION: ${gt.question}
+
+VERIFIED GROUND TRUTH (the actual call path + files):
+${gt.truth}
+
+AGENT'S ANSWER:
+${ans || '(empty)'}
+
+Score how correct the agent's answer is vs the ground truth. A "pass" means it identifies the core mechanism and the major hops with the right files/symbols and makes no materially wrong claim. "partial" = right area but misses major hops or has notable errors. "fail" = wrong layer, fabricated, or misses the mechanism.
+Output ONLY minified JSON, no prose, no code fences:
+{"verdict":"pass|partial|fail","score":<0-100>,"missedHops":["..."],"wrongClaims":["..."],"note":"<=20 words"}`;
+
+const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. The synthesized answer claims to trace a flow and cite file:line locations. Do NOT use any tools.
+
+QUESTION: ${gt.question}
+
+VERIFIED GROUND TRUTH (the actual call path + files):
+${gt.truth}
+
+SYNTHESIZED ANSWER (to score):
+${ans || '(empty)'}
+
+Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was that verdict honest about what it actually covered? A confident WRONG trace is the worst outcome — penalize it harder than an honest "partial/not found".
+Output ONLY minified JSON, no prose, no code fences:
+{"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"missedHops":["..."],"note":"<=20 words"}`;
+
+// Build the job list
+const jobs = [];
+for (const r of results) {
+  const gt = truth[r.repo];
+  if (!gt) { r._nojudge = true; continue; }
+  jobs.push({ r, kind: 'e2e', prompt: e2ePrompt(gt, r.finalAnswer) });
+  if (r.arm === 'offload' && Array.isArray(r.offloadAnswers))
+    r.offloadAnswers.forEach((ans, i) => { if (ans && ans.trim()) jobs.push({ r, kind: 'fid', idx: i, prompt: fidPrompt(gt, ans) }); });
+}
+console.error(`judging ${jobs.length} verdicts across ${results.length} runs (concurrency ${CONC})...`);
+
+let done = 0;
+async function worker(queue) {
+  while (queue.length) {
+    const job = queue.shift();
+    const v = await askJudge(job.prompt);
+    if (job.kind === 'e2e') job.r.e2e = v; else (job.r._fid ??= []).push(v);
+    console.error(`  [${++done}/${jobs.length}] ${job.r.repo}/${job.r.arm}#${job.r.rep} ${job.kind}: ${v.verdict}${v.score != null ? ' ' + v.score : ''}`);
+  }
+}
+const q = [...jobs];
+await Promise.all(Array.from({ length: CONC }, () => worker(q)));
+
+// Aggregate per-answer fidelity verdicts into one fidelity object per offload run.
+const medOf = (a) => { a = [...a].sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
+for (const r of results) {
+  if (r._fid?.length) {
+    const scores = r._fid.map(v => v.score).filter(x => x != null);
+    r.fidelity = {
+      n: r._fid.length, scores,
+      max: scores.length ? Math.max(...scores) : null,
+      min: scores.length ? Math.min(...scores) : null,
+      median: medOf(scores),
+      anyFabrication: r._fid.some(v => v.fabrication === true),
+      allCoverageHonest: r._fid.every(v => v.coverageHonest !== false),
+      verdicts: r._fid.map(v => v.verdict),
+    };
+  }
+  delete r._fid;
+}
+writeFileSync(OUT, results.map(r => JSON.stringify(r)).join('\n') + '\n');
+console.error(`wrote ${OUT}`);

package/scripts/agent-eval/offload-eval-matrix.sh

@@ -1,20 +1,20 @@
-#!/usr/bin/env bash
-# Drive the 3-arm campaign (offload/raw/nocg) across all 4 tiers, n reps each, into one
-# results.jsonl. Reads the canonical question per repo from offload-eval-ground-truth.json.
-# Env: REPS (default 3)  AGENT_EVAL_OUT=<scratch dir>
-set -uo pipefail
-HERE="$(cd "$(dirname "$0")" && pwd)"
-OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
-GT="$HERE/offload-eval-ground-truth.json"
-REPS="${REPS:-3}"
-export RESULTS="$OUT/results.jsonl"
-: > "$RESULTS"
-for repo in mtkruto postybirb shapeshift trezor; do
-  case "$repo" in mtkruto) tier=small;; postybirb) tier=medium;; shapeshift) tier=complex;; trezor) tier=large;; esac
-  Q=$(node -e "console.log(JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))[process.argv[2]].question)" "$GT" "$repo")
-  echo ""; echo "### $repo ($tier)  $(date +%H:%M:%S)"
-  bash "$HERE/offload-eval-3arm.sh" "$OUT/repos/$repo" "$tier" "$REPS" "$Q"
-done
-echo ""; echo "###### MATRIX DONE -> $RESULTS ($(wc -l < "$RESULTS") runs).  Judge + summarize with:"
-echo "  node $HERE/offload-eval-judge.mjs --results $RESULTS --truth $GT --out $OUT/judged.jsonl"
-echo "  node $HERE/offload-eval-summarize.mjs $OUT/judged.jsonl"
+#!/usr/bin/env bash
+# Drive the 3-arm campaign (offload/raw/nocg) across all 4 tiers, n reps each, into one
+# results.jsonl. Reads the canonical question per repo from offload-eval-ground-truth.json.
+# Env: REPS (default 3)  AGENT_EVAL_OUT=<scratch dir>
+set -uo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
+GT="$HERE/offload-eval-ground-truth.json"
+REPS="${REPS:-3}"
+export RESULTS="$OUT/results.jsonl"
+: > "$RESULTS"
+for repo in mtkruto postybirb shapeshift trezor; do
+  case "$repo" in mtkruto) tier=small;; postybirb) tier=medium;; shapeshift) tier=complex;; trezor) tier=large;; esac
+  Q=$(node -e "console.log(JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))[process.argv[2]].question)" "$GT" "$repo")
+  echo ""; echo "### $repo ($tier)  $(date +%H:%M:%S)"
+  bash "$HERE/offload-eval-3arm.sh" "$OUT/repos/$repo" "$tier" "$REPS" "$Q"
+done
+echo ""; echo "###### MATRIX DONE -> $RESULTS ($(wc -l < "$RESULTS") runs).  Judge + summarize with:"
+echo "  node $HERE/offload-eval-judge.mjs --results $RESULTS --truth $GT --out $OUT/judged.jsonl"
+echo "  node $HERE/offload-eval-summarize.mjs $OUT/judged.jsonl"

package/scripts/agent-eval/offload-eval-metrics.mjs

@@ -1,94 +1,94 @@
-#!/usr/bin/env node
-// Extract one eval run's metrics from its Claude stream-json transcript + the
-// offload usage sidecar log, emit ONE merged JSON line.
-//
-// Usage: extract-metrics.mjs --run <run.jsonl> --usage <usage.jsonl|-> \
-//          --arm <a> --rep <n> --repo <r> --tier <t> --q <question>
-import { readFileSync, existsSync } from 'fs';
-
-const args = {};
-for (let i = 2; i < process.argv.length; i += 2) args[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
-
-const runFile = args.run;
-const lines = existsSync(runFile) ? readFileSync(runFile, 'utf8').split('\n').filter(Boolean) : [];
-
-const toolCounts = {};
-let result = null;
-const tok = { gen: 0, fresh: 0, cached: 0 };
-const offloadAnswers = [];
-let exploreResults = 0; // tool_results from explore (offload or raw)
-let lastAssistantText = '';
-
-for (const line of lines) {
-  let ev; try { ev = JSON.parse(line); } catch { continue; }
-
-  // per-turn token usage (authoritative token measure; result.usage is last-turn only)
-  const u = ev.message?.usage;
-  if (u) {
-    tok.gen += u.output_tokens || 0;
-    tok.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
-    tok.cached += u.cache_read_input_tokens || 0;
-  }
-
-  if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
-    for (const b of ev.message.content) {
-      if (b.type === 'tool_use') toolCounts[b.name] = (toolCounts[b.name] || 0) + 1;
-      if (b.type === 'text' && b.text?.trim()) lastAssistantText = b.text.trim();
-    }
-  }
-  // tool_results arrive in user messages
-  if (ev.type === 'user' && Array.isArray(ev.message?.content)) {
-    for (const b of ev.message.content) {
-      if (b.type !== 'tool_result') continue;
-      const text = Array.isArray(b.content)
-        ? b.content.map(c => (typeof c === 'string' ? c : c.text || '')).join('')
-        : (typeof b.content === 'string' ? b.content : '');
-      // An offload answer is either the 'plain'/'report' synthesis (carries the
-      // "Synthesized by HomeGraph" footer) or a 'refs' answer (carries the re-expanded
-      // "### Referenced source — verbatim" appendix). A refs call that cited nothing
-      // valid falls back to RAW source, which is correctly counted as a raw explore below.
-      if (/Synthesized by HomeGraph|### Referenced source — verbatim/.test(text)) { offloadAnswers.push(text); exploreResults++; }
-      else if (/Found \d+ symbols? across|\*\*Exploration:/.test(text)) exploreResults++;
-    }
-  }
-  if (ev.type === 'result') result = ev;
-}
-
-// offload usage sidecar (HomeGraph AI tokens + cost) — one JSON line per offload call
-const ai = { calls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, credits: 0, costUsd: 0, ms: 0 };
-if (args.usage && args.usage !== '-' && existsSync(args.usage)) {
-  for (const line of readFileSync(args.usage, 'utf8').split('\n').filter(Boolean)) {
-    let e; try { e = JSON.parse(line); } catch { continue; }
-    ai.calls++;
-    ai.promptTokens += e.promptTokens || 0;
-    ai.completionTokens += e.completionTokens || 0;
-    ai.totalTokens += e.totalTokens || 0;
-    ai.credits += e.creditsCharged || 0;
-    ai.costUsd += e.costUsd || 0;
-    ai.ms += e.ms || 0;
-  }
-}
-
-// front-load hook fired iff its injected header appears in the transcript
-const frontload = lines.some(l => l.includes('auto-retrieved for this question'));
-const get = (n) => toolCounts[n] || 0;
-const read = get('Read');
-const grep = get('Grep') + get('Bash') + get('Glob');
-const explore = get('mcp__homegraph__homegraph_explore');
-const cgAny = Object.keys(toolCounts).filter(k => /mcp__homegraph__/.test(k)).reduce((s, k) => s + toolCounts[k], 0);
-
-const out = {
-  repo: args.repo, tier: args.tier, arm: args.arm, rep: Number(args.rep), question: args.q,
-  ok: result?.subtype === 'success',
-  durationSec: result ? +(result.duration_ms / 1000).toFixed(1) : null,
-  numTurns: result?.num_turns ?? null,
-  costUsdMain: result ? +(result.total_cost_usd || 0).toFixed(4) : null,
-  tokGen: tok.gen, tokFresh: tok.fresh, tokCached: tok.cached, tokBillable: tok.gen + tok.fresh,
-  read, grep, explore, cgAny, frontload,
-  offloadFired: offloadAnswers.length,
-  ai,
-  // text payloads for the accuracy judge (kept separate; large)
-  finalAnswer: (result?.result || lastAssistantText || '').slice(0, 8000),
-  offloadAnswers: offloadAnswers.map(a => a.slice(0, 6000)),
-};
-process.stdout.write(JSON.stringify(out) + '\n');
+#!/usr/bin/env node
+// Extract one eval run's metrics from its Claude stream-json transcript + the
+// offload usage sidecar log, emit ONE merged JSON line.
+//
+// Usage: extract-metrics.mjs --run <run.jsonl> --usage <usage.jsonl|-> \
+//          --arm <a> --rep <n> --repo <r> --tier <t> --q <question>
+import { readFileSync, existsSync } from 'fs';
+
+const args = {};
+for (let i = 2; i < process.argv.length; i += 2) args[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
+
+const runFile = args.run;
+const lines = existsSync(runFile) ? readFileSync(runFile, 'utf8').split('\n').filter(Boolean) : [];
+
+const toolCounts = {};
+let result = null;
+const tok = { gen: 0, fresh: 0, cached: 0 };
+const offloadAnswers = [];
+let exploreResults = 0; // tool_results from explore (offload or raw)
+let lastAssistantText = '';
+
+for (const line of lines) {
+  let ev; try { ev = JSON.parse(line); } catch { continue; }
+
+  // per-turn token usage (authoritative token measure; result.usage is last-turn only)
+  const u = ev.message?.usage;
+  if (u) {
+    tok.gen += u.output_tokens || 0;
+    tok.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+    tok.cached += u.cache_read_input_tokens || 0;
+  }
+
+  if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
+    for (const b of ev.message.content) {
+      if (b.type === 'tool_use') toolCounts[b.name] = (toolCounts[b.name] || 0) + 1;
+      if (b.type === 'text' && b.text?.trim()) lastAssistantText = b.text.trim();
+    }
+  }
+  // tool_results arrive in user messages
+  if (ev.type === 'user' && Array.isArray(ev.message?.content)) {
+    for (const b of ev.message.content) {
+      if (b.type !== 'tool_result') continue;
+      const text = Array.isArray(b.content)
+        ? b.content.map(c => (typeof c === 'string' ? c : c.text || '')).join('')
+        : (typeof b.content === 'string' ? b.content : '');
+      // An offload answer is either the 'plain'/'report' synthesis (carries the
+      // "Synthesized by HomeGraph" footer) or a 'refs' answer (carries the re-expanded
+      // "### Referenced source — verbatim" appendix). A refs call that cited nothing
+      // valid falls back to RAW source, which is correctly counted as a raw explore below.
+      if (/Synthesized by HomeGraph|### Referenced source — verbatim/.test(text)) { offloadAnswers.push(text); exploreResults++; }
+      else if (/Found \d+ symbols? across|\*\*Exploration:/.test(text)) exploreResults++;
+    }
+  }
+  if (ev.type === 'result') result = ev;
+}
+
+// offload usage sidecar (HomeGraph AI tokens + cost) — one JSON line per offload call
+const ai = { calls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, credits: 0, costUsd: 0, ms: 0 };
+if (args.usage && args.usage !== '-' && existsSync(args.usage)) {
+  for (const line of readFileSync(args.usage, 'utf8').split('\n').filter(Boolean)) {
+    let e; try { e = JSON.parse(line); } catch { continue; }
+    ai.calls++;
+    ai.promptTokens += e.promptTokens || 0;
+    ai.completionTokens += e.completionTokens || 0;
+    ai.totalTokens += e.totalTokens || 0;
+    ai.credits += e.creditsCharged || 0;
+    ai.costUsd += e.costUsd || 0;
+    ai.ms += e.ms || 0;
+  }
+}
+
+// front-load hook fired iff its injected header appears in the transcript
+const frontload = lines.some(l => l.includes('auto-retrieved for this question'));
+const get = (n) => toolCounts[n] || 0;
+const read = get('Read');
+const grep = get('Grep') + get('Bash') + get('Glob');
+const explore = get('mcp__homegraph__homegraph_explore');
+const cgAny = Object.keys(toolCounts).filter(k => /mcp__homegraph__/.test(k)).reduce((s, k) => s + toolCounts[k], 0);
+
+const out = {
+  repo: args.repo, tier: args.tier, arm: args.arm, rep: Number(args.rep), question: args.q,
+  ok: result?.subtype === 'success',
+  durationSec: result ? +(result.duration_ms / 1000).toFixed(1) : null,
+  numTurns: result?.num_turns ?? null,
+  costUsdMain: result ? +(result.total_cost_usd || 0).toFixed(4) : null,
+  tokGen: tok.gen, tokFresh: tok.fresh, tokCached: tok.cached, tokBillable: tok.gen + tok.fresh,
+  read, grep, explore, cgAny, frontload,
+  offloadFired: offloadAnswers.length,
+  ai,
+  // text payloads for the accuracy judge (kept separate; large)
+  finalAnswer: (result?.result || lastAssistantText || '').slice(0, 8000),
+  offloadAnswers: offloadAnswers.map(a => a.slice(0, 6000)),
+};
+process.stdout.write(JSON.stringify(out) + '\n');