homegraph 1.1.2 → 1.2.0

package/scripts/agent-eval/parse-bench-readme.mjs

@@ -1,84 +1,84 @@
-#!/usr/bin/env node
-// Aggregate the README A/B (bench-readme.sh output): per repo, median of N runs
-// per arm → time, tool calls, tokens, cost, and % saved. Plus an average row.
-//
-// Tokens = SUM of per-turn assistant `usage` (input + output + cache read +
-// cache creation) — the cumulative "total tokens processed". NOTE: `result.usage`
-// is last-turn-only in current Claude Code, so it under-counts badly; don't use it.
-// `total_cost_usd` and `duration_ms` are already cumulative.
-//
-// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
-import { readFileSync, existsSync, readdirSync } from 'fs';
-import { join } from 'path';
-const ROOT = process.argv[2] || '/tmp/ab-readme';
-const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
-
-function parse(file) {
-  if (!existsSync(file)) return null;
-  const L = readFileSync(file, 'utf8').split('\n').filter(Boolean);
-  let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null, raced = false;
-  for (const l of L) { let e; try { e = JSON.parse(l); } catch { continue; }
-    if (e.type === 'assistant') {
-      const u = e.message?.usage;
-      if (u) tokens += (u.input_tokens || 0) + (u.output_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
-      for (const b of (e.message?.content || [])) if (b.type === 'tool_use') {
-        const n = b.name;
-        if (n === 'ToolSearch') continue;
-        tools++;
-        if (n === 'Read') reads++;
-        else if (n === 'Grep' || n === 'Glob') grep++;
-        else if (/homegraph/.test(n)) cg++;
-      }
-    }
-    // MCP cold-start race: the headless agent fired before `homegraph serve --mcp`
-    // finished registering its tools, so early calls returned "No such tool
-    // available" and the agent floundered into grep/Read. That measures HomeGraph's
-    // startup latency, NOT its steady-state value — flag the run so the aggregate
-    // can exclude it (an artifact of headless first-turn timing, not the tool).
-    if (e.type === 'user') for (const b of (Array.isArray(e.message?.content) ? e.message.content : [])) {
-      if (b.type === 'tool_result') {
-        const t = Array.isArray(b.content) ? b.content.map(c => c.text || '').join('') : (b.content || '');
-        if (/No such tool available/.test(t)) raced = true;
-      }
-    }
-    if (e.type === 'result') r = e;
-  }
-  if (!r || r.subtype !== 'success') return null;
-  return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0, raced };
-}
-const median = (arr) => { const v = [...arr].sort((a, b) => a - b); const n = v.length; return n === 0 ? 0 : n % 2 ? v[(n - 1) / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; };
-const fmtTime = (s) => s >= 60 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${Math.round(s)}s`;
-const fmtTok = (t) => t >= 1e6 ? `${(t / 1e6).toFixed(1)}M` : `${Math.round(t / 1000)}k`;
-const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
-
-console.log('repo        n(w/wo)  time WITH→WITHOUT      tools W→WO   tokens W→WO (saved)     cost W→WO (saved)');
-const savings = { cost: [], tokens: [], time: [], tools: [] };
-for (const repo of REPOS) {
-  const dir = join(ROOT, repo);
-  const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)) : [];
-  // Exclude MCP-cold-start-raced WITH runs by default — they measure a startup
-  // race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw
-  // distribution). The WITHOUT arm has no MCP, so it's never raced.
-  const includeRaced = process.env.CG_INCLUDE_RACED === '1';
-  const W = [], WO = []; let racedExcluded = 0;
-  for (const rd of runDirs) {
-    const w = parse(join(dir, rd, 'run-headless-with.jsonl'));
-    if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); }
-    const wo = parse(join(dir, rd, 'run-headless-without.jsonl')); if (wo) WO.push(wo);
-  }
-  if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete: w=${W.length} wo=${WO.length})`); continue; }
-  const m = (arr, k) => median(arr.map(x => x[k]));
-  const wT = m(W, 'dur'), woT = m(WO, 'dur'), wTok = m(W, 'tokens'), woTok = m(WO, 'tokens');
-  const wC = m(W, 'cost'), woC = m(WO, 'cost'), wTl = m(W, 'tools'), woTl = m(WO, 'tools');
-  savings.time.push(pct(wT, woT)); savings.tokens.push(pct(wTok, woTok)); savings.cost.push(pct(wC, woC)); savings.tools.push(pct(wTl, woTl));
-  console.log(
-    `${repo.padEnd(11)} ${W.length}/${WO.length}      ` +
-    `${(fmtTime(wT) + '→' + fmtTime(woT)).padEnd(22)}` +
-    `${(Math.round(wTl) + '→' + Math.round(woTl)).padEnd(12)}` +
-    `${(fmtTok(wTok) + '→' + fmtTok(woTok) + ' (' + pct(wTok, woTok) + '%)').padEnd(24)}` +
-    `$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)` +
-    (racedExcluded ? `  [${racedExcluded} raced run${racedExcluded === 1 ? '' : 's'} excluded]` : '')
-  );
-}
-const avg = (a) => a.length ? Math.round(a.reduce((s, x) => s + x, 0) / a.length) : 0;
-console.log(`\nAVERAGE saved:  cost ${avg(savings.cost)}%  ·  tokens ${avg(savings.tokens)}%  ·  time ${avg(savings.time)}%  ·  tool calls ${avg(savings.tools)}%`);
+#!/usr/bin/env node
+// Aggregate the README A/B (bench-readme.sh output): per repo, median of N runs
+// per arm → time, tool calls, tokens, cost, and % saved. Plus an average row.
+//
+// Tokens = SUM of per-turn assistant `usage` (input + output + cache read +
+// cache creation) — the cumulative "total tokens processed". NOTE: `result.usage`
+// is last-turn-only in current Claude Code, so it under-counts badly; don't use it.
+// `total_cost_usd` and `duration_ms` are already cumulative.
+//
+// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+const ROOT = process.argv[2] || '/tmp/ab-readme';
+const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const L = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null, raced = false;
+  for (const l of L) { let e; try { e = JSON.parse(l); } catch { continue; }
+    if (e.type === 'assistant') {
+      const u = e.message?.usage;
+      if (u) tokens += (u.input_tokens || 0) + (u.output_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+      for (const b of (e.message?.content || [])) if (b.type === 'tool_use') {
+        const n = b.name;
+        if (n === 'ToolSearch') continue;
+        tools++;
+        if (n === 'Read') reads++;
+        else if (n === 'Grep' || n === 'Glob') grep++;
+        else if (/homegraph/.test(n)) cg++;
+      }
+    }
+    // MCP cold-start race: the headless agent fired before `homegraph serve --mcp`
+    // finished registering its tools, so early calls returned "No such tool
+    // available" and the agent floundered into grep/Read. That measures HomeGraph's
+    // startup latency, NOT its steady-state value — flag the run so the aggregate
+    // can exclude it (an artifact of headless first-turn timing, not the tool).
+    if (e.type === 'user') for (const b of (Array.isArray(e.message?.content) ? e.message.content : [])) {
+      if (b.type === 'tool_result') {
+        const t = Array.isArray(b.content) ? b.content.map(c => c.text || '').join('') : (b.content || '');
+        if (/No such tool available/.test(t)) raced = true;
+      }
+    }
+    if (e.type === 'result') r = e;
+  }
+  if (!r || r.subtype !== 'success') return null;
+  return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0, raced };
+}
+const median = (arr) => { const v = [...arr].sort((a, b) => a - b); const n = v.length; return n === 0 ? 0 : n % 2 ? v[(n - 1) / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; };
+const fmtTime = (s) => s >= 60 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${Math.round(s)}s`;
+const fmtTok = (t) => t >= 1e6 ? `${(t / 1e6).toFixed(1)}M` : `${Math.round(t / 1000)}k`;
+const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
+
+console.log('repo        n(w/wo)  time WITH→WITHOUT      tools W→WO   tokens W→WO (saved)     cost W→WO (saved)');
+const savings = { cost: [], tokens: [], time: [], tools: [] };
+for (const repo of REPOS) {
+  const dir = join(ROOT, repo);
+  const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)) : [];
+  // Exclude MCP-cold-start-raced WITH runs by default — they measure a startup
+  // race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw
+  // distribution). The WITHOUT arm has no MCP, so it's never raced.
+  const includeRaced = process.env.CG_INCLUDE_RACED === '1';
+  const W = [], WO = []; let racedExcluded = 0;
+  for (const rd of runDirs) {
+    const w = parse(join(dir, rd, 'run-headless-with.jsonl'));
+    if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); }
+    const wo = parse(join(dir, rd, 'run-headless-without.jsonl')); if (wo) WO.push(wo);
+  }
+  if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete: w=${W.length} wo=${WO.length})`); continue; }
+  const m = (arr, k) => median(arr.map(x => x[k]));
+  const wT = m(W, 'dur'), woT = m(WO, 'dur'), wTok = m(W, 'tokens'), woTok = m(WO, 'tokens');
+  const wC = m(W, 'cost'), woC = m(WO, 'cost'), wTl = m(W, 'tools'), woTl = m(WO, 'tools');
+  savings.time.push(pct(wT, woT)); savings.tokens.push(pct(wTok, woTok)); savings.cost.push(pct(wC, woC)); savings.tools.push(pct(wTl, woTl));
+  console.log(
+    `${repo.padEnd(11)} ${W.length}/${WO.length}      ` +
+    `${(fmtTime(wT) + '→' + fmtTime(woT)).padEnd(22)}` +
+    `${(Math.round(wTl) + '→' + Math.round(woTl)).padEnd(12)}` +
+    `${(fmtTok(wTok) + '→' + fmtTok(woTok) + ' (' + pct(wTok, woTok) + '%)').padEnd(24)}` +
+    `$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)` +
+    (racedExcluded ? `  [${racedExcluded} raced run${racedExcluded === 1 ? '' : 's'} excluded]` : '')
+  );
+}
+const avg = (a) => a.length ? Math.round(a.reduce((s, x) => s + x, 0) / a.length) : 0;
+console.log(`\nAVERAGE saved:  cost ${avg(savings.cost)}%  ·  tokens ${avg(savings.tokens)}%  ·  time ${avg(savings.time)}%  ·  tool calls ${avg(savings.tools)}%`);

package/scripts/agent-eval/parse-run.mjs

@@ -1,45 +1,45 @@
-#!/usr/bin/env node
-// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
-import { readFileSync } from 'fs';
-const file = process.argv[2];
-const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
-
-const toolCalls = [];
-let result = null;
-let initTools = null;
-
-for (const line of lines) {
-  let ev;
-  try { ev = JSON.parse(line); } catch { continue; }
-  if (ev.type === 'system' && ev.subtype === 'init') {
-    initTools = (ev.tools || []).filter(t => /homegraph/.test(t));
-  }
-  if (ev.type === 'assistant' && ev.message?.content) {
-    for (const block of ev.message.content) {
-      if (block.type === 'tool_use') {
-        let detail = '';
-        if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
-        else if (/homegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
-        else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
-        else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
-        toolCalls.push(`${block.name}${detail}`);
-      }
-    }
-  }
-  if (ev.type === 'result') result = ev;
-}
-
-console.log(`\n=== ${file.split('/').pop()} ===`);
-console.log(`homegraph tools exposed: ${initTools ? initTools.length : '?'}`);
-console.log(`\nTool calls (${toolCalls.length}):`);
-const counts = {};
-for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
-console.log('  by type:', JSON.stringify(counts));
-toolCalls.forEach((tc, i) => console.log(`  ${i+1}. ${tc}`));
-
-if (result) {
-  const u = result.usage || {};
-  const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
-  console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
-  console.log(`  tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
-}
+#!/usr/bin/env node
+// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
+import { readFileSync } from 'fs';
+const file = process.argv[2];
+const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+
+const toolCalls = [];
+let result = null;
+let initTools = null;
+
+for (const line of lines) {
+  let ev;
+  try { ev = JSON.parse(line); } catch { continue; }
+  if (ev.type === 'system' && ev.subtype === 'init') {
+    initTools = (ev.tools || []).filter(t => /homegraph/.test(t));
+  }
+  if (ev.type === 'assistant' && ev.message?.content) {
+    for (const block of ev.message.content) {
+      if (block.type === 'tool_use') {
+        let detail = '';
+        if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
+        else if (/homegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
+        else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
+        else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
+        toolCalls.push(`${block.name}${detail}`);
+      }
+    }
+  }
+  if (ev.type === 'result') result = ev;
+}
+
+console.log(`\n=== ${file.split('/').pop()} ===`);
+console.log(`homegraph tools exposed: ${initTools ? initTools.length : '?'}`);
+console.log(`\nTool calls (${toolCalls.length}):`);
+const counts = {};
+for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
+console.log('  by type:', JSON.stringify(counts));
+toolCalls.forEach((tc, i) => console.log(`  ${i+1}. ${tc}`));
+
+if (result) {
+  const u = result.usage || {};
+  const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
+  console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
+  console.log(`  tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
+}

package/scripts/agent-eval/parse-session.mjs

@@ -1,93 +1,93 @@
-#!/usr/bin/env node
-// Parse the newest Claude Code session log for a project + its subagent logs,
-// and report the tool-call breakdown (main + subagents). Works for interactive
-// runs (driven via itrun.sh) — Claude Code writes full transcripts to
-// ~/.claude/projects/<escaped-cwd>/<session>.jsonl with subagents/ alongside.
-import { readFileSync, readdirSync, statSync, existsSync, realpathSync } from 'fs';
-import { join } from 'path';
-import { homedir } from 'os';
-
-const projectArg = process.argv[2];
-if (!projectArg) { console.error('usage: parse-session.mjs <project-dir>'); process.exit(1); }
-
-// Claude Code escapes the (real) cwd by replacing every "/" with "-".
-const real = realpathSync(projectArg);
-const escaped = real.replace(/\//g, '-');
-const projDir = join(homedir(), '.claude', 'projects', escaped);
-if (!existsSync(projDir)) { console.error('no session logs at', projDir); process.exit(1); }
-
-// Newest top-level session .jsonl
-const sessions = readdirSync(projDir)
-  .filter(f => f.endsWith('.jsonl'))
-  .map(f => ({ f, m: statSync(join(projDir, f)).mtimeMs }))
-  .sort((a, b) => b.m - a.m);
-if (sessions.length === 0) { console.error('no .jsonl sessions in', projDir); process.exit(1); }
-const sessionId = sessions[0].f.replace('.jsonl', '');
-
-function tally(file) {
-  const counts = {};
-  for (const line of readFileSync(file, 'utf8').split('\n')) {
-    if (!line) continue;
-    let ev; try { ev = JSON.parse(line); } catch { continue; }
-    const content = ev.message?.content;
-    if (!Array.isArray(content)) continue;
-    for (const b of content) {
-      if (b.type === 'tool_use') counts[b.name] = (counts[b.name] || 0) + 1;
-    }
-  }
-  return counts;
-}
-
-// Sum token usage from a transcript. The TUI's "Done (…Xk tokens…)" line only
-// covers a subagent's throughput; this works for main-thread runs too and is
-// consistent across both paths. `gen` = output, `fresh` = uncached input
-// (input + cache_creation), `cached` = cache reads (≈free), `total` = all.
-function sumTokens(file) {
-  const t = { gen: 0, fresh: 0, cached: 0 };
-  for (const line of readFileSync(file, 'utf8').split('\n')) {
-    if (!line) continue;
-    let ev; try { ev = JSON.parse(line); } catch { continue; }
-    const u = ev.message?.usage;
-    if (!u) continue;
-    t.gen += u.output_tokens || 0;
-    t.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
-    t.cached += u.cache_read_input_tokens || 0;
-  }
-  return t;
-}
-
-const mainCounts = tally(join(projDir, sessionId + '.jsonl'));
-
-// Subagent transcripts live under <session>/subagents/*.jsonl
-const subDir = join(projDir, sessionId, 'subagents');
-const subCounts = {};
-let subAgentFiles = 0;
-if (existsSync(subDir)) {
-  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) {
-    subAgentFiles++;
-    const c = tally(join(subDir, f));
-    for (const [k, v] of Object.entries(c)) subCounts[k] = (subCounts[k] || 0) + v;
-  }
-}
-
-const fmt = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])
-  .map(([k, v]) => `    ${String(v).padStart(3)}  ${k}`).join('\n') || '    (none)';
-
-console.log(`session: ${sessionId}`);
-console.log(`\nMAIN thread tools:\n${fmt(mainCounts)}`);
-console.log(`\nSUBAGENT tools (${subAgentFiles} subagent transcript${subAgentFiles === 1 ? '' : 's'}):\n${fmt(subCounts)}`);
-
-const explore = subCounts['mcp__homegraph__homegraph_explore'] || mainCounts['mcp__homegraph__homegraph_explore'] || 0;
-const reads = (subCounts['Read'] || 0) + (mainCounts['Read'] || 0);
-const greps = (subCounts['Grep'] || 0) + (mainCounts['Grep'] || 0) + (subCounts['Bash'] || 0) + (mainCounts['Bash'] || 0);
-console.log(`\nVERDICT: homegraph_explore used ${explore}x | Read ${reads} | Grep/Bash ${greps}`);
-
-// Token totals (main + subagents), consistent across main-thread and subagent runs.
-const tok = { gen: 0, fresh: 0, cached: 0 };
-const addTok = (t) => { tok.gen += t.gen; tok.fresh += t.fresh; tok.cached += t.cached; };
-addTok(sumTokens(join(projDir, sessionId + '.jsonl')));
-if (existsSync(subDir)) {
-  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) addTok(sumTokens(join(subDir, f)));
-}
-const k = (n) => (n / 1000).toFixed(1) + 'k';
-console.log(`TOKENS: gen ${k(tok.gen)} | fresh-in ${k(tok.fresh)} | cached-in ${k(tok.cached)} | billable≈ ${k(tok.gen + tok.fresh)}`);
+#!/usr/bin/env node
+// Parse the newest Claude Code session log for a project + its subagent logs,
+// and report the tool-call breakdown (main + subagents). Works for interactive
+// runs (driven via itrun.sh) — Claude Code writes full transcripts to
+// ~/.claude/projects/<escaped-cwd>/<session>.jsonl with subagents/ alongside.
+import { readFileSync, readdirSync, statSync, existsSync, realpathSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+const projectArg = process.argv[2];
+if (!projectArg) { console.error('usage: parse-session.mjs <project-dir>'); process.exit(1); }
+
+// Claude Code escapes the (real) cwd by replacing every "/" with "-".
+const real = realpathSync(projectArg);
+const escaped = real.replace(/\//g, '-');
+const projDir = join(homedir(), '.claude', 'projects', escaped);
+if (!existsSync(projDir)) { console.error('no session logs at', projDir); process.exit(1); }
+
+// Newest top-level session .jsonl
+const sessions = readdirSync(projDir)
+  .filter(f => f.endsWith('.jsonl'))
+  .map(f => ({ f, m: statSync(join(projDir, f)).mtimeMs }))
+  .sort((a, b) => b.m - a.m);
+if (sessions.length === 0) { console.error('no .jsonl sessions in', projDir); process.exit(1); }
+const sessionId = sessions[0].f.replace('.jsonl', '');
+
+function tally(file) {
+  const counts = {};
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const content = ev.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const b of content) {
+      if (b.type === 'tool_use') counts[b.name] = (counts[b.name] || 0) + 1;
+    }
+  }
+  return counts;
+}
+
+// Sum token usage from a transcript. The TUI's "Done (…Xk tokens…)" line only
+// covers a subagent's throughput; this works for main-thread runs too and is
+// consistent across both paths. `gen` = output, `fresh` = uncached input
+// (input + cache_creation), `cached` = cache reads (≈free), `total` = all.
+function sumTokens(file) {
+  const t = { gen: 0, fresh: 0, cached: 0 };
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const u = ev.message?.usage;
+    if (!u) continue;
+    t.gen += u.output_tokens || 0;
+    t.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+    t.cached += u.cache_read_input_tokens || 0;
+  }
+  return t;
+}
+
+const mainCounts = tally(join(projDir, sessionId + '.jsonl'));
+
+// Subagent transcripts live under <session>/subagents/*.jsonl
+const subDir = join(projDir, sessionId, 'subagents');
+const subCounts = {};
+let subAgentFiles = 0;
+if (existsSync(subDir)) {
+  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) {
+    subAgentFiles++;
+    const c = tally(join(subDir, f));
+    for (const [k, v] of Object.entries(c)) subCounts[k] = (subCounts[k] || 0) + v;
+  }
+}
+
+const fmt = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])
+  .map(([k, v]) => `    ${String(v).padStart(3)}  ${k}`).join('\n') || '    (none)';
+
+console.log(`session: ${sessionId}`);
+console.log(`\nMAIN thread tools:\n${fmt(mainCounts)}`);
+console.log(`\nSUBAGENT tools (${subAgentFiles} subagent transcript${subAgentFiles === 1 ? '' : 's'}):\n${fmt(subCounts)}`);
+
+const explore = subCounts['mcp__homegraph__homegraph_explore'] || mainCounts['mcp__homegraph__homegraph_explore'] || 0;
+const reads = (subCounts['Read'] || 0) + (mainCounts['Read'] || 0);
+const greps = (subCounts['Grep'] || 0) + (mainCounts['Grep'] || 0) + (subCounts['Bash'] || 0) + (mainCounts['Bash'] || 0);
+console.log(`\nVERDICT: homegraph_explore used ${explore}x | Read ${reads} | Grep/Bash ${greps}`);
+
+// Token totals (main + subagents), consistent across main-thread and subagent runs.
+const tok = { gen: 0, fresh: 0, cached: 0 };
+const addTok = (t) => { tok.gen += t.gen; tok.fresh += t.fresh; tok.cached += t.cached; };
+addTok(sumTokens(join(projDir, sessionId + '.jsonl')));
+if (existsSync(subDir)) {
+  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) addTok(sumTokens(join(subDir, f)));
+}
+const k = (n) => (n / 1000).toFixed(1) + 'k';
+console.log(`TOKENS: gen ${k(tok.gen)} | fresh-in ${k(tok.fresh)} | cached-in ${k(tok.cached)} | billable≈ ${k(tok.gen + tok.fresh)}`);

package/scripts/agent-eval/probe-context.mjs

@@ -1,21 +1,21 @@
-#!/usr/bin/env node
-// Probe homegraph_context (with call-paths) against an index using the built dist.
-// Usage: node probe-context.mjs <repo-with-.homegraph> <task words...>
-import { pathToFileURL } from 'node:url';
-import { resolve } from 'node:path';
-
-const [, , repo, ...taskParts] = process.argv;
-const task = taskParts.join(' ');
-if (!repo || !task) { console.error('usage: probe-context.mjs <repo> <task...>'); process.exit(1); }
-
-const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
-const idx = await load('dist/index.js');
-const tools = await load('dist/mcp/tools.js');
-const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
-const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
-
-const cg = HomeGraph.openSync(repo);
-const h = new ToolHandler(cg);
-const res = await h.execute('homegraph_context', { task });
-console.log(res.content?.[0]?.text ?? '(no text)');
-try { cg.close?.(); } catch {}
+#!/usr/bin/env node
+// Probe homegraph_context (with call-paths) against an index using the built dist.
+// Usage: node probe-context.mjs <repo-with-.homegraph> <task words...>
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, ...taskParts] = process.argv;
+const task = taskParts.join(' ');
+if (!repo || !task) { console.error('usage: probe-context.mjs <repo> <task...>'); process.exit(1); }
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+const cg = HomeGraph.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('homegraph_context', { task });
+console.log(res.content?.[0]?.text ?? '(no text)');
+try { cg.close?.(); } catch {}

package/scripts/agent-eval/probe-explore.mjs

@@ -1,40 +1,40 @@
-#!/usr/bin/env node
-// One-shot probe: run handleExplore against an existing index using the built
-// dist, print the output + a few stats. Lets us verify explore's coverage fix
-// without a full agent run. Usage: node probe-explore.mjs <repo-with-.homegraph> "<query>"
-import { pathToFileURL } from 'node:url';
-import { resolve } from 'node:path';
-
-const [, , repo, query] = process.argv;
-if (!repo || !query) {
-  console.error('usage: probe-explore.mjs <repo> "<query>"');
-  process.exit(1);
-}
-
-const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
-const idx = await load('dist/index.js');
-const tools = await load('dist/mcp/tools.js');
-
-// esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
-const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
-const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
-
-if (typeof HomeGraph?.openSync !== 'function') {
-  console.error('could not resolve HomeGraph.openSync; index keys:', Object.keys(idx), 'default keys:', idx.default && Object.keys(idx.default));
-  process.exit(2);
-}
-if (typeof ToolHandler !== 'function') {
-  console.error('could not resolve ToolHandler; tools keys:', Object.keys(tools));
-  process.exit(2);
-}
-
-const cg = HomeGraph.openSync(repo);
-const h = new ToolHandler(cg);
-const res = await h.execute('homegraph_explore', { query });
-const text = res.content?.[0]?.text ?? '(no text)';
-console.log(text);
-console.error('\n--- PROBE STATS ---');
-console.error('output chars:', text.length);
-console.error('triggerRender body present (-> setState({})):', /triggerRender[\s\S]{0,400}setState\(\{\}\)/.test(text));
-console.error('App.tsx in source section:', /\*\*`.*App\.tsx`\*\* —/.test(text));
-try { cg.close?.(); } catch {}
+#!/usr/bin/env node
+// One-shot probe: run handleExplore against an existing index using the built
+// dist, print the output + a few stats. Lets us verify explore's coverage fix
+// without a full agent run. Usage: node probe-explore.mjs <repo-with-.homegraph> "<query>"
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, query] = process.argv;
+if (!repo || !query) {
+  console.error('usage: probe-explore.mjs <repo> "<query>"');
+  process.exit(1);
+}
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+
+// esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
+const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+if (typeof HomeGraph?.openSync !== 'function') {
+  console.error('could not resolve HomeGraph.openSync; index keys:', Object.keys(idx), 'default keys:', idx.default && Object.keys(idx.default));
+  process.exit(2);
+}
+if (typeof ToolHandler !== 'function') {
+  console.error('could not resolve ToolHandler; tools keys:', Object.keys(tools));
+  process.exit(2);
+}
+
+const cg = HomeGraph.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('homegraph_explore', { query });
+const text = res.content?.[0]?.text ?? '(no text)';
+console.log(text);
+console.error('\n--- PROBE STATS ---');
+console.error('output chars:', text.length);
+console.error('triggerRender body present (-> setState({})):', /triggerRender[\s\S]{0,400}setState\(\{\}\)/.test(text));
+console.error('App.tsx in source section:', /\*\*`.*App\.tsx`\*\* —/.test(text));
+try { cg.close?.(); } catch {}

package/scripts/agent-eval/probe-node.mjs

@@ -1,20 +1,20 @@
-#!/usr/bin/env node
-// Probe homegraph_node (with trail) against an index using the built dist.
-// Usage: node probe-node.mjs <repo-with-.homegraph> <symbol> [code]
-import { pathToFileURL } from 'node:url';
-import { resolve } from 'node:path';
-
-const [, , repo, symbol, code] = process.argv;
-if (!repo || !symbol) { console.error('usage: probe-node.mjs <repo> <symbol> [code]'); process.exit(1); }
-
-const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
-const idx = await load('dist/index.js');
-const tools = await load('dist/mcp/tools.js');
-const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
-const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
-
-const cg = HomeGraph.openSync(repo);
-const h = new ToolHandler(cg);
-const res = await h.execute('homegraph_node', { symbol, includeCode: code === 'code' });
-console.log(res.content?.[0]?.text ?? '(no text)');
-try { cg.close?.(); } catch {}
+#!/usr/bin/env node
+// Probe homegraph_node (with trail) against an index using the built dist.
+// Usage: node probe-node.mjs <repo-with-.homegraph> <symbol> [code]
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, symbol, code] = process.argv;
+if (!repo || !symbol) { console.error('usage: probe-node.mjs <repo> <symbol> [code]'); process.exit(1); }
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+const cg = HomeGraph.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('homegraph_node', { symbol, includeCode: code === 'code' });
+console.log(res.content?.[0]?.text ?? '(no text)');
+try { cg.close?.(); } catch {}