@agentllm/codegraph-arkts 0.7.12 → 0.9.4-arkts.0

package/scripts/agent-eval/audit.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# One-shot CodeGraph quality audit:
+#   set version -> ensure corpus repo -> wipe+reindex with that version ->
+#   run with/without A/B -> restore the local dev link.
+#
+# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
+#   <version>    "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
+#   <repo-name>  dir name under the corpus dir
+#   <repo-url>   git URL (cloned --depth 1 when the repo dir is missing)
+#   [mode]       headless (default) | all (also the interactive tmux arms)
+# Env: CORPUS  corpus dir (default: /tmp/codegraph-corpus)
+set -uo pipefail
+
+VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
+NAME="${2:?repo-name required}"
+URL="${3:?repo-url required}"
+Q="${4:?question required}"
+MODE="${5:-headless}"
+
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$HARNESS/../.." && pwd)"     # codegraph repo root
+CORPUS="${CORPUS:-/tmp/codegraph-corpus}"
+REPO="$CORPUS/$NAME"
+PKG="@colbymchenry/codegraph"
+
+echo "==================== CodeGraph audit ===================="
+echo "version=$VERSION  repo=$NAME  mode=$MODE  corpus=$CORPUS"
+echo
+
+# 1. Set the codegraph version under test (mutates the global install).
+if [ "$VERSION" = local ]; then
+  echo "→ [1/4] building + linking local dev build (local-install.sh)"
+  ( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
+else
+  echo "→ [1/4] installing $PKG@$VERSION globally"
+  npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
+fi
+ACTUAL="$(codegraph --version 2>/dev/null || echo '?')"
+echo "  codegraph on PATH: $(command -v codegraph) -> $ACTUAL"
+
+# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
+mkdir -p "$CORPUS"
+if [ -d "$REPO/.git" ]; then
+  echo "→ [2/4] reusing existing checkout: $REPO"
+else
+  echo "→ [2/4] cloning $URL"
+  git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
+fi
+
+# 3. Wipe + re-index with THIS version (the index must be built by the same
+#    binary that serves it — different versions extract differently).
+echo "→ [3/4] wiping .codegraph and re-indexing with $ACTUAL"
+rm -rf "$REPO/.codegraph"
+( cd "$REPO" && codegraph init -i ) || { echo "indexing failed"; exit 1; }
+
+# 4. Run the with/without A/B.
+echo "→ [4/4] running A/B harness (mode=$MODE)"
+bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
+
+# Restore the dev link (the normal working state in this repo).
+echo
+echo "→ restoring local dev link (local-install.sh)"
+if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
+  echo "  global codegraph restored to dev build"
+else
+  echo "  WARN: restore failed — run ./scripts/local-install.sh manually"
+fi
+echo "==================== audit complete ===================="

package/scripts/agent-eval/bench-readme.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Re-run the README "Benchmark Results" A/B (with vs without codegraph) on the
+# current build: the 7 README repos, same queries, RUNS per arm (default 4).
+# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
+# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
+# $CORPUS (default /tmp/codegraph-corpus) by the build under test.
+set -uo pipefail
+H="$(cd "$(dirname "$0")" && pwd)"
+C="${CORPUS:-/tmp/codegraph-corpus}"
+RUNS="${RUNS:-4}"
+ROWS=(
+"vscode|How does the extension host communicate with the main process?"
+"excalidraw|How does Excalidraw render and update canvas elements?"
+"django|How does Django's ORM build and execute a query from a QuerySet?"
+"tokio|How does tokio schedule and run async tasks on its runtime?"
+"okhttp|How does OkHttp process a request through its interceptor chain?"
+"gin|How does gin route requests through its middleware chain?"
+"alamofire|How does Alamofire build, send, and validate a request?"
+)
+echo "### README A/B START $(date) RUNS=$RUNS"
+for row in "${ROWS[@]}"; do
+  repo="${row%%|*}"; q="${row#*|}"
+  echo "===== $repo ====="
+  for run in $(seq 1 "$RUNS"); do
+    AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo "  run$run: (no exit line)"
+  done
+done
+echo "### README A/B DONE $(date)"

package/scripts/agent-eval/block-read-hook.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# PreToolUse hook (experiment): deny Read of codegraph-indexed source files and
+# steer the agent to codegraph_explore/codegraph_node instead. Tests whether
+# codegraph can FULLY replace Read for code-understanding once the escape hatch
+# is removed. Non-source reads (config, .env, markdown, new files) pass through.
+#
+# Wire via:  claude ... --settings scripts/agent-eval/hook-settings.json
+set -uo pipefail
+input="$(cat)"
+fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
+
+case "$fp" in
+  *.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
+    msg="Read is disabled for source files in this session — codegraph already has this file indexed (with line numbers, kept in sync on every change). Use codegraph_explore (several related symbols at once) or codegraph_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER codegraph_explore with its exact name instead of reading the file."
+    jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
+    exit 0
+    ;;
+esac
+exit 0

package/scripts/agent-eval/hook-settings.json

@@ -0,0 +1,15 @@
+{
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Read",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash /Users/colby/Development/Personal/codegraph/scripts/agent-eval/block-read-hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}

package/scripts/agent-eval/itrun.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
+# agent to finish, then print the tool-call breakdown from the session logs.
+#
+# Why interactive (not `claude -p`): headless print-mode picks the
+# general-purpose subagent, while real interactive sessions delegate to the
+# Explore subagent (or drive codegraph from the main thread). Only the
+# interactive TUI reproduces the behavior users actually see. (Idle-detection
+# technique borrowed from devpit's WaitForIdle.)
+#
+# Usage: itrun.sh <repo-path> <label> "<prompt>"
+# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
+# Requires: tmux 3.0+, a logged-in `claude` CLI, codegraph MCP configured.
+set -uo pipefail
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+SESSION="cgt_${LABEL}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/itrun-${LABEL}.txt"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
+
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Wide pane so the TUI doesn't hard-wrap tool lines.
+tmux new-session -d -s "$SESSION" -x 230 -y 60
+tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
+
+# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
+# welcome screen seconds before the input actually accepts keystrokes, so this is
+# necessary but NOT sufficient — the type-and-verify loop below is what proves
+# the input is live.
+ready=0
+for _ in $(seq 1 120); do
+  cap | grep -q "❯" && { ready=1; break; }
+  sleep 0.5
+done
+[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
+# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
+# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
+# cleared before the type-and-verify loop or keystrokes land on the menu.
+for _ in $(seq 1 20); do
+  cap | grep -q "trust this folder" || break
+  tmux send-keys -t "$SESSION" Enter
+  sleep 1
+done
+
+# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
+# landed in the input box, retry if it didn't (handles the early-❯ race where
+# the welcome screen shows the prompt glyph but MCP init is still eating keys).
+needle="${PROMPT:0:24}"
+typed=0
+for _ in $(seq 1 30); do
+  tmux send-keys -l -t "$SESSION" "$PROMPT"
+  sleep 1
+  if cap | grep -Fq "$needle"; then typed=1; break; fi
+  # Clear whatever partial text may have landed, then retry.
+  tmux send-keys -t "$SESSION" C-u
+  sleep 1
+done
+[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+sleep 0.5
+tmux send-keys -t "$SESSION" Enter
+
+# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
+# EVERY working state shows — both the pre-stream thinking phase
+# "(8s · thinking with max effort)" and the streaming phase
+# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
+# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
+# (some TUI versions/states show one but not the others).
+BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
+
+# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
+# fail loudly rather than silently reporting an empty run.
+started=0
+for _ in $(seq 1 120); do
+  cap | grep -qE "$BUSY_RE" && { started=1; break; }
+  sleep 0.5
+done
+[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Poll for idle: not busy AND ❯ present, for 10 consecutive polls (~5s) to ride
+# out mid-conversation thinking gaps that briefly drop the spinner. Up to ~15min.
+consec=0
+for _ in $(seq 1 1800); do
+  pane=$(cap)
+  if echo "$pane" | grep -qE "$BUSY_RE"; then
+    consec=0
+  elif echo "$pane" | grep -q "❯"; then
+    consec=$((consec+1)); [ "$consec" -ge 10 ] && break
+  else
+    consec=0
+  fi
+  sleep 0.5
+done
+sleep 1
+
+tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
+echo "captured $(wc -l < "$OUT") lines -> $OUT"
+grep -oE "Done \([^)]*\)" "$OUT" | tail -1
+grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Clean tool breakdown from the session logs (main + subagents).
+node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true

package/scripts/agent-eval/parse-arms.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+// Analyze the tool-surface ablation (/tmp/arms/<repo>/<arm>-r<n>.jsonl).
+// Compares arms A–E on trace adoption, Read/Grep fallback, codegraph payload,
+// round-trips, and duration — averaged across runs per arm.
+//
+// The decisive signal is READS: if removing a tool raises Reads on a question
+// class, that tool was load-bearing for it (not redundant). If removing it
+// changes nothing, it was redundant.
+//
+//   A control       all tools            no steering   (baseline)
+//   B steer         all tools            trace-first   (adoption)
+//   C no-explore    hide explore         trace-first   (is explore redundant?)
+//   D trace-centric hide explore+context trace-first   (is the survey pair redundant?)
+//   E control-probe hide explore+context trace-first   (NON-flow Q — should degrade)
+//
+// Usage: node scripts/agent-eval/parse-arms.mjs [/tmp/arms]
+import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
+import { join } from 'path';
+
+const ROOT = process.argv[2] || '/tmp/arms';
+const cgShort = (n) => n.replace('mcp__codegraph__codegraph_', '').replace('mcp__codegraph__', '');
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  const calls = []; let result = null, initCg = 0;
+  for (const l of lines) {
+    let ev; try { ev = JSON.parse(l); } catch { continue; }
+    if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /codegraph/.test(t)).length;
+    if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use')
+      calls.push({ id: b.id, name: b.name, out: 0 });
+    if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
+      const c = b.content;
+      const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
+      const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
+    }
+    if (ev.type === 'result') result = ev;
+  }
+  const cg = calls.filter(c => c.name.includes('codegraph'));
+  return {
+    initCg,
+    reads: calls.filter(c => c.name === 'Read').length,
+    greps: calls.filter(c => c.name === 'Grep').length + calls.filter(c => c.name === 'Glob').length,
+    cgCalls: cg.length,
+    cgSeq: cg.map(c => cgShort(c.name)),
+    cgOut: cg.reduce((s, c) => s + c.out, 0),
+    traceUsed: cg.some(c => c.name.includes('trace')),
+    turns: result?.num_turns ?? null,
+    dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
+    cost: result?.total_cost_usd || 0,
+    ok: result?.subtype === 'success',
+  };
+}
+
+// repo -> arm -> [runs]
+const data = {};
+if (!existsSync(ROOT)) { console.error(`no ${ROOT}`); process.exit(1); }
+for (const repo of readdirSync(ROOT)) {
+  const rdir = join(ROOT, repo);
+  if (!statSync(rdir).isDirectory()) continue;
+  for (const f of readdirSync(rdir)) {
+    const m = f.match(/^([A-I])-r(\d+)\.jsonl$/); if (!m) continue;
+    const p = parse(join(rdir, f)); if (!p || !p.ok) continue;
+    (((data[repo] ??= {})[m[1]]) ??= []).push(p);
+  }
+}
+
+const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
+const k = (n) => (n / 1000).toFixed(1);
+const pad = (s, n) => String(s).padEnd(n);
+const ARMS = ['A', 'H', 'I', 'B', 'F', 'G', 'C', 'D', 'E'];
+const LABEL = { A: 'A all/none(old)', H: 'H body-trace/none', I: 'I bodytrace+dest', B: 'B all/steer(thin)', F: 'F all/steer(body)', G: 'G ported(noprompt)', C: 'C no-explore', D: 'D trace-centric', E: 'E nonflow-probe' };
+
+// ---- per repo × arm ----
+console.log('\n=== PER REPO × ARM (avg over runs) ===');
+console.log(pad('repo', 22), pad('arm', 16), 'tools', 'trace', pad('reads', 6), pad('cgOutK', 7), pad('turns', 6), 'dur');
+for (const repo of Object.keys(data).sort()) {
+  for (const arm of ARMS) {
+    const runs = data[repo][arm]; if (!runs?.length) continue;
+    console.log(
+      pad(repo, 22), pad(LABEL[arm], 16),
+      pad(runs[0].initCg, 5),
+      pad(runs.filter(r => r.traceUsed).length + '/' + runs.length, 5),
+      pad(avg(runs, r => r.reads).toFixed(1), 6),
+      pad(k(avg(runs, r => r.cgOut)), 7),
+      pad(avg(runs, r => r.turns).toFixed(1), 6),
+      avg(runs, r => r.dur).toFixed(0) + 's',
+    );
+  }
+}
+
+// ---- aggregate per arm (flow arms A–D over the flow repos; E shown apart) ----
+console.log('\n=== AGGREGATE PER ARM (mean across repos) ===');
+console.log(pad('arm', 16), pad('adoption', 9), pad('reads', 7), pad('greps', 7), pad('cgOutK', 8), pad('turns', 7), pad('dur', 6), 'cost');
+for (const arm of ARMS) {
+  const all = [];
+  for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) all.push({ ...r, repo });
+  if (!all.length) continue;
+  const repos = new Set(all.map(r => r.repo)).size;
+  const adopt = all.filter(r => r.traceUsed).length;
+  console.log(
+    pad(LABEL[arm], 16),
+    pad(`${adopt}/${all.length}`, 9),
+    pad(avg(all, r => r.reads).toFixed(2), 7),
+    pad(avg(all, r => r.greps).toFixed(2), 7),
+    pad(k(avg(all, r => r.cgOut)), 8),
+    pad(avg(all, r => r.turns).toFixed(1), 7),
+    pad(avg(all, r => r.dur).toFixed(0) + 's', 6),
+    '$' + avg(all, r => r.cost).toFixed(3),
+    `  (${repos} repos)`,
+  );
+}
+
+console.log('\nRead the signal: B vs A = does steering alone fix adoption + cut payload.');
+console.log('C vs B = is explore redundant (reads should NOT jump). D vs C = is context redundant.');
+console.log('E = non-flow under trace-centric; reads SHOULD jump (proves survey tools are load-bearing).');

package/scripts/agent-eval/parse-bench-readme.mjs

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+// Aggregate the README A/B (bench-readme.sh output): per repo, median of N runs
+// per arm → time, tool calls, tokens, cost, and % saved. Plus an average row.
+//
+// Tokens = SUM of per-turn assistant `usage` (input + output + cache read +
+// cache creation) — the cumulative "total tokens processed". NOTE: `result.usage`
+// is last-turn-only in current Claude Code, so it under-counts badly; don't use it.
+// `total_cost_usd` and `duration_ms` are already cumulative.
+//
+// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+const ROOT = process.argv[2] || '/tmp/ab-readme';
+const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const L = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null;
+  for (const l of L) { let e; try { e = JSON.parse(l); } catch { continue; }
+    if (e.type === 'assistant') {
+      const u = e.message?.usage;
+      if (u) tokens += (u.input_tokens || 0) + (u.output_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+      for (const b of (e.message?.content || [])) if (b.type === 'tool_use') {
+        const n = b.name;
+        if (n === 'ToolSearch') continue;
+        tools++;
+        if (n === 'Read') reads++;
+        else if (n === 'Grep' || n === 'Glob') grep++;
+        else if (/codegraph/.test(n)) cg++;
+      }
+    }
+    if (e.type === 'result') r = e;
+  }
+  if (!r || r.subtype !== 'success') return null;
+  return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0 };
+}
+const median = (arr) => { const v = [...arr].sort((a, b) => a - b); const n = v.length; return n === 0 ? 0 : n % 2 ? v[(n - 1) / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; };
+const fmtTime = (s) => s >= 60 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${Math.round(s)}s`;
+const fmtTok = (t) => t >= 1e6 ? `${(t / 1e6).toFixed(1)}M` : `${Math.round(t / 1000)}k`;
+const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
+
+console.log('repo        n(w/wo)  time WITH→WITHOUT      tools W→WO   tokens W→WO (saved)     cost W→WO (saved)');
+const savings = { cost: [], tokens: [], time: [], tools: [] };
+for (const repo of REPOS) {
+  const dir = join(ROOT, repo);
+  const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)) : [];
+  const W = [], WO = [];
+  for (const rd of runDirs) {
+    const w = parse(join(dir, rd, 'run-headless-with.jsonl')); if (w) W.push(w);
+    const wo = parse(join(dir, rd, 'run-headless-without.jsonl')); if (wo) WO.push(wo);
+  }
+  if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete: w=${W.length} wo=${WO.length})`); continue; }
+  const m = (arr, k) => median(arr.map(x => x[k]));
+  const wT = m(W, 'dur'), woT = m(WO, 'dur'), wTok = m(W, 'tokens'), woTok = m(WO, 'tokens');
+  const wC = m(W, 'cost'), woC = m(WO, 'cost'), wTl = m(W, 'tools'), woTl = m(WO, 'tools');
+  savings.time.push(pct(wT, woT)); savings.tokens.push(pct(wTok, woTok)); savings.cost.push(pct(wC, woC)); savings.tools.push(pct(wTl, woTl));
+  console.log(
+    `${repo.padEnd(11)} ${W.length}/${WO.length}      ` +
+    `${(fmtTime(wT) + '→' + fmtTime(woT)).padEnd(22)}` +
+    `${(Math.round(wTl) + '→' + Math.round(woTl)).padEnd(12)}` +
+    `${(fmtTok(wTok) + '→' + fmtTok(woTok) + ' (' + pct(wTok, woTok) + '%)').padEnd(24)}` +
+    `$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)`
+  );
+}
+const avg = (a) => a.length ? Math.round(a.reduce((s, x) => s + x, 0) / a.length) : 0;
+console.log(`\nAVERAGE saved:  cost ${avg(savings.cost)}%  ·  tokens ${avg(savings.tokens)}%  ·  time ${avg(savings.time)}%  ·  tool calls ${avg(savings.tools)}%`);

package/scripts/agent-eval/parse-run.mjs

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
+import { readFileSync } from 'fs';
+const file = process.argv[2];
+const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+
+const toolCalls = [];
+let result = null;
+let initTools = null;
+
+for (const line of lines) {
+  let ev;
+  try { ev = JSON.parse(line); } catch { continue; }
+  if (ev.type === 'system' && ev.subtype === 'init') {
+    initTools = (ev.tools || []).filter(t => /codegraph/.test(t));
+  }
+  if (ev.type === 'assistant' && ev.message?.content) {
+    for (const block of ev.message.content) {
+      if (block.type === 'tool_use') {
+        let detail = '';
+        if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
+        else if (/codegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
+        else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
+        else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
+        toolCalls.push(`${block.name}${detail}`);
+      }
+    }
+  }
+  if (ev.type === 'result') result = ev;
+}
+
+console.log(`\n=== ${file.split('/').pop()} ===`);
+console.log(`codegraph tools exposed: ${initTools ? initTools.length : '?'}`);
+console.log(`\nTool calls (${toolCalls.length}):`);
+const counts = {};
+for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
+console.log('  by type:', JSON.stringify(counts));
+toolCalls.forEach((tc, i) => console.log(`  ${i+1}. ${tc}`));
+
+if (result) {
+  const u = result.usage || {};
+  const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
+  console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
+  console.log(`  tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
+}

package/scripts/agent-eval/parse-session.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+// Parse the newest Claude Code session log for a project + its subagent logs,
+// and report the tool-call breakdown (main + subagents). Works for interactive
+// runs (driven via itrun.sh) — Claude Code writes full transcripts to
+// ~/.claude/projects/<escaped-cwd>/<session>.jsonl with subagents/ alongside.
+import { readFileSync, readdirSync, statSync, existsSync, realpathSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+const projectArg = process.argv[2];
+if (!projectArg) { console.error('usage: parse-session.mjs <project-dir>'); process.exit(1); }
+
+// Claude Code escapes the (real) cwd by replacing every "/" with "-".
+const real = realpathSync(projectArg);
+const escaped = real.replace(/\//g, '-');
+const projDir = join(homedir(), '.claude', 'projects', escaped);
+if (!existsSync(projDir)) { console.error('no session logs at', projDir); process.exit(1); }
+
+// Newest top-level session .jsonl
+const sessions = readdirSync(projDir)
+  .filter(f => f.endsWith('.jsonl'))
+  .map(f => ({ f, m: statSync(join(projDir, f)).mtimeMs }))
+  .sort((a, b) => b.m - a.m);
+if (sessions.length === 0) { console.error('no .jsonl sessions in', projDir); process.exit(1); }
+const sessionId = sessions[0].f.replace('.jsonl', '');
+
+function tally(file) {
+  const counts = {};
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const content = ev.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const b of content) {
+      if (b.type === 'tool_use') counts[b.name] = (counts[b.name] || 0) + 1;
+    }
+  }
+  return counts;
+}
+
+// Sum token usage from a transcript. The TUI's "Done (…Xk tokens…)" line only
+// covers a subagent's throughput; this works for main-thread runs too and is
+// consistent across both paths. `gen` = output, `fresh` = uncached input
+// (input + cache_creation), `cached` = cache reads (≈free), `total` = all.
+function sumTokens(file) {
+  const t = { gen: 0, fresh: 0, cached: 0 };
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const u = ev.message?.usage;
+    if (!u) continue;
+    t.gen += u.output_tokens || 0;
+    t.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+    t.cached += u.cache_read_input_tokens || 0;
+  }
+  return t;
+}
+
+const mainCounts = tally(join(projDir, sessionId + '.jsonl'));
+
+// Subagent transcripts live under <session>/subagents/*.jsonl
+const subDir = join(projDir, sessionId, 'subagents');
+const subCounts = {};
+let subAgentFiles = 0;
+if (existsSync(subDir)) {
+  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) {
+    subAgentFiles++;
+    const c = tally(join(subDir, f));
+    for (const [k, v] of Object.entries(c)) subCounts[k] = (subCounts[k] || 0) + v;
+  }
+}
+
+const fmt = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])
+  .map(([k, v]) => `    ${String(v).padStart(3)}  ${k}`).join('\n') || '    (none)';
+
+console.log(`session: ${sessionId}`);
+console.log(`\nMAIN thread tools:\n${fmt(mainCounts)}`);
+console.log(`\nSUBAGENT tools (${subAgentFiles} subagent transcript${subAgentFiles === 1 ? '' : 's'}):\n${fmt(subCounts)}`);
+
+const explore = subCounts['mcp__codegraph__codegraph_explore'] || mainCounts['mcp__codegraph__codegraph_explore'] || 0;
+const reads = (subCounts['Read'] || 0) + (mainCounts['Read'] || 0);
+const greps = (subCounts['Grep'] || 0) + (mainCounts['Grep'] || 0) + (subCounts['Bash'] || 0) + (mainCounts['Bash'] || 0);
+console.log(`\nVERDICT: codegraph_explore used ${explore}x | Read ${reads} | Grep/Bash ${greps}`);
+
+// Token totals (main + subagents), consistent across main-thread and subagent runs.
+const tok = { gen: 0, fresh: 0, cached: 0 };
+const addTok = (t) => { tok.gen += t.gen; tok.fresh += t.fresh; tok.cached += t.cached; };
+addTok(sumTokens(join(projDir, sessionId + '.jsonl')));
+if (existsSync(subDir)) {
+  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) addTok(sumTokens(join(subDir, f)));
+}
+const k = (n) => (n / 1000).toFixed(1) + 'k';
+console.log(`TOKENS: gen ${k(tok.gen)} | fresh-in ${k(tok.fresh)} | cached-in ${k(tok.cached)} | billable≈ ${k(tok.gen + tok.fresh)}`);

package/scripts/agent-eval/probe-context.mjs

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+// Probe codegraph_context (with call-paths) against an index using the built dist.
+// Usage: node probe-context.mjs <repo-with-.codegraph> <task words...>
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, ...taskParts] = process.argv;
+const task = taskParts.join(' ');
+if (!repo || !task) { console.error('usage: probe-context.mjs <repo> <task...>'); process.exit(1); }
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+const cg = CodeGraph.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('codegraph_context', { task });
+console.log(res.content?.[0]?.text ?? '(no text)');
+try { cg.close?.(); } catch {}

package/scripts/agent-eval/probe-explore.mjs

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+// One-shot probe: run handleExplore against an existing index using the built
+// dist, print the output + a few stats. Lets us verify explore's coverage fix
+// without a full agent run. Usage: node probe-explore.mjs <repo-with-.codegraph> "<query>"
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, query] = process.argv;
+if (!repo || !query) {
+  console.error('usage: probe-explore.mjs <repo> "<query>"');
+  process.exit(1);
+}
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+
+// esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
+const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+if (typeof CodeGraph?.openSync !== 'function') {
+  console.error('could not resolve CodeGraph.openSync; index keys:', Object.keys(idx), 'default keys:', idx.default && Object.keys(idx.default));
+  process.exit(2);
+}
+if (typeof ToolHandler !== 'function') {
+  console.error('could not resolve ToolHandler; tools keys:', Object.keys(tools));
+  process.exit(2);
+}
+
+const cg = CodeGraph.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('codegraph_explore', { query });
+const text = res.content?.[0]?.text ?? '(no text)';
+console.log(text);
+console.error('\n--- PROBE STATS ---');
+console.error('output chars:', text.length);
+console.error('triggerRender body present (-> setState({})):', /triggerRender[\s\S]{0,400}setState\(\{\}\)/.test(text));
+console.error('App.tsx in source section:', /#### .*App\.tsx —/.test(text));
+try { cg.close?.(); } catch {}

package/scripts/agent-eval/probe-node.mjs

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+// Probe codegraph_node (with trail) against an index using the built dist.
+// Usage: node probe-node.mjs <repo-with-.codegraph> <symbol> [code]
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, symbol, code] = process.argv;
+if (!repo || !symbol) { console.error('usage: probe-node.mjs <repo> <symbol> [code]'); process.exit(1); }
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+const cg = CodeGraph.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('codegraph_node', { symbol, includeCode: code === 'code' });
+console.log(res.content?.[0]?.text ?? '(no text)');
+try { cg.close?.(); } catch {}