@ltm-alpha/alpha-cs 1.6.0

package/scripts/agent-eval/compare-arms.mjs

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+// One side-by-side table for the three feedback metrics, across the arms of a
+// single A/B output directory. This is the "did it move?" view — the per-run
+// blocks parse-run.mjs prints are the "why did it move?" view, and both are
+// printed by the harnesses (ab-new-vs-baseline.sh, run-all.sh).
+//
+//   residual context occupancy  how much window the arm's retrieval still holds
+//   explore sufficiency         whether a response was ENOUGH (agent's next act)
+//   allocation efficiency       what share of returned bytes the answer used
+//
+// Usage: compare-arms.mjs <out-dir> <label> [<label> ...]
+//   e.g.  compare-arms.mjs /tmp/ab-new-vs-baseline new baseline
+//         compare-arms.mjs /tmp/agent-eval headless-with headless-without
+//
+// Run discovery handles both shapes the harnesses write, per label:
+//   run-<label>-<i>.jsonl        N independent runs   (ab-new-vs-baseline, RUNS=N)
+//   run-<label>.jsonl + .tN      ONE session, N turns (run-all.sh multi-turn)
+// A `.tN` file is always a resumed SEGMENT of the run it hangs off, never a run
+// of its own — mixing those up would report a three-turn session as three runs
+// and average away the residual the later turns exist to charge.
+import { existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { parseSession, SUFFICIENCY } from './parse-run.mjs';
+
+/** Segment files of one run, in turn order: run-X.jsonl, run-X.t2.jsonl, … */
+function segmentsOf(dir, stem) {
+  const first = join(dir, `${stem}.jsonl`);
+  if (!existsSync(first)) return null;
+  const rest = readdirSync(dir)
+    .map((f) => [f, new RegExp(`^${stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.t(\\d+)\\.jsonl$`).exec(f)])
+    .filter(([, m]) => m)
+    .sort((a, b) => Number(a[1][1]) - Number(b[1][1]))
+    .map(([f]) => join(dir, f));
+  return [first, ...rest];
+}
+
+/** Every run of one arm, newest-numbering-first-run order. */
+export function discoverRuns(dir, label) {
+  const session = segmentsOf(dir, `run-${label}`);
+  if (session) return [{ name: label, files: session }];
+  const indexed = readdirSync(dir)
+    .map((f) => new RegExp(`^run-${label.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}-(\\d+)\\.jsonl$`).exec(f))
+    .filter(Boolean)
+    .map((m) => Number(m[1]))
+    .sort((a, b) => a - b);
+  return indexed.map((i) => ({ name: `${label}-${i}`, files: segmentsOf(dir, `run-${label}-${i}`) }));
+}
+
+const median = (xs) => {
+  if (!xs.length) return null;
+  const a = [...xs].sort((x, y) => x - y);
+  const m = a.length >> 1;
+  return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
+};
+
+/** The numbers one arm's runs contribute to the table. */
+function measure(run) {
+  const s = parseSession(run.files);
+  const o = s.occupancy;
+  const a = s.allocation;
+  return {
+    name: run.name,
+    ok: s.ok,
+    raced: s.raced,
+    turns: s.turns,
+    dur: s.dur,
+    tools: s.tools,
+    reads: s.reads,
+    grep: s.grep,
+    bash: s.counts.Bash || 0,
+    cg: s.cg,
+    cliCalls: s.cliCalls,
+    cliContaminated: s.cliContaminated,
+    ctx: o.ctxFinal,
+    occCg: o.residual.codegraph,
+    occFile: o.residualFileAccess,
+    // The arm's OWN retrieval residual: codegraph in a with-arm, Read/Grep/Bash
+    // in a without-arm. Comparing these two is the apples-to-apples pair.
+    occSelf: o.residual.codegraph + o.residualFileAccess,
+    occShare: o.ctxFinal > 0 ? ((o.residual.codegraph + o.residualFileAccess) / o.ctxFinal) * 100 : 0,
+    suffAnswered: s.sufficiency.answered,
+    suffCounts: s.sufficiency.counts,
+    suffErrors: s.sufficiency.errors,
+    // Allocation is byte-weighted, so a run with no explore contributes nothing
+    // rather than a zero — a zero would drag the pooled number toward "wasteful"
+    // for a run that never spent a byte.
+    allocUsed: a.envelope ? a.used : null,
+    allocEnvelope: a.envelope || null,
+    allocCalls: a.calls.length,
+  };
+}
+
+/** median [min–max] over runs; the range is the point — never quote one run. */
+function span(runs, pick, fmt = (x) => String(Math.round(x))) {
+  const xs = runs.map(pick).filter((x) => x !== null && x !== undefined && Number.isFinite(x));
+  if (!xs.length) return '—';
+  const m = fmt(median(xs));
+  if (xs.length === 1) return m;
+  const lo = fmt(Math.min(...xs)); const hi = fmt(Math.max(...xs));
+  return lo === hi ? m : `${m} [${lo}–${hi}]`;
+}
+
+const int = (x) => Math.round(x).toLocaleString('en-US');
+const pct1 = (x) => `${x.toFixed(1)}%`;
+
+export function formatComparison(arms) {
+  const W = 36; const C = 24;
+  const out = [];
+  // The leading space is a separator, not padding: a `median [min–max]` cell can
+  // fill its column, and two of those with only padStart between them run
+  // together into one unreadable number.
+  const row = (label, cells) => out.push('  ' + label.padEnd(W) + cells.map((c) => ' ' + String(c).padStart(C - 1)).join(''));
+  const rule = (title) => out.push(`  ${title}`);
+
+  row('', arms.map((a) => a.label));
+  row('runs', arms.map((a) => a.runs.length));
+  const anyFailed = arms.some((a) => a.runs.some((r) => !r.ok));
+  if (anyFailed) row('  of which non-success', arms.map((a) => a.runs.filter((r) => !r.ok).length));
+  if (arms.some((a) => a.runs.some((r) => r.raced))) {
+    row('  MCP cold-start race', arms.map((a) => a.runs.filter((r) => r.raced).length));
+  }
+  out.push('');
+
+  rule('behavior');
+  row('  duration (s)', arms.map((a) => span(a.runs, (r) => r.dur)));
+  row('  tool calls', arms.map((a) => span(a.runs, (r) => r.tools)));
+  row('  Read', arms.map((a) => span(a.runs, (r) => r.reads)));
+  row('  Grep/Glob', arms.map((a) => span(a.runs, (r) => r.grep)));
+  row('  Bash', arms.map((a) => span(a.runs, (r) => r.bash)));
+  row('  codegraph calls', arms.map((a) => span(a.runs, (r) => r.cg)));
+  out.push('');
+
+  rule('residual context occupancy (CG-7) — tokens still resident at end of run');
+  row('  final context (tok)', arms.map((a) => span(a.runs, (r) => r.ctx, int)));
+  row('  codegraph residual (tok)', arms.map((a) => span(a.runs, (r) => r.occCg, int)));
+  row('  file-access residual (tok)', arms.map((a) => span(a.runs, (r) => r.occFile, int)));
+  row('  → retrieval residual (tok)', arms.map((a) => span(a.runs, (r) => r.occSelf, int)));
+  row('  → share of final context', arms.map((a) => span(a.runs, (r) => r.occShare, pct1)));
+  out.push('');
+
+  rule('explore sufficiency (CG-8) — pooled over every answered explore call');
+  row('  answered explore calls', arms.map((a) => a.runs.reduce((s, r) => s + r.suffAnswered, 0)));
+  for (const [key, label] of SUFFICIENCY) {
+    row(`  ${label}`, arms.map((a) => {
+      const n = a.runs.reduce((s, r) => s + r.suffCounts[key], 0);
+      const tot = a.runs.reduce((s, r) => s + r.suffAnswered, 0);
+      return tot ? `${n}  ${((n / tot) * 100).toFixed(0)}%` : '—';
+    }));
+  }
+  if (arms.some((a) => a.runs.some((r) => r.suffErrors))) {
+    row('  errored/unanswered (not bucketed)', arms.map((a) => a.runs.reduce((s, r) => s + r.suffErrors, 0)));
+  }
+  out.push('');
+
+  rule('explore allocation efficiency (CG-9) — share of returned bytes the answer cited');
+  row('  explore calls with source', arms.map((a) => a.runs.reduce((s, r) => s + r.allocCalls, 0)));
+  row('  pooled efficiency', arms.map((a) => {
+    const env = a.runs.reduce((s, r) => s + (r.allocEnvelope || 0), 0);
+    const used = a.runs.reduce((s, r) => s + (r.allocUsed || 0), 0);
+    return env ? pct1((used / env) * 100) : '—';
+  }));
+  row('  per-run efficiency', arms.map((a) =>
+    span(a.runs, (r) => (r.allocEnvelope ? (r.allocUsed / r.allocEnvelope) * 100 : null), pct1)));
+  row('  envelope (chars)', arms.map((a) => int(a.runs.reduce((s, r) => s + (r.allocEnvelope || 0), 0))));
+  out.push('');
+
+  rule('contamination — the CLI must never be how codegraph is reached');
+  row('  CLI calls that RETURNED output', arms.map((a) => a.runs.reduce((s, r) => s + r.cliContaminated, 0)));
+  row('  CLI attempts blocked', arms.map((a) => a.runs.reduce((s, r) => s + r.cliCalls, 0)));
+  const contaminated = arms.filter((a) => a.runs.some((r) => r.cliContaminated));
+  if (contaminated.length) {
+    out.push(`  !! ${contaminated.map((a) => a.label).join(', ')} reached codegraph through Bash — those runs are CONTAMINATED`);
+    out.push('     (a without-arm was not without codegraph; a with-arm has bytes attributed to Bash, not codegraph)');
+  }
+  out.push('');
+
+  out.push('  how to read this');
+  out.push('    occupancy  compare each arm\'s RETRIEVAL residual (codegraph in a with-arm,');
+  out.push('               file-access in a without-arm). Shares are Claude Code on a 200k');
+  out.push('               window and do NOT transfer to another host; the ratio does.');
+  out.push('    sufficiency  pooled across runs because it is per-CALL. "explore again" is');
+  out.push('               ambiguous by construction; "Read a file we returned" is an');
+  out.push('               allocation miss, the two recall rows are recall misses.');
+  out.push('    allocation  RELATIVE, not absolute — attribution is by citation, and an agent');
+  out.push('               can use a file without naming it. Compare builds on the SAME');
+  out.push('               question; never quote it as "codegraph wastes N% of what it returns."');
+  out.push('    all three  small-n. Runs make 1–5 explore calls, so read the range, not the');
+  out.push('               median of one run. RUNS>=2, and the 7-repo campaign for a verdict.');
+  return out.join('\n');
+}
+
+const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) {
+  const [dir, ...labels] = process.argv.slice(2);
+  if (!dir || !labels.length) {
+    console.error('usage: compare-arms.mjs <out-dir> <label> [<label> ...]');
+    process.exit(1);
+  }
+  const arms = labels.map((label) => ({ label, runs: discoverRuns(dir, label).map(measure) }));
+  const empty = arms.filter((a) => !a.runs.length);
+  if (empty.length === arms.length) {
+    console.error(`no run logs for ${labels.join('/')} in ${dir}`);
+    process.exit(1);
+  }
+  for (const a of empty) console.error(`  WARN: no run logs for arm '${a.label}' in ${dir}`);
+  console.log(`\n====== ARM COMPARISON — ${dir} ======`);
+  console.log(formatComparison(arms.filter((a) => a.runs.length)));
+}

package/scripts/agent-eval/diff-index-drift.mjs

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+/**
+ * Diff two CodeGraph indexes of the SAME tree — typically a live,
+ * incrementally-synced `.codegraph/codegraph.db` against a clean full rebuild
+ * of the identical working tree (CG-33).
+ *
+ * Non-destructive: it only reads. Rebuilding is the caller's job, so the live
+ * index is never clobbered by the tool measuring it — the mistake that cost the
+ * original CG-33 artifact.
+ *
+ *   # snapshot the live index BEFORE touching it
+ *   cp .codegraph/codegraph.db /tmp/live.db
+ *   node dist/bin/codegraph.js index .
+ *   node scripts/agent-eval/diff-index-drift.mjs /tmp/live.db .codegraph/codegraph.db
+ *
+ * Edges are compared as distinct `(source, target, kind)` triples. Raw row
+ * counts are NOT a drift signal: a bidirectional divergence nets out. On the
+ * codegraph repo the raw counts differed by +0.7% while 4.3% of distinct edges
+ * were actually wrong.
+ */
+import { DatabaseSync } from 'node:sqlite';
+import { existsSync } from 'node:fs';
+
+const [livePath, rebuiltPath] = process.argv.slice(2);
+if (!livePath || !rebuiltPath) {
+  console.error('usage: diff-index-drift.mjs <live.db> <rebuilt.db>');
+  process.exit(2);
+}
+for (const p of [livePath, rebuiltPath]) {
+  if (!existsSync(p)) {
+    // node:sqlite CREATES a missing file rather than failing, which silently
+    // yields an empty schema and a confident, wrong conclusion. Refuse first.
+    console.error(`not found: ${p}`);
+    process.exit(2);
+  }
+}
+
+const open = (p) => new DatabaseSync(p, { readOnly: true });
+const live = open(livePath);
+const rebuilt = open(rebuiltPath);
+
+const scalar = (db, q) => db.prepare(q).get().n;
+const edgeKey = (r) => `${r.source}\u0000${r.target}\u0000${r.kind}`;
+
+const liveEdges = live.prepare('select source, target, kind from edges').all();
+const rebuiltEdges = rebuilt.prepare('select source, target, kind from edges').all();
+const liveSet = new Set(liveEdges.map(edgeKey));
+const rebuiltSet = new Set(rebuiltEdges.map(edgeKey));
+
+const missing = rebuiltEdges.filter((r) => !liveSet.has(edgeKey(r)));   // should exist, doesn't
+const stale = liveEdges.filter((r) => !rebuiltSet.has(edgeKey(r)));      // exists, shouldn't
+const divergent = missing.length + stale.length;
+
+const byKind = (rows) => {
+  const m = new Map();
+  for (const r of rows) m.set(r.kind, (m.get(r.kind) ?? 0) + 1);
+  return [...m].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(', ') || '(none)';
+};
+
+const pct = (n, d) => (d ? ((n / d) * 100).toFixed(1) : '0.0');
+
+console.log(`live    ${livePath}`);
+console.log(`rebuilt ${rebuiltPath}`);
+console.log('');
+console.log('counts                 live      rebuilt');
+for (const [label, q] of [
+  ['files', 'select count(*) n from files'],
+  ['nodes', 'select count(*) n from nodes'],
+  ['edges (rows)', 'select count(*) n from edges'],
+  // Grouped rather than `count(distinct a || b || c)`: bare concatenation has no
+  // separator, so `(ab, c)` and `(a, bc)` would collapse into one.
+  ['edges (distinct)', 'select count(*) n from (select distinct source, target, kind from edges)'],
+  ['heuristic edges', "select count(*) n from edges where provenance='heuristic'"],
+]) {
+  console.log(`  ${label.padEnd(20)} ${String(scalar(live, q)).padEnd(9)} ${scalar(rebuilt, q)}`);
+}
+
+console.log('');
+console.log('edge divergence (distinct triples)');
+console.log(`  missing from live: ${missing.length}  — ${byKind(missing)}`);
+console.log(`  stale in live:     ${stale.length}  — ${byKind(stale)}`);
+console.log(`  TOTAL divergent:   ${divergent}  (${pct(divergent, rebuiltSet.size)}% of ${rebuiltSet.size})`);
+
+// Integrity checks — these separate "resolution went stale" (edges wrong, nodes
+// identical) from "residue accumulated" (duplicate/orphan rows). CG-33 is the
+// former: on the codegraph repo every check below was 0 on BOTH indexes.
+console.log('');
+console.log('integrity                        live   rebuilt');
+for (const [label, q] of [
+  ['duplicate nodes', 'select count(*) n from (select file_path,name,kind,start_line from nodes group by 1,2,3,4 having count(*)>1)'],
+  ['orphan edges', 'select count(*) n from edges e where not exists(select 1 from nodes where id=e.source) or not exists(select 1 from nodes where id=e.target)'],
+  ['nodes w/ missing file row', 'select count(*) n from nodes nd where not exists(select 1 from files f where f.path=nd.file_path)'],
+]) {
+  console.log(`  ${label.padEnd(30)} ${String(scalar(live, q)).padEnd(6)} ${scalar(rebuilt, q)}`);
+}
+
+console.log('');
+console.log(divergent === 0
+  ? 'CONVERGED — the synced index matches a full rebuild.'
+  : `DRIFTED — ${divergent} edges differ. Rebuild-vs-rebuild is 0 (the indexer is deterministic), so this is sync divergence, not noise.`);
+
+process.exitCode = divergent === 0 ? 0 : 1;

package/scripts/agent-eval/hook-settings.json

@@ -0,0 +1,15 @@
+{
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Read",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash /Users/colby/Development/Personal/codegraph/scripts/agent-eval/block-read-hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}

package/scripts/agent-eval/itrun.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
+# agent to finish, then print the tool-call breakdown from the session logs.
+#
+# Why interactive (not `claude -p`): headless print-mode picks the
+# general-purpose subagent, while real interactive sessions delegate to the
+# Explore subagent (or drive codegraph from the main thread). Only the
+# interactive TUI reproduces the behavior users actually see. (Idle-detection
+# technique borrowed from devpit's WaitForIdle.)
+#
+# Usage: itrun.sh <repo-path> <label> "<prompt>"
+# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
+# Requires: tmux 3.0+, a logged-in `claude` CLI, codegraph MCP configured.
+set -uo pipefail
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+SESSION="cgt_${LABEL}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/itrun-${LABEL}.txt"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
+
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Wide pane so the TUI doesn't hard-wrap tool lines.
+tmux new-session -d -s "$SESSION" -x 230 -y 60
+tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
+
+# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
+# welcome screen seconds before the input actually accepts keystrokes, so this is
+# necessary but NOT sufficient — the type-and-verify loop below is what proves
+# the input is live.
+ready=0
+for _ in $(seq 1 120); do
+  cap | grep -q "❯" && { ready=1; break; }
+  sleep 0.5
+done
+[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
+# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
+# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
+# cleared before the type-and-verify loop or keystrokes land on the menu.
+for _ in $(seq 1 20); do
+  cap | grep -q "trust this folder" || break
+  tmux send-keys -t "$SESSION" Enter
+  sleep 1
+done
+
+# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
+# landed in the input box, retry if it didn't (handles the early-❯ race where
+# the welcome screen shows the prompt glyph but MCP init is still eating keys).
+needle="${PROMPT:0:24}"
+typed=0
+for _ in $(seq 1 30); do
+  tmux send-keys -l -t "$SESSION" "$PROMPT"
+  sleep 1
+  if cap | grep -Fq "$needle"; then typed=1; break; fi
+  # Clear whatever partial text may have landed, then retry.
+  tmux send-keys -t "$SESSION" C-u
+  sleep 1
+done
+[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+sleep 0.5
+tmux send-keys -t "$SESSION" Enter
+
+# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
+# EVERY working state shows — both the pre-stream thinking phase
+# "(8s · thinking with max effort)" and the streaming phase
+# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
+# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
+# (some TUI versions/states show one but not the others).
+BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
+
+# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
+# fail loudly rather than silently reporting an empty run.
+started=0
+for _ in $(seq 1 120); do
+  cap | grep -qE "$BUSY_RE" && { started=1; break; }
+  sleep 0.5
+done
+[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Poll for idle. CRITICAL: Opus 4.8 (extended thinking) renders NO spinner /
+# "esc to interrupt" / timer while it STREAMS its final answer — those appear
+# only during the thinking + tool-use phases ("✻ Marinating… (32s · ↓ 1.3k
+# tokens · thinking with max effort)"). So BUSY_RE reads "not busy" for the whole
+# 10-30s answer stream, and any short not-busy threshold kills the run mid-answer
+# (the truncation bug). We therefore detect "done" by CONTENT STABILITY, not by a
+# spinner string: while the agent streams, the captured pane changes every poll,
+# so stability never accrues; it accrues only once the agent has finished and the
+# static "✻ Brewed for 1m 9s" summary is all that is left. BUSY_RE still hard-
+# resets stability (covers thinking/tool-use/live-timer, where text can briefly
+# sit still). Need STABLE_NEEDED polls (~8s) of zero pane change + ❯ present.
+# Content-stability is model-agnostic — it survives future spinner re-wordings.
+STABLE_NEEDED=16
+prev=""; stable=0
+for _ in $(seq 1 2400); do            # up to ~20 min
+  pane="$(cap)"
+  sig="$(printf '%s' "$pane" | tr -s '[:space:]' ' ')"
+  if printf '%s' "$pane" | grep -qE "$BUSY_RE"; then
+    stable=0                          # thinking / tool use / live timer → busy
+  elif [ -n "$sig" ] && [ "$sig" = "$prev" ] && printf '%s' "$pane" | grep -q "❯"; then
+    stable=$((stable+1)); [ "$stable" -ge "$STABLE_NEEDED" ] && break
+  else
+    stable=0                          # answer still streaming → pane changing
+  fi
+  prev="$sig"
+  sleep 0.5
+done
+sleep 1
+
+tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
+echo "captured $(wc -l < "$OUT") lines -> $OUT"
+grep -oE "Done \([^)]*\)|[A-Z][a-z]+ for ([0-9]+m )?[0-9]+s" "$OUT" | tail -1
+grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Clean tool breakdown from the session logs (main + subagents).
+node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true

package/scripts/agent-eval/no-cli-shim.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Keep the codegraph CLI out of an eval arm, so the MCP server is the ONLY way
+# the agent can reach codegraph. Sourced by run-all.sh and ab-new-vs-baseline.sh.
+#
+#   . "$HARNESS/no-cli-shim.sh"
+#   cg_no_cli_setup "$OUT"       # -> sets $ARM_PATH and $ARM_SETTINGS
+#   PATH="$ARM_PATH" claude … --settings "$ARM_SETTINGS"
+#
+# Why this exists, in both harnesses:
+#
+#   with/without (run-all.sh)  The without-arm gets an empty MCP config but still
+#   has Bash, and the target repo carries the .codegraph/ index. Agents find that:
+#   14 of 15 without-arm runs in one 7-repo pass ran `codegraph explore` through
+#   Bash (one via `ls .codegraph && codegraph explore …`), so that arm was
+#   measuring codegraph-over-CLI, not codegraph-absent.
+#
+#   new/baseline (ab-new-vs-baseline.sh)  Both arms are codegraph-on, so a CLI
+#   call is not a with/without leak — it is an ATTRIBUTION leak, and it breaks all
+#   three feedback metrics at once. Output that arrives through Bash is charged to
+#   Bash (understating occupancy), and an explore issued through the CLI is not a
+#   tool call at all, so it never reaches the sufficiency classifier or the
+#   allocation parse. A run that shells out silently drops calls from the numbers.
+#
+# Two layers, because one was not enough:
+#
+#   1. PATH. The binary usually shares a directory with tools the run needs
+#      (claude itself lives next to it here), so dropping the whole directory is
+#      not an option. Substitute an equivalent directory IN PLACE: symlinks to
+#      every entry except codegraph, keeping PATH order and precedence intact.
+#   2. A PreToolUse hook. An agent denied `codegraph` ran
+#      `find / -maxdepth 4 -iname "*codegraph*"`, found the binary, and invoked it
+#      by ABSOLUTE PATH — so block the invocation itself. Written into the output
+#      dir as a run artifact rather than a repo file, same as the MCP configs.
+#
+# Neither layer is a substitute for the counter: parse-run.mjs flags any Bash
+# command that named codegraph, separating attempts it blocked (no output entered
+# the window) from calls that RETURNED output. Prevention fails silently the next
+# time the binary lands somewhere new; the counter does not.
+
+# Command positions only: `grep codegraph x`, `ls .codegraph` and
+# `which codegraph` are looking, not using, and pass through.
+CG_CMD_RE='(^|[;&|(]|&&|\|\||\$\(|`)[[:space:]]*([A-Za-z_][A-Za-z0-9_]*=[^[:space:]]*[[:space:]]+)*[A-Za-z0-9_./~-]*codegraph([[:space:]]|$)'
+
+cg_no_cli_setup() {
+  local out="${1:?cg_no_cli_setup <out-dir>}"
+  local shim="$out/nocg-bin"
+
+  rm -rf "$shim"; mkdir -p "$shim"
+  local built="" d e
+  local IFS=:
+  for d in $PATH; do
+    [ -n "$d" ] || continue
+    if [ -x "$d/codegraph" ]; then
+      for e in "$d"/*; do
+        [ "$(basename "$e")" = codegraph ] && continue
+        ln -sf "$e" "$shim/" 2>/dev/null
+      done
+      d="$shim"
+    fi
+    built="${built:+$built:}$d"
+  done
+  unset IFS
+  ARM_PATH="$built"
+
+  if PATH="$ARM_PATH" command -v codegraph >/dev/null 2>&1; then
+    echo "WARNING: 'codegraph' is still on the arm PATH — runs will be contaminated"
+  fi
+  for e in claude node; do
+    PATH="$ARM_PATH" command -v "$e" >/dev/null || { echo "sanitized PATH lost '$e' — refusing to run"; return 1; }
+  done
+
+  command -v jq >/dev/null || { echo "jq is required for the CLI-block hook — install it or the arms will be contaminated"; return 1; }
+  cat > "$out/no-cli-hook.sh" <<HOOK
+#!/usr/bin/env bash
+# Deny Bash invocations of the codegraph CLI so the MCP server stays the A/B's
+# single variable. Looking for it is fine; running it is not.
+set -uo pipefail
+cmd="\$(cat | jq -r '.tool_input.command // empty' 2>/dev/null)"
+if printf '%s' "\$cmd" | grep -Eq '$CG_CMD_RE'; then
+  msg="The codegraph CLI is not available in this session. Answer using the tools you have."
+  jq -n --arg m "\$msg" '{reason:\$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:\$m}}'
+fi
+exit 0
+HOOK
+  chmod +x "$out/no-cli-hook.sh"
+  cat > "$out/hook-settings.json" <<JSON
+{"hooks":{"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"bash $out/no-cli-hook.sh"}]}]}}
+JSON
+  ARM_SETTINGS="$out/hook-settings.json"
+
+  # Prove the hook denies a real invocation and lets a mere mention through.
+  cg_no_cli_probe() { printf '{"tool_input":{"command":%s}}' "$2" | bash "$1/no-cli-hook.sh" | grep -c deny; }
+  [ "$(cg_no_cli_probe "$out" '"/Users/x/.local/bin/codegraph explore \"q\""')" = 1 ] || { echo "hook fails to block an absolute-path invocation"; return 1; }
+  [ "$(cg_no_cli_probe "$out" '"grep -rn codegraph src/"')" = 0 ] || { echo "hook over-blocks a plain mention"; return 1; }
+  return 0
+}

package/scripts/agent-eval/offload-eval-3arm.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# 3-arm offload eval for ONE indexed repo + ONE question, n reps each.
+#   ARM offload : codegraph attached, managed offload ON  (per-run AI usage log)
+#   ARM raw     : codegraph attached, CODEGRAPH_OFFLOAD_DISABLE=1 (raw source)
+#   ARM nocg    : no codegraph (empty MCP config) -> Read/Grep baseline
+# All arms: claude -p sonnet --effort high. One JSON metrics line/run -> $RESULTS.
+#
+# Usage: offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"
+# Env:   MODEL=sonnet EFFORT=high  RESULTS=<file>  AGENT_EVAL_OUT=<scratch dir>
+set -uo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ENGINE="$(cd "$HERE/../.." && pwd)"
+BIN="$ENGINE/dist/bin/codegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
+TARGET="${1:?usage: offload-eval-3arm.sh <indexed-repo> <tier> <reps> \"<question>\"}"
+TIER="${2:?tier}"; REPS="${3:?reps}"; Q="${4:?question}"
+RUNS="$OUT/runs"
+EXTRACT="$HERE/offload-eval-metrics.mjs"
+RESULTS="${RESULTS:-$OUT/results.jsonl}"
+REPO=$(basename "$TARGET")
+mkdir -p "$RUNS"
+command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
+[ -d "$TARGET/.codegraph" ] || { echo "not indexed: $TARGET (run offload-eval-setup.sh first)"; exit 1; }
+# Physical path so pkill matches the daemon's real cmdline (macOS /tmp->/private/tmp symlink
+# otherwise makes the kill miss the daemon, and the next arm connects to the SURVIVING daemon
+# — contaminating the raw arm with offload).
+TARGET=$(cd "$TARGET" && pwd -P)
+
+prewarm() { # path  extra-env (e.g. "FOO=bar")
+  pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
+  env ${2:-} CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
+    && echo "  daemon warm" || echo "  WARN daemon never bound"
+}
+
+run() { # arm rep mcp-config usage-log-or-dash
+  local arm="$1" rep="$2" cfg="$3" usage="$4" tag="$REPO-$1-$2"
+  [ "$usage" != "-" ] && : > "$usage"
+  # DISALLOW (optional): block sub-agent delegation across all arms so the A/B
+  # measures the retrieval mode, not whether Sonnet decides to spawn a codegraph-blind
+  # Explore subagent (which thrashes regardless and adds huge variance).
+  ( cd "$TARGET" && claude -p "$Q" \
+      --output-format stream-json --verbose --permission-mode bypassPermissions \
+      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
+      ${DISALLOW:+--disallowedTools "$DISALLOW"} \
+      --strict-mcp-config --mcp-config "$cfg" \
+      </dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
+  node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$usage" --arm "$arm" --rep "$rep" \
+      --repo "$REPO" --tier "$TIER" --q "$Q" >> "$RESULTS"
+  node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(`  [${o.arm} #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable} tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}call/${o.ai.totalTokens}tok/$${o.ai.costUsd.toFixed(4)} | ok=${o.ok}`)' "$RESULTS"
+}
+
+CFG_OFF="$RUNS/mcp-offload-$REPO.json"; CFG_RAW="$RUNS/mcp-raw-$REPO.json"; CFG_NOCG="$RUNS/mcp-nocg.json"
+USAGE="$RUNS/$REPO-usage.jsonl"
+printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_USAGE_LOG=%s","node","%s","serve","--mcp","--path","%s"]}}}' "$USAGE" "$BIN" "$TARGET" > "$CFG_OFF"
+printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TARGET" > "$CFG_RAW"
+printf '{"mcpServers":{}}' > "$CFG_NOCG"
+
+# REP_START lets a later batch ADD reps without clobbering earlier jsonls
+# (e.g. REP_START=4 REPS=3 -> reps 4,5,6; default starts at 1).
+START="${REP_START:-1}"; END=$((START + REPS - 1))
+echo "###### repo=$REPO tier=$TIER reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high}"
+echo "###### Q=$Q"
+echo "== ARM offload =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
+for r in $(seq "$START" "$END"); do run offload "$r" "$CFG_OFF" "$USAGE"; done
+pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 1
+echo "== ARM raw =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_DISABLE=1"
+for r in $(seq "$START" "$END"); do run raw "$r" "$CFG_RAW" "-"; done
+pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 1
+echo "== ARM nocg =="
+for r in $(seq "$START" "$END"); do run nocg "$r" "$CFG_NOCG" "-"; done
+echo "###### DONE $REPO"