homegraph 1.5.0 → 1.5.1

package/scripts/agent-eval/repro-daemon-clients.mjs

@@ -1,125 +1,125 @@
-#!/usr/bin/env node
-// Reproduction harness B — the FAITHFUL opencode scenario.
-//
-// Spawns N real `codegraph serve --mcp --path <repo>` processes (each becomes a
-// proxy that attaches to ONE shared daemon — exactly what opencode does with N
-// subagents), drives clean MCP JSON-RPC over each child's stdio, then fires ONE
-// concurrent wave of codegraph_explore tools/call across all N and measures
-// end-to-end latency + timeouts. This captures transport-flush starvation: a
-// daemon event-loop blocked in synchronous explore compute can neither read the
-// next request nor flush a finished response.
-//
-// Usage: node repro-daemon-clients.mjs <repo> <N=10> [perCallTimeoutMs=60000] [warm=1]
-import { spawn } from 'node:child_process';
-import { performance } from 'node:perf_hooks';
-import { resolve } from 'node:path';
-
-const [, , repoRaw, nRaw, timeoutRaw, warmRaw] = process.argv;
-const repo = resolve(repoRaw || '.');
-const N = Number(nRaw) || 10;
-const TIMEOUT_MS = Number(timeoutRaw) || 60000;
-const WARM = warmRaw === undefined ? true : warmRaw !== '0';
-const CLI = resolve('dist/bin/codegraph.js');
-
-const QUERIES = [
-  'how does the text model handle edits and undo',
-  'how does the file service watch for changes on disk',
-  'how does the keybinding service resolve a chord to a command',
-  'how does the extension host activate an extension',
-  'how does the editor render decorations in the viewport',
-  'how does the search service stream results to the UI',
-  'how does the terminal process manager spawn a shell',
-  'how does the configuration service merge user and workspace settings',
-  'how does the debug adapter forward breakpoints to the runtime',
-  'how does the quick input widget filter its items',
-  'how does the notification service queue and show toasts',
-  'how does the git extension compute the diff for a file',
-];
-
-function makeClient(id) {
-  const child = spawn('node', [CLI, 'serve', '--mcp', '--path', repo], {
-    env: { ...process.env, CODEGRAPH_TELEMETRY: '0', DO_NOT_TRACK: '1', CODEGRAPH_MCP_LOG_ATTACH: '0' },
-    stdio: ['pipe', 'pipe', 'inherit'],
-  });
-  let buf = '';
-  const waiters = new Map(); // id -> resolve
-  child.stdout.setEncoding('utf8');
-  child.stdout.on('data', (chunk) => {
-    buf += chunk;
-    let idx;
-    while ((idx = buf.indexOf('\n')) !== -1) {
-      const line = buf.slice(0, idx).trim();
-      buf = buf.slice(idx + 1);
-      if (!line) continue;
-      let msg; try { msg = JSON.parse(line); } catch { continue; }
-      if (msg.id !== undefined && waiters.has(msg.id)) {
-        waiters.get(msg.id)(msg);
-        waiters.delete(msg.id);
-      }
-    }
-  });
-  const send = (obj) => child.stdin.write(JSON.stringify(obj) + '\n');
-  const request = (method, params, rpcId, timeoutMs) =>
-    new Promise((res) => {
-      let timer;
-      if (timeoutMs) timer = setTimeout(() => { waiters.delete(rpcId); res({ __timeout: true }); }, timeoutMs);
-      waiters.set(rpcId, (m) => { if (timer) clearTimeout(timer); res(m); });
-      send({ jsonrpc: '2.0', id: rpcId, method, params });
-    });
-  return { id, child, send, request };
-}
-
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-
-const clients = Array.from({ length: N }, (_, i) => makeClient(i));
-
-// Initialize every client (handshake is answered locally by each proxy, instant).
-await Promise.all(clients.map((c) =>
-  c.request('initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'repro', version: '1' } }, `init-${c.id}`, 10000)
-    .then(() => c.send({ jsonrpc: '2.0', method: 'initialized' }))
-));
-
-// Warm the daemon: one explore through client 0 forces daemon spawn + project
-// open + catch-up gate to complete, so the concurrent wave measures the STEADY
-// state (the user's real scenario after the first call), not cold start.
-if (WARM) {
-  process.stderr.write('[repro] warming daemon (first explore triggers spawn+open+catchup)...\n');
-  const t0 = performance.now();
-  const r = await clients[0].request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[0] } }, 'warm-0', 120000);
-  process.stderr.write(`[repro] warm explore took ${Math.round(performance.now() - t0)}ms (timeout=${!!r.__timeout})\n`);
-  await sleep(500);
-}
-
-// THE WAVE: fire one explore on every client as simultaneously as possible.
-process.stderr.write(`[repro] firing ${N} concurrent explores...\n`);
-const waveStart = performance.now();
-const results = await Promise.all(clients.map((c, i) => {
-  const started = performance.now();
-  return c.request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[i % QUERIES.length] } }, `call-${c.id}`, TIMEOUT_MS)
-    .then((m) => ({
-      id: c.id,
-      ms: Math.round(performance.now() - started),
-      timedOut: !!m.__timeout,
-      ok: !!m.result && !m.result.isError,
-      chars: m.result?.content?.[0]?.text?.length ?? 0,
-    }));
-}));
-const waveMs = Math.round(performance.now() - waveStart);
-
-const lat = results.map((r) => r.ms).sort((a, b) => a - b);
-const timeouts = results.filter((r) => r.timedOut).length;
-const p = (q) => lat[Math.min(lat.length - 1, Math.floor(q * lat.length))];
-
-console.log('='.repeat(64));
-console.log(`HARNESS B (real daemon + ${N} proxies)   repo=${repo}`);
-console.log(`warm=${WARM}  perCallTimeout=${TIMEOUT_MS}ms`);
-console.log('-'.repeat(64));
-console.log(`wave wall-clock: ${waveMs}ms`);
-console.log(`per-call latency  min=${lat[0]}  p50=${p(0.5)}  p90=${p(0.9)}  max=${lat[lat.length - 1]}  (ms)`);
-console.log(`TIMEOUTS (>${TIMEOUT_MS}ms): ${timeouts} / ${N}`);
-console.log(`completion order (id:ms): ${results.slice().sort((a,b)=>a.ms-b.ms).map(r=>`${r.id}:${r.ms}`).join('  ')}`);
-console.log('='.repeat(64));
-
-for (const c of clients) { try { c.child.stdin.end(); c.child.kill('SIGTERM'); } catch {} }
-await sleep(300);
-process.exit(0);
+#!/usr/bin/env node
+// Reproduction harness B — the FAITHFUL opencode scenario.
+//
+// Spawns N real `codegraph serve --mcp --path <repo>` processes (each becomes a
+// proxy that attaches to ONE shared daemon — exactly what opencode does with N
+// subagents), drives clean MCP JSON-RPC over each child's stdio, then fires ONE
+// concurrent wave of codegraph_explore tools/call across all N and measures
+// end-to-end latency + timeouts. This captures transport-flush starvation: a
+// daemon event-loop blocked in synchronous explore compute can neither read the
+// next request nor flush a finished response.
+//
+// Usage: node repro-daemon-clients.mjs <repo> <N=10> [perCallTimeoutMs=60000] [warm=1]
+import { spawn } from 'node:child_process';
+import { performance } from 'node:perf_hooks';
+import { resolve } from 'node:path';
+
+const [, , repoRaw, nRaw, timeoutRaw, warmRaw] = process.argv;
+const repo = resolve(repoRaw || '.');
+const N = Number(nRaw) || 10;
+const TIMEOUT_MS = Number(timeoutRaw) || 60000;
+const WARM = warmRaw === undefined ? true : warmRaw !== '0';
+const CLI = resolve('dist/bin/codegraph.js');
+
+const QUERIES = [
+  'how does the text model handle edits and undo',
+  'how does the file service watch for changes on disk',
+  'how does the keybinding service resolve a chord to a command',
+  'how does the extension host activate an extension',
+  'how does the editor render decorations in the viewport',
+  'how does the search service stream results to the UI',
+  'how does the terminal process manager spawn a shell',
+  'how does the configuration service merge user and workspace settings',
+  'how does the debug adapter forward breakpoints to the runtime',
+  'how does the quick input widget filter its items',
+  'how does the notification service queue and show toasts',
+  'how does the git extension compute the diff for a file',
+];
+
+function makeClient(id) {
+  const child = spawn('node', [CLI, 'serve', '--mcp', '--path', repo], {
+    env: { ...process.env, CODEGRAPH_TELEMETRY: '0', DO_NOT_TRACK: '1', CODEGRAPH_MCP_LOG_ATTACH: '0' },
+    stdio: ['pipe', 'pipe', 'inherit'],
+  });
+  let buf = '';
+  const waiters = new Map(); // id -> resolve
+  child.stdout.setEncoding('utf8');
+  child.stdout.on('data', (chunk) => {
+    buf += chunk;
+    let idx;
+    while ((idx = buf.indexOf('\n')) !== -1) {
+      const line = buf.slice(0, idx).trim();
+      buf = buf.slice(idx + 1);
+      if (!line) continue;
+      let msg; try { msg = JSON.parse(line); } catch { continue; }
+      if (msg.id !== undefined && waiters.has(msg.id)) {
+        waiters.get(msg.id)(msg);
+        waiters.delete(msg.id);
+      }
+    }
+  });
+  const send = (obj) => child.stdin.write(JSON.stringify(obj) + '\n');
+  const request = (method, params, rpcId, timeoutMs) =>
+    new Promise((res) => {
+      let timer;
+      if (timeoutMs) timer = setTimeout(() => { waiters.delete(rpcId); res({ __timeout: true }); }, timeoutMs);
+      waiters.set(rpcId, (m) => { if (timer) clearTimeout(timer); res(m); });
+      send({ jsonrpc: '2.0', id: rpcId, method, params });
+    });
+  return { id, child, send, request };
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+const clients = Array.from({ length: N }, (_, i) => makeClient(i));
+
+// Initialize every client (handshake is answered locally by each proxy, instant).
+await Promise.all(clients.map((c) =>
+  c.request('initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'repro', version: '1' } }, `init-${c.id}`, 10000)
+    .then(() => c.send({ jsonrpc: '2.0', method: 'initialized' }))
+));
+
+// Warm the daemon: one explore through client 0 forces daemon spawn + project
+// open + catch-up gate to complete, so the concurrent wave measures the STEADY
+// state (the user's real scenario after the first call), not cold start.
+if (WARM) {
+  process.stderr.write('[repro] warming daemon (first explore triggers spawn+open+catchup)...\n');
+  const t0 = performance.now();
+  const r = await clients[0].request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[0] } }, 'warm-0', 120000);
+  process.stderr.write(`[repro] warm explore took ${Math.round(performance.now() - t0)}ms (timeout=${!!r.__timeout})\n`);
+  await sleep(500);
+}
+
+// THE WAVE: fire one explore on every client as simultaneously as possible.
+process.stderr.write(`[repro] firing ${N} concurrent explores...\n`);
+const waveStart = performance.now();
+const results = await Promise.all(clients.map((c, i) => {
+  const started = performance.now();
+  return c.request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[i % QUERIES.length] } }, `call-${c.id}`, TIMEOUT_MS)
+    .then((m) => ({
+      id: c.id,
+      ms: Math.round(performance.now() - started),
+      timedOut: !!m.__timeout,
+      ok: !!m.result && !m.result.isError,
+      chars: m.result?.content?.[0]?.text?.length ?? 0,
+    }));
+}));
+const waveMs = Math.round(performance.now() - waveStart);
+
+const lat = results.map((r) => r.ms).sort((a, b) => a - b);
+const timeouts = results.filter((r) => r.timedOut).length;
+const p = (q) => lat[Math.min(lat.length - 1, Math.floor(q * lat.length))];
+
+console.log('='.repeat(64));
+console.log(`HARNESS B (real daemon + ${N} proxies)   repo=${repo}`);
+console.log(`warm=${WARM}  perCallTimeout=${TIMEOUT_MS}ms`);
+console.log('-'.repeat(64));
+console.log(`wave wall-clock: ${waveMs}ms`);
+console.log(`per-call latency  min=${lat[0]}  p50=${p(0.5)}  p90=${p(0.9)}  max=${lat[lat.length - 1]}  (ms)`);
+console.log(`TIMEOUTS (>${TIMEOUT_MS}ms): ${timeouts} / ${N}`);
+console.log(`completion order (id:ms): ${results.slice().sort((a,b)=>a.ms-b.ms).map(r=>`${r.id}:${r.ms}`).join('  ')}`);
+console.log('='.repeat(64));
+
+for (const c of clients) { try { c.child.stdin.end(); c.child.kill('SIGTERM'); } catch {} }
+await sleep(300);
+process.exit(0);

package/scripts/agent-eval/run-agent.sh

@@ -1,34 +1,34 @@
-#!/usr/bin/env bash
-# Headless Claude Code run against a repo with homegraph MCP, capturing the
-# full stream-json so we can see tool calls + token usage. Complements the
-# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
-# tokens/cost, but defaults to the general-purpose subagent (not Explore).
-# To force the Explore path, ask for it in the prompt.
-#
-# Usage: run-agent.sh <repo-path> <label> "<prompt>"
-# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (homegraph dist binary)
-set -uo pipefail
-
-REPO="$1"; LABEL="$2"; PROMPT="$3"
-CG_BIN="${CG_BIN:-$(command -v homegraph || echo /usr/local/bin/homegraph)}"
-OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
-OUT="$OUT_DIR/run-${LABEL}.jsonl"
-
-MCP_CONFIG=$(cat <<JSON
-{"mcpServers":{"homegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
-JSON
-)
-
-echo "→ running [$LABEL] in $REPO"
-cd "$REPO" || exit 1
-
-claude -p "$PROMPT" \
-  --output-format stream-json --verbose \
-  --permission-mode bypassPermissions \
-  --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
-  --max-budget-usd 2 \
-  --strict-mcp-config --mcp-config "$MCP_CONFIG" \
-  > "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
-
-echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
-node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true
+#!/usr/bin/env bash
+# Headless Claude Code run against a repo with homegraph MCP, capturing the
+# full stream-json so we can see tool calls + token usage. Complements the
+# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
+# tokens/cost, but defaults to the general-purpose subagent (not Explore).
+# To force the Explore path, ask for it in the prompt.
+#
+# Usage: run-agent.sh <repo-path> <label> "<prompt>"
+# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (homegraph dist binary)
+set -uo pipefail
+
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+CG_BIN="${CG_BIN:-$(command -v homegraph || echo /usr/local/bin/homegraph)}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/run-${LABEL}.jsonl"
+
+MCP_CONFIG=$(cat <<JSON
+{"mcpServers":{"homegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
+JSON
+)
+
+echo "→ running [$LABEL] in $REPO"
+cd "$REPO" || exit 1
+
+claude -p "$PROMPT" \
+  --output-format stream-json --verbose \
+  --permission-mode bypassPermissions \
+  --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
+  --max-budget-usd 2 \
+  --strict-mcp-config --mcp-config "$MCP_CONFIG" \
+  > "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
+
+echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
+node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true

package/scripts/agent-eval/run-all.sh

@@ -1,75 +1,75 @@
-#!/usr/bin/env bash
-# With/without A/B (and optional interactive) eval for a homegraph version on a
-# repo. Codegraph is the ONLY variable: both arms launch claude with
-# --strict-mcp-config — with = homegraph-only MCP (pointed at $CG_BIN),
-# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
-#
-# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
-# Env:   CG_BIN          homegraph binary (default: command -v homegraph)
-#        AGENT_EVAL_OUT  output dir (default: /tmp/agent-eval)
-#        MODEL / EFFORT  claude model/effort (default: sonnet / high — the
-#                        standing A/B policy; see CLAUDE.md, don't raise)
-set -uo pipefail
-
-REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
-Q="${2:?question required}"
-MODE="${3:-headless}"
-CG_BIN="${CG_BIN:-$(command -v homegraph)}"
-OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
-HARNESS="$(cd "$(dirname "$0")" && pwd)"
-mkdir -p "$OUT"
-
-# Neutralize any ambient HomeGraph prompt-hook (~/.claude) in BOTH arms:
-# the hook injects homegraph context into every prompt, which contaminates
-# the without-arm (free structural context) and double-counts the with-arm.
-# The A/B's only variable must be the MCP server wired below.
-export HOMEGRAPH_NO_PROMPT_HOOK=1
-
-[ -n "$CG_BIN" ] || { echo "no homegraph binary on PATH (set CG_BIN)"; exit 1; }
-[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO — index it first"; exit 1; }
-case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
-
-# MCP config files (path form avoids inline-JSON quoting through tmux).
-cat > "$OUT/mcp-homegraph.json" <<JSON
-{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
-JSON
-echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
-
-echo "###### homegraph: $CG_BIN"
-echo "###### repo:      $REPO"
-echo "###### question:  $Q"
-echo
-
-# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
-headless() {
-  local label="$1" cfg="$2"
-  echo "############################## HEADLESS [$label] ##############################"
-  ( cd "$REPO" && claude -p "$Q" \
-      --output-format stream-json --verbose \
-      --permission-mode bypassPermissions \
-      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
-      --max-budget-usd 4 \
-      --strict-mcp-config --mcp-config "$cfg" \
-      > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
-  echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
-  tail -2 "$OUT/run-$label.err" 2>/dev/null
-  node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
-  echo
-}
-
-if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
-  headless "headless-with"    "$OUT/mcp-homegraph.json"
-  headless "headless-without" "$OUT/mcp-empty.json"
-fi
-
-if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
-  echo "############################## INTERACTIVE [with] ##############################"
-  CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-homegraph.json" \
-    bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
-  echo
-  echo "############################## INTERACTIVE [without] ##############################"
-  CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
-    bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
-  echo
-fi
-echo "############################## RUN-ALL COMPLETE ##############################"
+#!/usr/bin/env bash
+# With/without A/B (and optional interactive) eval for a homegraph version on a
+# repo. Codegraph is the ONLY variable: both arms launch claude with
+# --strict-mcp-config — with = homegraph-only MCP (pointed at $CG_BIN),
+# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
+#
+# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
+# Env:   CG_BIN          homegraph binary (default: command -v homegraph)
+#        AGENT_EVAL_OUT  output dir (default: /tmp/agent-eval)
+#        MODEL / EFFORT  claude model/effort (default: sonnet / high — the
+#                        standing A/B policy; see CLAUDE.md, don't raise)
+set -uo pipefail
+
+REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
+Q="${2:?question required}"
+MODE="${3:-headless}"
+CG_BIN="${CG_BIN:-$(command -v homegraph)}"
+OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+mkdir -p "$OUT"
+
+# Neutralize any ambient HomeGraph prompt-hook (~/.claude) in BOTH arms:
+# the hook injects homegraph context into every prompt, which contaminates
+# the without-arm (free structural context) and double-counts the with-arm.
+# The A/B's only variable must be the MCP server wired below.
+export HOMEGRAPH_NO_PROMPT_HOOK=1
+
+[ -n "$CG_BIN" ] || { echo "no homegraph binary on PATH (set CG_BIN)"; exit 1; }
+[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO — index it first"; exit 1; }
+case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
+
+# MCP config files (path form avoids inline-JSON quoting through tmux).
+cat > "$OUT/mcp-homegraph.json" <<JSON
+{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
+JSON
+echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
+
+echo "###### homegraph: $CG_BIN"
+echo "###### repo:      $REPO"
+echo "###### question:  $Q"
+echo
+
+# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
+headless() {
+  local label="$1" cfg="$2"
+  echo "############################## HEADLESS [$label] ##############################"
+  ( cd "$REPO" && claude -p "$Q" \
+      --output-format stream-json --verbose \
+      --permission-mode bypassPermissions \
+      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
+      --max-budget-usd 4 \
+      --strict-mcp-config --mcp-config "$cfg" \
+      > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
+  echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
+  tail -2 "$OUT/run-$label.err" 2>/dev/null
+  node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
+  echo
+}
+
+if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
+  headless "headless-with"    "$OUT/mcp-homegraph.json"
+  headless "headless-without" "$OUT/mcp-empty.json"
+fi
+
+if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
+  echo "############################## INTERACTIVE [with] ##############################"
+  CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-homegraph.json" \
+    bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
+  echo
+  echo "############################## INTERACTIVE [without] ##############################"
+  CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
+    bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
+  echo
+fi
+echo "############################## RUN-ALL COMPLETE ##############################"

package/scripts/agent-eval/run-arms.sh

@@ -1,56 +1,56 @@
-#!/usr/bin/env bash
-# Tool-surface ablation — run ONE repo+question under ONE arm.
-#
-# Arms vary (exposed homegraph tools, trace-first steering). Tools are trimmed
-# SERVER-SIDE via HOMEGRAPH_MCP_TOOLS in the MCP config's `env` block, so an
-# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
-# denied-call confound (which --disallowedTools would introduce). Steering is
-# injected with --append-system-prompt, so no rebuild of the shipped
-# server-instructions is needed to A/B it.
-#
-#   A control       all tools            no steering
-#   B steer         all tools            trace-first
-#   C no-explore    hide explore         trace-first
-#   D trace-centric hide explore+context trace-first
-#   E control-probe hide explore+context trace-first  (caller passes a NON-flow Q)
-#
-# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
-set -uo pipefail
-REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
-CG_BIN="${CG_BIN:-$(command -v homegraph)}"
-OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
-mkdir -p "$OUT"
-[ -n "$CG_BIN" ] || { echo "no homegraph binary (set CG_BIN)"; exit 1; }
-[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO"; exit 1; }
-
-STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call homegraph_trace(from,to) FIRST — one call returns the whole path. Use homegraph_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
-KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
-KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
-
-case "$ARM" in
-  A|G|H|I) TOOLS="";            STEERING="" ;;  # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
-  B|F) TOOLS="";                STEERING="$STEER" ;;  # F = B's surface, run on the body-inlining trace build
-  C) TOOLS="$KEEP_NO_EXPLORE";  STEERING="$STEER" ;;
-  D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
-  *) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
-esac
-
-CFG="$OUT/mcp-$ARM.json"
-if [ -n "$TOOLS" ]; then
-  cat > "$CFG" <<JSON
-{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"HOMEGRAPH_MCP_TOOLS":"$TOOLS"}}}}
-JSON
-else
-  cat > "$CFG" <<JSON
-{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
-JSON
-fi
-
-LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
-ARGS=( -p "$Q" --output-format stream-json --verbose
-       --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
-       --strict-mcp-config --mcp-config "$CFG" )
-[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
-
-( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
-echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"
+#!/usr/bin/env bash
+# Tool-surface ablation — run ONE repo+question under ONE arm.
+#
+# Arms vary (exposed homegraph tools, trace-first steering). Tools are trimmed
+# SERVER-SIDE via HOMEGRAPH_MCP_TOOLS in the MCP config's `env` block, so an
+# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
+# denied-call confound (which --disallowedTools would introduce). Steering is
+# injected with --append-system-prompt, so no rebuild of the shipped
+# server-instructions is needed to A/B it.
+#
+#   A control       all tools            no steering
+#   B steer         all tools            trace-first
+#   C no-explore    hide explore         trace-first
+#   D trace-centric hide explore+context trace-first
+#   E control-probe hide explore+context trace-first  (caller passes a NON-flow Q)
+#
+# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
+set -uo pipefail
+REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
+CG_BIN="${CG_BIN:-$(command -v homegraph)}"
+OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
+mkdir -p "$OUT"
+[ -n "$CG_BIN" ] || { echo "no homegraph binary (set CG_BIN)"; exit 1; }
+[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO"; exit 1; }
+
+STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call homegraph_trace(from,to) FIRST — one call returns the whole path. Use homegraph_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
+KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
+KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
+
+case "$ARM" in
+  A|G|H|I) TOOLS="";            STEERING="" ;;  # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
+  B|F) TOOLS="";                STEERING="$STEER" ;;  # F = B's surface, run on the body-inlining trace build
+  C) TOOLS="$KEEP_NO_EXPLORE";  STEERING="$STEER" ;;
+  D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
+  *) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
+esac
+
+CFG="$OUT/mcp-$ARM.json"
+if [ -n "$TOOLS" ]; then
+  cat > "$CFG" <<JSON
+{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"HOMEGRAPH_MCP_TOOLS":"$TOOLS"}}}}
+JSON
+else
+  cat > "$CFG" <<JSON
+{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
+JSON
+fi
+
+LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
+ARGS=( -p "$Q" --output-format stream-json --verbose
+       --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
+       --strict-mcp-config --mcp-config "$CFG" )
+[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
+
+( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
+echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"