@specship/specship 0.11.0

package/scripts/agent-eval/probe-sweep.mjs

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+// probe-sweep — direct MCP test across N repos × N tools, no claude needed.
+//
+// Measures response characteristics (size, sections present, signals fired)
+// for each (repo, query) pair against the built dist/. Sub-second per probe;
+// the full sweep below runs in ~10-30s vs hours for a real claude audit.
+//
+// Use this to iterate on backend changes rapidly: change tools.ts /
+// context-builder, npm run build, re-run probe-sweep, compare. Once a
+// change looks good on probe metrics, run a focused claude audit for the
+// few repos that matter to confirm end-to-end cost behavior.
+//
+// Usage: node scripts/agent-eval/probe-sweep.mjs [--tool=context|explore|trace] [--repos=a,b,c]
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const args = Object.fromEntries(
+  process.argv.slice(2).map(a => a.startsWith('--') ? a.slice(2).split('=') : [a, true])
+);
+const TOOL = args.tool ?? 'context';
+
+const load = (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+const SpecShip = idx.default?.default ?? idx.default ?? idx.SpecShip;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+// Each entry: repo, query, optional 2nd arg for trace (from, to).
+// The query is the same prompt used in the real claude audits, so probe
+// output is directly comparable to the agent's would-be input.
+const SWEEP = [
+  // Small realworld template repos (the loss cases from the cross-language sweep)
+  { id: 'gin-rw',        repo: '/tmp/specship-corpus/gin-realworld',         q: 'How does this Gin app route a request through its middleware chain to a handler?' },
+  { id: 'go-mux',        repo: '/tmp/specship-corpus/go-mux',                q: 'How does this gorilla/mux app route a request to its handler?' },
+  { id: 'fastapi-rw',    repo: '/tmp/specship-corpus/fastapi-realworld',     q: 'How does FastAPI route a request through its dependencies to a handler?' },
+  { id: 'spring-pc',     repo: '/tmp/specship-corpus/spring-petclinic',      q: 'How does Spring route an HTTP request to a controller method?' },
+  { id: 'axum-rw',       repo: '/tmp/specship-corpus/rust-axum-realworld',   q: 'How does Axum route a request to its handler in this app?' },
+  { id: 'express-rw',    repo: '/tmp/specship-corpus/express-realworld',     q: 'How does this Express app route a request through middleware to a handler?' },
+  { id: 'kotlin-pc',     repo: '/tmp/specship-corpus/kotlin-petclinic',      q: 'How does the Kotlin Spring app route an HTTP request to its handler?' },
+  { id: 'flask-mb',      repo: '/tmp/specship-corpus/flask-microblog',       q: 'How does this Flask app route a request to a view function?' },
+  { id: 'vapor-tpl',     repo: '/tmp/specship-corpus/vapor-template',        q: 'How does Vapor route an HTTP request to its handler?' },
+  { id: 'cpp-leveldb',   repo: '/tmp/specship-corpus/cpp-leveldb',           q: 'How does LevelDB handle a Put operation through to disk?' },
+  { id: 'lualine',       repo: '/tmp/specship-corpus/lualine.nvim',          q: 'How does lualine assemble and render the statusline?' },
+  { id: 'drupal-admin',  repo: '/tmp/specship-corpus/drupal-admintoolbar',   q: 'How does the Drupal admin toolbar module render its toolbar?' },
+  { id: 'svelte-rw',     repo: '/tmp/specship-corpus/svelte-realworld',      q: 'How does this SvelteKit app route a request to a handler?' },
+  { id: 'react-rw',      repo: '/tmp/specship-corpus/react-realworld',       q: 'How does this React app fetch and display articles?' },
+  { id: 'rails-rw',      repo: '/tmp/specship-corpus/rails-realworld',       q: 'How does Rails route a request to a controller action?' },
+  { id: 'flask-rest',    repo: '/tmp/specship-corpus/flask-restful-realworld', q: 'How does Flask-RESTful route a request to a resource method?' },
+  { id: 'laravel-rw',    repo: '/tmp/specship-corpus/laravel-realworld',     q: 'How does Laravel route a request to the controller method?' },
+  { id: 'aspnet-rw',     repo: '/tmp/specship-corpus/aspnet-realworld',      q: 'How does ASP.NET route a request to the controller action?' },
+  // The iter7 wins/ties (to make sure we don't regress)
+  { id: 'cobra',         repo: '/tmp/specship-corpus/cobra',                 q: 'How does cobra parse commands and flags?' },
+  { id: 'sinatra',       repo: '/tmp/specship-corpus/sinatra',               q: 'How does sinatra route a request to its handler?' },
+  { id: 'slim',          repo: '/tmp/specship-corpus/slim',                  q: 'How does slim route a request and apply middleware?' },
+];
+
+// Detect signals in response text — these are the levers we've added that
+// otherwise only show up via "agent ran X more tool calls" downstream.
+const detect = (text) => ({
+  hasEntryPoints: /^### Entry Points/m.test(text),
+  hasRelatedSymbols: /^### Related Symbols/m.test(text),
+  hasFlowTrace: /^## Inline flow trace/m.test(text),
+  hasRouteManifest: /^## Routing manifest/m.test(text),
+  hasTopHandler: /^### Top handler file/m.test(text),
+  hasSmallRepoTail: /This project is small/.test(text),
+});
+
+const filterRepos = args.repos ? new Set(String(args.repos).split(',')) : null;
+const subjects = SWEEP.filter(s => !filterRepos || filterRepos.has(s.id));
+
+const t0 = Date.now();
+const rows = [];
+for (const s of subjects) {
+  try {
+    const cg = SpecShip.openSync(s.repo);
+    const handler = new ToolHandler(cg);
+    const t1 = Date.now();
+    const res = await handler.execute('specship_' + TOOL,
+      TOOL === 'context' ? { task: s.q } :
+      TOOL === 'explore' ? { query: s.q } : { from: 'main', to: 'main' });
+    const text = res.content?.[0]?.text ?? '';
+    const signals = detect(text);
+    rows.push({
+      id: s.id,
+      ms: Date.now() - t1,
+      chars: text.length,
+      lines: text.split('\n').length,
+      ...signals,
+    });
+    try { cg.close?.(); } catch {}
+  } catch (e) {
+    rows.push({ id: s.id, error: String(e).slice(0, 80) });
+  }
+}
+
+// Pretty-print as a compact table.
+const fmt = (r) =>
+  r.error
+    ? `  ${r.id.padEnd(13)} ERROR: ${r.error}`
+    : `  ${r.id.padEnd(13)} ${String(r.chars).padStart(6)}c ${String(r.lines).padStart(4)}L ${String(r.ms).padStart(4)}ms` +
+      ` ${r.hasEntryPoints ? 'EP ' : '   '}` +
+      `${r.hasFlowTrace ? 'TRC ' : '    '}` +
+      `${r.hasRouteManifest ? 'MAN ' : '    '}` +
+      `${r.hasTopHandler ? 'HND ' : '    '}` +
+      `${r.hasSmallRepoTail ? 'TAIL' : '    '}`;
+console.log(`=== probe-sweep tool=${TOOL} n=${subjects.length} (${Date.now() - t0}ms total) ===`);
+console.log('  id            chars  lines    ms signals');
+console.log('  ' + '-'.repeat(56));
+for (const r of rows) console.log(fmt(r));
+
+// Sum + medians for the size pillar
+const sizes = rows.filter(r => !r.error).map(r => r.chars);
+sizes.sort((a, b) => a - b);
+const median = sizes[Math.floor(sizes.length / 2)];
+const sum = sizes.reduce((a, b) => a + b, 0);
+console.log(`  ${'-'.repeat(64)}`);
+console.log(`  median=${median}c  total=${sum}c  ` +
+  `manifest=${rows.filter(r => r.hasRouteManifest).length}/${rows.filter(r => !r.error).length}  ` +
+  `top-handler=${rows.filter(r => r.hasTopHandler).length}/${rows.filter(r => !r.error).length}`);

package/scripts/agent-eval/probe-trace.mjs

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+// Probe specship_trace against an index using the built dist.
+// Usage: node probe-trace.mjs <repo-with-.specship> <from> <to>
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+
+const [, , repo, from, to] = process.argv;
+if (!repo || !from || !to) { console.error('usage: probe-trace.mjs <repo> <from> <to>'); process.exit(1); }
+
+const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
+const idx = await load('dist/index.js');
+const tools = await load('dist/mcp/tools.js');
+const SpecShip = idx.default?.default ?? idx.default ?? idx.SpecShip;
+const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
+
+const cg = SpecShip.openSync(repo);
+const h = new ToolHandler(cg);
+const res = await h.execute('specship_trace', { from, to });
+console.log(res.content?.[0]?.text ?? '(no text)');
+try { cg.close?.(); } catch {}

package/scripts/agent-eval/run-agent.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Headless Claude Code run against a repo with specship MCP, capturing the
+# full stream-json so we can see tool calls + token usage. Complements the
+# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
+# tokens/cost, but defaults to the general-purpose subagent (not Explore).
+# To force the Explore path, ask for it in the prompt.
+#
+# Usage: run-agent.sh <repo-path> <label> "<prompt>"
+# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (specship dist binary)
+set -uo pipefail
+
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+CG_BIN="${CG_BIN:-$(command -v specship || echo /usr/local/bin/specship)}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/run-${LABEL}.jsonl"
+
+MCP_CONFIG=$(cat <<JSON
+{"mcpServers":{"specship":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
+JSON
+)
+
+echo "→ running [$LABEL] in $REPO"
+cd "$REPO" || exit 1
+
+claude -p "$PROMPT" \
+  --output-format stream-json --verbose \
+  --permission-mode bypassPermissions \
+  --model opus \
+  --max-budget-usd 2 \
+  --strict-mcp-config --mcp-config "$MCP_CONFIG" \
+  > "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
+
+echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
+node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true

package/scripts/agent-eval/run-all.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# With/without A/B (and optional interactive) eval for a specship version on a
+# repo. Specship is the ONLY variable: both arms launch claude with
+# --strict-mcp-config — with = specship-only MCP (pointed at $CG_BIN),
+# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
+#
+# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
+# Env:   CG_BIN          specship binary (default: command -v specship)
+#        AGENT_EVAL_OUT  output dir (default: /tmp/agent-eval)
+set -uo pipefail
+
+REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
+Q="${2:?question required}"
+MODE="${3:-headless}"
+CG_BIN="${CG_BIN:-$(command -v specship)}"
+OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+mkdir -p "$OUT"
+
+[ -n "$CG_BIN" ] || { echo "no specship binary on PATH (set CG_BIN)"; exit 1; }
+[ -d "$REPO/.specship" ] || { echo "no .specship index at $REPO — index it first"; exit 1; }
+case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
+
+# MCP config files (path form avoids inline-JSON quoting through tmux).
+cat > "$OUT/mcp-specship.json" <<JSON
+{"mcpServers":{"specship":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
+JSON
+echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
+
+echo "###### specship: $CG_BIN"
+echo "###### repo:      $REPO"
+echo "###### question:  $Q"
+echo
+
+# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
+headless() {
+  local label="$1" cfg="$2"
+  echo "############################## HEADLESS [$label] ##############################"
+  ( cd "$REPO" && claude -p "$Q" \
+      --output-format stream-json --verbose \
+      --permission-mode bypassPermissions \
+      --model opus \
+      --max-budget-usd 4 \
+      --strict-mcp-config --mcp-config "$cfg" \
+      > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
+  echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
+  tail -2 "$OUT/run-$label.err" 2>/dev/null
+  node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
+  echo
+}
+
+if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
+  headless "headless-with"    "$OUT/mcp-specship.json"
+  headless "headless-without" "$OUT/mcp-empty.json"
+fi
+
+if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
+  echo "############################## INTERACTIVE [with] ##############################"
+  CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-specship.json" \
+    bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
+  echo
+  echo "############################## INTERACTIVE [without] ##############################"
+  CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
+    bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
+  echo
+fi
+echo "############################## RUN-ALL COMPLETE ##############################"

package/scripts/agent-eval/run-arms.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Tool-surface ablation — run ONE repo+question under ONE arm.
+#
+# Arms vary (exposed specship tools, trace-first steering). Tools are trimmed
+# SERVER-SIDE via SPECSHIP_MCP_TOOLS in the MCP config's `env` block, so an
+# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
+# denied-call confound (which --disallowedTools would introduce). Steering is
+# injected with --append-system-prompt, so no rebuild of the shipped
+# server-instructions is needed to A/B it.
+#
+#   A control       all tools            no steering
+#   B steer         all tools            trace-first
+#   C no-explore    hide explore         trace-first
+#   D trace-centric hide explore+context trace-first
+#   E control-probe hide explore+context trace-first  (caller passes a NON-flow Q)
+#
+# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
+set -uo pipefail
+REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
+CG_BIN="${CG_BIN:-$(command -v specship)}"
+OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
+mkdir -p "$OUT"
+[ -n "$CG_BIN" ] || { echo "no specship binary (set CG_BIN)"; exit 1; }
+[ -d "$REPO/.specship" ] || { echo "no .specship index at $REPO"; exit 1; }
+
+STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call specship_trace(from,to) FIRST — one call returns the whole path. Use specship_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
+KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
+KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
+
+case "$ARM" in
+  A|G|H|I) TOOLS="";            STEERING="" ;;  # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
+  B|F) TOOLS="";                STEERING="$STEER" ;;  # F = B's surface, run on the body-inlining trace build
+  C) TOOLS="$KEEP_NO_EXPLORE";  STEERING="$STEER" ;;
+  D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
+  *) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
+esac
+
+CFG="$OUT/mcp-$ARM.json"
+if [ -n "$TOOLS" ]; then
+  cat > "$CFG" <<JSON
+{"mcpServers":{"specship":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"SPECSHIP_MCP_TOOLS":"$TOOLS"}}}}
+JSON
+else
+  cat > "$CFG" <<JSON
+{"mcpServers":{"specship":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
+JSON
+fi
+
+LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
+ARGS=( -p "$Q" --output-format stream-json --verbose
+       --permission-mode bypassPermissions --model opus --max-budget-usd 4
+       --strict-mcp-config --mcp-config "$CFG" )
+[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
+
+( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
+echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"

package/scripts/agent-eval/seq-matrix.mjs

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+// Mine the surviving A/B stream-json logs (/tmp/ab-matrix/<Cell>/run-headless-*.jsonl)
+// for what the aggregate matrix can't see: the call SEQUENCE and per-call output SIZE.
+//
+// Answers three questions:
+//   1. Trace adoption — on a flow question, does the with-arm actually call specship_trace?
+//   2. Payload size vs repo size — is trace path-scoped (tiny, size-independent) while
+//      explore is breadth-scoped (grows with the repo / over-returns on small repos)?
+//   3. Round-trips — num_turns with vs without (the real wall-clock driver).
+//
+// Usage: node scripts/agent-eval/seq-matrix.mjs [/tmp/ab-matrix]
+import { readFileSync, readdirSync, existsSync } from 'fs';
+import { join } from 'path';
+
+const AB = process.argv[2] || '/tmp/ab-matrix';
+const MD = new URL('../../docs/benchmarks/specship-ab-matrix.md', import.meta.url).pathname;
+
+// repo -> {lang,size,files} from the published matrix table
+const repoMeta = {};
+if (existsSync(MD)) for (const line of readFileSync(MD, 'utf8').split('\n')) {
+  const m = line.match(/^\|\s*([^|]+?)\s*\|\s*(S|M|L)\s*\|\s*`([^`]+)`\s*\|\s*(\d+)\s*\|/);
+  if (m) repoMeta[m[3]] = { lang: m[1].trim(), size: m[2], files: +m[4] };
+}
+
+const cgShort = (n) => n.replace('mcp__specship__specship_', '').replace('mcp__specship__', '');
+const tag = (n) => n === 'Read' ? 'R' : n === 'Grep' ? 'G' : n === 'Glob' ? 'Gl'
+  : n === 'Bash' ? 'B' : n === 'Task' ? 'Ag' : n === 'ToolSearch' ? 'TS'
+  : n.includes('specship') ? cgShort(n) : n;
+
+function parse(file) {
+  if (!existsSync(file)) return null;
+  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+  const calls = []; let result = null, initCg = 0;
+  for (const l of lines) {
+    let ev; try { ev = JSON.parse(l); } catch { continue; }
+    if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /specship/.test(t)).length;
+    if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use') {
+      const i = b.input || {};
+      const q = i.query ?? i.symbol ?? i.task ?? (i.from && i.to ? `${i.from}->${i.to}` : (i.file_path || i.command || ''));
+      calls.push({ id: b.id, name: b.name, q: String(q ?? '').slice(0, 38), out: 0 });
+    }
+    if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
+      const c = b.content;
+      const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
+      const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
+    }
+    if (ev.type === 'result') result = ev;
+  }
+  const cg = calls.filter(c => c.name.includes('specship'));
+  const perTool = {};
+  for (const c of cg) { const k = cgShort(c.name); (perTool[k] ??= { n: 0, out: 0 }); perTool[k].n++; perTool[k].out += c.out; }
+  const traceIdx = cg.findIndex(c => c.name.includes('trace'));
+  const u = result?.usage || {};
+  return {
+    initCg, cg, perTool,
+    cgSeq: cg.map(c => cgShort(c.name)),
+    seq: calls.map(c => tag(c.name)),
+    reads: calls.filter(c => c.name === 'Read').length,
+    greps: calls.filter(c => c.name === 'Grep').length,
+    cgOut: cg.reduce((s, c) => s + c.out, 0),
+    traceUsed: traceIdx >= 0,
+    afterTrace: traceIdx >= 0 ? cg.slice(traceIdx + 1).map(c => cgShort(c.name)) : null,
+    turns: result?.num_turns ?? null,
+    dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
+    cost: result?.total_cost_usd || 0,
+  };
+}
+
+const cells = [];
+for (const d of readdirSync(AB)) {
+  const dir = join(AB, d);
+  if (!existsSync(join(dir, 'run-headless-with.jsonl'))) continue;
+  const log = existsSync(join(AB, d + '.log')) ? readFileSync(join(AB, d + '.log'), 'utf8') : '';
+  const repo = (log.match(/repo:\s*\S*\/([^\s/]+)/) || [])[1] || d;
+  const question = (log.match(/question:\s*(.+)/) || [])[1] || '';
+  cells.push({ cell: d, repo, question, ...(repoMeta[repo] || {}),
+    with: parse(join(dir, 'run-headless-with.jsonl')),
+    without: parse(join(dir, 'run-headless-without.jsonl')) });
+}
+cells.sort((a, b) => (a.files || 0) - (b.files || 0));
+
+const k = (n) => (n / 1000).toFixed(1);
+const pad = (s, n) => String(s).padEnd(n);
+
+// ---- per-cell sequence table ----
+console.log('\n=== PER-CELL: with-arm specship sequence + payload (sorted by repo size) ===');
+console.log(pad('repo', 22), pad('files', 6), 'trace', pad('cg-call sequence', 40), pad('cgOutK', 7), 'turns(w/wo)');
+for (const c of cells) {
+  const w = c.with;
+  console.log(
+    pad(c.repo, 22), pad(c.files ?? '?', 6),
+    pad(w.traceUsed ? 'YES' : 'no', 5),
+    pad(w.cgSeq.join(',') || '(none)', 40),
+    pad(k(w.cgOut), 7),
+    `${w.turns}/${c.without?.turns}`,
+  );
+}
+
+// ---- trace adoption ----
+const flow = cells; // every matrix question is a canonical flow question by design
+const used = flow.filter(c => c.with.traceUsed);
+console.log(`\n=== TRACE ADOPTION (all ${flow.length} cells are flow questions) ===`);
+console.log(`trace called in ${used.length}/${flow.length} cells`);
+console.log('used trace:', used.map(c => c.repo).join(', ') || '(none)');
+if (used.length) console.log('after-trace follow-ups:', used.map(c => `${c.repo}[${c.with.afterTrace.join(',') || 'none'}]`).join('  '));
+
+// ---- payload size by repo-size tier ----
+const tier = (f) => f < 200 ? 'S(<200)' : f < 2000 ? 'M(<2000)' : 'L(>=2000)';
+const byTier = {};
+for (const c of cells) { (byTier[tier(c.files || 0)] ??= []).push(c.with.cgOut); }
+console.log('\n=== with-arm TOTAL specship payload by repo-size tier ===');
+for (const t of ['S(<200)', 'M(<2000)', 'L(>=2000)']) {
+  const a = byTier[t] || []; if (!a.length) continue;
+  const avg = a.reduce((s, x) => s + x, 0) / a.length;
+  console.log(`  ${pad(t, 10)} n=${a.length}  avg cgOut=${k(avg)}K  range ${k(Math.min(...a))}-${k(Math.max(...a))}K`);
+}
+
+// ---- per-tool usage + avg payload (breadth vs path evidence) ----
+const tot = {};
+for (const c of cells) for (const [name, v] of Object.entries(c.with.perTool)) {
+  (tot[name] ??= { n: 0, out: 0 }); tot[name].n += v.n; tot[name].out += v.out;
+}
+console.log('\n=== specship tool usage across all cells (n calls, avg payload/call) ===');
+for (const [name, v] of Object.entries(tot).sort((a, b) => b[1].n - a[1].n)) {
+  console.log(`  ${pad(name, 10)} calls=${pad(v.n, 4)} avg=${k(v.out / v.n)}K/call  total=${k(v.out)}K`);
+}
+
+// ---- round-trips ----
+const sum = (arr, f) => arr.reduce((s, x) => s + (f(x) || 0), 0);
+const wTurns = sum(cells, c => c.with.turns), woTurns = sum(cells, c => c.without?.turns);
+const wCalls = sum(cells, c => c.with.cg.length);
+const tsAll = cells.every(c => c.with.seq[0] === 'TS');
+console.log('\n=== ROUND-TRIPS ===');
+console.log(`turns: with=${wTurns}  without=${woTurns}  (${((1 - wTurns / woTurns) * 100).toFixed(0)}% fewer with)`);
+console.log(`avg turns/cell: with=${(wTurns / cells.length).toFixed(1)}  without=${(woTurns / cells.length).toFixed(1)}`);
+console.log(`total specship calls=${wCalls} (avg ${(wCalls / cells.length).toFixed(1)}/cell)`);
+console.log(`every with-arm opens with a ToolSearch round-trip (deferred tools): ${tsAll ? 'YES — 1 fixed tax/run' : 'no'}`);

package/scripts/build-bundle.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+#
+# Build a self-contained SpecShip bundle: an official Node runtime + the
+# compiled app + its production deps, so SpecShip runs with NO system Node and
+# NO native build — node:sqlite is built into the bundled Node. One archive per
+# platform.
+#
+# Because dropping better-sqlite3 left zero native addons, the recipe is pure
+# file-packaging (download the target's Node, copy the app, archive) — so any
+# platform's bundle can be built on any OS. No cross-compile, no native runners.
+#
+# Usage:
+#   scripts/build-bundle.sh <target> [node-version]
+#     target:        darwin-arm64 | darwin-x64 | linux-x64 | linux-arm64
+#                  | win32-x64 | win32-arm64
+#     node-version:  e.g. v24.16.0 (default below; pin for reproducible builds)
+#
+# Output:
+#   unix:    release/specship-<target>.tar.gz   (launcher: bin/specship)
+#   windows: release/specship-<target>.zip      (launcher: bin/specship.cmd)
+set -euo pipefail
+
+TARGET="${1:?usage: build-bundle.sh <target> [node-version]}"
+NODE_VERSION="${2:-v24.16.0}"
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+OUT="$ROOT/release"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+ARCH="${TARGET##*-}"   # x64 | arm64
+OSFAM="${TARGET%-*}"   # darwin | linux | win32
+
+echo "[bundle] target=${TARGET} node=${NODE_VERSION}"
+
+# 1. Download + extract the official Node runtime for the target platform.
+if [ "$OSFAM" = "win32" ]; then
+  NODE_DIST="node-${NODE_VERSION}-win-${ARCH}"
+  NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.zip"
+  echo "[bundle] downloading ${NODE_URL}"
+  curl -fsSL "$NODE_URL" -o "$WORK/node.zip"
+  if command -v unzip >/dev/null 2>&1; then
+    unzip -q "$WORK/node.zip" -d "$WORK"
+  else
+    tar -xf "$WORK/node.zip" -C "$WORK"   # bsdtar can read zip
+  fi
+  NODE_BIN="$WORK/${NODE_DIST}/node.exe"
+else
+  NODE_DIST="node-${NODE_VERSION}-${TARGET}"
+  NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.tar.gz"
+  echo "[bundle] downloading ${NODE_URL}"
+  curl -fsSL "$NODE_URL" -o "$WORK/node.tar.gz"
+  tar -xzf "$WORK/node.tar.gz" -C "$WORK"
+  NODE_BIN="$WORK/${NODE_DIST}/bin/node"
+fi
+[ -f "$NODE_BIN" ] || { echo "[bundle] error: node binary not found ($NODE_BIN)" >&2; exit 1; }
+
+# 2. Build the app (compiled JS + copied wasm/schema assets).
+echo "[bundle] building app"
+( cd "$ROOT" && npm run build >/dev/null )
+
+# 3. Stage: app + production-only deps (pure JS/wasm → portable across platforms).
+STAGE="$WORK/specship-${TARGET}"
+mkdir -p "$STAGE/lib" "$STAGE/bin"
+cp -R "$ROOT/dist" "$STAGE/lib/dist"
+cp "$ROOT/package.json" "$ROOT/package-lock.json" "$STAGE/lib/"
+echo "[bundle] installing production dependencies"
+( cd "$STAGE/lib" && npm ci --omit=dev --ignore-scripts >/dev/null 2>&1 )
+rm -f "$STAGE/lib/package-lock.json"
+
+# 4. Vendored Node + launcher (the launcher uses the bundled Node by relative
+#    path, so no system Node is ever needed).
+#
+# `--liftoff-only`: keep tree-sitter's large WASM grammars on V8's Liftoff
+# baseline compiler so they never reach the turboshaft optimizing tier, whose
+# per-compilation Zone arena OOMs the whole process (`Fatal process out of
+# memory: Zone`) on Node >= 22 — even with tens of GB free. The flag is read at
+# V8 engine init so it must be on node's command line; the parse worker inherits
+# it. See issues #293/#298 and src/extraction/wasm-runtime-flags.ts. (The CLI
+# also self-relaunches with this flag when launched without it, so non-bundled
+# runs are covered too; passing it here avoids that extra spawn.)
+if [ "$OSFAM" = "win32" ]; then
+  cp "$NODE_BIN" "$STAGE/node.exe"
+  printf '@"%%~dp0..\\node.exe" --liftoff-only "%%~dp0..\\lib\\dist\\bin\\specship.js" %%*\r\n' \
+    > "$STAGE/bin/specship.cmd"
+else
+  cp "$NODE_BIN" "$STAGE/node"
+  cat > "$STAGE/bin/specship" <<'LAUNCH'
+#!/bin/sh
+# Resolve symlinks (e.g. the ~/.local/bin/specship link install.sh creates) so
+# we find the real bundle dir, not the symlink's location.
+SELF="$0"
+while [ -L "$SELF" ]; do
+  target="$(readlink "$SELF")"
+  case "$target" in
+    /*) SELF="$target" ;;
+    *) SELF="$(dirname "$SELF")/$target" ;;
+  esac
+done
+DIR="$(cd "$(dirname "$SELF")/.." && pwd)"
+# --liftoff-only: avoid the V8 turboshaft WASM Zone OOM (issues #293/#298).
+exec "$DIR/node" --liftoff-only "$DIR/lib/dist/bin/specship.js" "$@"
+LAUNCH
+  chmod +x "$STAGE/bin/specship"
+fi
+
+# 5. Archive (.zip for Windows, .tar.gz otherwise).
+mkdir -p "$OUT"
+if [ "$OSFAM" = "win32" ]; then
+  ARCHIVE="$OUT/specship-${TARGET}.zip"
+  rm -f "$ARCHIVE"
+  ( cd "$WORK" && zip -rqX "$ARCHIVE" "specship-${TARGET}" )
+else
+  ARCHIVE="$OUT/specship-${TARGET}.tar.gz"
+  # --no-xattrs: don't embed macOS xattrs that make GNU tar warn on Linux.
+  tar --no-xattrs -czf "$ARCHIVE" -C "$WORK" "specship-${TARGET}"
+fi
+echo "[bundle] wrote ${ARCHIVE} ($(du -h "$ARCHIVE" | cut -f1))"

package/scripts/build-server-bundle.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+/**
+ * Compile the HTTP server (`packages/server/src/**`) into the root's
+ * `dist/server/` so it ships as part of the single npm tarball.
+ *
+ * Why this lives at root scope: the publish pipeline (build-bundle.sh →
+ * pack-npm.sh) copies the root `dist/` + the root `package.json` into each
+ * per-platform tarball. Anything below `dist/server/` rides along; anything
+ * NOT below `dist/` is invisible to the published artifact. Shipping the
+ * server as its own package would mean a second release flow — this avoids
+ * it.
+ *
+ * Runs `tsc` against `packages/server/tsconfig.json` with `--outDir` pointed
+ * at the root's `dist/server/`. The server's own `npm run build` is left
+ * untouched so the workspace dev path keeps working.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync, statSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const root = path.resolve(here, '..');
+const serverPkg = path.resolve(root, 'packages', 'server');
+const serverTsconfig = path.join(serverPkg, 'tsconfig.json');
+const outDir = path.join(root, 'dist', 'server');
+
+if (!existsSync(serverTsconfig)) {
+  console.error(`[build-server-bundle] expected ${serverTsconfig} but not found`);
+  process.exit(1);
+}
+
+// Prefer the workspace's typescript install so the version is consistent.
+const tscBin = process.platform === 'win32'
+  ? path.join(root, 'node_modules', '.bin', 'tsc.cmd')
+  : path.join(root, 'node_modules', '.bin', 'tsc');
+
+if (!existsSync(tscBin)) {
+  console.error(`[build-server-bundle] tsc not found at ${tscBin} — run npm install at the repo root first`);
+  process.exit(1);
+}
+
+console.log(`[build-server-bundle] compiling ${path.relative(root, serverPkg)} → ${path.relative(root, outDir)}`);
+
+// Use absolute path for --outDir so tsc resolves it the same regardless of cwd.
+execFileSync(
+  tscBin,
+  ['-p', serverTsconfig, '--outDir', outDir, '--declaration', 'false', '--declarationMap', 'false', '--sourceMap', 'false'],
+  { stdio: 'inherit', cwd: serverPkg },
+);
+
+// Make the CLI executable (its shebang is preserved by tsc, but the file
+// mode isn't automatic on POSIX).
+import('node:fs').then((m) => {
+  try { m.chmodSync(path.join(outDir, 'cli.js'), 0o755); } catch { /* ignore */ }
+  // Drop a nested package.json so Node treats every .js file under
+  // dist/server/ as ESM — the root package.json has no "type": "module",
+  // so without this Node would parse these files as CommonJS and the
+  // `import`/`import.meta.url` syntax would throw.
+  try {
+    m.writeFileSync(
+      path.join(outDir, 'package.json'),
+      JSON.stringify({ type: 'module' }, null, 2) + '\n',
+    );
+  } catch (e) {
+    console.error('[build-server-bundle] could not write dist/server/package.json:', e);
+    process.exit(1);
+  }
+});
+
+// Sanity: the entry points should exist.
+for (const f of ['index.js', 'server.js', 'cli.js']) {
+  const p = path.join(outDir, f);
+  if (!existsSync(p) || !statSync(p).isFile()) {
+    console.error(`[build-server-bundle] expected ${path.relative(root, p)} after compile`);
+    process.exit(1);
+  }
+}
+
+console.log(`[build-server-bundle] OK — dist/server/{index,server,cli}.js present`);