@specship/specship 0.11.0

package/hooks/hooks.json

@@ -0,0 +1,38 @@
+{
+  "hooks": {
+    "PostToolUse": [
+      {
+        "matcher": "Edit|Write|MultiEdit",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "specship sync --quiet",
+            "async": true
+          }
+        ]
+      }
+    ],
+    "SessionStart": [
+      {
+        "matcher": "startup|resume",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "specship sync --quiet"
+          }
+        ]
+      }
+    ],
+    "UserPromptSubmit": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "specship spec-nudge"
+          }
+        ]
+      }
+    ]
+  }
+}

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@specship/specship",
+  "version": "0.11.0",
+  "description": "Supercharge Claude Code with semantic code intelligence. 94% fewer tool calls • 77% faster exploration • 100% local.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "specship": "./dist/bin/specship.js"
+  },
+  "files": [
+    "dist",
+    "scripts",
+    "README.md",
+    ".claude-plugin",
+    "commands",
+    "agents",
+    "hooks",
+    "selectors.json"
+  ],
+  "scripts": {
+    "build": "tsc && tsc -p src/designer/tsconfig.json && npm run copy-assets && npm run build:server && npm run build:web && node -e \"require('fs').chmodSync('dist/bin/specship.js', 0o755)\"",
+    "build:server": "node scripts/build-server-bundle.mjs",
+    "build:web": "node scripts/build-web-bundle.mjs",
+    "preuninstall": "node dist/bin/uninstall.js",
+    "copy-assets": "node -e \"const fs=require('fs');fs.mkdirSync('dist/db',{recursive:true});fs.copyFileSync('src/db/schema.sql','dist/db/schema.sql');fs.mkdirSync('dist/extraction/wasm',{recursive:true});fs.readdirSync('src/extraction/wasm').filter(f=>f.endsWith('.wasm')).forEach(f=>fs.copyFileSync('src/extraction/wasm/'+f,'dist/extraction/wasm/'+f));try{fs.mkdirSync('dist/workflows/defaults',{recursive:true});fs.readdirSync('src/workflows/defaults').filter(f=>/\\.ya?ml$/i.test(f)).forEach(f=>fs.copyFileSync('src/workflows/defaults/'+f,'dist/workflows/defaults/'+f))}catch(e){}\"",
+    "dev": "tsc --watch",
+    "cli": "npm run build && node dist/bin/specship.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:eval": "vitest run __tests__/evaluation/",
+    "eval": "npm run build && npx tsx __tests__/evaluation/runner.ts",
+    "clean": "node -e \"const fs=require('fs');fs.rmSync('dist',{recursive:true,force:true})\"",
+    "sync:shim": "node scripts/sync-shim-version.mjs",
+    "version": "npm run sync:shim",
+    "release:shim": "node scripts/sync-shim-version.mjs && cd packages/specship-shim && npm publish --access public"
+  },
+  "keywords": [
+    "code-intelligence",
+    "knowledge-graph",
+    "static-analysis"
+  ],
+  "author": "",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@clack/prompts": "^1.3.0",
+    "@fastify/cors": "^9.0.1",
+    "commander": "^14.0.2",
+    "cross-spawn": "^7.0.6",
+    "fast-string-width": "^3.0.2",
+    "fast-wrap-ansi": "^0.2.0",
+    "fastify": "^4.28.1",
+    "fflate": "^0.8.3",
+    "ignore": "^7.0.5",
+    "jsonc-parser": "^3.3.1",
+    "picomatch": "^4.0.3",
+    "sisteransi": "^1.0.5",
+    "tree-sitter-wasms": "^0.1.11",
+    "web-tree-sitter": "^0.25.3",
+    "yaml": "^2.5.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.0",
+    "@types/cross-spawn": "^6.0.6",
+    "@types/node": "^20.19.30",
+    "@types/picomatch": "^4.0.2",
+    "typescript": "^5.0.0",
+    "vitest": "^2.1.9"
+  },
+  "optionalDependencies": {
+    "better-sqlite3": "^11.10.0"
+  },
+  "engines": {
+    "node": ">=20.0.0 <25.0.0"
+  }
+}

package/scripts/add-lang/bench.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Add-lang benchmark for ONE repo:
+#   clone -> wipe+index (with the specship on PATH) -> verify extraction ->
+#   with/without retrieval A/B (reuses scripts/agent-eval/run-all.sh).
+#
+# Assumes the specship dev build is already built + linked on PATH — the skill
+# runs `npm run build && ./scripts/local-install.sh` ONCE before looping repos.
+# The A/B is skipped if extraction fails its critical checks (don't burn $ on a
+# broken extractor); set FORCE_AB=1 to run it anyway.
+#
+# Usage: bench.sh <lang> <repo-name> <repo-url> "<question>" [headless|tmux|all]
+# Env:   CORPUS   corpus dir (default /tmp/specship-corpus, shared with agent-eval)
+set -uo pipefail
+
+LANG_TOKEN="${1:?usage: bench.sh <lang> <repo-name> <repo-url> \"<question>\" [mode]}"
+NAME="${2:?repo-name required}"
+URL="${3:?repo-url required}"
+Q="${4:?question required}"
+MODE="${5:-headless}"
+
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+AGENT_EVAL="$(cd "$HARNESS/../agent-eval" && pwd)"
+CORPUS="${CORPUS:-/tmp/specship-corpus}"
+REPO="$CORPUS/$NAME"
+
+command -v specship >/dev/null || { echo "no specship on PATH (build + ./scripts/local-install.sh first)"; exit 1; }
+
+echo "==================== add-lang bench: $NAME ($LANG_TOKEN) ===================="
+echo "specship: $(command -v specship) -> $(specship --version 2>/dev/null || echo '?')"
+
+# 1. Ensure the repo (shallow clone, reuse if present).
+mkdir -p "$CORPUS"
+if [ -d "$REPO/.git" ]; then
+  echo "→ reusing checkout: $REPO"
+else
+  echo "→ cloning $URL"
+  git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
+fi
+
+# 2. Wipe + index with the binary under test.
+echo "→ wiping .specship and indexing"
+rm -rf "$REPO/.specship"
+( cd "$REPO" && specship init -i ) || { echo "indexing failed"; exit 1; }
+
+# 3. Verify extraction (cheap guard before the paid A/B).
+echo "→ verifying extraction"
+node "$HARNESS/verify-extraction.mjs" "$REPO" "$LANG_TOKEN"
+VERIFY=$?
+
+# 4. Retrieval A/B (skipped if extraction is broken, unless FORCE_AB=1).
+if [ "$VERIFY" -ne 0 ] && [ "${FORCE_AB:-0}" != "1" ]; then
+  echo "→ SKIPPING A/B — extraction failed critical checks (set FORCE_AB=1 to override)"
+else
+  echo "→ retrieval A/B (mode=$MODE)"
+  bash "$AGENT_EVAL/run-all.sh" "$REPO" "$Q" "$MODE"
+fi
+
+echo "==================== bench complete: $NAME (verify exit=$VERIFY) ===================="
+# Exit reflects extraction: 0 = pass/warn, 1 = critical fail, 2 = couldn't read status.
+exit "$VERIFY"

package/scripts/add-lang/check-grammar.mjs

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+// Verify a tree-sitter grammar wasm is HEALTHY under the project's web-tree-sitter
+// runtime BEFORE writing an extractor. Prints the ABI version and parses a valid
+// sample many times in a multi-grammar context, to catch heap-corruption bugs
+// that silently drop nodes on every parse after the first.
+//
+// Why this exists: the tree-sitter-wasms Lua grammar is ABI 13 and corrupts the
+// shared WASM heap under web-tree-sitter 0.25 — Lua extraction degraded on every
+// file after the first (nested calls/imports vanished). The fix was to vendor the
+// upstream ABI-15 wasm. Run this on any new grammar first; if it FAILs, vendor a
+// newer build instead of using the tree-sitter-wasms one.
+//
+// Usage: node scripts/add-lang/check-grammar.mjs <lang|wasm-path> <valid-sample> [iterations]
+// Exit: 0 healthy, 1 corruption / parse errors, 2 could not run.
+// NOTE: the sample must be SYNTACTICALLY VALID — a broken sample fails for the
+//       wrong reason.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { Parser, Language } from 'web-tree-sitter';
+
+const require = createRequire(import.meta.url);
+const fail = (code, msg) => { console.error(`[check-grammar] ${msg}`); process.exit(code); };
+
+const [token, sample, iterArg] = process.argv.slice(2);
+if (!token || !sample) fail(2, 'usage: check-grammar.mjs <lang|wasm-path> <valid-sample> [iterations]');
+if (!existsSync(sample)) fail(2, `sample not found: ${sample}`);
+const iters = iterArg ? parseInt(iterArg, 10) : 20;
+
+const SPECIAL = { csharp: 'c_sharp', 'c#': 'c_sharp' };
+function resolveWasm(t) {
+  if (t.endsWith('.wasm')) return existsSync(t) ? t : fail(2, `wasm not found: ${t}`);
+  const base = SPECIAL[t.toLowerCase()] ?? t.toLowerCase();
+  try { return require.resolve(`tree-sitter-wasms/out/tree-sitter-${base}.wasm`); } catch { /* try vendored */ }
+  const vendored = `src/extraction/wasm/tree-sitter-${base}.wasm`;
+  if (existsSync(vendored)) return vendored;
+  return fail(2, `no grammar for "${t}" — not in tree-sitter-wasms and not vendored`);
+}
+
+const wasmPath = resolveWasm(token);
+const source = readFileSync(sample, 'utf8');
+
+try { await Parser.init(); }
+catch { await Parser.init({ locateFile: () => require.resolve('web-tree-sitter/tree-sitter.wasm') }); }
+
+// Load a second, known-good grammar — the corruption surfaces under the
+// multi-grammar runtime that real indexing uses, not a single grammar in isolation.
+try { await Language.load(require.resolve('tree-sitter-wasms/out/tree-sitter-python.wasm')); } catch { /* ok */ }
+
+let language;
+try { language = await Language.load(wasmPath); }
+catch (e) { fail(2, `failed to load ${wasmPath}: ${e.message}`); }
+
+const parser = new Parser();
+parser.setLanguage(language);
+
+let ok = 0, err = 0;
+for (let i = 0; i < iters; i++) {
+  const tree = parser.parse(source);
+  if (tree.rootNode.hasError) err++; else ok++;
+}
+
+console.log(`grammar: ${wasmPath.split('/').pop()}`);
+console.log(`  ABI version: ${language.abiVersion}`);
+console.log(`  parses: ${ok} clean / ${err} with errors (of ${iters})`);
+if (err > 0) {
+  console.log(
+    `RESULT: FAIL — ${err}/${iters} parses produced ERROR trees on a valid sample. ` +
+    `This grammar corrupts under web-tree-sitter; vendor a newer (ABI 14/15) wasm ` +
+    `(see SKILL.md "Find a grammar"). Confirm your sample is syntactically valid first.`
+  );
+  process.exit(1);
+}
+console.log('RESULT: PASS — grammar parses cleanly and reuses safely.');
+process.exit(0);

package/scripts/add-lang/dump-ast.mjs

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+// Dump the tree-sitter AST for a sample file so you can write a LanguageExtractor
+// mapping. Loads a grammar .wasm directly via web-tree-sitter (the same runtime
+// specship uses) — you do NOT need to register the language first.
+//
+// Usage:
+//   node scripts/add-lang/dump-ast.mjs <lang|wasm-path> <sample-file> [--depth=N] [--full]
+// Examples:
+//   node scripts/add-lang/dump-ast.mjs lua sample.lua
+//   node scripts/add-lang/dump-ast.mjs src/extraction/wasm/tree-sitter-zig.wasm a.zig --depth=4
+//
+// Output: an indented AST (named nodes, with field names) followed by a
+// node-type FREQUENCY table. The frequency table is the payoff — it tells you
+// which node types to map to functionTypes / classTypes / importTypes / etc.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { Parser, Language } from 'web-tree-sitter';
+
+const require = createRequire(import.meta.url);
+const fail = (msg) => { console.error(`[dump-ast] ${msg}`); process.exit(1); };
+
+const argv = process.argv.slice(2);
+const positional = argv.filter((a) => !a.startsWith('--'));
+const [langOrWasm, sampleFile] = positional;
+const depthFlag = argv.find((a) => a.startsWith('--depth='));
+const showAll = argv.includes('--full'); // also print anonymous (token) nodes
+const maxDepth = depthFlag ? parseInt(depthFlag.split('=')[1], 10) : (showAll ? Infinity : 8);
+
+if (!langOrWasm || !sampleFile) {
+  fail('usage: dump-ast.mjs <lang|wasm-path> <sample-file> [--depth=N] [--full]');
+}
+if (!existsSync(sampleFile)) fail(`sample file not found: ${sampleFile}`);
+
+// Language tokens whose tree-sitter-wasms filename differs from the token.
+const WASM_SPECIAL = { csharp: 'c_sharp', 'c#': 'c_sharp' };
+
+function resolveWasm(token) {
+  if (token.endsWith('.wasm')) {
+    if (!existsSync(token)) fail(`wasm not found: ${token}`);
+    return token;
+  }
+  const base = WASM_SPECIAL[token.toLowerCase()] ?? token.toLowerCase();
+  try {
+    return require.resolve(`tree-sitter-wasms/out/tree-sitter-${base}.wasm`);
+  } catch {
+    /* not in tree-sitter-wasms — try a vendored copy */
+  }
+  const vendored = `src/extraction/wasm/tree-sitter-${base}.wasm`;
+  if (existsSync(vendored)) return vendored;
+  fail(
+    `no grammar for "${token}" — not in tree-sitter-wasms and not vendored at ` +
+      `${vendored}. Pass an explicit .wasm path, or vendor one (see SKILL.md "Find a grammar").`
+  );
+}
+
+const wasmPath = resolveWasm(langOrWasm);
+const source = readFileSync(sampleFile, 'utf8');
+
+try {
+  await Parser.init();
+} catch {
+  await Parser.init({ locateFile: () => require.resolve('web-tree-sitter/tree-sitter.wasm') });
+}
+
+let language;
+try {
+  language = await Language.load(wasmPath);
+} catch (e) {
+  fail(`failed to load grammar ${wasmPath}: ${e.message}`);
+}
+
+const parser = new Parser();
+parser.setLanguage(language);
+const tree = parser.parse(source);
+
+const freq = new Map();
+const snippet = (node) => {
+  const t = node.text.replace(/\s+/g, ' ').trim();
+  return t.length > 48 ? `${t.slice(0, 48)}…` : t;
+};
+
+function walk(node, depth, fieldName) {
+  if (node.isNamed) freq.set(node.type, (freq.get(node.type) || 0) + 1);
+  if ((node.isNamed || showAll) && depth <= maxDepth) {
+    const field = fieldName ? `${fieldName}: ` : '';
+    const leaf = node.childCount === 0 ? `  "${snippet(node)}"` : '';
+    console.log(`${'  '.repeat(depth)}${field}${node.type}  @${node.startPosition.row + 1}:${node.startPosition.column}${leaf}`);
+  }
+  for (let i = 0; i < node.childCount; i++) {
+    const child = node.child(i);
+    if (child) walk(child, depth + 1, node.fieldNameForChild(i));
+  }
+}
+
+console.log(`\n# AST for ${sampleFile}  (grammar: ${wasmPath.split('/').pop()})\n`);
+walk(tree.rootNode, 0, null);
+
+console.log('\n# Node-type frequency (named nodes) — map the relevant ones in your extractor:\n');
+[...freq.entries()]
+  .sort((a, b) => b[1] - a[1])
+  .forEach(([type, n]) => console.log(`  ${String(n).padStart(5)}  ${type}`));
+console.log();

package/scripts/add-lang/verify-extraction.mjs

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+// Sanity-check that specship extracted REAL symbols (not just file/import nodes)
+// from a repo for a given language. Exits non-zero on a critical failure so it
+// can drive a write-extractor -> build -> re-check loop.
+//
+// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
+// Reads `specship status <repo> --json` using whatever specship is on PATH,
+// so it reflects the binary that built the index.
+//
+// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
+
+import { execFileSync } from 'node:child_process';
+
+const [repo, lang] = process.argv.slice(2);
+if (!repo || !lang) {
+  console.error('usage: verify-extraction.mjs <repo-path> <lang>');
+  process.exit(2);
+}
+
+let status;
+try {
+  const out = execFileSync('specship', ['status', repo, '--json'], { encoding: 'utf8' });
+  status = JSON.parse(out);
+} catch (e) {
+  console.error(`[verify] could not read specship status for ${repo}: ${e.message}`);
+  process.exit(2);
+}
+
+// Kinds that prove the extractor mapped AST node types (everything except
+// 'file' and 'import', which specship creates structurally for any language).
+const SYMBOL_KINDS = new Set([
+  'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
+  'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
+  'type_alias', 'namespace', 'route', 'component',
+]);
+
+const byKind = status.nodesByKind || {};
+const langs = status.languages || [];
+const files = status.fileCount || 0;
+const edges = status.edgeCount || 0;
+const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
+const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
+
+const checks = [];
+const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
+
+add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
+add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
+add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
+add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
+add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
+
+console.log(`\n# Extraction check — ${repo}  (lang=${lang}, backend=${status.backend})`);
+console.log(`  files=${files} nodes=${status.nodeCount} edges=${edges}`);
+console.log(`  nodesByKind: ${JSON.stringify(byKind)}\n`);
+for (const c of checks) console.log(`  ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
+
+const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
+const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
+console.log();
+if (critical.length) {
+  console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
+  process.exit(1);
+}
+if (soft.length) {
+  console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
+  process.exit(0);
+}
+console.log('RESULT: PASS — extraction looks healthy.');
+process.exit(0);

package/scripts/agent-eval/arms-F.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Arm F (body-inlining trace + trace-first steering) across the same 6 repos as
+# arms-matrix.sh, so F vs B isolates the trace-enrichment effect (same surface,
+# old thin trace in B vs body-inlining trace here).
+set -uo pipefail
+H="$(cd "$(dirname "$0")" && pwd)"; RUNS="${RUNS:-2}"; C="${CORPUS:-/tmp/specship-corpus}"
+ROWS=(
+"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
+"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
+"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
+"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
+"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
+"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
+)
+ARM="${ARM:-F}"
+echo "### ARM $ARM START $(date) RUNS=$RUNS"
+for row in "${ROWS[@]}"; do
+  repo="${row%%|*}"; q="${row#*|}"
+  for r in $(seq 1 "$RUNS"); do bash "$H/run-arms.sh" "$repo" "$q" "$ARM" "$r"; done
+done
+echo "### ARM $ARM COMPLETE $(date)"

package/scripts/agent-eval/arms-matrix.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Drive the tool-surface ablation across the chosen repos × arms (A–E).
+# Arms A–D ask the canonical FLOW question; arm E asks a NON-flow survey
+# question (the control probe — should degrade without explore+context).
+# Output: /tmp/arms/<repo>/<arm>-r<n>.jsonl  (parse with parse-arms.mjs).
+set -uo pipefail
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+RUNS="${RUNS:-2}"
+C="${CORPUS:-/tmp/specship-corpus}"
+NFQ='What are the main modules/components of this codebase and what does each one do? Give an overview of how it is organized.'
+
+# repo-path|flow-question  (2 small, 2 medium, 2 large — spans the size range)
+ROWS=(
+"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
+"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
+"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
+"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
+"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
+"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
+)
+
+echo "### ARMS MATRIX START $(date) RUNS=$RUNS"
+for row in "${ROWS[@]}"; do
+  repo="${row%%|*}"; q="${row#*|}"
+  for arm in A B C D; do
+    for r in $(seq 1 "$RUNS"); do
+      bash "$HARNESS/run-arms.sh" "$repo" "$q" "$arm" "$r"
+    done
+  done
+done
+# E: non-flow control probe on two repos (must degrade without explore+context)
+for repo in "$C/excalidraw" "$C/spring-mall"; do
+  for r in $(seq 1 "$RUNS"); do
+    bash "$HARNESS/run-arms.sh" "$repo" "$NFQ" E "$r"
+  done
+done
+echo "### ARMS MATRIX COMPLETE $(date)"

package/scripts/agent-eval/audit.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# One-shot SpecShip quality audit:
+#   set version -> ensure corpus repo -> wipe+reindex with that version ->
+#   run with/without A/B -> restore the local dev link.
+#
+# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
+#   <version>    "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
+#   <repo-name>  dir name under the corpus dir
+#   <repo-url>   git URL (cloned --depth 1 when the repo dir is missing)
+#   [mode]       headless (default) | all (also the interactive tmux arms)
+# Env: CORPUS  corpus dir (default: /tmp/specship-corpus)
+set -uo pipefail
+
+VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
+NAME="${2:?repo-name required}"
+URL="${3:?repo-url required}"
+Q="${4:?question required}"
+MODE="${5:-headless}"
+
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$HARNESS/../.." && pwd)"     # specship repo root
+CORPUS="${CORPUS:-/tmp/specship-corpus}"
+REPO="$CORPUS/$NAME"
+PKG="@specship/specship"
+
+echo "==================== SpecShip audit ===================="
+echo "version=$VERSION  repo=$NAME  mode=$MODE  corpus=$CORPUS"
+echo
+
+# 1. Set the specship version under test (mutates the global install).
+if [ "$VERSION" = local ]; then
+  echo "→ [1/4] building + linking local dev build (local-install.sh)"
+  ( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
+else
+  echo "→ [1/4] installing $PKG@$VERSION globally"
+  npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
+fi
+ACTUAL="$(specship --version 2>/dev/null || echo '?')"
+echo "  specship on PATH: $(command -v specship) -> $ACTUAL"
+
+# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
+mkdir -p "$CORPUS"
+if [ -d "$REPO/.git" ]; then
+  echo "→ [2/4] reusing existing checkout: $REPO"
+else
+  echo "→ [2/4] cloning $URL"
+  git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
+fi
+
+# 3. Wipe + re-index with THIS version (the index must be built by the same
+#    binary that serves it — different versions extract differently).
+echo "→ [3/4] wiping .specship and re-indexing with $ACTUAL"
+rm -rf "$REPO/.specship"
+( cd "$REPO" && specship init -i ) || { echo "indexing failed"; exit 1; }
+
+# 4. Run the with/without A/B.
+echo "→ [4/4] running A/B harness (mode=$MODE)"
+bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
+
+# Restore the dev link (the normal working state in this repo).
+echo
+echo "→ restoring local dev link (local-install.sh)"
+if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
+  echo "  global specship restored to dev build"
+else
+  echo "  WARN: restore failed — run ./scripts/local-install.sh manually"
+fi
+echo "==================== audit complete ===================="

package/scripts/agent-eval/bench-readme.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Re-run the README "Benchmark Results" A/B (with vs without specship) on the
+# current build: the 7 README repos, same queries, RUNS per arm (default 4).
+# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
+# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
+# $CORPUS (default /tmp/specship-corpus) by the build under test.
+set -uo pipefail
+H="$(cd "$(dirname "$0")" && pwd)"
+C="${CORPUS:-/tmp/specship-corpus}"
+RUNS="${RUNS:-4}"
+ROWS=(
+"vscode|How does the extension host communicate with the main process?"
+"excalidraw|How does Excalidraw render and update canvas elements?"
+"django|How does Django's ORM build and execute a query from a QuerySet?"
+"tokio|How does tokio schedule and run async tasks on its runtime?"
+"okhttp|How does OkHttp process a request through its interceptor chain?"
+"gin|How does gin route requests through its middleware chain?"
+"alamofire|How does Alamofire build, send, and validate a request?"
+)
+echo "### README A/B START $(date) RUNS=$RUNS"
+for row in "${ROWS[@]}"; do
+  repo="${row%%|*}"; q="${row#*|}"
+  echo "===== $repo ====="
+  for run in $(seq 1 "$RUNS"); do
+    AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo "  run$run: (no exit line)"
+  done
+done
+echo "### README A/B DONE $(date)"

package/scripts/agent-eval/bench-why-repo.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# One README repo, WITH-specship only, N runs. Each run appends a why-Read
+# diagnostic so the agent explains any Read/Grep. (The WITHOUT baseline is
+# specship-independent and already in the README — no point re-running it.)
+# Output -> /tmp/ab-why/<repo>/with<n>.jsonl
+# Usage: bench-why-repo.sh <repo-path> "<query>" [N]
+set -uo pipefail
+REPO="$1"; Q="$2"; N="${3:-4}"
+NAME="$(basename "$REPO")"
+CG="/Users/colby/Development/Personal/specship/dist/bin/specship.js"
+OUT="/tmp/ab-why/$NAME"; mkdir -p "$OUT"
+WHY=$'\n\nIMPORTANT — diagnostic: if you use the Read or Grep tool at ANY point, for EACH such call explain why specship_explore / specship_node did not already give you what you needed. End your entire answer with a section titled exactly "## Why I read" listing every Read and Grep you made and the precise reason specship fell short for it. If you used neither, write "## Why I read" then "none — specship was sufficient."'
+printf '{"mcpServers":{"specship":{"command":"%s","args":["serve","--mcp","--path","%s"]}}}' "$CG" "$REPO" > "$OUT/cg.json"
+
+for i in $(seq 1 "$N"); do
+  pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.specship/daemon.sock"
+  ( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
+      --permission-mode bypassPermissions --model opus --effort "${EFFORT:-high}" --max-budget-usd 4 \
+      --strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
+  echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
+done
+echo "DONE $NAME"

package/scripts/agent-eval/block-read-hook.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# PreToolUse hook (experiment): deny Read of specship-indexed source files and
+# steer the agent to specship_explore/specship_node instead. Tests whether
+# specship can FULLY replace Read for code-understanding once the escape hatch
+# is removed. Non-source reads (config, .env, markdown, new files) pass through.
+#
+# Wire via:  claude ... --settings scripts/agent-eval/hook-settings.json
+set -uo pipefail
+input="$(cat)"
+fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
+
+case "$fp" in
+  *.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
+    msg="Read is disabled for source files in this session — specship already has this file indexed (with line numbers, kept in sync on every change). Use specship_explore (several related symbols at once) or specship_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER specship_explore with its exact name instead of reading the file."
+    jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
+    exit 0
+    ;;
+esac
+exit 0

package/scripts/agent-eval/hook-settings.json

@@ -0,0 +1,15 @@
+{
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Read",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash /Users/colby/Development/Personal/specship/scripts/agent-eval/block-read-hook.sh"
+          }
+        ]
+      }
+    ]
+  }
+}