homegraph 0.9.10 → 1.1.1

package/scripts/add-lang/verify-extraction.mjs

@@ -1,70 +1,70 @@
-#!/usr/bin/env node
-// Sanity-check that codegraph extracted REAL symbols (not just file/import nodes)
-// from a repo for a given language. Exits non-zero on a critical failure so it
-// can drive a write-extractor -> build -> re-check loop.
-//
-// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
-// Reads `codegraph status <repo> --json` using whatever codegraph is on PATH,
-// so it reflects the binary that built the index.
-//
-// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
-
-import { execFileSync } from 'node:child_process';
-
-const [repo, lang] = process.argv.slice(2);
-if (!repo || !lang) {
-  console.error('usage: verify-extraction.mjs <repo-path> <lang>');
-  process.exit(2);
-}
-
-let status;
-try {
-  const out = execFileSync('codegraph', ['status', repo, '--json'], { encoding: 'utf8' });
-  status = JSON.parse(out);
-} catch (e) {
-  console.error(`[verify] could not read codegraph status for ${repo}: ${e.message}`);
-  process.exit(2);
-}
-
-// Kinds that prove the extractor mapped AST node types (everything except
-// 'file' and 'import', which codegraph creates structurally for any language).
-const SYMBOL_KINDS = new Set([
-  'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
-  'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
-  'type_alias', 'namespace', 'route', 'component',
-]);
-
-const byKind = status.nodesByKind || {};
-const langs = status.languages || [];
-const files = status.fileCount || 0;
-const edges = status.edgeCount || 0;
-const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
-const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
-
-const checks = [];
-const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
-
-add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
-add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
-add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
-add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
-add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
-
-console.log(`\n# Extraction check — ${repo}  (lang=${lang}, backend=${status.backend})`);
-console.log(`  files=${files} nodes=${status.nodeCount} edges=${edges}`);
-console.log(`  nodesByKind: ${JSON.stringify(byKind)}\n`);
-for (const c of checks) console.log(`  ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
-
-const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
-const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
-console.log();
-if (critical.length) {
-  console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
-  process.exit(1);
-}
-if (soft.length) {
-  console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
-  process.exit(0);
-}
-console.log('RESULT: PASS — extraction looks healthy.');
-process.exit(0);
+#!/usr/bin/env node
+// Sanity-check that homegraph extracted REAL symbols (not just file/import nodes)
+// from a repo for a given language. Exits non-zero on a critical failure so it
+// can drive a write-extractor -> build -> re-check loop.
+//
+// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
+// Reads `homegraph status <repo> --json` using whatever homegraph is on PATH,
+// so it reflects the binary that built the index.
+//
+// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
+
+import { execFileSync } from 'node:child_process';
+
+const [repo, lang] = process.argv.slice(2);
+if (!repo || !lang) {
+  console.error('usage: verify-extraction.mjs <repo-path> <lang>');
+  process.exit(2);
+}
+
+let status;
+try {
+  const out = execFileSync('homegraph', ['status', repo, '--json'], { encoding: 'utf8' });
+  status = JSON.parse(out);
+} catch (e) {
+  console.error(`[verify] could not read homegraph status for ${repo}: ${e.message}`);
+  process.exit(2);
+}
+
+// Kinds that prove the extractor mapped AST node types (everything except
+// 'file' and 'import', which homegraph creates structurally for any language).
+const SYMBOL_KINDS = new Set([
+  'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
+  'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
+  'type_alias', 'namespace', 'route', 'component',
+]);
+
+const byKind = status.nodesByKind || {};
+const langs = status.languages || [];
+const files = status.fileCount || 0;
+const edges = status.edgeCount || 0;
+const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
+const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
+
+const checks = [];
+const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
+
+add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
+add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
+add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
+add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
+add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
+
+console.log(`\n# Extraction check — ${repo}  (lang=${lang}, backend=${status.backend})`);
+console.log(`  files=${files} nodes=${status.nodeCount} edges=${edges}`);
+console.log(`  nodesByKind: ${JSON.stringify(byKind)}\n`);
+for (const c of checks) console.log(`  ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
+
+const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
+const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
+console.log();
+if (critical.length) {
+  console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
+  process.exit(1);
+}
+if (soft.length) {
+  console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
+  process.exit(0);
+}
+console.log('RESULT: PASS — extraction looks healthy.');
+process.exit(0);

package/scripts/agent-eval/ab-adoption.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Does the agent PICK homegraph_node to read a file, vs the built-in Read tool?
+# Build A/B: NEW build (HEAD, homegraph_node has Read parity) vs BASELINE build
+# (a ref where it doesn't), BOTH homegraph-attached + pre-warmed, same task. The
+# metric is tool CHOICE: Read calls vs homegraph_node[file] calls per run.
+#
+# Usage: ab-adoption.sh <indexed-repo> "<task>" [runs-per-arm] [baseline-ref]
+# Env: AGENT_EVAL_OUT (default: /tmp/ab-adoption)
+set -uo pipefail
+TARGET="${1:?usage: ab-adoption.sh <indexed-repo> \"<task>\" [runs] [baseline-ref]}"
+TASK="${2:?task required}"
+RUNS="${3:-2}"
+BASE_REF="${4:-HEAD~1}"
+ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
+BIN="$ENGINE/dist/bin/homegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/ab-adoption}"
+
+command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
+[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
+git -C "$ENGINE" diff --quiet && git -C "$ENGINE" diff --cached --quiet || { echo "engine has uncommitted changes — commit/stash first"; exit 1; }
+CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
+[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD"; exit 1; }
+
+cleanup() {
+  pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
+  git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
+  ( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
+}
+trap cleanup EXIT
+mkdir -p "$OUT"
+echo "###### target=$TARGET  runs/arm=$RUNS  baseline=$BASE_REF"
+echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
+echo "###### task=$TASK"; echo
+
+prewarm() {
+  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
+  HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
+}
+
+# Per-run tool-choice counts: Read vs homegraph_node[file] vs [symbol].
+count() {
+  node -e '
+    const fs=require("fs");
+    const lines=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
+    let read=0,cgFile=0,cgSym=0,cgOther=0,exposed="?";
+    for(const l of lines){try{const o=JSON.parse(l);
+      if(o.type==="system"&&o.subtype==="init"){exposed=(o.tools||[]).filter(t=>/homegraph/.test(t)).length;}
+      const blocks=o.message?.content||[];
+      for(const b of (Array.isArray(blocks)?blocks:[])){
+        if(b.type!=="tool_use")continue;
+        if(b.name==="Read")read++;
+        else if(b.name==="mcp__homegraph__homegraph_node"){ if(b.input&&b.input.symbol)cgSym++; else cgFile++; }
+        else if(/mcp__homegraph__/.test(b.name))cgOther++;
+      }
+    }catch{}}
+    console.log(`    Read=${read}  homegraph_node[file]=${cgFile}  homegraph_node[symbol]=${cgSym}  other_cg=${cgOther}  (cg exposed=${exposed})`);
+  ' "$1"
+}
+
+run_arm() { # label, N
+  local label="$1" n="$2"
+  local c="$OUT/mcp-$label.json"
+  for i in $(seq 1 "$n"); do
+    local tgt="$OUT/t-$label-$i"
+    rm -rf "$tgt"
+    rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$tgt/"
+    node "$BIN" init "$tgt" >/dev/null 2>&1
+    printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
+    prewarm "$tgt"
+    echo "----- [$label] run $i -----"
+    ( cd "$tgt" && claude -p "$TASK" \
+        --output-format stream-json --verbose --permission-mode bypassPermissions \
+        --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
+        </dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
+    count "$OUT/run-$label-$i.jsonl"
+    pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
+  done
+  echo
+}
+
+echo "== NEW build (HEAD: homegraph_node has Read parity) =="
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
+run_arm new "$RUNS"
+
+echo "== BASELINE build ($BASE_REF) =="
+git -C "$ENGINE" checkout "$BASE_REF" -- $CHANGED
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
+run_arm baseline "$RUNS"
+
+echo "###### DONE — compare [new] vs [baseline]: does homegraph_node[file] rise / Read fall? Logs: $OUT"

package/scripts/agent-eval/ab-hook.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# A/B the PreToolUse(Read) REDIRECT hook (P1): does steering Read → homegraph_node
+# file-view actually move the agent off Read during implementation? BOTH arms use
+# the CURRENT build with homegraph attached and pre-warmed; the only difference is
+# the hook. Isolates the hook's behavioral effect from the build/file-view change
+# (use ab-new-vs-baseline.sh for the build A/B).
+#
+#   arm [nohook] — homegraph on, no hook   (does the better file-view get picked on its own?)
+#   arm [hook]   — homegraph on, + redirect hook   (does routing close it?)
+#
+# Reliable attach (works nested): each arm pre-warms a persistent daemon and skips
+# the startup re-exec (HOMEGRAPH_WASM_RELAUNCHED=1), so claude connects before the
+# agent's first turn. Judge by ACTUAL homegraph usage in parse-run.mjs's "by type",
+# not claude's init snapshot (which can read pending even when it then connects).
+#
+# Usage: ab-hook.sh <indexed-repo> "<implementation task>" [runs-per-arm]
+#   <indexed-repo>  a repo with a .homegraph index (copied per arm; never mutated)
+#   "<task>"        a GENUINELY-NEW implementation task (verify it isn't already done)
+#   [runs-per-arm]  default 2 (n=1 is noisy — the doctrine says >=2)
+# Env: AGENT_EVAL_OUT (default: /tmp/ab-hook)
+set -uo pipefail
+
+TARGET="${1:?usage: ab-hook.sh <indexed-repo> \"<task>\" [runs-per-arm]}"
+TASK="${2:?task required}"
+RUNS="${3:-2}"
+ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
+BIN="$ENGINE/dist/bin/homegraph.js"
+HOOK="$ENGINE/scripts/agent-eval/redirect-read-hook.sh"
+OUT="${AGENT_EVAL_OUT:-/tmp/ab-hook}"
+PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
+
+command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
+command -v jq >/dev/null || { echo "jq not on PATH (the hook needs it)"; exit 1; }
+[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
+chmod +x "$HOOK"
+
+cleanup() { pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
+trap cleanup EXIT
+
+mkdir -p "$OUT"
+echo "###### engine=$ENGINE"
+echo "###### target=$TARGET   runs/arm=$RUNS"
+echo "###### task=$TASK"
+echo
+
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
+
+# A settings file carrying ONLY the PreToolUse(Read) redirect hook.
+HOOK_SETTINGS="$OUT/hook-settings.json"
+jq -n --arg cmd "bash $HOOK" \
+  '{hooks:{PreToolUse:[{matcher:"Read",hooks:[{type:"command",command:$cmd}]}]}}' > "$HOOK_SETTINGS"
+
+prewarm() { # target — spawn a persistent daemon and wait for its socket
+  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
+  HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
+    && echo "  daemon warm: $1" || echo "  WARN: daemon never bound for $1"
+}
+
+run_one() { # arm-label, run-index, use-hook(0|1)
+  local label="$1" idx="$2" hook="$3"
+  local tgt="$OUT/t-$label-$idx" c="$OUT/mcp-$label.json"
+  rm -rf "$tgt"
+  rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$tgt/"
+  node "$BIN" init "$tgt" >/dev/null 2>&1
+  printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
+  prewarm "$tgt"
+  local extra=()
+  [ "$hook" = "1" ] && extra=(--settings "$HOOK_SETTINGS")
+  echo "----- [$label] run $idx -----"
+  # ${extra[@]+...} guard: bash 3.2 (macOS) under `set -u` errors on an empty
+  # array expansion otherwise, which would skip the no-hook arm's claude run.
+  ( cd "$tgt" && claude -p "$TASK" \
+      --output-format stream-json --verbose --permission-mode bypassPermissions \
+      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
+      </dev/null > "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" )
+  node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo "  (parse failed — see $OUT/run-$label-$idx.jsonl)"
+  pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
+  echo
+}
+
+for i in $(seq 1 "$RUNS"); do run_one nohook "$i" 0; done
+for i in $(seq 1 "$RUNS"); do run_one hook   "$i" 1; done
+
+echo "###### DONE. Compare [nohook] vs [hook] 'by type' — Read should fall and"
+echo "###### mcp__homegraph__homegraph_node should rise in the [hook] arm. Logs: $OUT"

package/scripts/agent-eval/ab-impl.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Sufficiency A/B for an IMPLEMENTATION task (the agent edits): when it uses
+# homegraph (explore/node) to understand before editing, does it still Read? Like
+# ab-sufficiency.sh but copies+indexes a FRESH target per run (the agent mutates
+# it), so runs don't see each other's edits.
+#
+# WITH homegraph (pre-warmed) vs WITHOUT (empty MCP), N runs each. Reports
+# explore/node vs Read/Grep + the files Read, and whether the build still passes.
+#
+# Usage: ab-impl.sh <indexed-repo> "<task>" [runs] [build-cmd]
+# Env: AGENT_EVAL_OUT (default: /tmp/ab-impl)
+set -uo pipefail
+REPO="${1:?usage: ab-impl.sh <indexed-repo> \"<task>\" [runs] [build-cmd]}"
+Q="${2:?task required}"
+RUNS="${3:-2}"
+BUILD_CMD="${4:-}"
+ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
+BIN="$ENGINE/dist/bin/homegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/ab-impl}"
+command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
+[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO"; exit 1; }
+cleanup(){ pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
+trap cleanup EXIT
+mkdir -p "$OUT"
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built engine"
+echo "###### repo=$REPO  runs/arm=$RUNS"
+echo "###### task=$Q"; echo
+echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
+
+prewarm(){
+  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
+  HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
+}
+
+analyze(){
+  node -e '
+    const fs=require("fs");
+    const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
+    let ex=0,nf=0,ns=0,oc=0,gr=0,ed=0,exposed="?";const reads=[];
+    for(const l of L){try{const o=JSON.parse(l);
+      if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/homegraph/.test(t)).length;
+      for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
+        if(b.name==="mcp__homegraph__homegraph_explore")ex++;
+        else if(b.name==="mcp__homegraph__homegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
+        else if(/mcp__homegraph__/.test(b.name))oc++;
+        else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
+        else if(b.name==="Grep")gr++;
+        else if(b.name==="Edit"||b.name==="Write")ed++;
+      }}catch{}}
+    console.log(`    explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr} Edit=${ed}  [cg exposed=${exposed}]`);
+  ' "$1"
+}
+
+run(){ # label, withCodegraph(0/1)
+  local label="$1" wcg="$2"
+  for i in $(seq 1 "$RUNS"); do
+    local tgt="$OUT/t-$label-$i" cfg="$OUT/mcp-$label.json"
+    rm -rf "$tgt"
+    rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$REPO/" "$tgt/"
+    node "$BIN" init "$tgt" >/dev/null 2>&1
+    if [ "$wcg" = "1" ]; then
+      printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$cfg"
+      prewarm "$tgt"
+    else cp "$OUT/mcp-empty.json" "$cfg"; fi
+    ( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \
+        --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
+        --strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
+    echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
+    if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo "      build: PASS" || echo "      build: FAIL" ); fi
+    pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
+  done
+  echo
+}
+
+echo "== WITH homegraph =="; run with 1
+echo "== WITHOUT (Read/Grep only) =="; run without 0
+echo "###### DONE: $OUT"

package/scripts/agent-eval/ab-new-vs-baseline.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# A/B a homegraph retrieval/steering change: the NEW build (current HEAD) vs a
+# BASELINE build (a git ref) — BOTH with homegraph attached — on the same
+# implementation task, measuring how many Read vs homegraph calls the agent
+# makes. ISOLATES the change (unlike run-all.sh's with-vs-without). The agent
+# works on a throwaway copy of the target, so your repos are never touched.
+#
+# Reliable attach (works even when this is itself run nested inside a Claude
+# session): each arm PRE-WARMS a persistent homegraph daemon for its target so
+# claude connects to an already-bound, index-loaded daemon instantly — before
+# the agent's first turn — and SKIPS homegraph's startup re-exec via
+# HOMEGRAPH_WASM_RELAUNCHED=1. Without this, on a multi-step task the agent
+# dives into Read/grep before homegraph finishes its ~2-3s startup (worse under
+# the CPU contention of a nested run) and runs with NO homegraph.
+#
+# Gotcha: claude's `system/init` snapshot can read status:"pending" / 0 tools
+# even when the server then connects fine — judge by ACTUAL homegraph usage in
+# parse-run.mjs's "by type", not the init line.
+#
+# Usage: ab-new-vs-baseline.sh <indexed-repo> "<task>" [baseline-ref]
+#   <indexed-repo>  a repo with a .homegraph index (copied per arm)
+#   "<task>"        an implementation task, e.g. "Add X to Y and wire it through"
+#   [baseline-ref]  git ref for the BEFORE build (default: HEAD~1)
+# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline)
+set -uo pipefail
+
+TARGET="${1:?usage: ab-new-vs-baseline.sh <indexed-repo> \"<task>\" [baseline-ref]}"
+TASK="${2:?task required}"
+BASE_REF="${3:-HEAD~1}"
+ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
+BIN="$ENGINE/dist/bin/homegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/ab-new-vs-baseline}"
+PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
+
+command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
+[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
+if ! git -C "$ENGINE" diff --quiet || ! git -C "$ENGINE" diff --cached --quiet; then
+  echo "engine repo has uncommitted changes — commit or stash first (this script checks files out)"; exit 1
+fi
+CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
+[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD — nothing to A/B"; exit 1; }
+
+# On exit: kill any eval daemons + restore the engine to HEAD.
+cleanup() {
+  pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
+  git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
+  ( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
+}
+trap cleanup EXIT
+
+mkdir -p "$OUT"
+echo "###### engine=$ENGINE  baseline=$BASE_REF"
+echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
+echo "###### target=$TARGET"
+echo "###### task=$TASK"
+echo
+
+# Two pristine copies so each arm starts clean (the agent edits its own copy).
+rm -rf "$OUT/t-new" "$OUT/t-base"
+rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$OUT/t-new/"
+cp -R "$OUT/t-new" "$OUT/t-base"
+
+prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for its socket
+  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
+  HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
+    && echo "  daemon warm: $1" || echo "  WARN: daemon never bound for $1 (arm may run without homegraph)"
+}
+
+run_arm() { # label, target-copy
+  local label="$1" tgt="$2" c="$OUT/mcp-$1.json"
+  # Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach.
+  printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
+  prewarm "$tgt"
+  echo "############## ARM [$label] ##############"
+  ( cd "$tgt" && claude -p "$TASK" \
+      --output-format stream-json --verbose --permission-mode bypassPermissions \
+      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
+      </dev/null > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
+  node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo "  (parse failed — see $OUT/run-$label.jsonl)"
+  pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
+  echo
+}
+
+echo "== NEW build (HEAD) =="
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "  built"
+node "$BIN" init "$OUT/t-new" >/dev/null 2>&1 && echo "  indexed t-new"
+run_arm new "$OUT/t-new"
+
+echo "== BASELINE build ($BASE_REF) =="
+# Per-file: a file ADDED since baseline has no pathspec on the ref — and a
+# single multi-file checkout with one bad pathspec checks out NOTHING, which
+# silently ran the NEW build in the baseline arm. Absent-on-baseline → remove.
+for f in $CHANGED; do
+  git -C "$ENGINE" checkout "$BASE_REF" -- "$f" 2>/dev/null || rm -f "$ENGINE/$f"
+done
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "  built"
+node "$BIN" init "$OUT/t-base" >/dev/null 2>&1 && echo "  indexed t-base"
+run_arm baseline "$OUT/t-base"
+
+echo "###### DONE. Compare the [new] vs [baseline] 'by type' counts above"
+echo "###### (especially Read vs mcp__homegraph__*). Full logs in: $OUT"

package/scripts/agent-eval/ab-sufficiency.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Sufficiency A/B: on a real understanding/flow question, WHEN the agent uses
+# homegraph (explore/node), does it still Read? Premise under test: explore/node
+# return source WITH line numbers, so a Read should not be needed.
+#
+# WITH homegraph (pre-warmed daemon, reliable nested attach) vs WITHOUT (empty
+# MCP, Read/Grep only), N runs each, on a throwaway copy of the repo. Reports
+# explore/node vs Read/Grep, and LISTS the files Read in the WITH arm so a true
+# sufficiency gap (an indexed source file) is distinguishable from out-of-scope
+# (configs, docs, a file homegraph didn't index).
+#
+# Usage: ab-sufficiency.sh <indexed-repo> "<question>" [runs-per-arm]
+# Env: AGENT_EVAL_OUT (default: /tmp/ab-sufficiency)
+set -uo pipefail
+REPO="${1:?usage: ab-sufficiency.sh <indexed-repo> \"<question>\" [runs]}"
+Q="${2:?question required}"
+RUNS="${3:-2}"
+ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
+BIN="$ENGINE/dist/bin/homegraph.js"
+OUT="${AGENT_EVAL_OUT:-/tmp/ab-sufficiency}"
+TGT="$OUT/target"
+command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
+[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO"; exit 1; }
+cleanup(){ pkill -9 -f "serve --mcp --path $TGT" 2>/dev/null; }
+trap cleanup EXIT
+mkdir -p "$OUT"
+( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
+
+# Throwaway copy + fresh index (the agent works here; a read-only question won't
+# edit, but isolate anyway). Excludes the source repo's index/build/vcs.
+rm -rf "$TGT"
+rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$REPO/" "$TGT/"
+node "$BIN" init "$TGT" >/dev/null 2>&1 && echo "indexed copy ($(node "$BIN" status --json 2>/dev/null | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{console.log(JSON.parse(s).fileCount+" files")}catch{console.log("?")}})' 2>/dev/null || echo '?'))"
+
+echo "###### repo=$REPO  runs/arm=$RUNS"
+echo "###### Q=$Q"; echo
+echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
+printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TGT" > "$OUT/mcp-cg.json"
+
+prewarm(){
+  pkill -9 -f "serve --mcp --path $TGT" 2>/dev/null
+  HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$TGT" </dev/null >/dev/null 2>&1 &
+  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$TGT" >/dev/null 2>&1
+}
+
+analyze(){
+  node -e '
+    const fs=require("fs");
+    const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
+    let ex=0,nf=0,ns=0,oc=0,gr=0,exposed="?";const reads=[];
+    for(const l of L){try{const o=JSON.parse(l);
+      if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/homegraph/.test(t)).length;
+      for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
+        if(b.name==="mcp__homegraph__homegraph_explore")ex++;
+        else if(b.name==="mcp__homegraph__homegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
+        else if(/mcp__homegraph__/.test(b.name))oc++;
+        else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
+        else if(b.name==="Grep")gr++;
+      }}catch{}}
+    console.log(`    explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr}  [cg exposed=${exposed}]`);
+  ' "$1"
+}
+
+run(){ # label, cfg, prewarm(0/1)
+  local label="$1" cfg="$2" pw="$3"
+  for i in $(seq 1 "$RUNS"); do
+    [ "$pw" = "1" ] && prewarm
+    ( cd "$TGT" && claude -p "$Q" --output-format stream-json --verbose \
+        --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
+        --strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
+    echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
+  done
+  echo
+}
+
+echo "== WITH homegraph (premise: explore/node used -> Read ~0) =="; run with "$OUT/mcp-cg.json" 1
+echo "== WITHOUT (Read/Grep only — the contrast) =="; run without "$OUT/mcp-empty.json" 0
+echo "###### DONE. In the WITH arm: are explore/node>0 and Read~0? Any Read of an INDEXED source file = sufficiency gap. Logs: $OUT"

package/scripts/agent-eval/arms-F.sh

@@ -1,21 +1,21 @@
-#!/usr/bin/env bash
-# Arm F (body-inlining trace + trace-first steering) across the same 6 repos as
-# arms-matrix.sh, so F vs B isolates the trace-enrichment effect (same surface,
-# old thin trace in B vs body-inlining trace here).
-set -uo pipefail
-H="$(cd "$(dirname "$0")" && pwd)"; RUNS="${RUNS:-2}"; C="${CORPUS:-/tmp/homegraph-corpus}"
-ROWS=(
-"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
-"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
-"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
-"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
-"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
-"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
-)
-ARM="${ARM:-F}"
-echo "### ARM $ARM START $(date) RUNS=$RUNS"
-for row in "${ROWS[@]}"; do
-  repo="${row%%|*}"; q="${row#*|}"
-  for r in $(seq 1 "$RUNS"); do bash "$H/run-arms.sh" "$repo" "$q" "$ARM" "$r"; done
-done
-echo "### ARM $ARM COMPLETE $(date)"
+#!/usr/bin/env bash
+# Arm F (body-inlining trace + trace-first steering) across the same 6 repos as
+# arms-matrix.sh, so F vs B isolates the trace-enrichment effect (same surface,
+# old thin trace in B vs body-inlining trace here).
+set -uo pipefail
+H="$(cd "$(dirname "$0")" && pwd)"; RUNS="${RUNS:-2}"; C="${CORPUS:-/tmp/homegraph-corpus}"
+ROWS=(
+"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
+"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
+"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
+"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
+"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
+"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
+)
+ARM="${ARM:-F}"
+echo "### ARM $ARM START $(date) RUNS=$RUNS"
+for row in "${ROWS[@]}"; do
+  repo="${row%%|*}"; q="${row#*|}"
+  for r in $(seq 1 "$RUNS"); do bash "$H/run-arms.sh" "$repo" "$q" "$ARM" "$r"; done
+done
+echo "### ARM $ARM COMPLETE $(date)"