cgraphx 1.4.1 → 2.0.0

package/scripts/add-lang/check-grammar.mjs

@@ -1,75 +0,0 @@
-#!/usr/bin/env node
-// Verify a tree-sitter grammar wasm is HEALTHY under the project's web-tree-sitter
-// runtime BEFORE writing an extractor. Prints the ABI version and parses a valid
-// sample many times in a multi-grammar context, to catch heap-corruption bugs
-// that silently drop nodes on every parse after the first.
-//
-// Why this exists: the tree-sitter-wasms Lua grammar is ABI 13 and corrupts the
-// shared WASM heap under web-tree-sitter 0.25 — Lua extraction degraded on every
-// file after the first (nested calls/imports vanished). The fix was to vendor the
-// upstream ABI-15 wasm. Run this on any new grammar first; if it FAILs, vendor a
-// newer build instead of using the tree-sitter-wasms one.
-//
-// Usage: node scripts/add-lang/check-grammar.mjs <lang|wasm-path> <valid-sample> [iterations]
-// Exit: 0 healthy, 1 corruption / parse errors, 2 could not run.
-// NOTE: the sample must be SYNTACTICALLY VALID — a broken sample fails for the
-//       wrong reason.
-
-import { readFileSync, existsSync } from 'node:fs';
-import { createRequire } from 'node:module';
-import { Parser, Language } from 'web-tree-sitter';
-
-const require = createRequire(import.meta.url);
-const fail = (code, msg) => { console.error(`[check-grammar] ${msg}`); process.exit(code); };
-
-const [token, sample, iterArg] = process.argv.slice(2);
-if (!token || !sample) fail(2, 'usage: check-grammar.mjs <lang|wasm-path> <valid-sample> [iterations]');
-if (!existsSync(sample)) fail(2, `sample not found: ${sample}`);
-const iters = iterArg ? parseInt(iterArg, 10) : 20;
-
-const SPECIAL = { csharp: 'c_sharp', 'c#': 'c_sharp' };
-function resolveWasm(t) {
-  if (t.endsWith('.wasm')) return existsSync(t) ? t : fail(2, `wasm not found: ${t}`);
-  const base = SPECIAL[t.toLowerCase()] ?? t.toLowerCase();
-  try { return require.resolve(`tree-sitter-wasms/out/tree-sitter-${base}.wasm`); } catch { /* try vendored */ }
-  const vendored = `src/extraction/wasm/tree-sitter-${base}.wasm`;
-  if (existsSync(vendored)) return vendored;
-  return fail(2, `no grammar for "${t}" — not in tree-sitter-wasms and not vendored`);
-}
-
-const wasmPath = resolveWasm(token);
-const source = readFileSync(sample, 'utf8');
-
-try { await Parser.init(); }
-catch { await Parser.init({ locateFile: () => require.resolve('web-tree-sitter/tree-sitter.wasm') }); }
-
-// Load a second, known-good grammar — the corruption surfaces under the
-// multi-grammar runtime that real indexing uses, not a single grammar in isolation.
-try { await Language.load(require.resolve('tree-sitter-wasms/out/tree-sitter-python.wasm')); } catch { /* ok */ }
-
-let language;
-try { language = await Language.load(wasmPath); }
-catch (e) { fail(2, `failed to load ${wasmPath}: ${e.message}`); }
-
-const parser = new Parser();
-parser.setLanguage(language);
-
-let ok = 0, err = 0;
-for (let i = 0; i < iters; i++) {
-  const tree = parser.parse(source);
-  if (tree.rootNode.hasError) err++; else ok++;
-}
-
-console.log(`grammar: ${wasmPath.split('/').pop()}`);
-console.log(`  ABI version: ${language.abiVersion}`);
-console.log(`  parses: ${ok} clean / ${err} with errors (of ${iters})`);
-if (err > 0) {
-  console.log(
-    `RESULT: FAIL — ${err}/${iters} parses produced ERROR trees on a valid sample. ` +
-    `This grammar corrupts under web-tree-sitter; vendor a newer (ABI 14/15) wasm ` +
-    `(see SKILL.md "Find a grammar"). Confirm your sample is syntactically valid first.`
-  );
-  process.exit(1);
-}
-console.log('RESULT: PASS — grammar parses cleanly and reuses safely.');
-process.exit(0);

package/scripts/add-lang/dump-ast.mjs

@@ -1,103 +0,0 @@
-#!/usr/bin/env node
-// Dump the tree-sitter AST for a sample file so you can write a LanguageExtractor
-// mapping. Loads a grammar .wasm directly via web-tree-sitter (the same runtime
-// codegraph uses) — you do NOT need to register the language first.
-//
-// Usage:
-//   node scripts/add-lang/dump-ast.mjs <lang|wasm-path> <sample-file> [--depth=N] [--full]
-// Examples:
-//   node scripts/add-lang/dump-ast.mjs lua sample.lua
-//   node scripts/add-lang/dump-ast.mjs src/extraction/wasm/tree-sitter-zig.wasm a.zig --depth=4
-//
-// Output: an indented AST (named nodes, with field names) followed by a
-// node-type FREQUENCY table. The frequency table is the payoff — it tells you
-// which node types to map to functionTypes / classTypes / importTypes / etc.
-
-import { readFileSync, existsSync } from 'node:fs';
-import { createRequire } from 'node:module';
-import { Parser, Language } from 'web-tree-sitter';
-
-const require = createRequire(import.meta.url);
-const fail = (msg) => { console.error(`[dump-ast] ${msg}`); process.exit(1); };
-
-const argv = process.argv.slice(2);
-const positional = argv.filter((a) => !a.startsWith('--'));
-const [langOrWasm, sampleFile] = positional;
-const depthFlag = argv.find((a) => a.startsWith('--depth='));
-const showAll = argv.includes('--full'); // also print anonymous (token) nodes
-const maxDepth = depthFlag ? parseInt(depthFlag.split('=')[1], 10) : (showAll ? Infinity : 8);
-
-if (!langOrWasm || !sampleFile) {
-  fail('usage: dump-ast.mjs <lang|wasm-path> <sample-file> [--depth=N] [--full]');
-}
-if (!existsSync(sampleFile)) fail(`sample file not found: ${sampleFile}`);
-
-// Language tokens whose tree-sitter-wasms filename differs from the token.
-const WASM_SPECIAL = { csharp: 'c_sharp', 'c#': 'c_sharp' };
-
-function resolveWasm(token) {
-  if (token.endsWith('.wasm')) {
-    if (!existsSync(token)) fail(`wasm not found: ${token}`);
-    return token;
-  }
-  const base = WASM_SPECIAL[token.toLowerCase()] ?? token.toLowerCase();
-  try {
-    return require.resolve(`tree-sitter-wasms/out/tree-sitter-${base}.wasm`);
-  } catch {
-    /* not in tree-sitter-wasms — try a vendored copy */
-  }
-  const vendored = `src/extraction/wasm/tree-sitter-${base}.wasm`;
-  if (existsSync(vendored)) return vendored;
-  fail(
-    `no grammar for "${token}" — not in tree-sitter-wasms and not vendored at ` +
-      `${vendored}. Pass an explicit .wasm path, or vendor one (see SKILL.md "Find a grammar").`
-  );
-}
-
-const wasmPath = resolveWasm(langOrWasm);
-const source = readFileSync(sampleFile, 'utf8');
-
-try {
-  await Parser.init();
-} catch {
-  await Parser.init({ locateFile: () => require.resolve('web-tree-sitter/tree-sitter.wasm') });
-}
-
-let language;
-try {
-  language = await Language.load(wasmPath);
-} catch (e) {
-  fail(`failed to load grammar ${wasmPath}: ${e.message}`);
-}
-
-const parser = new Parser();
-parser.setLanguage(language);
-const tree = parser.parse(source);
-
-const freq = new Map();
-const snippet = (node) => {
-  const t = node.text.replace(/\s+/g, ' ').trim();
-  return t.length > 48 ? `${t.slice(0, 48)}…` : t;
-};
-
-function walk(node, depth, fieldName) {
-  if (node.isNamed) freq.set(node.type, (freq.get(node.type) || 0) + 1);
-  if ((node.isNamed || showAll) && depth <= maxDepth) {
-    const field = fieldName ? `${fieldName}: ` : '';
-    const leaf = node.childCount === 0 ? `  "${snippet(node)}"` : '';
-    console.log(`${'  '.repeat(depth)}${field}${node.type}  @${node.startPosition.row + 1}:${node.startPosition.column}${leaf}`);
-  }
-  for (let i = 0; i < node.childCount; i++) {
-    const child = node.child(i);
-    if (child) walk(child, depth + 1, node.fieldNameForChild(i));
-  }
-}
-
-console.log(`\n# AST for ${sampleFile}  (grammar: ${wasmPath.split('/').pop()})\n`);
-walk(tree.rootNode, 0, null);
-
-console.log('\n# Node-type frequency (named nodes) — map the relevant ones in your extractor:\n');
-[...freq.entries()]
-  .sort((a, b) => b[1] - a[1])
-  .forEach(([type, n]) => console.log(`  ${String(n).padStart(5)}  ${type}`));
-console.log();

package/scripts/add-lang/verify-extraction.mjs

@@ -1,70 +0,0 @@
-#!/usr/bin/env node
-// Sanity-check that codegraph extracted REAL symbols (not just file/import nodes)
-// from a repo for a given language. Exits non-zero on a critical failure so it
-// can drive a write-extractor -> build -> re-check loop.
-//
-// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
-// Reads `cgraphx status <repo> --json` using whatever cgraphx is on PATH,
-// so it reflects the binary that built the index.
-//
-// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
-
-import { execFileSync } from 'node:child_process';
-
-const [repo, lang] = process.argv.slice(2);
-if (!repo || !lang) {
-  console.error('usage: verify-extraction.mjs <repo-path> <lang>');
-  process.exit(2);
-}
-
-let status;
-try {
-  const out = execFileSync('cgraphx', ['status', repo, '--json'], { encoding: 'utf8' });
-  status = JSON.parse(out);
-} catch (e) {
-  console.error(`[verify] could not read cgraphx status for ${repo}: ${e.message}`);
-  process.exit(2);
-}
-
-// Kinds that prove the extractor mapped AST node types (everything except
-// 'file' and 'import', which codegraph creates structurally for any language).
-const SYMBOL_KINDS = new Set([
-  'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
-  'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
-  'type_alias', 'namespace', 'route', 'component',
-]);
-
-const byKind = status.nodesByKind || {};
-const langs = status.languages || [];
-const files = status.fileCount || 0;
-const edges = status.edgeCount || 0;
-const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
-const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
-
-const checks = [];
-const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
-
-add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
-add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
-add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
-add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
-add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
-
-console.log(`\n# Extraction check — ${repo}  (lang=${lang}, backend=${status.backend})`);
-console.log(`  files=${files} nodes=${status.nodeCount} edges=${edges}`);
-console.log(`  nodesByKind: ${JSON.stringify(byKind)}\n`);
-for (const c of checks) console.log(`  ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
-
-const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
-const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
-console.log();
-if (critical.length) {
-  console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
-  process.exit(1);
-}
-if (soft.length) {
-  console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
-  process.exit(0);
-}
-console.log('RESULT: PASS — extraction looks healthy.');
-process.exit(0);

package/scripts/agent-eval/ab-adoption.sh

@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-# Does the agent PICK codegraph_node to read a file, vs the built-in Read tool?
-# Build A/B: NEW build (HEAD, codegraph_node has Read parity) vs BASELINE build
-# (a ref where it doesn't), BOTH cgraphx-attached + pre-warmed, same task. The
-# metric is tool CHOICE: Read calls vs codegraph_node[file] calls per run.
-#
-# Usage: ab-adoption.sh <indexed-repo> "<task>" [runs-per-arm] [baseline-ref]
-# Env: AGENT_EVAL_OUT (default: /tmp/ab-adoption)
-set -uo pipefail
-TARGET="${1:?usage: ab-adoption.sh <indexed-repo> \"<task>\" [runs] [baseline-ref]}"
-TASK="${2:?task required}"
-RUNS="${3:-2}"
-BASE_REF="${4:-HEAD~1}"
-ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
-BIN="$ENGINE/dist/bin/codegraph.js"
-OUT="${AGENT_EVAL_OUT:-/tmp/ab-adoption}"
-
-command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
-[ -d "$TARGET/.cgraphx" ] || { echo "target not indexed: run 'cgraphx init $TARGET' first"; exit 1; }
-git -C "$ENGINE" diff --quiet && git -C "$ENGINE" diff --cached --quiet || { echo "engine has uncommitted changes — commit/stash first"; exit 1; }
-CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
-[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD"; exit 1; }
-
-cleanup() {
-  pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
-  git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
-  ( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
-}
-trap cleanup EXIT
-mkdir -p "$OUT"
-echo "###### target=$TARGET  runs/arm=$RUNS  baseline=$BASE_REF"
-echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
-echo "###### task=$TASK"; echo
-
-prewarm() {
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
-  CGRAPHX_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.cgraphx/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
-}
-
-# Per-run tool-choice counts: Read vs codegraph_node[file] vs [symbol].
-count() {
-  node -e '
-    const fs=require("fs");
-    const lines=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
-    let read=0,cgFile=0,cgSym=0,cgOther=0,exposed="?";
-    for(const l of lines){try{const o=JSON.parse(l);
-      if(o.type==="system"&&o.subtype==="init"){exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;}
-      const blocks=o.message?.content||[];
-      for(const b of (Array.isArray(blocks)?blocks:[])){
-        if(b.type!=="tool_use")continue;
-        if(b.name==="Read")read++;
-        else if(b.name==="mcp__cgraphx__codegraph_node"){ if(b.input&&b.input.symbol)cgSym++; else cgFile++; }
-        else if(/mcp__cgraphx__/.test(b.name))cgOther++;
-      }
-    }catch{}}
-    console.log(`    Read=${read}  codegraph_node[file]=${cgFile}  codegraph_node[symbol]=${cgSym}  other_cg=${cgOther}  (cg exposed=${exposed})`);
-  ' "$1"
-}
-
-run_arm() { # label, N
-  local label="$1" n="$2"
-  local c="$OUT/mcp-$label.json"
-  for i in $(seq 1 "$n"); do
-    local tgt="$OUT/t-$label-$i"
-    rm -rf "$tgt"
-    rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .cgraphx "$TARGET/" "$tgt/"
-    node "$BIN" init "$tgt" >/dev/null 2>&1
-    printf '{"mcpServers":{"cgraphx":{"command":"env","args":["CGRAPHX_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
-    prewarm "$tgt"
-    echo "----- [$label] run $i -----"
-    ( cd "$tgt" && claude -p "$TASK" \
-        --output-format stream-json --verbose --permission-mode bypassPermissions \
-        --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
-        </dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
-    count "$OUT/run-$label-$i.jsonl"
-    pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
-  done
-  echo
-}
-
-echo "== NEW build (HEAD: codegraph_node has Read parity) =="
-( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
-run_arm new "$RUNS"
-
-echo "== BASELINE build ($BASE_REF) =="
-git -C "$ENGINE" checkout "$BASE_REF" -- $CHANGED
-( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
-run_arm baseline "$RUNS"
-
-echo "###### DONE — compare [new] vs [baseline]: does codegraph_node[file] rise / Read fall? Logs: $OUT"

package/scripts/agent-eval/ab-hook.sh

@@ -1,86 +0,0 @@
-#!/usr/bin/env bash
-# A/B the PreToolUse(Read) REDIRECT hook (P1): does steering Read → codegraph_node
-# file-view actually move the agent off Read during implementation? BOTH arms use
-# the CURRENT build with cgraphx attached and pre-warmed; the only difference is
-# the hook. Isolates the hook's behavioral effect from the build/file-view change
-# (use ab-new-vs-baseline.sh for the build A/B).
-#
-#   arm [nohook] — cgraphx on, no hook   (does the better file-view get picked on its own?)
-#   arm [hook]   — cgraphx on, + redirect hook   (does routing close it?)
-#
-# Reliable attach (works nested): each arm pre-warms a persistent daemon and skips
-# the startup re-exec (CGRAPHX_WASM_RELAUNCHED=1), so claude connects before the
-# agent's first turn. Judge by ACTUAL cgraphx usage in parse-run.mjs's "by type",
-# not claude's init snapshot (which can read pending even when it then connects).
-#
-# Usage: ab-hook.sh <indexed-repo> "<implementation task>" [runs-per-arm]
-#   <indexed-repo>  a repo with a .cgraphx index (copied per arm; never mutated)
-#   "<task>"        a GENUINELY-NEW implementation task (verify it isn't already done)
-#   [runs-per-arm]  default 2 (n=1 is noisy — the doctrine says >=2)
-# Env: AGENT_EVAL_OUT (default: /tmp/ab-hook)
-set -uo pipefail
-
-TARGET="${1:?usage: ab-hook.sh <indexed-repo> \"<task>\" [runs-per-arm]}"
-TASK="${2:?task required}"
-RUNS="${3:-2}"
-ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
-BIN="$ENGINE/dist/bin/codegraph.js"
-HOOK="$ENGINE/scripts/agent-eval/redirect-read-hook.sh"
-OUT="${AGENT_EVAL_OUT:-/tmp/ab-hook}"
-PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
-
-command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
-command -v jq >/dev/null || { echo "jq not on PATH (the hook needs it)"; exit 1; }
-[ -d "$TARGET/.cgraphx" ] || { echo "target not indexed: run 'cgraphx init $TARGET' first"; exit 1; }
-chmod +x "$HOOK"
-
-cleanup() { pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
-trap cleanup EXIT
-
-mkdir -p "$OUT"
-echo "###### engine=$ENGINE"
-echo "###### target=$TARGET   runs/arm=$RUNS"
-echo "###### task=$TASK"
-echo
-
-( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
-
-# A settings file carrying ONLY the PreToolUse(Read) redirect hook.
-HOOK_SETTINGS="$OUT/hook-settings.json"
-jq -n --arg cmd "bash $HOOK" \
-  '{hooks:{PreToolUse:[{matcher:"Read",hooks:[{type:"command",command:$cmd}]}]}}' > "$HOOK_SETTINGS"
-
-prewarm() { # target — spawn a persistent daemon and wait for its socket
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
-  CGRAPHX_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.cgraphx/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
-    && echo "  daemon warm: $1" || echo "  WARN: daemon never bound for $1"
-}
-
-run_one() { # arm-label, run-index, use-hook(0|1)
-  local label="$1" idx="$2" hook="$3"
-  local tgt="$OUT/t-$label-$idx" c="$OUT/mcp-$label.json"
-  rm -rf "$tgt"
-  rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .cgraphx "$TARGET/" "$tgt/"
-  node "$BIN" init "$tgt" >/dev/null 2>&1
-  printf '{"mcpServers":{"cgraphx":{"command":"env","args":["CGRAPHX_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
-  prewarm "$tgt"
-  local extra=()
-  [ "$hook" = "1" ] && extra=(--settings "$HOOK_SETTINGS")
-  echo "----- [$label] run $idx -----"
-  # ${extra[@]+...} guard: bash 3.2 (macOS) under `set -u` errors on an empty
-  # array expansion otherwise, which would skip the no-hook arm's claude run.
-  ( cd "$tgt" && claude -p "$TASK" \
-      --output-format stream-json --verbose --permission-mode bypassPermissions \
-      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
-      </dev/null > "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" )
-  node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo "  (parse failed — see $OUT/run-$label-$idx.jsonl)"
-  pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
-  echo
-}
-
-for i in $(seq 1 "$RUNS"); do run_one nohook "$i" 0; done
-for i in $(seq 1 "$RUNS"); do run_one hook   "$i" 1; done
-
-echo "###### DONE. Compare [nohook] vs [hook] 'by type' — Read should fall and"
-echo "###### mcp__cgraphx__codegraph_node should rise in the [hook] arm. Logs: $OUT"

package/scripts/agent-eval/ab-impl.sh

@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-# Sufficiency A/B for an IMPLEMENTATION task (the agent edits): when it uses
-# codegraph (explore/node) to understand before editing, does it still Read? Like
-# ab-sufficiency.sh but copies+indexes a FRESH target per run (the agent mutates
-# it), so runs don't see each other's edits.
-#
-# WITH codegraph (pre-warmed) vs WITHOUT (empty MCP), N runs each. Reports
-# explore/node vs Read/Grep + the files Read, and whether the build still passes.
-#
-# Usage: ab-impl.sh <indexed-repo> "<task>" [runs] [build-cmd]
-# Env: AGENT_EVAL_OUT (default: /tmp/ab-impl)
-set -uo pipefail
-REPO="${1:?usage: ab-impl.sh <indexed-repo> \"<task>\" [runs] [build-cmd]}"
-Q="${2:?task required}"
-RUNS="${3:-2}"
-BUILD_CMD="${4:-}"
-ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
-BIN="$ENGINE/dist/bin/codegraph.js"
-OUT="${AGENT_EVAL_OUT:-/tmp/ab-impl}"
-command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
-[ -d "$REPO/.cgraphx" ] || { echo "no .cgraphx index at $REPO"; exit 1; }
-cleanup(){ pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
-trap cleanup EXIT
-mkdir -p "$OUT"
-( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built engine"
-echo "###### repo=$REPO  runs/arm=$RUNS"
-echo "###### task=$Q"; echo
-echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
-
-prewarm(){
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
-  CGRAPHX_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.cgraphx/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
-}
-
-analyze(){
-  node -e '
-    const fs=require("fs");
-    const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
-    let ex=0,nf=0,ns=0,oc=0,gr=0,ed=0,exposed="?";const reads=[];
-    for(const l of L){try{const o=JSON.parse(l);
-      if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;
-      for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
-        if(b.name==="mcp__cgraphx__codegraph_explore")ex++;
-        else if(b.name==="mcp__cgraphx__codegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
-        else if(/mcp__cgraphx__/.test(b.name))oc++;
-        else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
-        else if(b.name==="Grep")gr++;
-        else if(b.name==="Edit"||b.name==="Write")ed++;
-      }}catch{}}
-    console.log(`    explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr} Edit=${ed}  [cg exposed=${exposed}]`);
-  ' "$1"
-}
-
-run(){ # label, withCodegraph(0/1)
-  local label="$1" wcg="$2"
-  for i in $(seq 1 "$RUNS"); do
-    local tgt="$OUT/t-$label-$i" cfg="$OUT/mcp-$label.json"
-    rm -rf "$tgt"
-    rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .cgraphx "$REPO/" "$tgt/"
-    node "$BIN" init "$tgt" >/dev/null 2>&1
-    if [ "$wcg" = "1" ]; then
-      printf '{"mcpServers":{"cgraphx":{"command":"env","args":["CGRAPHX_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$cfg"
-      prewarm "$tgt"
-    else cp "$OUT/mcp-empty.json" "$cfg"; fi
-    ( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \
-        --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
-        --strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
-    echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
-    if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo "      build: PASS" || echo "      build: FAIL" ); fi
-    pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
-  done
-  echo
-}
-
-echo "== WITH codegraph =="; run with 1
-echo "== WITHOUT (Read/Grep only) =="; run without 0
-echo "###### DONE: $OUT"

package/scripts/agent-eval/ab-new-vs-baseline.sh

@@ -1,102 +0,0 @@
-#!/usr/bin/env bash
-# A/B a cgraphx retrieval/steering change: the NEW build (current HEAD) vs a
-# BASELINE build (a git ref) — BOTH with cgraphx attached — on the same
-# implementation task, measuring how many Read vs cgraphx calls the agent
-# makes. ISOLATES the change (unlike run-all.sh's with-vs-without). The agent
-# works on a throwaway copy of the target, so your repos are never touched.
-#
-# Reliable attach (works even when this is itself run nested inside a Claude
-# session): each arm PRE-WARMS a persistent cgraphx daemon for its target so
-# claude connects to an already-bound, index-loaded daemon instantly — before
-# the agent's first turn — and SKIPS cgraphx's startup re-exec via
-# CGRAPHX_WASM_RELAUNCHED=1. Without this, on a multi-step task the agent
-# dives into Read/grep before cgraphx finishes its ~2-3s startup (worse under
-# the CPU contention of a nested run) and runs with NO codegraph.
-#
-# Gotcha: claude's `system/init` snapshot can read status:"pending" / 0 tools
-# even when the server then connects fine — judge by ACTUAL cgraphx usage in
-# parse-run.mjs's "by type", not the init line.
-#
-# Usage: ab-new-vs-baseline.sh <indexed-repo> "<task>" [baseline-ref]
-#   <indexed-repo>  a repo with a .cgraphx index (copied per arm)
-#   "<task>"        an implementation task, e.g. "Add X to Y and wire it through"
-#   [baseline-ref]  git ref for the BEFORE build (default: HEAD~1)
-# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline)
-set -uo pipefail
-
-TARGET="${1:?usage: ab-new-vs-baseline.sh <indexed-repo> \"<task>\" [baseline-ref]}"
-TASK="${2:?task required}"
-BASE_REF="${3:-HEAD~1}"
-ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
-BIN="$ENGINE/dist/bin/codegraph.js"
-OUT="${AGENT_EVAL_OUT:-/tmp/ab-new-vs-baseline}"
-PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
-
-command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
-[ -d "$TARGET/.cgraphx" ] || { echo "target not indexed: run 'cgraphx init $TARGET' first"; exit 1; }
-if ! git -C "$ENGINE" diff --quiet || ! git -C "$ENGINE" diff --cached --quiet; then
-  echo "engine repo has uncommitted changes — commit or stash first (this script checks files out)"; exit 1
-fi
-CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
-[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD — nothing to A/B"; exit 1; }
-
-# On exit: kill any eval daemons + restore the engine to HEAD.
-cleanup() {
-  pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
-  git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
-  ( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
-}
-trap cleanup EXIT
-
-mkdir -p "$OUT"
-echo "###### engine=$ENGINE  baseline=$BASE_REF"
-echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
-echo "###### target=$TARGET"
-echo "###### task=$TASK"
-echo
-
-# Two pristine copies so each arm starts clean (the agent edits its own copy).
-rm -rf "$OUT/t-new" "$OUT/t-base"
-rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .cgraphx "$TARGET/" "$OUT/t-new/"
-cp -R "$OUT/t-new" "$OUT/t-base"
-
-prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for its socket
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null
-  CGRAPHX_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.cgraphx/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
-    && echo "  daemon warm: $1" || echo "  WARN: daemon never bound for $1 (arm may run without cgraphx)"
-}
-
-run_arm() { # label, target-copy
-  local label="$1" tgt="$2" c="$OUT/mcp-$1.json"
-  # Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach.
-  printf '{"mcpServers":{"cgraphx":{"command":"env","args":["CGRAPHX_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
-  prewarm "$tgt"
-  echo "############## ARM [$label] ##############"
-  ( cd "$tgt" && claude -p "$TASK" \
-      --output-format stream-json --verbose --permission-mode bypassPermissions \
-      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
-      </dev/null > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
-  node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo "  (parse failed — see $OUT/run-$label.jsonl)"
-  pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
-  echo
-}
-
-echo "== NEW build (HEAD) =="
-( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "  built"
-node "$BIN" init "$OUT/t-new" >/dev/null 2>&1 && echo "  indexed t-new"
-run_arm new "$OUT/t-new"
-
-echo "== BASELINE build ($BASE_REF) =="
-# Per-file: a file ADDED since baseline has no pathspec on the ref — and a
-# single multi-file checkout with one bad pathspec checks out NOTHING, which
-# silently ran the NEW build in the baseline arm. Absent-on-baseline → remove.
-for f in $CHANGED; do
-  git -C "$ENGINE" checkout "$BASE_REF" -- "$f" 2>/dev/null || rm -f "$ENGINE/$f"
-done
-( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "  built"
-node "$BIN" init "$OUT/t-base" >/dev/null 2>&1 && echo "  indexed t-base"
-run_arm baseline "$OUT/t-base"
-
-echo "###### DONE. Compare the [new] vs [baseline] 'by type' counts above"
-echo "###### (especially Read vs mcp__cgraphx__*). Full logs in: $OUT"