@colbymchenry/codegraph 0.7.10 → 0.8.0

package/scripts/agent-eval/itrun.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
+# agent to finish, then print the tool-call breakdown from the session logs.
+#
+# Why interactive (not `claude -p`): headless print-mode picks the
+# general-purpose subagent, while real interactive sessions delegate to the
+# Explore subagent (or drive codegraph from the main thread). Only the
+# interactive TUI reproduces the behavior users actually see. (Idle-detection
+# technique borrowed from devpit's WaitForIdle.)
+#
+# Usage: itrun.sh <repo-path> <label> "<prompt>"
+# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
+# Requires: tmux 3.0+, a logged-in `claude` CLI, codegraph MCP configured.
+set -uo pipefail
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+SESSION="cgt_${LABEL}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/itrun-${LABEL}.txt"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
+
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Wide pane so the TUI doesn't hard-wrap tool lines.
+tmux new-session -d -s "$SESSION" -x 230 -y 60
+tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
+
+# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
+# welcome screen seconds before the input actually accepts keystrokes, so this is
+# necessary but NOT sufficient — the type-and-verify loop below is what proves
+# the input is live.
+ready=0
+for _ in $(seq 1 120); do
+  cap | grep -q "❯" && { ready=1; break; }
+  sleep 0.5
+done
+[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
+# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
+# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
+# cleared before the type-and-verify loop or keystrokes land on the menu.
+for _ in $(seq 1 20); do
+  cap | grep -q "trust this folder" || break
+  tmux send-keys -t "$SESSION" Enter
+  sleep 1
+done
+
+# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
+# landed in the input box, retry if it didn't (handles the early-❯ race where
+# the welcome screen shows the prompt glyph but MCP init is still eating keys).
+needle="${PROMPT:0:24}"
+typed=0
+for _ in $(seq 1 30); do
+  tmux send-keys -l -t "$SESSION" "$PROMPT"
+  sleep 1
+  if cap | grep -Fq "$needle"; then typed=1; break; fi
+  # Clear whatever partial text may have landed, then retry.
+  tmux send-keys -t "$SESSION" C-u
+  sleep 1
+done
+[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+sleep 0.5
+tmux send-keys -t "$SESSION" Enter
+
+# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
+# EVERY working state shows — both the pre-stream thinking phase
+# "(8s · thinking with max effort)" and the streaming phase
+# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
+# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
+# (some TUI versions/states show one but not the others).
+BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
+
+# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
+# fail loudly rather than silently reporting an empty run.
+started=0
+for _ in $(seq 1 120); do
+  cap | grep -qE "$BUSY_RE" && { started=1; break; }
+  sleep 0.5
+done
+[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
+
+# Poll for idle: not busy AND ❯ present, for 10 consecutive polls (~5s) to ride
+# out mid-conversation thinking gaps that briefly drop the spinner. Up to ~15min.
+consec=0
+for _ in $(seq 1 1800); do
+  pane=$(cap)
+  if echo "$pane" | grep -qE "$BUSY_RE"; then
+    consec=0
+  elif echo "$pane" | grep -q "❯"; then
+    consec=$((consec+1)); [ "$consec" -ge 10 ] && break
+  else
+    consec=0
+  fi
+  sleep 0.5
+done
+sleep 1
+
+tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
+echo "captured $(wc -l < "$OUT") lines -> $OUT"
+grep -oE "Done \([^)]*\)" "$OUT" | tail -1
+grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
+tmux kill-session -t "$SESSION" 2>/dev/null
+
+# Clean tool breakdown from the session logs (main + subagents).
+node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true

package/scripts/agent-eval/parse-run.mjs

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
+import { readFileSync } from 'fs';
+const file = process.argv[2];
+const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+
+const toolCalls = [];
+let result = null;
+let initTools = null;
+
+for (const line of lines) {
+  let ev;
+  try { ev = JSON.parse(line); } catch { continue; }
+  if (ev.type === 'system' && ev.subtype === 'init') {
+    initTools = (ev.tools || []).filter(t => /codegraph/.test(t));
+  }
+  if (ev.type === 'assistant' && ev.message?.content) {
+    for (const block of ev.message.content) {
+      if (block.type === 'tool_use') {
+        let detail = '';
+        if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
+        else if (/codegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
+        else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
+        else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
+        toolCalls.push(`${block.name}${detail}`);
+      }
+    }
+  }
+  if (ev.type === 'result') result = ev;
+}
+
+console.log(`\n=== ${file.split('/').pop()} ===`);
+console.log(`codegraph tools exposed: ${initTools ? initTools.length : '?'}`);
+console.log(`\nTool calls (${toolCalls.length}):`);
+const counts = {};
+for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
+console.log('  by type:', JSON.stringify(counts));
+toolCalls.forEach((tc, i) => console.log(`  ${i+1}. ${tc}`));
+
+if (result) {
+  const u = result.usage || {};
+  const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
+  console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
+  console.log(`  tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
+}

package/scripts/agent-eval/parse-session.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+// Parse the newest Claude Code session log for a project + its subagent logs,
+// and report the tool-call breakdown (main + subagents). Works for interactive
+// runs (driven via itrun.sh) — Claude Code writes full transcripts to
+// ~/.claude/projects/<escaped-cwd>/<session>.jsonl with subagents/ alongside.
+import { readFileSync, readdirSync, statSync, existsSync, realpathSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+const projectArg = process.argv[2];
+if (!projectArg) { console.error('usage: parse-session.mjs <project-dir>'); process.exit(1); }
+
+// Claude Code escapes the (real) cwd by replacing every "/" with "-".
+const real = realpathSync(projectArg);
+const escaped = real.replace(/\//g, '-');
+const projDir = join(homedir(), '.claude', 'projects', escaped);
+if (!existsSync(projDir)) { console.error('no session logs at', projDir); process.exit(1); }
+
+// Newest top-level session .jsonl
+const sessions = readdirSync(projDir)
+  .filter(f => f.endsWith('.jsonl'))
+  .map(f => ({ f, m: statSync(join(projDir, f)).mtimeMs }))
+  .sort((a, b) => b.m - a.m);
+if (sessions.length === 0) { console.error('no .jsonl sessions in', projDir); process.exit(1); }
+const sessionId = sessions[0].f.replace('.jsonl', '');
+
+function tally(file) {
+  const counts = {};
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const content = ev.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const b of content) {
+      if (b.type === 'tool_use') counts[b.name] = (counts[b.name] || 0) + 1;
+    }
+  }
+  return counts;
+}
+
+// Sum token usage from a transcript. The TUI's "Done (…Xk tokens…)" line only
+// covers a subagent's throughput; this works for main-thread runs too and is
+// consistent across both paths. `gen` = output, `fresh` = uncached input
+// (input + cache_creation), `cached` = cache reads (≈free), `total` = all.
+function sumTokens(file) {
+  const t = { gen: 0, fresh: 0, cached: 0 };
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    let ev; try { ev = JSON.parse(line); } catch { continue; }
+    const u = ev.message?.usage;
+    if (!u) continue;
+    t.gen += u.output_tokens || 0;
+    t.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+    t.cached += u.cache_read_input_tokens || 0;
+  }
+  return t;
+}
+
+const mainCounts = tally(join(projDir, sessionId + '.jsonl'));
+
+// Subagent transcripts live under <session>/subagents/*.jsonl
+const subDir = join(projDir, sessionId, 'subagents');
+const subCounts = {};
+let subAgentFiles = 0;
+if (existsSync(subDir)) {
+  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) {
+    subAgentFiles++;
+    const c = tally(join(subDir, f));
+    for (const [k, v] of Object.entries(c)) subCounts[k] = (subCounts[k] || 0) + v;
+  }
+}
+
+const fmt = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])
+  .map(([k, v]) => `    ${String(v).padStart(3)}  ${k}`).join('\n') || '    (none)';
+
+console.log(`session: ${sessionId}`);
+console.log(`\nMAIN thread tools:\n${fmt(mainCounts)}`);
+console.log(`\nSUBAGENT tools (${subAgentFiles} subagent transcript${subAgentFiles === 1 ? '' : 's'}):\n${fmt(subCounts)}`);
+
+const explore = subCounts['mcp__codegraph__codegraph_explore'] || mainCounts['mcp__codegraph__codegraph_explore'] || 0;
+const reads = (subCounts['Read'] || 0) + (mainCounts['Read'] || 0);
+const greps = (subCounts['Grep'] || 0) + (mainCounts['Grep'] || 0) + (subCounts['Bash'] || 0) + (mainCounts['Bash'] || 0);
+console.log(`\nVERDICT: codegraph_explore used ${explore}x | Read ${reads} | Grep/Bash ${greps}`);
+
+// Token totals (main + subagents), consistent across main-thread and subagent runs.
+const tok = { gen: 0, fresh: 0, cached: 0 };
+const addTok = (t) => { tok.gen += t.gen; tok.fresh += t.fresh; tok.cached += t.cached; };
+addTok(sumTokens(join(projDir, sessionId + '.jsonl')));
+if (existsSync(subDir)) {
+  for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) addTok(sumTokens(join(subDir, f)));
+}
+const k = (n) => (n / 1000).toFixed(1) + 'k';
+console.log(`TOKENS: gen ${k(tok.gen)} | fresh-in ${k(tok.fresh)} | cached-in ${k(tok.cached)} | billable≈ ${k(tok.gen + tok.fresh)}`);

package/scripts/agent-eval/run-agent.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Headless Claude Code run against a repo with codegraph MCP, capturing the
+# full stream-json so we can see tool calls + token usage. Complements the
+# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
+# tokens/cost, but defaults to the general-purpose subagent (not Explore).
+# To force the Explore path, ask for it in the prompt.
+#
+# Usage: run-agent.sh <repo-path> <label> "<prompt>"
+# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (codegraph dist binary)
+set -uo pipefail
+
+REPO="$1"; LABEL="$2"; PROMPT="$3"
+CG_BIN="${CG_BIN:-$(command -v codegraph || echo /usr/local/bin/codegraph)}"
+OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
+OUT="$OUT_DIR/run-${LABEL}.jsonl"
+
+MCP_CONFIG=$(cat <<JSON
+{"mcpServers":{"codegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
+JSON
+)
+
+echo "→ running [$LABEL] in $REPO"
+cd "$REPO" || exit 1
+
+claude -p "$PROMPT" \
+  --output-format stream-json --verbose \
+  --permission-mode bypassPermissions \
+  --model opus \
+  --max-budget-usd 2 \
+  --strict-mcp-config --mcp-config "$MCP_CONFIG" \
+  > "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
+
+echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
+node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true

package/scripts/agent-eval/run-all.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# With/without A/B (and optional interactive) eval for a codegraph version on a
+# repo. Codegraph is the ONLY variable: both arms launch claude with
+# --strict-mcp-config — with = codegraph-only MCP (pointed at $CG_BIN),
+# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
+#
+# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
+# Env:   CG_BIN          codegraph binary (default: command -v codegraph)
+#        AGENT_EVAL_OUT  output dir (default: /tmp/agent-eval)
+set -uo pipefail
+
+REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
+Q="${2:?question required}"
+MODE="${3:-headless}"
+CG_BIN="${CG_BIN:-$(command -v codegraph)}"
+OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
+HARNESS="$(cd "$(dirname "$0")" && pwd)"
+mkdir -p "$OUT"
+
+[ -n "$CG_BIN" ] || { echo "no codegraph binary on PATH (set CG_BIN)"; exit 1; }
+[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO — index it first"; exit 1; }
+case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
+
+# MCP config files (path form avoids inline-JSON quoting through tmux).
+cat > "$OUT/mcp-codegraph.json" <<JSON
+{"mcpServers":{"codegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
+JSON
+echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
+
+echo "###### codegraph: $CG_BIN"
+echo "###### repo:      $REPO"
+echo "###### question:  $Q"
+echo
+
+# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
+headless() {
+  local label="$1" cfg="$2"
+  echo "############################## HEADLESS [$label] ##############################"
+  ( cd "$REPO" && claude -p "$Q" \
+      --output-format stream-json --verbose \
+      --permission-mode bypassPermissions \
+      --model opus \
+      --max-budget-usd 4 \
+      --strict-mcp-config --mcp-config "$cfg" \
+      > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
+  echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
+  tail -2 "$OUT/run-$label.err" 2>/dev/null
+  node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
+  echo
+}
+
+if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
+  headless "headless-with"    "$OUT/mcp-codegraph.json"
+  headless "headless-without" "$OUT/mcp-empty.json"
+fi
+
+if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
+  echo "############################## INTERACTIVE [with] ##############################"
+  CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-codegraph.json" \
+    bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
+  echo
+  echo "############################## INTERACTIVE [without] ##############################"
+  CLAUDE_EXTRA_ARGS="--model opus --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
+    bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
+  echo
+fi
+echo "############################## RUN-ALL COMPLETE ##############################"

package/scripts/extract-release-notes.mjs

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+/**
+ * Extract a release-notes block from CHANGELOG.md for a given version
+ * (or unwrap text supplied on stdin), then join hard-wrapped paragraphs.
+ *
+ * Why: GitHub renders release-note Markdown with GFM hard breaks, so
+ * every `\n` becomes `<br>`. The CHANGELOG is hard-wrapped at ~75
+ * chars for readable diffs, which then renders as awkward visible
+ * line breaks on the release page. This script joins indented
+ * continuation lines into a single line per bullet so the GFM
+ * renderer produces clean paragraphs.
+ *
+ * Repo-level CHANGELOG.md viewing is unaffected (CommonMark treats
+ * newlines as spaces there).
+ *
+ * Usage:
+ *   extract-release-notes.mjs <version>     # read CHANGELOG.md
+ *   extract-release-notes.mjs --stdin       # read from stdin (any text)
+ */
+
+import { readFileSync } from 'fs';
+
+const arg = process.argv[2];
+if (!arg) {
+  console.error('usage: extract-release-notes.mjs <version> | --stdin');
+  process.exit(1);
+}
+
+let block;
+if (arg === '--stdin') {
+  block = readFileSync(0, 'utf8').replace(/\r\n?/g, '\n').split('\n');
+} else {
+  const version = arg;
+  const escaped = version.replace(/\./g, '\\.');
+  const headerRe = new RegExp(`^## \\[${escaped}\\]`);
+  const anyHeaderRe = /^## \[/;
+  const lines = readFileSync('CHANGELOG.md', 'utf8').split('\n');
+  const start = lines.findIndex((l) => headerRe.test(l));
+  if (start === -1) {
+    console.error(`no '## [${version}]' entry found in CHANGELOG.md`);
+    process.exit(1);
+  }
+  const after = lines.findIndex((l, i) => i > start && anyHeaderRe.test(l));
+  block = lines.slice(start, after === -1 ? lines.length : after);
+}
+
+// Track a stack of `{ indent: number }` frames so a continuation line
+// can attach to the right ancestor. Handles the post-nested-list
+// continuation pattern:
+//
+//     - top-level
+//         - nested
+//       back to top-level  <- 2-space indent, joins the top-level bullet
+const out = [];
+let buf = '';
+let stack = [];
+
+function flushBuf() {
+  if (buf !== '') {
+    out.push(buf);
+    buf = '';
+  }
+}
+
+function leadingSpaces(s) {
+  const m = s.match(/^(\s*)/);
+  return m ? m[1].length : 0;
+}
+
+// Bullets: `-`, `*`, `digit.` only. `+` is intentionally excluded — the
+// CHANGELOG uses literal `+` inline (`config + instructions`) and we
+// don't want to misread those as nested bullets.
+const listItemRe = /^(\s*)([-*]|\d+\.)\s+/;
+const fenceRe = /^\s*```/;
+
+let inFence = false;
+
+for (const line of block) {
+  // Fenced code blocks: pass through verbatim, no joining.
+  if (fenceRe.test(line)) {
+    flushBuf();
+    stack = [];
+    out.push(line);
+    inFence = !inFence;
+    continue;
+  }
+  if (inFence) {
+    out.push(line);
+    continue;
+  }
+  if (/^\s*$/.test(line)) {
+    flushBuf();
+    out.push('');
+    continue;
+  }
+  if (/^#/.test(line)) {
+    flushBuf();
+    stack = [];
+    out.push(line);
+    continue;
+  }
+  const itemMatch = line.match(listItemRe);
+  if (itemMatch) {
+    flushBuf();
+    const indent = itemMatch[1].length;
+    while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
+      stack.pop();
+    }
+    stack.push({ indent });
+    buf = line;
+    continue;
+  }
+  if (/^\s/.test(line)) {
+    const indent = leadingSpaces(line);
+    while (stack.length > 1 && stack[stack.length - 1].indent >= indent) {
+      flushBuf();
+      stack.pop();
+    }
+    const trimmed = line.replace(/^\s+/, '');
+    buf = buf === '' ? trimmed : `${buf} ${trimmed}`;
+    continue;
+  }
+  flushBuf();
+  stack = [];
+  out.push(line);
+}
+flushBuf();
+
+process.stdout.write(out.join('\n'));
+if (!out[out.length - 1]?.endsWith('\n')) process.stdout.write('\n');

package/scripts/release.sh

@@ -30,13 +30,11 @@ if ! grep -q "^## \[${VERSION}\]" CHANGELOG.md; then
   exit 1
 fi
 
-NOTES=$(awk -v v="${VERSION}" '
-  /^## \[/ {
-    if (p) exit
-    if ($0 ~ "^## \\[" v "\\]") p = 1
-  }
-  p
-' CHANGELOG.md)
+# Extract notes with paragraph unwrapping — GitHub Releases render with
+# GFM hard-breaks, so the CHANGELOG's hard-wrapped lines would show as
+# visible `<br>` breaks otherwise. The helper joins continuation lines
+# into a single line per bullet.
+NOTES=$(node scripts/extract-release-notes.mjs "${VERSION}")
 
 if [ -z "${NOTES}" ]; then
   echo "error: failed to extract changelog notes for ${VERSION}" >&2