cgraphx 1.2.0 → 1.3.0

package/scripts/agent-eval/block-cgraphx-cli-hook.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# PreToolUse hook: deny Bash invocations of the `cgraphx` CLI (or `node .../codegraph.js`).
-# Used by the cli-vs-mcp-3arm eval to enforce "no CLI" in the baseline and mcp-only arms.
-#
-# Match rules (any one → deny):
-#   - command starts with `cgraphx `         (e.g. `cgraphx calls Foo#bar`)
-#   - command contains token ` cgraphx `     (e.g. `sudo cgraphx ...`)
-#   - command contains `codegraph.js`        (e.g. `node /path/codegraph.js serve`)
-#
-# Not matched (allowed):
-#   - `cat README.md | grep cgraphx`         (grep the literal word, not invoke binary)
-#   - `ls .cgraphx/`                         (read dir, not run binary)
-#   - `echo "cgraphx"`                       (literal text)
-#
-# Wire via:  claude ... --settings scripts/agent-eval/block-cgraphx-cli-settings.json
-set -uo pipefail
-input="$(cat)"
-cmd="$(printf '%s' "$input" | jq -r '.tool_input.command // empty' 2>/dev/null)"
-
-[ -z "$cmd" ] && exit 0
-
-deny=0
-case "$cmd" in
-  cgraphx\ *|*\ cgraphx\ *|*codegraph.js*) deny=1 ;;
-esac
-
-if [ "$deny" = "1" ]; then
-  msg="该 arm 禁用 cgraphx CLI(Bash 调用被 hook 拦截)。改用:Read/Grep/Glob(baseline arm)、codegraph_explore 等 MCP 工具(mcp-only arm)。本评估旨在对比三 arm 在禁/允许 CLI 下的真实表现,请勿尝试绕过。"
-  jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
-  exit 0
-fi
-exit 0

package/scripts/agent-eval/block-cgraphx-cli-settings.json

@@ -1,16 +0,0 @@
-{
-  "_comment": "PreToolUse hook config: deny Bash invocations of `cgraphx` CLI. Used by cli-vs-mcp-3arm.sh for the baseline and mcp-only arms. The cli-vs-mcp-3arm.sh script regenerates an absolute-path copy at $OUT/runs/ at runtime, so this static file is mostly for reference / standalone hook testing. If you want to use it directly with claude --settings, edit the path below to your local absolute path.",
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "bash /Users/smdnk/objects/ai/codegraph/scripts/agent-eval/block-cgraphx-cli-hook.sh"
-          }
-        ]
-      }
-    ]
-  }
-}

package/scripts/agent-eval/cli-vs-mcp-3arm.sh

@@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-# CLI-vs-MCP 3-arm eval for ONE indexed repo + ONE flow question, n reps each.
-#   ARM baseline : cgraphx CLI禁用(hook) + 空 mcp-config      -> Read/Grep baseline
-#   ARM cli      : cgraphx CLI 允许 + 空 mcp-config + 显式提示用 cgraphx-search skill
-#   ARM mcp      : cgraphx CLI 禁用(hook) + 完整 cgraphx mcp-config -> MCP codegraph_explore
-# All arms: claude -p sonnet --effort high. One JSON metrics line/run -> $RESULTS.
-#
-# Usage: cli-vs-mcp-3arm.sh <reps> "<question>"
-# Env:   TARGET=<indexed-repo path, default /Users/smdnk/objects/java/hx>
-#        MODEL=sonnet EFFORT=high  RESULTS=<file>  AGENT_EVAL_OUT=<scratch dir>
-set -uo pipefail
-HERE="$(cd "$(dirname "$0")" && pwd)"
-ENGINE="$(cd "$HERE/../.." && pwd)"
-BIN="$ENGINE/dist/bin/codegraph.js"
-HOOK="$HERE/block-cgraphx-cli-hook.sh"
-OUT="${AGENT_EVAL_OUT:-/tmp/cg-cli-vs-mcp}"
-TARGET="${TARGET:-/Users/smdnk/objects/java/hx}"
-REPS="${1:?usage: cli-vs-mcp-3arm.sh <reps> \"<question>\"}"
-Q="${2:?question required}"
-RUNS="$OUT/runs"
-EXTRACT="$HERE/parse-cli-vs-mcp.mjs"
-RESULTS="${RESULTS:-$OUT/results.jsonl}"
-REPO=$(basename "$TARGET")
-mkdir -p "$RUNS"
-command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
-[ -d "$TARGET/.cgraphx" ] || { echo "not indexed: $TARGET"; exit 1; }
-[ -f "$BIN" ] || { echo "engine not built: run 'npm run build' in $ENGINE"; exit 1; }
-# Physical path so pkill matches the daemon's real cmdline (macOS /tmp->/private/tmp symlink
-# otherwise makes the kill miss the daemon — contaminating the next arm).
-TARGET=$(cd "$TARGET" && pwd -P)
-REPO=$(basename "$TARGET")
-
-# Regenerate the settings.json with the absolute hook path (avoids hard-coding user paths
-# in the checked-in static settings file).
-SETTINGS="$RUNS/block-cgraphx-cli-settings.json"
-cat > "$SETTINGS" <<JSON
-{
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          { "type": "command", "command": "bash $HOOK" }
-        ]
-      }
-    ]
-  }
-}
-JSON
-
-# A2 cli-only system prompt — explicitly steer to invoke the cgraphx-search skill
-# (hx/.claude/skills/cgraphx-search exists; auto-discovery works, but the explicit prompt
-#  guarantees the agent loads the skill content). Do NOT list CLI syntax here — that would
-# double up with the skill's own content and bias the measurement.
-CLI_PROMPT="在该项目里理解或定位 Java 代码之前,先用 Skill tool 调用 cgraphx-search skill,获取基于 AST 的代码搜索 CLI 套件(calls / context / impact / related / slice)的完整用法、决策树和 hx 项目实战例子。然后按 skill 指引选择合适的 cgraphx CLI 命令(Bash 调用)做任何代码理解任务,而不是默认用 grep+Read。"
-
-# mcp-configs
-CFG_EMPTY="$RUNS/mcp-empty.json"
-CFG_CGRAPHX="$RUNS/mcp-cgraphx.json"
-printf '{"mcpServers":{}}' > "$CFG_EMPTY"
-printf '{"mcpServers":{"cgraphx":{"command":"node","args":["%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TARGET" > "$CFG_CGRAPHX"
-
-prewarm() { # path
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.cgraphx/daemon.sock" 2>/dev/null; sleep 0.6
-  env CGRAPHX_DAEMON_IDLE_TIMEOUT_MS=1800000 CGRAPHX_WASM_RELAUNCHED=1 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.cgraphx/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
-    && echo "  daemon warm" || { echo "  ERROR daemon never bound"; exit 1; }
-}
-
-run() { # arm rep mcp-config extra-args...
-  local arm="$1" rep="$2" cfg="$3"; shift 3
-  local tag="$REPO-$arm-$rep" extra_args=("$@")
-  local log="$RUNS/$tag.jsonl" err="$RUNS/$tag.err"
-  echo "  → [$arm #$rep] running..."
-  ( cd "$TARGET" && claude -p "$Q" \
-      --output-format stream-json --verbose \
-      --permission-mode bypassPermissions \
-      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
-      --strict-mcp-config --mcp-config "$cfg" \
-      "${extra_args[@]}" \
-      </dev/null > "$log" 2>"$err" )
-  local rc=$?
-  if [ $rc -ne 0 ]; then
-    echo "  ← [$arm #$rep] claude exit $rc (err tail:)"; tail -5 "$err" 2>/dev/null
-  fi
-  # Progress line via jq — robust against missing result event
-  if [ -s "$log" ]; then
-    local last_line; last_line=$(tail -1 "$log")
-    local rtype; rtype=$(printf '%s' "$last_line" | jq -r '.type // "?"' 2>/dev/null)
-    if [ "$rtype" = "result" ]; then
-      local dur cost sub
-      dur=$(printf '%s' "$last_line" | jq -r '(.duration_ms/1000|floor)' 2>/dev/null)
-      cost=$(printf '%s' "$last_line" | jq -r '(.total_cost_usd//0|tostring)' 2>/dev/null)
-      sub=$(printf '%s' "$last_line" | jq -r '.subtype // "?"' 2>/dev/null)
-      echo "  ← [$arm #$rep] ${dur}s | \$${cost} | ok=${sub}"
-    else
-      echo "  ← [$arm #$rep] (last event type=${rtype}, no result yet)"
-    fi
-  fi
-}
-
-# REP_START lets a later batch ADD reps without clobbering earlier jsonls.
-START="${REP_START:-1}"; END=$((START + REPS - 1))
-echo "###### repo=$REPO reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high}"
-echo "###### Q=$Q"
-echo "== ARM baseline (no cgraphx, hook denies CLI, empty mcp) =="
-for r in $(seq "$START" "$END"); do run baseline "$r" "$CFG_EMPTY" --settings "$SETTINGS"; done
-
-echo "== ARM cli-only (cgraphx CLI allowed, empty mcp, explicit skill prompt) =="
-for r in $(seq "$START" "$END"); do run cli "$r" "$CFG_EMPTY" --append-system-prompt "$CLI_PROMPT"; done
-
-echo "== ARM mcp-only (cgraphx MCP allowed, hook denies CLI) =="
-prewarm "$TARGET"
-for r in $(seq "$START" "$END"); do run mcp "$r" "$CFG_CGRAPHX" --settings "$SETTINGS"; done
-pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.cgraphx/daemon.sock" 2>/dev/null
-
-echo "###### DONE $REPO"
-echo "== parsing all transcripts =="
-: > "$RESULTS"
-node "$EXTRACT" "$OUT"
-echo "== results.jsonl: $RESULTS =="

package/scripts/agent-eval/multi-tool-eval.sh

@@ -1,171 +0,0 @@
-#!/usr/bin/env bash
-# Multi-tool eval: cgraphx CLI vs cgraphx MCP vs gitnexus MCP, across 3 task types.
-#
-# Task types (each agent gets ONE question, no tool-presetting in the wording):
-#   flow-trace   — trace a call chain through the codebase
-#   onboarding   — give a new-engineer overview of a module
-#   impact       — assess refactor blast radius of a specific symbol
-#
-# Arms:
-#   cgraphx-cli    : cgraphx CLI allowed + cgraphx-search skill prompt
-#                    hook denies cgraphx MCP is empty + gitnexus CLI/MCP both empty
-#   cgraphx-mcp    : cgraphx MCP allowed + hook denies BOTH cgraphx CLI and gitnexus CLI
-#   gitnexus-mcp   : gitnexus MCP allowed + hook denies BOTH cgraphx CLI and gitnexus CLI
-#
-# Usage: multi-tool-eval.sh <reps> [<task>...]
-#   task defaults to "flow onboarding impact" (all three)
-# Env:   TARGET=<indexed-repo path, default /Users/smdnk/objects/java/hx>
-#        MODEL=sonnet EFFORT=high  AGENT_EVAL_OUT=<scratch dir>
-set -uo pipefail
-HERE="$(cd "$(dirname "$0")" && pwd)"
-ENGINE="$(cd "$HERE/../.." && pwd)"
-CGRAPHX_BIN="$ENGINE/dist/bin/codegraph.js"
-GITNEXUS_BIN="/Users/smdnk/objects/ai/code-workflow/gitnexus/gitnexus/dist/cli/index.js"
-HOOK="$HERE/block-cgraphx-and-gitnexus-cli-hook.sh"
-OUT="${AGENT_EVAL_OUT:-/tmp/cg-multi-tool}"
-TARGET="${TARGET:-/Users/smdnk/objects/java/hx}"
-REPS="${1:?usage: multi-tool-eval.sh <reps> [task...]}"
-shift
-TASKS="${*:-flow onboarding impact}"
-RUNS="$OUT/runs"
-EXTRACT="$HERE/parse-multi-tool.mjs"
-RESULTS="${RESULTS:-$OUT/results.jsonl}"
-REPO=$(basename "$TARGET")
-mkdir -p "$RUNS"
-command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
-[ -d "$TARGET/.cgraphx" ] || { echo "not cgraphx-indexed: $TARGET"; exit 1; }
-[ -d "$TARGET/.gitnexus" ] || { echo "not gitnexus-indexed: $TARGET"; exit 1; }
-[ -f "$CGRAPHX_BIN" ] || { echo "cgraphx engine not built: run 'npm run build' in $ENGINE"; exit 1; }
-[ -f "$GITNEXUS_BIN" ] || { echo "gitnexus binary missing at $GITNEXUS_BIN"; exit 1; }
-TARGET=$(cd "$TARGET" && pwd -P)
-REPO=$(basename "$TARGET")
-
-# Regenerate settings.json with absolute hook path.
-SETTINGS="$RUNS/block-cli-settings.json"
-cat > "$SETTINGS" <<JSON
-{
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          { "type": "command", "command": "bash $HOOK" }
-        ]
-      }
-    ]
-  }
-}
-JSON
-
-# A1 cgraphx-cli: explicit skill prompt to invoke cgraphx-search skill
-CGRAPHX_CLI_PROMPT="在该项目里理解或定位 Java 代码之前,先用 Skill tool 调用 cgraphx-search skill,获取基于 AST 的代码搜索 CLI 套件(calls / context / impact / related / slice)的完整用法、决策树和 hx 项目实战例子。然后按 skill 指引选择合适的 cgraphx CLI 命令(Bash 调用)做任何代码理解任务,而不是默认用 grep+Read。"
-
-# mcp-configs
-CFG_EMPTY="$RUNS/mcp-empty.json"
-CFG_CGRAPHX="$RUNS/mcp-cgraphx.json"
-CFG_GITNEXUS="$RUNS/mcp-gitnexus.json"
-printf '{"mcpServers":{}}' > "$CFG_EMPTY"
-# cgraphx-only: ONLY cgraphx MCP, NO gitnexus MCP
-printf '{"mcpServers":{"cgraphx":{"command":"node","args":["%s","serve","--mcp","--path","%s"]}}}' "$CGRAPHX_BIN" "$TARGET" > "$CFG_CGRAPHX"
-# gitnexus-only: ONLY gitnexus MCP, NO cgraphx MCP
-printf '{"mcpServers":{"gitnexus":{"command":"node","args":["%s","mcp"]}}}' "$GITNEXUS_BIN" > "$CFG_GITNEXUS"
-
-# Task questions (locked wording — DO NOT preset tools)
-# Use a function instead of associative array (macOS bash 3.2 doesn't support declare -A).
-get_q() {
-  case "$1" in
-    flow)
-      echo '在 hx 项目里追踪 so-service 的订单提交流程:从外部接口入口,经过 SMO/BUSI 层业务编排,到调用 order-service 或落库的完整调用链。给出每一步的类#方法和文件路径。'
-      ;;
-    onboarding)
-      echo '给一个新接手 hx 项目 so-service 模块的工程师做 onboarding 概览:它有哪些核心业务能力?对外暴露的接口有哪些?内部分哪些子模块?跟其他服务(order-service / inst-service)有什么调用关系?'
-      ;;
-    impact)
-      echo '在 hx 项目里,如果要修改 SceneCommitServiceImpl#sceneCommit 方法的签名(增加一个参数),会影响哪些调用方?涉及哪些跨服务调用?需要更新哪些测试?给一个重构风险评估。'
-      ;;
-    *) echo "ERROR: unknown task: $1" >&2; exit 1 ;;
-  esac
-}
-
-prewarm_cgraphx() {
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.cgraphx/daemon.sock" 2>/dev/null; sleep 0.6
-  env CGRAPHX_DAEMON_IDLE_TIMEOUT_MS=1800000 CGRAPHX_WASM_RELAUNCHED=1 node "$CGRAPHX_BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.cgraphx/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
-    && echo "  cgraphx daemon warm" || { echo "  ERROR cgraphx daemon never bound"; exit 1; }
-}
-
-prewarm_gitnexus() {
-  # gitnexus mcp is stateless HTTP-less stdio; no daemon needed, just verify it starts.
-  # The MCP server is invoked per claude-session via stdio, no prewarm needed.
-  echo "  gitnexus mcp is stdio (no prewarm needed)"
-}
-
-run() { # task arm rep mcp-config extra-args...
-  local task="$1" arm="$2" rep="$3" cfg="$4"; shift 4
-  local tag="$REPO-$task-$arm-$rep" extra_args=("$@")
-  local log="$RUNS/$tag.jsonl" err="$RUNS/$tag.err"
-  local q; q=$(get_q "$task")
-  echo "  → [$task/$arm #$rep] running..."
-  ( cd "$TARGET" && claude -p "$q" \
-      --output-format stream-json --verbose \
-      --permission-mode bypassPermissions \
-      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
-      --strict-mcp-config --mcp-config "$cfg" \
-      "${extra_args[@]}" \
-      </dev/null > "$log" 2>"$err" )
-  local rc=$?
-  if [ $rc -ne 0 ]; then
-    echo "  ← [$task/$arm #$rep] claude exit $rc (err tail:)"; tail -5 "$err" 2>/dev/null
-  fi
-  if [ -s "$log" ]; then
-    local last_line rtype
-    last_line=$(tail -1 "$log")
-    rtype=$(printf '%s' "$last_line" | jq -r '.type // "?"' 2>/dev/null)
-    if [ "$rtype" = "result" ]; then
-      local dur cost sub
-      dur=$(printf '%s' "$last_line" | jq -r '(.duration_ms/1000|floor)' 2>/dev/null)
-      cost=$(printf '%s' "$last_line" | jq -r '(.total_cost_usd//0*1000|floor)' 2>/dev/null)
-      cost=$(printf '%s' "$last_line" | jq -r '(.total_cost_usd//0|tostring)' 2>/dev/null)
-      sub=$(printf '%s' "$last_line" | jq -r '.subtype // "?"' 2>/dev/null)
-      echo "  ← [$task/$arm #$rep] ${dur}s | \$${cost} | ok=${sub}"
-    else
-      echo "  ← [$task/$arm #$rep] (last event type=${rtype}, no result yet)"
-    fi
-  fi
-}
-
-# REP_START lets a later batch ADD reps without clobbering earlier jsonls.
-START="${REP_START:-1}"; END=$((START + REPS - 1))
-echo "###### repo=$REPO tasks=[ $TASKS ] reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high}"
-echo "###### arms: cgraphx-cli / cgraphx-mcp / gitnexus-mcp"
-
-for task in $TASKS; do
-  echo ""
-  echo "===== TASK: $task ====="
-  echo "Q: $(get_q "$task")"
-  echo ""
-
-  echo "-- ARM cgraphx-cli (cgraphx CLI allowed, both MCPs empty, skill prompt) --"
-  for r in $(seq "$START" "$END"); do
-    run "$task" cgraphx-cli "$r" "$CFG_EMPTY" --append-system-prompt "$CGRAPHX_CLI_PROMPT"
-  done
-
-  echo "-- ARM cgraphx-mcp (cgraphx MCP, hook denies both CLIs) --"
-  prewarm_cgraphx "$TARGET"
-  for r in $(seq "$START" "$END"); do
-    run "$task" cgraphx-mcp "$r" "$CFG_CGRAPHX" --settings "$SETTINGS"
-  done
-  pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.cgraphx/daemon.sock" 2>/dev/null
-
-  echo "-- ARM gitnexus-mcp (gitnexus MCP, hook denies both CLIs) --"
-  prewarm_gitnexus
-  for r in $(seq "$START" "$END"); do
-    run "$task" gitnexus-mcp "$r" "$CFG_GITNEXUS" --settings "$SETTINGS"
-  done
-done
-
-echo ""
-echo "###### DONE — parsing all transcripts"
-: > "$RESULTS"
-node "$EXTRACT" "$OUT"
-echo "== results.jsonl: $RESULTS =="

package/scripts/agent-eval/parse-cli-vs-mcp.mjs

@@ -1,232 +0,0 @@
-#!/usr/bin/env node
-// Parse cli-vs-mcp-3arm eval transcripts into:
-//   1. one JSON metrics line per transcript -> <out-dir>/results.jsonl
-//   2. stdout: per-arm × per-rep detail table + per-arm aggregate (mean across reps)
-//
-// File-name contract: <repo>-(baseline|cli|mcp)-<rep>.jsonl under <out-dir>/runs/
-//
-// Per-transcript metrics:
-//   read / grep (Grep+Glob, NOT Bash) / bash / cliCalls (Bash cgraphx invocations)
-//   mcpCalls (mcp__cgraphx__*) / skillInvokes (Skill tool, skill=cgraphx-search)
-//   hookDenies (PreToolUse deny events for Bash cgraphx)
-//   durationSec / numTurns / costUsdMain / tokGen / tokFresh / tokCached / tokBillable / ok
-import { readFileSync, readdirSync, existsSync, writeFileSync } from 'fs';
-import { join } from 'path';
-
-const OUT = process.argv[2] || '/tmp/cg-cli-vs-mcp';
-const RUNS_DIR = join(OUT, 'runs');
-const RESULTS = join(OUT, 'results.jsonl');
-
-if (!existsSync(RUNS_DIR)) {
-  console.error(`no runs dir at ${RUNS_DIR}`);
-  process.exit(1);
-}
-
-function parse(file) {
-  const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
-  const toolCounts = {};
-  const cliCommands = [];
-  const hookDenies = [];
-  let result = null;
-
-  for (const line of lines) {
-    let ev;
-    try { ev = JSON.parse(line); } catch { continue; }
-
-    // NOTE: per-turn assistant.message.usage is placeholder 0s in stream-json;
-    // authoritative token totals live only in the final result.usage. So we
-    // don't accumulate per-turn — we read result.usage at the end.
-
-    if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
-      for (const b of ev.message.content) {
-        if (b.type !== 'tool_use') continue;
-        toolCounts[b.name] = (toolCounts[b.name] || 0) + 1;
-        if (b.name === 'Bash' && typeof b.input?.command === 'string') {
-          const cmd = b.input.command;
-          // cliCalls = Bash commands that invoke the cgraphx binary (matches hook matcher).
-          if (/^cgraphx /.test(cmd) || / cgraphx /.test(cmd) || /codegraph\.js/.test(cmd)) {
-            cliCommands.push(cmd.slice(0, 80));
-          }
-        }
-        if (b.name === 'Skill' && b.input?.skill === 'cgraphx-search') {
-          // counted via toolCounts['Skill'] + we filter by skill name below
-        }
-      }
-    }
-
-    // Hook denies come back as tool_result with is_error + content matching the deny message,
-    // OR as a hookSpecificOutput denial. In stream-json they show up in user messages
-    // (tool_result content) when permissionDecision:"deny" — the agent sees the reason text.
-    if (ev.type === 'user' && Array.isArray(ev.message?.content)) {
-      for (const b of ev.message.content) {
-        if (b.type !== 'tool_result') continue;
-        const text = typeof b.content === 'string'
-          ? b.content
-          : Array.isArray(b.content)
-            ? b.content.map(c => (typeof c === 'string' ? c : c?.text || '')).join('')
-            : '';
-        if (/该 arm 禁用 cgraphx CLI/.test(text)) {
-          hookDenies.push(text.slice(0, 100));
-        }
-      }
-    }
-
-    if (ev.type === 'result') result = ev;
-  }
-
-  // Skill invokes: count tool_use blocks where name=Skill AND input.skill=cgraphx-search.
-  // Re-scan to be precise (toolCounts['Skill'] would count ALL skill invokes).
-  let skillInvokes = 0;
-  for (const line of lines) {
-    let ev;
-    try { ev = JSON.parse(line); } catch { continue; }
-    if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
-      for (const b of ev.message.content) {
-        if (b.type === 'tool_use' && b.name === 'Skill' && b.input?.skill === 'cgraphx-search') {
-          skillInvokes++;
-        }
-      }
-    }
-  }
-
-  const get = (n) => toolCounts[n] || 0;
-  // Token usage from the final result event (authoritative — see note above).
-  const u = result?.usage || {};
-  const tokGen = u.output_tokens || 0;
-  const tokFresh = (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
-  const tokCached = u.cache_read_input_tokens || 0;
-  return {
-    read: get('Read'),
-    grep: get('Grep') + get('Glob'),
-    bash: get('Bash'),
-    cliCalls: cliCommands.length,
-    cliCommands,
-    mcpCalls: Object.keys(toolCounts)
-      .filter(k => /mcp__cgraphx__/.test(k))
-      .reduce((s, k) => s + toolCounts[k], 0),
-    skillInvokes,
-    hookDenies: hookDenies.length,
-    durationSec: result ? Math.floor(result.duration_ms / 1000) : null,
-    numTurns: result?.num_turns ?? null,
-    costUsdMain: result ? +(result.total_cost_usd || 0).toFixed(4) : null,
-    tokGen,
-    tokFresh,
-    tokCached,
-    tokBillable: tokGen + tokFresh,
-    ok: result?.subtype === 'success',
-  };
-}
-
-// repo -> arm -> [runs]
-const data = {};
-const allMetrics = [];
-for (const f of readdirSync(RUNS_DIR)) {
-  const m = f.match(/^(.+)-(baseline|cli|mcp)-(\d+)\.jsonl$/);
-  if (!m) continue;
-  const [, repo, arm, repStr] = m;
-  const parsed = parse(join(RUNS_DIR, f));
-  const rep = Number(repStr);
-  const record = { repo, arm, rep, file: f, ...parsed };
-  allMetrics.push(record);
-  (((data[repo] ??= {})[arm]) ??= []).push(record);
-}
-
-// Write all metrics to results.jsonl
-writeFileSync(RESULTS, allMetrics
-  .sort((a, b) => a.repo.localeCompare(b.repo) || a.arm.localeCompare(b.arm) || a.rep - b.rep)
-  .map(r => JSON.stringify(r))
-  .join('\n') + '\n'
-);
-
-// ---- Detail table ----
-const pad = (s, n) => String(s).padEnd(n);
-const ARMS = ['baseline', 'cli', 'mcp'];
-const LABEL = { baseline: 'A1 baseline', cli: 'A2 cli-only', mcp: 'A3 mcp-only' };
-
-console.log('\n=== PER ARM × REP (detail) ===');
-console.log(
-  pad('arm', 14), pad('rep', 4),
-  pad('dur', 6), pad('read', 5), pad('grep', 5), pad('bash', 5),
-  pad('cli', 4), pad('mcp', 4), pad('skill', 5), pad('deny', 5),
-  pad('tokBill', 9), pad('cost', 8), 'ok'
-);
-for (const repo of Object.keys(data).sort()) {
-  for (const arm of ARMS) {
-    const runs = data[repo][arm] || [];
-    for (const r of runs) {
-      console.log(
-        pad(LABEL[arm], 14), pad(r.rep, 4),
-        pad(r.durationSec ?? '?', 6),
-        pad(r.read, 5), pad(r.grep, 5), pad(r.bash, 5),
-        pad(r.cliCalls, 4), pad(r.mcpCalls, 4), pad(r.skillInvokes, 5), pad(r.hookDenies, 5),
-        pad(r.tokBillable, 9),
-        pad('$' + (r.costUsdMain ?? 0).toFixed(4), 8),
-        r.ok ? '✓' : '✗'
-      );
-    }
-  }
-}
-
-// ---- Aggregate table (full mean across all reps) ----
-console.log('\n=== AGGREGATE PER ARM (mean across ALL reps) ===');
-const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
-console.log(
-  pad('arm', 14),
-  pad('reads', 7), pad('grep', 7), pad('bash', 7),
-  pad('cli', 7), pad('mcp', 7), pad('skill', 7), pad('deny', 7),
-  pad('dur', 7), pad('tokBill', 9), pad('cost', 8), 'n'
-);
-for (const arm of ARMS) {
-  const all = [];
-  for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) all.push(r);
-  if (!all.length) continue;
-  console.log(
-    pad(LABEL[arm], 14),
-    pad(avg(all, r => r.read).toFixed(1), 7),
-    pad(avg(all, r => r.grep).toFixed(1), 7),
-    pad(avg(all, r => r.bash).toFixed(1), 7),
-    pad(avg(all, r => r.cliCalls).toFixed(1), 7),
-    pad(avg(all, r => r.mcpCalls).toFixed(1), 7),
-    pad(avg(all, r => r.skillInvokes).toFixed(1), 7),
-    pad(avg(all, r => r.hookDenies).toFixed(1), 7),
-    pad(avg(all, r => r.durationSec).toFixed(0) + 's', 7),
-    pad(avg(all, r => r.tokBillable).toFixed(0), 9),
-    pad('$' + avg(all, r => r.costUsdMain).toFixed(4), 8),
-    all.length
-  );
-}
-
-// ---- Stability table: mean for each n-prefix (n=2, n=3, n=4 if data permits) ----
-// Shows how the mean shifts as we add more reps. If a metric is stable across n-prefixes,
-// n=2 was already enough; if it shifts a lot, more reps are needed.
-console.log('\n=== STABILITY BY N-PREFIX (mean of first N reps per arm) ===');
-console.log(pad('arm', 14) + pad('metric', 10) + pad('n=2', 12) + pad('n=3', 12) + pad('n=4', 12));
-const METRICS = [
-  ['cost', r => r.costUsdMain, v => '$' + v.toFixed(4)],
-  ['tokBill', r => r.tokBillable, v => v.toFixed(0)],
-  ['dur', r => r.durationSec, v => v.toFixed(0) + 's'],
-  ['reads', r => r.read, v => v.toFixed(1)],
-  ['cli', r => r.cliCalls, v => v.toFixed(1)],
-  ['mcp', r => r.mcpCalls, v => v.toFixed(1)],
-];
-for (const arm of ARMS) {
-  const runs = [];
-  for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) runs.push(r);
-  runs.sort((a, b) => a.rep - b.rep);
-  if (!runs.length) continue;
-  for (const [name, fn, fmt] of METRICS) {
-    const line = pad(LABEL[arm], 14) + pad(name, 10);
-    const cells = [];
-    for (const n of [2, 3, 4]) {
-      if (runs.length >= n) {
-        const sub = runs.slice(0, n);
-        cells.push(pad(fmt(avg(sub, fn)), 12));
-      } else {
-        cells.push(pad('-', 12));
-      }
-    }
-    console.log(line + cells.join(''));
-  }
-}
-
-console.log(`\nresults.jsonl: ${RESULTS}`);