homegraph 1.5.1 → 1.5.2

package/scripts/agent-eval/arms-matrix.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-# Drive the tool-surface ablation across the chosen repos × arms (A–E).
-# Arms A–D ask the canonical FLOW question; arm E asks a NON-flow survey
-# question (the control probe — should degrade without explore+context).
-# Output: /tmp/arms/<repo>/<arm>-r<n>.jsonl  (parse with parse-arms.mjs).
-set -uo pipefail
-HARNESS="$(cd "$(dirname "$0")" && pwd)"
-RUNS="${RUNS:-2}"
-C="${CORPUS:-/tmp/homegraph-corpus}"
-NFQ='What are the main modules/components of this codebase and what does each one do? Give an overview of how it is organized.'
-
-# repo-path|flow-question  (2 small, 2 medium, 2 large — spans the size range)
-ROWS=(
-"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
-"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
-"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
-"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
-"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
-"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
-)
-
-echo "### ARMS MATRIX START $(date) RUNS=$RUNS"
-for row in "${ROWS[@]}"; do
-  repo="${row%%|*}"; q="${row#*|}"
-  for arm in A B C D; do
-    for r in $(seq 1 "$RUNS"); do
-      bash "$HARNESS/run-arms.sh" "$repo" "$q" "$arm" "$r"
-    done
-  done
-done
-# E: non-flow control probe on two repos (must degrade without explore+context)
-for repo in "$C/excalidraw" "$C/spring-mall"; do
-  for r in $(seq 1 "$RUNS"); do
-    bash "$HARNESS/run-arms.sh" "$repo" "$NFQ" E "$r"
-  done
-done
-echo "### ARMS MATRIX COMPLETE $(date)"

package/scripts/agent-eval/audit.sh

@@ -1,68 +0,0 @@
-#!/usr/bin/env bash
-# One-shot HomeGraph quality audit:
-#   set version -> ensure corpus repo -> wipe+reindex with that version ->
-#   run with/without A/B -> restore the local dev link.
-#
-# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
-#   <version>    "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
-#   <repo-name>  dir name under the corpus dir
-#   <repo-url>   git URL (cloned --depth 1 when the repo dir is missing)
-#   [mode]       headless (default) | all (also the interactive tmux arms)
-# Env: CORPUS  corpus dir (default: /tmp/homegraph-corpus)
-set -uo pipefail
-
-VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
-NAME="${2:?repo-name required}"
-URL="${3:?repo-url required}"
-Q="${4:?question required}"
-MODE="${5:-headless}"
-
-HARNESS="$(cd "$(dirname "$0")" && pwd)"
-REPO_ROOT="$(cd "$HARNESS/../.." && pwd)"     # homegraph repo root
-CORPUS="${CORPUS:-/tmp/homegraph-corpus}"
-REPO="$CORPUS/$NAME"
-PKG="homegraph"
-
-echo "==================== HomeGraph audit ===================="
-echo "version=$VERSION  repo=$NAME  mode=$MODE  corpus=$CORPUS"
-echo
-
-# 1. Set the homegraph version under test (mutates the global install).
-if [ "$VERSION" = local ]; then
-  echo "→ [1/4] building + linking local dev build (local-install.sh)"
-  ( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
-else
-  echo "→ [1/4] installing $PKG@$VERSION globally"
-  npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
-fi
-ACTUAL="$(homegraph --version 2>/dev/null || echo '?')"
-echo "  homegraph on PATH: $(command -v homegraph) -> $ACTUAL"
-
-# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
-mkdir -p "$CORPUS"
-if [ -d "$REPO/.git" ]; then
-  echo "→ [2/4] reusing existing checkout: $REPO"
-else
-  echo "→ [2/4] cloning $URL"
-  git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
-fi
-
-# 3. Wipe + re-index with THIS version (the index must be built by the same
-#    binary that serves it — different versions extract differently).
-echo "→ [3/4] wiping .homegraph and re-indexing with $ACTUAL"
-rm -rf "$REPO/.homegraph"
-( cd "$REPO" && homegraph init -i ) || { echo "indexing failed"; exit 1; }
-
-# 4. Run the with/without A/B.
-echo "→ [4/4] running A/B harness (mode=$MODE)"
-bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
-
-# Restore the dev link (the normal working state in this repo).
-echo
-echo "→ restoring local dev link (local-install.sh)"
-if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
-  echo "  global homegraph restored to dev build"
-else
-  echo "  WARN: restore failed — run ./scripts/local-install.sh manually"
-fi
-echo "==================== audit complete ===================="

package/scripts/agent-eval/bench-readme.sh

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Re-run the README "Benchmark Results" A/B (with vs without homegraph) on the
-# current build: the 7 README repos, same queries, RUNS per arm (default 4).
-# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
-# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
-# $CORPUS (default /tmp/homegraph-corpus) by the build under test.
-set -uo pipefail
-H="$(cd "$(dirname "$0")" && pwd)"
-C="${CORPUS:-/tmp/homegraph-corpus}"
-RUNS="${RUNS:-4}"
-ROWS=(
-"vscode|How does the extension host communicate with the main process?"
-"excalidraw|How does Excalidraw render and update canvas elements?"
-"django|How does Django's ORM build and execute a query from a QuerySet?"
-"tokio|How does tokio schedule and run async tasks on its runtime?"
-"okhttp|How does OkHttp process a request through its interceptor chain?"
-"gin|How does gin route requests through its middleware chain?"
-"alamofire|How does Alamofire build, send, and validate a request?"
-)
-echo "### README A/B START $(date) RUNS=$RUNS"
-for row in "${ROWS[@]}"; do
-  repo="${row%%|*}"; q="${row#*|}"
-  echo "===== $repo ====="
-  for run in $(seq 1 "$RUNS"); do
-    AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo "  run$run: (no exit line)"
-  done
-done
-echo "### README A/B DONE $(date)"

package/scripts/agent-eval/bench-why-repo.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-# One README repo, WITH-homegraph only, N runs. Each run appends a why-Read
-# diagnostic so the agent explains any Read/Grep. (The WITHOUT baseline is
-# homegraph-independent and already in the README — no point re-running it.)
-# Output -> /tmp/ab-why/<repo>/with<n>.jsonl
-# Usage: bench-why-repo.sh <repo-path> "<query>" [N]
-set -uo pipefail
-REPO="$1"; Q="$2"; N="${3:-4}"
-NAME="$(basename "$REPO")"
-CG="/Users/colby/Development/Personal/homegraph/dist/bin/homegraph.js"
-OUT="/tmp/ab-why/$NAME"; mkdir -p "$OUT"
-WHY=$'\n\nIMPORTANT — diagnostic: if you use the Read or Grep tool at ANY point, for EACH such call explain why homegraph_explore / homegraph_node did not already give you what you needed. End your entire answer with a section titled exactly "## Why I read" listing every Read and Grep you made and the precise reason homegraph fell short for it. If you used neither, write "## Why I read" then "none — homegraph was sufficient."'
-printf '{"mcpServers":{"homegraph":{"command":"%s","args":["serve","--mcp","--path","%s"]}}}' "$CG" "$REPO" > "$OUT/cg.json"
-
-for i in $(seq 1 "$N"); do
-  pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.homegraph/daemon.sock"
-  ( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
-      --permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
-      --strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
-  echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
-done
-echo "DONE $NAME"

package/scripts/agent-eval/block-read-hook.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-# PreToolUse hook (experiment): deny Read of homegraph-indexed source files and
-# steer the agent to homegraph_explore/homegraph_node instead. Tests whether
-# homegraph can FULLY replace Read for code-understanding once the escape hatch
-# is removed. Non-source reads (config, .env, markdown, new files) pass through.
-#
-# Wire via:  claude ... --settings scripts/agent-eval/hook-settings.json
-set -uo pipefail
-input="$(cat)"
-fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
-
-case "$fp" in
-  *.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
-    msg="Read is disabled for source files in this session — homegraph already has this file indexed (with line numbers, kept in sync on every change). Use homegraph_explore (several related symbols at once) or homegraph_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER homegraph_explore with its exact name instead of reading the file."
-    jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
-    exit 0
-    ;;
-esac
-exit 0

package/scripts/agent-eval/hook-settings.json

@@ -1,15 +0,0 @@
-{
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Read",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "bash /Users/colby/Development/Personal/homegraph/scripts/agent-eval/block-read-hook.sh"
-          }
-        ]
-      }
-    ]
-  }
-}

package/scripts/agent-eval/itrun.sh

@@ -1,120 +0,0 @@
-#!/usr/bin/env bash
-# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
-# agent to finish, then print the tool-call breakdown from the session logs.
-#
-# Why interactive (not `claude -p`): headless print-mode picks the
-# general-purpose subagent, while real interactive sessions delegate to the
-# Explore subagent (or drive homegraph from the main thread). Only the
-# interactive TUI reproduces the behavior users actually see. (Idle-detection
-# technique borrowed from devpit's WaitForIdle.)
-#
-# Usage: itrun.sh <repo-path> <label> "<prompt>"
-# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
-# Requires: tmux 3.0+, a logged-in `claude` CLI, homegraph MCP configured.
-set -uo pipefail
-REPO="$1"; LABEL="$2"; PROMPT="$3"
-SESSION="cgt_${LABEL}"
-OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
-OUT="$OUT_DIR/itrun-${LABEL}.txt"
-HERE="$(cd "$(dirname "$0")" && pwd)"
-
-cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
-
-tmux kill-session -t "$SESSION" 2>/dev/null
-
-# Wide pane so the TUI doesn't hard-wrap tool lines.
-tmux new-session -d -s "$SESSION" -x 230 -y 60
-tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
-
-# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
-# welcome screen seconds before the input actually accepts keystrokes, so this is
-# necessary but NOT sufficient — the type-and-verify loop below is what proves
-# the input is live.
-ready=0
-for _ in $(seq 1 120); do
-  cap | grep -q "❯" && { ready=1; break; }
-  sleep 0.5
-done
-[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
-
-# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
-# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
-# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
-# cleared before the type-and-verify loop or keystrokes land on the menu.
-for _ in $(seq 1 20); do
-  cap | grep -q "trust this folder" || break
-  tmux send-keys -t "$SESSION" Enter
-  sleep 1
-done
-
-# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
-# landed in the input box, retry if it didn't (handles the early-❯ race where
-# the welcome screen shows the prompt glyph but MCP init is still eating keys).
-needle="${PROMPT:0:24}"
-typed=0
-for _ in $(seq 1 30); do
-  tmux send-keys -l -t "$SESSION" "$PROMPT"
-  sleep 1
-  if cap | grep -Fq "$needle"; then typed=1; break; fi
-  # Clear whatever partial text may have landed, then retry.
-  tmux send-keys -t "$SESSION" C-u
-  sleep 1
-done
-[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
-sleep 0.5
-tmux send-keys -t "$SESSION" Enter
-
-# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
-# EVERY working state shows — both the pre-stream thinking phase
-# "(8s · thinking with max effort)" and the streaming phase
-# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
-# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
-# (some TUI versions/states show one but not the others).
-BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
-
-# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
-# fail loudly rather than silently reporting an empty run.
-started=0
-for _ in $(seq 1 120); do
-  cap | grep -qE "$BUSY_RE" && { started=1; break; }
-  sleep 0.5
-done
-[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
-
-# Poll for idle. CRITICAL: Opus 4.8 (extended thinking) renders NO spinner /
-# "esc to interrupt" / timer while it STREAMS its final answer — those appear
-# only during the thinking + tool-use phases ("✻ Marinating… (32s · ↓ 1.3k
-# tokens · thinking with max effort)"). So BUSY_RE reads "not busy" for the whole
-# 10-30s answer stream, and any short not-busy threshold kills the run mid-answer
-# (the truncation bug). We therefore detect "done" by CONTENT STABILITY, not by a
-# spinner string: while the agent streams, the captured pane changes every poll,
-# so stability never accrues; it accrues only once the agent has finished and the
-# static "✻ Brewed for 1m 9s" summary is all that is left. BUSY_RE still hard-
-# resets stability (covers thinking/tool-use/live-timer, where text can briefly
-# sit still). Need STABLE_NEEDED polls (~8s) of zero pane change + ❯ present.
-# Content-stability is model-agnostic — it survives future spinner re-wordings.
-STABLE_NEEDED=16
-prev=""; stable=0
-for _ in $(seq 1 2400); do            # up to ~20 min
-  pane="$(cap)"
-  sig="$(printf '%s' "$pane" | tr -s '[:space:]' ' ')"
-  if printf '%s' "$pane" | grep -qE "$BUSY_RE"; then
-    stable=0                          # thinking / tool use / live timer → busy
-  elif [ -n "$sig" ] && [ "$sig" = "$prev" ] && printf '%s' "$pane" | grep -q "❯"; then
-    stable=$((stable+1)); [ "$stable" -ge "$STABLE_NEEDED" ] && break
-  else
-    stable=0                          # answer still streaming → pane changing
-  fi
-  prev="$sig"
-  sleep 0.5
-done
-sleep 1
-
-tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
-echo "captured $(wc -l < "$OUT") lines -> $OUT"
-grep -oE "Done \([^)]*\)|[A-Z][a-z]+ for ([0-9]+m )?[0-9]+s" "$OUT" | tail -1
-grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
-tmux kill-session -t "$SESSION" 2>/dev/null
-
-# Clean tool breakdown from the session logs (main + subagents).
-node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true

package/scripts/agent-eval/offload-eval-3arm.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-# 3-arm offload eval for ONE indexed repo + ONE question, n reps each.
-#   ARM offload : homegraph attached, managed offload ON  (per-run AI usage log)
-#   ARM raw     : homegraph attached, HOMEGRAPH_OFFLOAD_DISABLE=1 (raw source)
-#   ARM nocg    : no homegraph (empty MCP config) -> Read/Grep baseline
-# All arms: claude -p sonnet --effort high. One JSON metrics line/run -> $RESULTS.
-#
-# Usage: offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"
-# Env:   MODEL=sonnet EFFORT=high  RESULTS=<file>  AGENT_EVAL_OUT=<scratch dir>
-set -uo pipefail
-HERE="$(cd "$(dirname "$0")" && pwd)"
-ENGINE="$(cd "$HERE/../.." && pwd)"
-BIN="$ENGINE/dist/bin/homegraph.js"
-OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
-TARGET="${1:?usage: offload-eval-3arm.sh <indexed-repo> <tier> <reps> \"<question>\"}"
-TIER="${2:?tier}"; REPS="${3:?reps}"; Q="${4:?question}"
-RUNS="$OUT/runs"
-EXTRACT="$HERE/offload-eval-metrics.mjs"
-RESULTS="${RESULTS:-$OUT/results.jsonl}"
-REPO=$(basename "$TARGET")
-mkdir -p "$RUNS"
-command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
-[ -d "$TARGET/.homegraph" ] || { echo "not indexed: $TARGET (run offload-eval-setup.sh first)"; exit 1; }
-# Physical path so pkill matches the daemon's real cmdline (macOS /tmp->/private/tmp symlink
-# otherwise makes the kill miss the daemon, and the next arm connects to the SURVIVING daemon
-# — contaminating the raw arm with offload).
-TARGET=$(cd "$TARGET" && pwd -P)
-
-prewarm() { # path  extra-env (e.g. "FOO=bar")
-  pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.homegraph/daemon.sock" 2>/dev/null; sleep 0.6
-  env ${2:-} HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
-  node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
-    && echo "  daemon warm" || echo "  WARN daemon never bound"
-}
-
-run() { # arm rep mcp-config usage-log-or-dash
-  local arm="$1" rep="$2" cfg="$3" usage="$4" tag="$REPO-$1-$2"
-  [ "$usage" != "-" ] && : > "$usage"
-  # DISALLOW (optional): block sub-agent delegation across all arms so the A/B
-  # measures the retrieval mode, not whether Sonnet decides to spawn a homegraph-blind
-  # Explore subagent (which thrashes regardless and adds huge variance).
-  ( cd "$TARGET" && claude -p "$Q" \
-      --output-format stream-json --verbose --permission-mode bypassPermissions \
-      --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
-      ${DISALLOW:+--disallowedTools "$DISALLOW"} \
-      --strict-mcp-config --mcp-config "$cfg" \
-      </dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
-  node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$usage" --arm "$arm" --rep "$rep" \
-      --repo "$REPO" --tier "$TIER" --q "$Q" >> "$RESULTS"
-  node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(`  [${o.arm} #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable} tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}call/${o.ai.totalTokens}tok/$${o.ai.costUsd.toFixed(4)} | ok=${o.ok}`)' "$RESULTS"
-}
-
-CFG_OFF="$RUNS/mcp-offload-$REPO.json"; CFG_RAW="$RUNS/mcp-raw-$REPO.json"; CFG_NOCG="$RUNS/mcp-nocg.json"
-USAGE="$RUNS/$REPO-usage.jsonl"
-printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","HOMEGRAPH_OFFLOAD_USAGE_LOG=%s","node","%s","serve","--mcp","--path","%s"]}}}' "$USAGE" "$BIN" "$TARGET" > "$CFG_OFF"
-printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","HOMEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TARGET" > "$CFG_RAW"
-printf '{"mcpServers":{}}' > "$CFG_NOCG"
-
-# REP_START lets a later batch ADD reps without clobbering earlier jsonls
-# (e.g. REP_START=4 REPS=3 -> reps 4,5,6; default starts at 1).
-START="${REP_START:-1}"; END=$((START + REPS - 1))
-echo "###### repo=$REPO tier=$TIER reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high}"
-echo "###### Q=$Q"
-echo "== ARM offload =="; prewarm "$TARGET" "HOMEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
-for r in $(seq "$START" "$END"); do run offload "$r" "$CFG_OFF" "$USAGE"; done
-pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.homegraph/daemon.sock" 2>/dev/null; sleep 1
-echo "== ARM raw =="; prewarm "$TARGET" "HOMEGRAPH_OFFLOAD_DISABLE=1"
-for r in $(seq "$START" "$END"); do run raw "$r" "$CFG_RAW" "-"; done
-pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.homegraph/daemon.sock" 2>/dev/null; sleep 1
-echo "== ARM nocg =="
-for r in $(seq "$START" "$END"); do run nocg "$r" "$CFG_NOCG" "-"; done
-echo "###### DONE $REPO"

package/scripts/agent-eval/offload-eval-cost.mjs

@@ -1,133 +0,0 @@
-#!/usr/bin/env node
-// Cost/token analysis for the 3-arm offload eval, with a MAIN-vs-SUBAGENT split.
-//
-// The explore-subagent question. With delegation ALLOWED, the nocg arm spawns a
-// Claude Code Explore subagent; the homegraph arms do all work in the main agent.
-// Two facts make naive accounting wrong:
-//   1. The Explore subagent runs on HAIKU 4.5; the main agent on SONNET 4.6.
-//      So per-token cost differs ~3x between them — you cannot price both the same.
-//   2. The subagent's consumption is ~95% cache-reads. At Haiku's $0.10/MTok
-//      cache-read rate, a huge TOKEN volume is a small DOLLAR cost.
-//
-// Rather than re-derive cost from raw token counts (and guess the cache TTL —
-// Claude Code uses 1-hour ephemeral cache here, 2x write, not 5-min), we read
-// Claude Code's OWN authoritative accounting from the `result` event:
-//   result.modelUsage[model].costUSD  — per-model cost CC itself billed
-//   result.total_cost_usd             — their sum (INCLUDES the Haiku subagent;
-//                                       the handoff's "excludes subagent" was wrong)
-// The model split IS the agent split here: sonnet => main, haiku => Explore subagent
-// (only nocg spawns one, and only nocg shows haiku usage). Token volume is still
-// summed per-model from modelUsage for the separate "tokens" story.
-//
-// Usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]
-//   e.g. offload-eval-cost.mjs /tmp/cg-offload-eval/runs trezor 3
-import { readFileSync, existsSync } from 'fs';
-
-const MAIN_TIER = /sonnet/;   // main agent
-const SUB_TIER  = /haiku/;    // Claude Code Explore subagent
-
-const [,, runsDir, repo, repsArg] = process.argv;
-if (!runsDir || !repo) { console.error('usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]   (env ARMS=nocg,raw,offload)'); process.exit(1); }
-const REPS = Number(repsArg || 3);
-// Arms to analyze (file stems `<repo>-<arm>-<rep>.jsonl`). Override for the style A/B:
-// ARMS=raw,refs,map,src. nocg's Haiku subagent is the only sub-tier; the rest are main-only.
-const ARMS = (process.env.ARMS || 'nocg,raw,offload').split(',').map((s) => s.trim()).filter(Boolean);
-
-const toks = (u) => (u.inputTokens||0)+(u.outputTokens||0)+(u.cacheReadInputTokens||0)+(u.cacheCreationInputTokens||0);
-
-function analyzeRun(file) {
-  let result = null, agentCalls = 0;
-  const tools = {}, subPids = new Set();
-  for (const line of readFileSync(file, 'utf8').split('\n')) {
-    if (!line) continue;
-    let e; try { e = JSON.parse(line); } catch { continue; }
-    if (e.parent_tool_use_id && e.message?.usage) subPids.add(e.parent_tool_use_id);
-    if (e.type === 'assistant' && Array.isArray(e.message?.content))
-      for (const b of e.message.content)
-        if (b.type === 'tool_use') { tools[b.name] = (tools[b.name]||0)+1; if (b.name === 'Agent') agentCalls++; }
-    if (e.type === 'result') result = e;
-  }
-  // Authoritative cost + tokens from Claude Code's per-model accounting.
-  const mu = result?.modelUsage || {};
-  const main = { cost: 0, tok: 0 }, sub = { cost: 0, tok: 0 };
-  for (const [model, u] of Object.entries(mu)) {
-    const bucket = SUB_TIER.test(model) ? sub : main; // sonnet/anything-else => main
-    bucket.cost += u.costUSD || 0;
-    bucket.tok  += toks(u);
-  }
-  return {
-    main, sub, subagents: subPids.size, agentCalls,
-    ccTotal: result?.total_cost_usd ?? null,
-    ok: result?.subtype === 'success',
-    durationSec: result?.duration_ms ? +(result.duration_ms/1000).toFixed(1) : null,
-    models: Object.keys(mu), tools,
-  };
-}
-
-const k = (n) => (n/1000).toFixed(0).padStart(5) + 'K';
-const d = (n) => '$' + n.toFixed(3);
-const cost = (b) => b.cost;
-const tot  = (b) => b.tok;
-
-const byArm = {};
-for (const arm of ARMS) {
-  const runs = [];
-  for (let r = 1; r <= REPS; r++) {
-    const f = `${runsDir}/${repo}-${arm}-${r}.jsonl`;
-    if (existsSync(f)) runs.push({ rep: r, ...analyzeRun(f) });
-  }
-  byArm[arm] = runs;
-}
-
-// Per-run detail. Cost is Claude Code's own modelUsage.costUSD (authoritative,
-// per-model pricing + correct cache TTL). MAIN=Sonnet, SUB=Haiku Explore subagent.
-// cc-check: main$+sub$ must equal result.total_cost_usd (delta should be ~0).
-console.log(`\n=== ${repo}: per-run main(Sonnet)/sub(Haiku) split — Claude Code's own cost accounting ===`);
-console.log('arm      rep | subAg | MAIN(sonnet) tok / $ | SUB(haiku) tok / $   | TOTAL tok / $   | cc_total Δ | dur  reads');
-for (const arm of ARMS) for (const r of byArm[arm]) {
-  const mC = cost(r.main), sC = cost(r.sub), mT = tot(r.main), sT = tot(r.sub);
-  const reads = r.tools['Read'] || 0, grep = (r.tools['Grep']||0)+(r.tools['Bash']||0)+(r.tools['Glob']||0);
-  const explore = r.tools['mcp__homegraph__homegraph_explore'] || 0;
-  const delta = (mC + sC) - (r.ccTotal || 0); // should be ~0
-  console.log(
-    `${arm.padEnd(8)} #${r.rep} | ${String(r.subagents).padStart(2)}    | ${k(mT)} ${d(mC).padStart(7)}     | ${k(sT)} ${d(sC).padStart(7)}     | ${k(mT+sT)} ${d(mC+sC).padStart(7)} | ${d(r.ccTotal||0).padStart(7)} ${(delta>=0?'+':'')+delta.toFixed(4)} | ${String(r.durationSec).padStart(5)} r=${reads} g=${grep} x=${explore}`
-  );
-}
-
-// Per-arm means
-const mean = (arr, f) => arr.length ? arr.reduce((s,x)=>s+f(x),0)/arr.length : 0;
-console.log(`\n=== ${repo}: per-arm MEANS (n per arm) ===`);
-console.log('arm      n | main $   sub $    TOTAL $  | main tok   sub tok    TOTAL tok | %$ in sub | %tok in sub');
-for (const arm of ARMS) {
-  const runs = byArm[arm]; if (!runs.length) continue;
-  const mC = mean(runs, r=>cost(r.main)), sC = mean(runs, r=>cost(r.sub));
-  const mT = mean(runs, r=>tot(r.main)),  sT = mean(runs, r=>tot(r.sub));
-  const pctSubC = (mC+sC) ? (100*sC/(mC+sC)) : 0;
-  const pctSubT = (mT+sT) ? (100*sT/(mT+sT)) : 0;
-  console.log(
-    `${arm.padEnd(8)} ${runs.length} | ${d(mC).padStart(7)} ${d(sC).padStart(7)} ${d(mC+sC).padStart(7)} | ${k(mT)} ${k(sT)} ${k(mT+sT)} | ${pctSubC.toFixed(0).padStart(3)}%      | ${pctSubT.toFixed(0).padStart(3)}%`
-  );
-}
-
-// Headline ladders — cost, tokens, duration, all vs a baseline (nocg if present, else first arm).
-console.log(`\n=== Ladders (mean, incl. subagent) ===`);
-const totals = ARMS.map(a => ({ a, c: mean(byArm[a], r=>cost(r.main)+cost(r.sub)), t: mean(byArm[a], r=>tot(r.main)+tot(r.sub)) })).filter(x=>byArm[x.a].length);
-const base = totals.find(x=>x.a==='nocg') ?? totals[0];
-const bn = base?.a ?? '?';
-console.log(`  COST (vs ${bn}):`);
-for (const x of totals) {
-  const vs = base && base.c ? ` (${((x.c/base.c-1)*100>=0?'+':'')}${((x.c/base.c-1)*100).toFixed(0)}%)` : '';
-  console.log(`    ${x.a.padEnd(8)} ${d(x.c)}${vs}`);
-}
-console.log(`  TOKENS (vs ${bn}):`);
-for (const x of totals) {
-  const vs = base && base.t ? ` (${((x.t/base.t-1)*100>=0?'+':'')}${((x.t/base.t-1)*100).toFixed(0)}%)` : '';
-  console.log(`    ${x.a.padEnd(8)} ${k(x.t)}${vs}`);
-}
-console.log(`  DURATION (wall-clock, vs ${bn}):`);
-const durs = ARMS.map(a => ({ a, s: mean(byArm[a].filter(r=>r.durationSec!=null), r=>r.durationSec) })).filter(x=>byArm[x.a].length);
-const dbase = durs.find(x=>x.a==='nocg') ?? durs[0];
-for (const x of durs) {
-  const vs = dbase && dbase.s ? ` (${((x.s/dbase.s-1)*100>=0?'+':'')}${((x.s/dbase.s-1)*100).toFixed(0)}%)` : '';
-  console.log(`    ${x.a.padEnd(8)} ${x.s.toFixed(0)}s${vs}`);
-}

package/scripts/agent-eval/offload-eval-effort.mjs

@@ -1,108 +0,0 @@
-#!/usr/bin/env node
-// Effort A/B — does HOMEGRAPH_OFFLOAD_EFFORT=high improve offload SYNTHESIS FIDELITY vs low?
-// Probe-based (no agent): for each repo × effort × rep, run homegraph_explore with the offload
-// ON on the canonical question, capture the synthesized answer + AI tokens/cost/latency, then
-// Sonnet-judge that answer's fidelity vs source-verified ground truth. Isolates the synthesis
-// from agent/adoption noise. Requires `homegraph login` (managed offload) + indexed repos.
-//
-// Env: REPS (default 3) · CG_ENGINE (engine repo) · AGENT_EVAL_OUT (repos under /repos) · CONC (judge concurrency)
-import { pathToFileURL, fileURLToPath } from 'node:url';
-import { resolve, dirname, join } from 'node:path';
-import { readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
-import { execFile } from 'node:child_process';
-import { tmpdir } from 'node:os';
-
-const HERE = dirname(fileURLToPath(import.meta.url));
-const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
-const OUT = process.env.AGENT_EVAL_OUT || '/tmp/cg-offload-eval';
-const REPOS = join(OUT, 'repos');
-const GT = JSON.parse(readFileSync(resolve(HERE, 'offload-eval-ground-truth.json'), 'utf8'));
-const REPS = Number(process.env.REPS || 3);
-const CONC = Number(process.env.CONC || 4);
-const EFFORTS = (process.env.EFFORTS_FILTER || 'low,high').split(',');
-const ONLY = process.env.REPOS_FILTER ? new Set(process.env.REPOS_FILTER.split(',')) : null;
-const TIER = { mtkruto: 'small', postybirb: 'medium', shapeshift: 'complex', trezor: 'large' };
-
-const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
-const idx = await load('dist/index.js');
-const toolsMod = await load('dist/mcp/tools.js');
-const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
-const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler;
-if (typeof HomeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
-  console.error('could not load engine from', ENGINE); process.exit(2);
-}
-
-const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. Do NOT use any tools.
-
-QUESTION: ${gt.question}
-
-VERIFIED GROUND TRUTH (the actual call path + files):
-${gt.truth}
-
-SYNTHESIZED ANSWER (to score):
-${ans || '(empty)'}
-
-Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was it honest? A confident WRONG trace is the worst outcome — penalize it harder than an honest partial.
-Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`;
-
-const askJudge = (prompt) => new Promise((res) => {
-  execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5',
-    '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
-    { cwd: OUT, maxBuffer: 1 << 24, timeout: 120000 }, (err, stdout) => {
-      const m = (stdout || '').match(/\{[\s\S]*\}/);
-      if (!m) return res({ verdict: 'error', score: null, note: (err ? err.message : 'no json').slice(0, 60) });
-      try { res(JSON.parse(m[0])); } catch { res({ verdict: 'error', score: null }); }
-    });
-});
-
-// ---- 1. Probe: collect synthesized answers at each effort -------------------
-const records = [];
-for (const repo of Object.keys(GT)) {
-  if (ONLY && !ONLY.has(repo)) continue;
-  const dir = join(REPOS, repo);
-  if (!existsSync(join(dir, '.homegraph'))) { console.error('skip (not indexed):', repo); continue; }
-  const cg = HomeGraph.openSync(dir);
-  const h = new ToolHandler(cg);
-  for (const effort of EFFORTS) {
-    for (let rep = 1; rep <= REPS; rep++) {
-      process.env.HOMEGRAPH_OFFLOAD_EFFORT = effort;
-      const usageLog = join(tmpdir(), `effort-${repo}-${effort}-${rep}.jsonl`);
-      try { rmSync(usageLog); } catch { /* none */ }
-      process.env.HOMEGRAPH_OFFLOAD_USAGE_LOG = usageLog;
-      let answer = '';
-      try { answer = (await h.execute('homegraph_explore', { query: GT[repo].question }))?.content?.[0]?.text ?? ''; }
-      catch (e) { console.error(`  ${repo}/${effort}#${rep} explore failed: ${e?.message}`); }
-      const fired = /Synthesized by HomeGraph/.test(answer);
-      const ai = { tokens: 0, cost: 0, ms: 0 };
-      if (existsSync(usageLog)) for (const e of readFileSync(usageLog, 'utf8').split('\n').filter(Boolean).map(JSON.parse)) {
-        ai.tokens += e.totalTokens || 0; ai.cost += e.costUsd || 0; ai.ms += e.ms || 0;
-      }
-      records.push({ repo, tier: TIER[repo], effort, rep, fired, ai, answer });
-      console.error(`  ${repo}/${effort}#${rep}: fired=${fired} ${ai.tokens}tok $${ai.cost.toFixed(4)} ${ai.ms}ms`);
-    }
-  }
-  try { cg.close?.(); } catch { /* none */ }
-}
-
-// ---- 2. Judge fidelity (concurrency) ---------------------------------------
-console.error(`\njudging ${records.length} answers (concurrency ${CONC})...`);
-let done = 0;
-const q = [...records];
-async function worker() { while (q.length) { const r = q.shift(); r.fid = await askJudge(fidPrompt(GT[r.repo], r.answer)); console.error(`  [${++done}/${records.length}] ${r.repo}/${r.effort}#${r.rep}: ${r.fid.verdict} ${r.fid.score ?? ''}`); } }
-await Promise.all(Array.from({ length: CONC }, worker));
-writeFileSync(join(OUT, 'effort-results.jsonl'), records.map((r) => JSON.stringify(r)).join('\n') + '\n');
-
-// ---- 3. Aggregate: low vs high per repo ------------------------------------
-const med = (a) => { a = a.filter((x) => x != null).sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
-console.log(`\n${'='.repeat(80)}\nEFFORT A/B — offload synthesis fidelity (probe, n=${REPS}/cell)\n${'='.repeat(80)}`);
-console.log(`${'repo'.padEnd(11)} ${'tier'.padEnd(8)} ${'effort'.padEnd(6)} fired  ${'fid(med)'.padStart(8)} ${'fab%'.padStart(5)} ${'AItok'.padStart(7)} ${'AIcost'.padStart(8)} ${'ms(med)'.padStart(8)}`);
-for (const repo of Object.keys(GT)) {
-  for (const effort of EFFORTS) {
-    const rs = records.filter((r) => r.repo === repo && r.effort === effort);
-    if (!rs.length) continue;
-    const fids = rs.map((r) => r.fid?.score).filter((x) => x != null);
-    const fab = rs.filter((r) => r.fid?.fabrication === true).length;
-    console.log(`${repo.padEnd(11)} ${TIER[repo].padEnd(8)} ${effort.padEnd(6)} ${rs.filter((r) => r.fired).length}/${rs.length}   ${String(med(fids) ?? '—').padStart(8)} ${String(Math.round(100 * fab / rs.length) + '%').padStart(5)} ${String(Math.round(med(rs.map((r) => r.ai.tokens)) / 1000) + 'k').padStart(7)} ${('$' + (med(rs.map((r) => r.ai.cost)) ?? 0).toFixed(4)).padStart(8)} ${String(med(rs.map((r) => r.ai.ms)) ?? '—').padStart(8)}`);
-  }
-}
-console.log('');