homegraph 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +243 -243
- package/dist/bin/homegraph.js +371 -9
- package/dist/bin/homegraph.js.map +1 -1
- package/dist/db/migrations.js +18 -18
- package/dist/db/queries.js +140 -140
- package/dist/db/schema.sql +152 -152
- package/dist/directory.js +5 -5
- package/dist/extraction/wasm/tree-sitter-c_sharp.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-pascal.wasm +0 -0
- package/dist/installer/instructions-template.js +9 -9
- package/dist/installer/targets/shared.d.ts +5 -6
- package/dist/installer/targets/shared.d.ts.map +1 -1
- package/dist/installer/targets/shared.js +5 -6
- package/dist/installer/targets/shared.js.map +1 -1
- package/dist/mcp/liveness-watchdog.js +16 -16
- package/dist/mcp/server-instructions.js +69 -69
- package/dist/mcp/tools.d.ts +17 -4
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +192 -22
- package/dist/mcp/tools.js.map +1 -1
- package/dist/reasoning/reasoner.js +32 -32
- package/dist/spec/config.d.ts +43 -0
- package/dist/spec/config.d.ts.map +1 -0
- package/dist/spec/config.js +275 -0
- package/dist/spec/config.js.map +1 -0
- package/dist/spec/db/commit-node.d.ts +23 -0
- package/dist/spec/db/commit-node.d.ts.map +1 -0
- package/dist/spec/db/commit-node.js +62 -0
- package/dist/spec/db/commit-node.js.map +1 -0
- package/dist/spec/db/fragment-node.d.ts +23 -0
- package/dist/spec/db/fragment-node.d.ts.map +1 -0
- package/dist/spec/db/fragment-node.js +120 -0
- package/dist/spec/db/fragment-node.js.map +1 -0
- package/dist/spec/db/fts.d.ts +60 -0
- package/dist/spec/db/fts.d.ts.map +1 -0
- package/dist/spec/db/fts.js +285 -0
- package/dist/spec/db/fts.js.map +1 -0
- package/dist/spec/db/index.d.ts +13 -0
- package/dist/spec/db/index.d.ts.map +1 -0
- package/dist/spec/db/index.js +50 -0
- package/dist/spec/db/index.js.map +1 -0
- package/dist/spec/db/relations.d.ts +55 -0
- package/dist/spec/db/relations.d.ts.map +1 -0
- package/dist/spec/db/relations.js +158 -0
- package/dist/spec/db/relations.js.map +1 -0
- package/dist/spec/db/schema.d.ts +30 -0
- package/dist/spec/db/schema.d.ts.map +1 -0
- package/dist/spec/db/schema.js +87 -0
- package/dist/spec/db/schema.js.map +1 -0
- package/dist/spec/db/schema.sql +107 -0
- package/dist/spec/db/spec-node.d.ts +41 -0
- package/dist/spec/db/spec-node.d.ts.map +1 -0
- package/dist/spec/db/spec-node.js +114 -0
- package/dist/spec/db/spec-node.js.map +1 -0
- package/dist/spec/evolve/impact-locator.d.ts +13 -0
- package/dist/spec/evolve/impact-locator.d.ts.map +1 -0
- package/dist/spec/evolve/impact-locator.js +25 -0
- package/dist/spec/evolve/impact-locator.js.map +1 -0
- package/dist/spec/evolve/llm-client.d.ts +50 -0
- package/dist/spec/evolve/llm-client.d.ts.map +1 -0
- package/dist/spec/evolve/llm-client.js +176 -0
- package/dist/spec/evolve/llm-client.js.map +1 -0
- package/dist/spec/evolve/logic-checker.d.ts +12 -0
- package/dist/spec/evolve/logic-checker.d.ts.map +1 -0
- package/dist/spec/evolve/logic-checker.js +48 -0
- package/dist/spec/evolve/logic-checker.js.map +1 -0
- package/dist/spec/evolve/pipeline.d.ts +40 -0
- package/dist/spec/evolve/pipeline.d.ts.map +1 -0
- package/dist/spec/evolve/pipeline.js +314 -0
- package/dist/spec/evolve/pipeline.js.map +1 -0
- package/dist/spec/evolve/spec-rewriter.d.ts +42 -0
- package/dist/spec/evolve/spec-rewriter.d.ts.map +1 -0
- package/dist/spec/evolve/spec-rewriter.js +254 -0
- package/dist/spec/evolve/spec-rewriter.js.map +1 -0
- package/dist/spec/graph/queries.d.ts +58 -0
- package/dist/spec/graph/queries.d.ts.map +1 -0
- package/dist/spec/graph/queries.js +215 -0
- package/dist/spec/graph/queries.js.map +1 -0
- package/dist/spec/mining/diff-parser.d.ts +33 -0
- package/dist/spec/mining/diff-parser.d.ts.map +1 -0
- package/dist/spec/mining/diff-parser.js +166 -0
- package/dist/spec/mining/diff-parser.js.map +1 -0
- package/dist/spec/mining/git-scanner.d.ts +91 -0
- package/dist/spec/mining/git-scanner.d.ts.map +1 -0
- package/dist/spec/mining/git-scanner.js +266 -0
- package/dist/spec/mining/git-scanner.js.map +1 -0
- package/dist/spec/mining/pipeline.d.ts +53 -0
- package/dist/spec/mining/pipeline.d.ts.map +1 -0
- package/dist/spec/mining/pipeline.js +165 -0
- package/dist/spec/mining/pipeline.js.map +1 -0
- package/dist/spec/mining/scope-resolver.d.ts +45 -0
- package/dist/spec/mining/scope-resolver.d.ts.map +1 -0
- package/dist/spec/mining/scope-resolver.js +103 -0
- package/dist/spec/mining/scope-resolver.js.map +1 -0
- package/dist/spec/mining/spec-extractor.d.ts +69 -0
- package/dist/spec/mining/spec-extractor.d.ts.map +1 -0
- package/dist/spec/mining/spec-extractor.js +369 -0
- package/dist/spec/mining/spec-extractor.js.map +1 -0
- package/dist/spec/types.d.ts +149 -0
- package/dist/spec/types.d.ts.map +1 -0
- package/dist/spec/types.js +15 -0
- package/dist/spec/types.js.map +1 -0
- package/dist/spec/utils.d.ts +166 -0
- package/dist/spec/utils.d.ts.map +1 -0
- package/dist/spec/utils.js +461 -0
- package/dist/spec/utils.js.map +1 -0
- package/package.json +58 -57
- package/scripts/add-lang/bench.sh +60 -60
- package/scripts/add-lang/check-grammar.mjs +75 -75
- package/scripts/add-lang/dump-ast.mjs +103 -103
- package/scripts/add-lang/verify-extraction.mjs +70 -70
- package/scripts/agent-eval/ab-adoption.sh +91 -91
- package/scripts/agent-eval/ab-hook.sh +86 -86
- package/scripts/agent-eval/ab-impl.sh +78 -78
- package/scripts/agent-eval/ab-new-vs-baseline.sh +102 -102
- package/scripts/agent-eval/ab-sufficiency.sh +78 -78
- package/scripts/agent-eval/arms-F.sh +21 -21
- package/scripts/agent-eval/arms-matrix.sh +37 -37
- package/scripts/agent-eval/audit.sh +68 -68
- package/scripts/agent-eval/bench-readme.sh +28 -28
- package/scripts/agent-eval/bench-why-repo.sh +22 -22
- package/scripts/agent-eval/block-read-hook.sh +19 -19
- package/scripts/agent-eval/hook-settings.json +15 -15
- package/scripts/agent-eval/itrun.sh +120 -120
- package/scripts/agent-eval/offload-eval-3arm.sh +72 -72
- package/scripts/agent-eval/offload-eval-cost.mjs +133 -133
- package/scripts/agent-eval/offload-eval-effort.mjs +108 -108
- package/scripts/agent-eval/offload-eval-frontload-matrix.sh +25 -25
- package/scripts/agent-eval/offload-eval-frontload.sh +47 -47
- package/scripts/agent-eval/offload-eval-ground-truth.json +18 -18
- package/scripts/agent-eval/offload-eval-hook.mjs +84 -84
- package/scripts/agent-eval/offload-eval-judge.mjs +103 -103
- package/scripts/agent-eval/offload-eval-matrix.sh +20 -20
- package/scripts/agent-eval/offload-eval-metrics.mjs +94 -94
- package/scripts/agent-eval/offload-eval-refs1.sh +50 -50
- package/scripts/agent-eval/offload-eval-setup.sh +24 -24
- package/scripts/agent-eval/offload-eval-styles.sh +71 -71
- package/scripts/agent-eval/offload-eval-summarize.mjs +68 -68
- package/scripts/agent-eval/offload-eval.md +76 -76
- package/scripts/agent-eval/parse-arms.mjs +116 -116
- package/scripts/agent-eval/parse-bench-readme.mjs +84 -84
- package/scripts/agent-eval/parse-run.mjs +45 -45
- package/scripts/agent-eval/parse-session.mjs +93 -93
- package/scripts/agent-eval/probe-context.mjs +21 -21
- package/scripts/agent-eval/probe-explore.mjs +40 -40
- package/scripts/agent-eval/probe-node.mjs +20 -20
- package/scripts/agent-eval/probe-sweep.mjs +119 -119
- package/scripts/agent-eval/probe-trace.mjs +20 -20
- package/scripts/agent-eval/redirect-read-hook.sh +38 -38
- package/scripts/agent-eval/run-agent.sh +34 -34
- package/scripts/agent-eval/run-all.sh +69 -69
- package/scripts/agent-eval/run-arms.sh +56 -56
- package/scripts/agent-eval/seq-matrix.mjs +137 -137
- package/scripts/build-bundle.sh +118 -118
- package/scripts/extract-release-notes.mjs +130 -130
- package/scripts/local-install.sh +41 -41
- package/scripts/npm-sdk.js +75 -75
- package/scripts/npm-shim.js +246 -246
- package/scripts/pack-npm.sh +119 -119
- package/scripts/prepare-release.mjs +270 -270
- package/scripts/qa_eval/README.md +407 -404
- package/scripts/qa_eval/_test_deveco_probe.py +41 -41
- package/scripts/qa_eval/agent_runner.py +526 -526
- package/scripts/qa_eval/data/.gitignore +4 -4
- package/scripts/qa_eval/data/test-set.jsonl +2 -22
- package/scripts/qa_eval/eval_metrics.py +274 -233
- package/scripts/qa_eval/external_agent.py +976 -671
- package/scripts/qa_eval/llm_config.py +92 -92
- package/scripts/qa_eval/memory_monitor.py +132 -132
- package/scripts/qa_eval/my_answer_accuracy.py +187 -187
- package/scripts/qa_eval/requirements.txt +2 -2
- package/scripts/qa_eval/run_pipeline.py +804 -711
- package/scripts/qa_eval/stats_efficiency.py +279 -279
- package/scripts/qa_eval/stats_scores.py +207 -207
|
@@ -1,37 +1,37 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Drive the tool-surface ablation across the chosen repos × arms (A–E).
|
|
3
|
-
# Arms A–D ask the canonical FLOW question; arm E asks a NON-flow survey
|
|
4
|
-
# question (the control probe — should degrade without explore+context).
|
|
5
|
-
# Output: /tmp/arms/<repo>/<arm>-r<n>.jsonl (parse with parse-arms.mjs).
|
|
6
|
-
set -uo pipefail
|
|
7
|
-
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
|
8
|
-
RUNS="${RUNS:-2}"
|
|
9
|
-
C="${CORPUS:-/tmp/homegraph-corpus}"
|
|
10
|
-
NFQ='What are the main modules/components of this codebase and what does each one do? Give an overview of how it is organized.'
|
|
11
|
-
|
|
12
|
-
# repo-path|flow-question (2 small, 2 medium, 2 large — spans the size range)
|
|
13
|
-
ROWS=(
|
|
14
|
-
"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
|
|
15
|
-
"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
|
|
16
|
-
"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
|
|
17
|
-
"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
|
|
18
|
-
"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
|
|
19
|
-
"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
echo "### ARMS MATRIX START $(date) RUNS=$RUNS"
|
|
23
|
-
for row in "${ROWS[@]}"; do
|
|
24
|
-
repo="${row%%|*}"; q="${row#*|}"
|
|
25
|
-
for arm in A B C D; do
|
|
26
|
-
for r in $(seq 1 "$RUNS"); do
|
|
27
|
-
bash "$HARNESS/run-arms.sh" "$repo" "$q" "$arm" "$r"
|
|
28
|
-
done
|
|
29
|
-
done
|
|
30
|
-
done
|
|
31
|
-
# E: non-flow control probe on two repos (must degrade without explore+context)
|
|
32
|
-
for repo in "$C/excalidraw" "$C/spring-mall"; do
|
|
33
|
-
for r in $(seq 1 "$RUNS"); do
|
|
34
|
-
bash "$HARNESS/run-arms.sh" "$repo" "$NFQ" E "$r"
|
|
35
|
-
done
|
|
36
|
-
done
|
|
37
|
-
echo "### ARMS MATRIX COMPLETE $(date)"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Drive the tool-surface ablation across the chosen repos × arms (A–E).
|
|
3
|
+
# Arms A–D ask the canonical FLOW question; arm E asks a NON-flow survey
|
|
4
|
+
# question (the control probe — should degrade without explore+context).
|
|
5
|
+
# Output: /tmp/arms/<repo>/<arm>-r<n>.jsonl (parse with parse-arms.mjs).
|
|
6
|
+
set -uo pipefail
|
|
7
|
+
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
|
8
|
+
RUNS="${RUNS:-2}"
|
|
9
|
+
C="${CORPUS:-/tmp/homegraph-corpus}"
|
|
10
|
+
NFQ='What are the main modules/components of this codebase and what does each one do? Give an overview of how it is organized.'
|
|
11
|
+
|
|
12
|
+
# repo-path|flow-question (2 small, 2 medium, 2 large — spans the size range)
|
|
13
|
+
ROWS=(
|
|
14
|
+
"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
|
|
15
|
+
"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
|
|
16
|
+
"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
|
|
17
|
+
"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
|
|
18
|
+
"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
|
|
19
|
+
"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
echo "### ARMS MATRIX START $(date) RUNS=$RUNS"
|
|
23
|
+
for row in "${ROWS[@]}"; do
|
|
24
|
+
repo="${row%%|*}"; q="${row#*|}"
|
|
25
|
+
for arm in A B C D; do
|
|
26
|
+
for r in $(seq 1 "$RUNS"); do
|
|
27
|
+
bash "$HARNESS/run-arms.sh" "$repo" "$q" "$arm" "$r"
|
|
28
|
+
done
|
|
29
|
+
done
|
|
30
|
+
done
|
|
31
|
+
# E: non-flow control probe on two repos (must degrade without explore+context)
|
|
32
|
+
for repo in "$C/excalidraw" "$C/spring-mall"; do
|
|
33
|
+
for r in $(seq 1 "$RUNS"); do
|
|
34
|
+
bash "$HARNESS/run-arms.sh" "$repo" "$NFQ" E "$r"
|
|
35
|
+
done
|
|
36
|
+
done
|
|
37
|
+
echo "### ARMS MATRIX COMPLETE $(date)"
|
|
@@ -1,68 +1,68 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# One-shot HomeGraph quality audit:
|
|
3
|
-
# set version -> ensure corpus repo -> wipe+reindex with that version ->
|
|
4
|
-
# run with/without A/B -> restore the local dev link.
|
|
5
|
-
#
|
|
6
|
-
# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
|
|
7
|
-
# <version> "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
|
|
8
|
-
# <repo-name> dir name under the corpus dir
|
|
9
|
-
# <repo-url> git URL (cloned --depth 1 when the repo dir is missing)
|
|
10
|
-
# [mode] headless (default) | all (also the interactive tmux arms)
|
|
11
|
-
# Env: CORPUS corpus dir (default: /tmp/homegraph-corpus)
|
|
12
|
-
set -uo pipefail
|
|
13
|
-
|
|
14
|
-
VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
|
|
15
|
-
NAME="${2:?repo-name required}"
|
|
16
|
-
URL="${3:?repo-url required}"
|
|
17
|
-
Q="${4:?question required}"
|
|
18
|
-
MODE="${5:-headless}"
|
|
19
|
-
|
|
20
|
-
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
|
21
|
-
REPO_ROOT="$(cd "$HARNESS/../.." && pwd)" # homegraph repo root
|
|
22
|
-
CORPUS="${CORPUS:-/tmp/homegraph-corpus}"
|
|
23
|
-
REPO="$CORPUS/$NAME"
|
|
24
|
-
PKG="homegraph"
|
|
25
|
-
|
|
26
|
-
echo "==================== HomeGraph audit ===================="
|
|
27
|
-
echo "version=$VERSION repo=$NAME mode=$MODE corpus=$CORPUS"
|
|
28
|
-
echo
|
|
29
|
-
|
|
30
|
-
# 1. Set the homegraph version under test (mutates the global install).
|
|
31
|
-
if [ "$VERSION" = local ]; then
|
|
32
|
-
echo "→ [1/4] building + linking local dev build (local-install.sh)"
|
|
33
|
-
( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
|
|
34
|
-
else
|
|
35
|
-
echo "→ [1/4] installing $PKG@$VERSION globally"
|
|
36
|
-
npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
|
|
37
|
-
fi
|
|
38
|
-
ACTUAL="$(homegraph --version 2>/dev/null || echo '?')"
|
|
39
|
-
echo " homegraph on PATH: $(command -v homegraph) -> $ACTUAL"
|
|
40
|
-
|
|
41
|
-
# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
|
|
42
|
-
mkdir -p "$CORPUS"
|
|
43
|
-
if [ -d "$REPO/.git" ]; then
|
|
44
|
-
echo "→ [2/4] reusing existing checkout: $REPO"
|
|
45
|
-
else
|
|
46
|
-
echo "→ [2/4] cloning $URL"
|
|
47
|
-
git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
|
|
48
|
-
fi
|
|
49
|
-
|
|
50
|
-
# 3. Wipe + re-index with THIS version (the index must be built by the same
|
|
51
|
-
# binary that serves it — different versions extract differently).
|
|
52
|
-
echo "→ [3/4] wiping .homegraph and re-indexing with $ACTUAL"
|
|
53
|
-
rm -rf "$REPO/.homegraph"
|
|
54
|
-
( cd "$REPO" && homegraph init -i ) || { echo "indexing failed"; exit 1; }
|
|
55
|
-
|
|
56
|
-
# 4. Run the with/without A/B.
|
|
57
|
-
echo "→ [4/4] running A/B harness (mode=$MODE)"
|
|
58
|
-
bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
|
|
59
|
-
|
|
60
|
-
# Restore the dev link (the normal working state in this repo).
|
|
61
|
-
echo
|
|
62
|
-
echo "→ restoring local dev link (local-install.sh)"
|
|
63
|
-
if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
|
|
64
|
-
echo " global homegraph restored to dev build"
|
|
65
|
-
else
|
|
66
|
-
echo " WARN: restore failed — run ./scripts/local-install.sh manually"
|
|
67
|
-
fi
|
|
68
|
-
echo "==================== audit complete ===================="
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# One-shot HomeGraph quality audit:
|
|
3
|
+
# set version -> ensure corpus repo -> wipe+reindex with that version ->
|
|
4
|
+
# run with/without A/B -> restore the local dev link.
|
|
5
|
+
#
|
|
6
|
+
# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
|
|
7
|
+
# <version> "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
|
|
8
|
+
# <repo-name> dir name under the corpus dir
|
|
9
|
+
# <repo-url> git URL (cloned --depth 1 when the repo dir is missing)
|
|
10
|
+
# [mode] headless (default) | all (also the interactive tmux arms)
|
|
11
|
+
# Env: CORPUS corpus dir (default: /tmp/homegraph-corpus)
|
|
12
|
+
set -uo pipefail
|
|
13
|
+
|
|
14
|
+
VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
|
|
15
|
+
NAME="${2:?repo-name required}"
|
|
16
|
+
URL="${3:?repo-url required}"
|
|
17
|
+
Q="${4:?question required}"
|
|
18
|
+
MODE="${5:-headless}"
|
|
19
|
+
|
|
20
|
+
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
|
21
|
+
REPO_ROOT="$(cd "$HARNESS/../.." && pwd)" # homegraph repo root
|
|
22
|
+
CORPUS="${CORPUS:-/tmp/homegraph-corpus}"
|
|
23
|
+
REPO="$CORPUS/$NAME"
|
|
24
|
+
PKG="homegraph"
|
|
25
|
+
|
|
26
|
+
echo "==================== HomeGraph audit ===================="
|
|
27
|
+
echo "version=$VERSION repo=$NAME mode=$MODE corpus=$CORPUS"
|
|
28
|
+
echo
|
|
29
|
+
|
|
30
|
+
# 1. Set the homegraph version under test (mutates the global install).
|
|
31
|
+
if [ "$VERSION" = local ]; then
|
|
32
|
+
echo "→ [1/4] building + linking local dev build (local-install.sh)"
|
|
33
|
+
( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
|
|
34
|
+
else
|
|
35
|
+
echo "→ [1/4] installing $PKG@$VERSION globally"
|
|
36
|
+
npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
|
|
37
|
+
fi
|
|
38
|
+
ACTUAL="$(homegraph --version 2>/dev/null || echo '?')"
|
|
39
|
+
echo " homegraph on PATH: $(command -v homegraph) -> $ACTUAL"
|
|
40
|
+
|
|
41
|
+
# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
|
|
42
|
+
mkdir -p "$CORPUS"
|
|
43
|
+
if [ -d "$REPO/.git" ]; then
|
|
44
|
+
echo "→ [2/4] reusing existing checkout: $REPO"
|
|
45
|
+
else
|
|
46
|
+
echo "→ [2/4] cloning $URL"
|
|
47
|
+
git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
|
|
48
|
+
fi
|
|
49
|
+
|
|
50
|
+
# 3. Wipe + re-index with THIS version (the index must be built by the same
|
|
51
|
+
# binary that serves it — different versions extract differently).
|
|
52
|
+
echo "→ [3/4] wiping .homegraph and re-indexing with $ACTUAL"
|
|
53
|
+
rm -rf "$REPO/.homegraph"
|
|
54
|
+
( cd "$REPO" && homegraph init -i ) || { echo "indexing failed"; exit 1; }
|
|
55
|
+
|
|
56
|
+
# 4. Run the with/without A/B.
|
|
57
|
+
echo "→ [4/4] running A/B harness (mode=$MODE)"
|
|
58
|
+
bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
|
|
59
|
+
|
|
60
|
+
# Restore the dev link (the normal working state in this repo).
|
|
61
|
+
echo
|
|
62
|
+
echo "→ restoring local dev link (local-install.sh)"
|
|
63
|
+
if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
|
|
64
|
+
echo " global homegraph restored to dev build"
|
|
65
|
+
else
|
|
66
|
+
echo " WARN: restore failed — run ./scripts/local-install.sh manually"
|
|
67
|
+
fi
|
|
68
|
+
echo "==================== audit complete ===================="
|
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Re-run the README "Benchmark Results" A/B (with vs without homegraph) on the
|
|
3
|
-
# current build: the 7 README repos, same queries, RUNS per arm (default 4).
|
|
4
|
-
# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
|
|
5
|
-
# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
|
|
6
|
-
# $CORPUS (default /tmp/homegraph-corpus) by the build under test.
|
|
7
|
-
set -uo pipefail
|
|
8
|
-
H="$(cd "$(dirname "$0")" && pwd)"
|
|
9
|
-
C="${CORPUS:-/tmp/homegraph-corpus}"
|
|
10
|
-
RUNS="${RUNS:-4}"
|
|
11
|
-
ROWS=(
|
|
12
|
-
"vscode|How does the extension host communicate with the main process?"
|
|
13
|
-
"excalidraw|How does Excalidraw render and update canvas elements?"
|
|
14
|
-
"django|How does Django's ORM build and execute a query from a QuerySet?"
|
|
15
|
-
"tokio|How does tokio schedule and run async tasks on its runtime?"
|
|
16
|
-
"okhttp|How does OkHttp process a request through its interceptor chain?"
|
|
17
|
-
"gin|How does gin route requests through its middleware chain?"
|
|
18
|
-
"alamofire|How does Alamofire build, send, and validate a request?"
|
|
19
|
-
)
|
|
20
|
-
echo "### README A/B START $(date) RUNS=$RUNS"
|
|
21
|
-
for row in "${ROWS[@]}"; do
|
|
22
|
-
repo="${row%%|*}"; q="${row#*|}"
|
|
23
|
-
echo "===== $repo ====="
|
|
24
|
-
for run in $(seq 1 "$RUNS"); do
|
|
25
|
-
AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo " run$run: (no exit line)"
|
|
26
|
-
done
|
|
27
|
-
done
|
|
28
|
-
echo "### README A/B DONE $(date)"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Re-run the README "Benchmark Results" A/B (with vs without homegraph) on the
|
|
3
|
+
# current build: the 7 README repos, same queries, RUNS per arm (default 4).
|
|
4
|
+
# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
|
|
5
|
+
# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
|
|
6
|
+
# $CORPUS (default /tmp/homegraph-corpus) by the build under test.
|
|
7
|
+
set -uo pipefail
|
|
8
|
+
H="$(cd "$(dirname "$0")" && pwd)"
|
|
9
|
+
C="${CORPUS:-/tmp/homegraph-corpus}"
|
|
10
|
+
RUNS="${RUNS:-4}"
|
|
11
|
+
ROWS=(
|
|
12
|
+
"vscode|How does the extension host communicate with the main process?"
|
|
13
|
+
"excalidraw|How does Excalidraw render and update canvas elements?"
|
|
14
|
+
"django|How does Django's ORM build and execute a query from a QuerySet?"
|
|
15
|
+
"tokio|How does tokio schedule and run async tasks on its runtime?"
|
|
16
|
+
"okhttp|How does OkHttp process a request through its interceptor chain?"
|
|
17
|
+
"gin|How does gin route requests through its middleware chain?"
|
|
18
|
+
"alamofire|How does Alamofire build, send, and validate a request?"
|
|
19
|
+
)
|
|
20
|
+
echo "### README A/B START $(date) RUNS=$RUNS"
|
|
21
|
+
for row in "${ROWS[@]}"; do
|
|
22
|
+
repo="${row%%|*}"; q="${row#*|}"
|
|
23
|
+
echo "===== $repo ====="
|
|
24
|
+
for run in $(seq 1 "$RUNS"); do
|
|
25
|
+
AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo " run$run: (no exit line)"
|
|
26
|
+
done
|
|
27
|
+
done
|
|
28
|
+
echo "### README A/B DONE $(date)"
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# One README repo, WITH-homegraph only, N runs. Each run appends a why-Read
|
|
3
|
-
# diagnostic so the agent explains any Read/Grep. (The WITHOUT baseline is
|
|
4
|
-
# homegraph-independent and already in the README — no point re-running it.)
|
|
5
|
-
# Output -> /tmp/ab-why/<repo>/with<n>.jsonl
|
|
6
|
-
# Usage: bench-why-repo.sh <repo-path> "<query>" [N]
|
|
7
|
-
set -uo pipefail
|
|
8
|
-
REPO="$1"; Q="$2"; N="${3:-4}"
|
|
9
|
-
NAME="$(basename "$REPO")"
|
|
10
|
-
CG="/Users/colby/Development/Personal/homegraph/dist/bin/homegraph.js"
|
|
11
|
-
OUT="/tmp/ab-why/$NAME"; mkdir -p "$OUT"
|
|
12
|
-
WHY=$'\n\nIMPORTANT — diagnostic: if you use the Read or Grep tool at ANY point, for EACH such call explain why homegraph_explore / homegraph_node did not already give you what you needed. End your entire answer with a section titled exactly "## Why I read" listing every Read and Grep you made and the precise reason homegraph fell short for it. If you used neither, write "## Why I read" then "none — homegraph was sufficient."'
|
|
13
|
-
printf '{"mcpServers":{"homegraph":{"command":"%s","args":["serve","--mcp","--path","%s"]}}}' "$CG" "$REPO" > "$OUT/cg.json"
|
|
14
|
-
|
|
15
|
-
for i in $(seq 1 "$N"); do
|
|
16
|
-
pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.homegraph/daemon.sock"
|
|
17
|
-
( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
|
|
18
|
-
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
|
19
|
-
--strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
|
|
20
|
-
echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
|
|
21
|
-
done
|
|
22
|
-
echo "DONE $NAME"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# One README repo, WITH-homegraph only, N runs. Each run appends a why-Read
|
|
3
|
+
# diagnostic so the agent explains any Read/Grep. (The WITHOUT baseline is
|
|
4
|
+
# homegraph-independent and already in the README — no point re-running it.)
|
|
5
|
+
# Output -> /tmp/ab-why/<repo>/with<n>.jsonl
|
|
6
|
+
# Usage: bench-why-repo.sh <repo-path> "<query>" [N]
|
|
7
|
+
set -uo pipefail
|
|
8
|
+
REPO="$1"; Q="$2"; N="${3:-4}"
|
|
9
|
+
NAME="$(basename "$REPO")"
|
|
10
|
+
CG="/Users/colby/Development/Personal/homegraph/dist/bin/homegraph.js"
|
|
11
|
+
OUT="/tmp/ab-why/$NAME"; mkdir -p "$OUT"
|
|
12
|
+
WHY=$'\n\nIMPORTANT — diagnostic: if you use the Read or Grep tool at ANY point, for EACH such call explain why homegraph_explore / homegraph_node did not already give you what you needed. End your entire answer with a section titled exactly "## Why I read" listing every Read and Grep you made and the precise reason homegraph fell short for it. If you used neither, write "## Why I read" then "none — homegraph was sufficient."'
|
|
13
|
+
printf '{"mcpServers":{"homegraph":{"command":"%s","args":["serve","--mcp","--path","%s"]}}}' "$CG" "$REPO" > "$OUT/cg.json"
|
|
14
|
+
|
|
15
|
+
for i in $(seq 1 "$N"); do
|
|
16
|
+
pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.homegraph/daemon.sock"
|
|
17
|
+
( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
|
|
18
|
+
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
|
19
|
+
--strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
|
|
20
|
+
echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
|
|
21
|
+
done
|
|
22
|
+
echo "DONE $NAME"
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# PreToolUse hook (experiment): deny Read of homegraph-indexed source files and
|
|
3
|
-
# steer the agent to homegraph_explore/homegraph_node instead. Tests whether
|
|
4
|
-
# homegraph can FULLY replace Read for code-understanding once the escape hatch
|
|
5
|
-
# is removed. Non-source reads (config, .env, markdown, new files) pass through.
|
|
6
|
-
#
|
|
7
|
-
# Wire via: claude ... --settings scripts/agent-eval/hook-settings.json
|
|
8
|
-
set -uo pipefail
|
|
9
|
-
input="$(cat)"
|
|
10
|
-
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
|
11
|
-
|
|
12
|
-
case "$fp" in
|
|
13
|
-
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
|
|
14
|
-
msg="Read is disabled for source files in this session — homegraph already has this file indexed (with line numbers, kept in sync on every change). Use homegraph_explore (several related symbols at once) or homegraph_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER homegraph_explore with its exact name instead of reading the file."
|
|
15
|
-
jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
|
16
|
-
exit 0
|
|
17
|
-
;;
|
|
18
|
-
esac
|
|
19
|
-
exit 0
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# PreToolUse hook (experiment): deny Read of homegraph-indexed source files and
|
|
3
|
+
# steer the agent to homegraph_explore/homegraph_node instead. Tests whether
|
|
4
|
+
# homegraph can FULLY replace Read for code-understanding once the escape hatch
|
|
5
|
+
# is removed. Non-source reads (config, .env, markdown, new files) pass through.
|
|
6
|
+
#
|
|
7
|
+
# Wire via: claude ... --settings scripts/agent-eval/hook-settings.json
|
|
8
|
+
set -uo pipefail
|
|
9
|
+
input="$(cat)"
|
|
10
|
+
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
|
11
|
+
|
|
12
|
+
case "$fp" in
|
|
13
|
+
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
|
|
14
|
+
msg="Read is disabled for source files in this session — homegraph already has this file indexed (with line numbers, kept in sync on every change). Use homegraph_explore (several related symbols at once) or homegraph_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER homegraph_explore with its exact name instead of reading the file."
|
|
15
|
+
jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
|
16
|
+
exit 0
|
|
17
|
+
;;
|
|
18
|
+
esac
|
|
19
|
+
exit 0
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
{
|
|
2
|
-
"hooks": {
|
|
3
|
-
"PreToolUse": [
|
|
4
|
-
{
|
|
5
|
-
"matcher": "Read",
|
|
6
|
-
"hooks": [
|
|
7
|
-
{
|
|
8
|
-
"type": "command",
|
|
9
|
-
"command": "bash /Users/colby/Development/Personal/homegraph/scripts/agent-eval/block-read-hook.sh"
|
|
10
|
-
}
|
|
11
|
-
]
|
|
12
|
-
}
|
|
13
|
-
]
|
|
14
|
-
}
|
|
15
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"hooks": {
|
|
3
|
+
"PreToolUse": [
|
|
4
|
+
{
|
|
5
|
+
"matcher": "Read",
|
|
6
|
+
"hooks": [
|
|
7
|
+
{
|
|
8
|
+
"type": "command",
|
|
9
|
+
"command": "bash /Users/colby/Development/Personal/homegraph/scripts/agent-eval/block-read-hook.sh"
|
|
10
|
+
}
|
|
11
|
+
]
|
|
12
|
+
}
|
|
13
|
+
]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -1,120 +1,120 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
|
|
3
|
-
# agent to finish, then print the tool-call breakdown from the session logs.
|
|
4
|
-
#
|
|
5
|
-
# Why interactive (not `claude -p`): headless print-mode picks the
|
|
6
|
-
# general-purpose subagent, while real interactive sessions delegate to the
|
|
7
|
-
# Explore subagent (or drive homegraph from the main thread). Only the
|
|
8
|
-
# interactive TUI reproduces the behavior users actually see. (Idle-detection
|
|
9
|
-
# technique borrowed from devpit's WaitForIdle.)
|
|
10
|
-
#
|
|
11
|
-
# Usage: itrun.sh <repo-path> <label> "<prompt>"
|
|
12
|
-
# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
|
|
13
|
-
# Requires: tmux 3.0+, a logged-in `claude` CLI, homegraph MCP configured.
|
|
14
|
-
set -uo pipefail
|
|
15
|
-
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
|
16
|
-
SESSION="cgt_${LABEL}"
|
|
17
|
-
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
|
18
|
-
OUT="$OUT_DIR/itrun-${LABEL}.txt"
|
|
19
|
-
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
20
|
-
|
|
21
|
-
cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
|
|
22
|
-
|
|
23
|
-
tmux kill-session -t "$SESSION" 2>/dev/null
|
|
24
|
-
|
|
25
|
-
# Wide pane so the TUI doesn't hard-wrap tool lines.
|
|
26
|
-
tmux new-session -d -s "$SESSION" -x 230 -y 60
|
|
27
|
-
tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
|
|
28
|
-
|
|
29
|
-
# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
|
|
30
|
-
# welcome screen seconds before the input actually accepts keystrokes, so this is
|
|
31
|
-
# necessary but NOT sufficient — the type-and-verify loop below is what proves
|
|
32
|
-
# the input is live.
|
|
33
|
-
ready=0
|
|
34
|
-
for _ in $(seq 1 120); do
|
|
35
|
-
cap | grep -q "❯" && { ready=1; break; }
|
|
36
|
-
sleep 0.5
|
|
37
|
-
done
|
|
38
|
-
[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
|
39
|
-
|
|
40
|
-
# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
|
|
41
|
-
# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
|
|
42
|
-
# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
|
|
43
|
-
# cleared before the type-and-verify loop or keystrokes land on the menu.
|
|
44
|
-
for _ in $(seq 1 20); do
|
|
45
|
-
cap | grep -q "trust this folder" || break
|
|
46
|
-
tmux send-keys -t "$SESSION" Enter
|
|
47
|
-
sleep 1
|
|
48
|
-
done
|
|
49
|
-
|
|
50
|
-
# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
|
|
51
|
-
# landed in the input box, retry if it didn't (handles the early-❯ race where
|
|
52
|
-
# the welcome screen shows the prompt glyph but MCP init is still eating keys).
|
|
53
|
-
needle="${PROMPT:0:24}"
|
|
54
|
-
typed=0
|
|
55
|
-
for _ in $(seq 1 30); do
|
|
56
|
-
tmux send-keys -l -t "$SESSION" "$PROMPT"
|
|
57
|
-
sleep 1
|
|
58
|
-
if cap | grep -Fq "$needle"; then typed=1; break; fi
|
|
59
|
-
# Clear whatever partial text may have landed, then retry.
|
|
60
|
-
tmux send-keys -t "$SESSION" C-u
|
|
61
|
-
sleep 1
|
|
62
|
-
done
|
|
63
|
-
[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
|
64
|
-
sleep 0.5
|
|
65
|
-
tmux send-keys -t "$SESSION" Enter
|
|
66
|
-
|
|
67
|
-
# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
|
|
68
|
-
# EVERY working state shows — both the pre-stream thinking phase
|
|
69
|
-
# "(8s · thinking with max effort)" and the streaming phase
|
|
70
|
-
# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
|
|
71
|
-
# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
|
|
72
|
-
# (some TUI versions/states show one but not the others).
|
|
73
|
-
BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
|
|
74
|
-
|
|
75
|
-
# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
|
|
76
|
-
# fail loudly rather than silently reporting an empty run.
|
|
77
|
-
started=0
|
|
78
|
-
for _ in $(seq 1 120); do
|
|
79
|
-
cap | grep -qE "$BUSY_RE" && { started=1; break; }
|
|
80
|
-
sleep 0.5
|
|
81
|
-
done
|
|
82
|
-
[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
|
83
|
-
|
|
84
|
-
# Poll for idle. CRITICAL: Opus 4.8 (extended thinking) renders NO spinner /
|
|
85
|
-
# "esc to interrupt" / timer while it STREAMS its final answer — those appear
|
|
86
|
-
# only during the thinking + tool-use phases ("✻ Marinating… (32s · ↓ 1.3k
|
|
87
|
-
# tokens · thinking with max effort)"). So BUSY_RE reads "not busy" for the whole
|
|
88
|
-
# 10-30s answer stream, and any short not-busy threshold kills the run mid-answer
|
|
89
|
-
# (the truncation bug). We therefore detect "done" by CONTENT STABILITY, not by a
|
|
90
|
-
# spinner string: while the agent streams, the captured pane changes every poll,
|
|
91
|
-
# so stability never accrues; it accrues only once the agent has finished and the
|
|
92
|
-
# static "✻ Brewed for 1m 9s" summary is all that is left. BUSY_RE still hard-
|
|
93
|
-
# resets stability (covers thinking/tool-use/live-timer, where text can briefly
|
|
94
|
-
# sit still). Need STABLE_NEEDED polls (~8s) of zero pane change + ❯ present.
|
|
95
|
-
# Content-stability is model-agnostic — it survives future spinner re-wordings.
|
|
96
|
-
STABLE_NEEDED=16
|
|
97
|
-
prev=""; stable=0
|
|
98
|
-
for _ in $(seq 1 2400); do # up to ~20 min
|
|
99
|
-
pane="$(cap)"
|
|
100
|
-
sig="$(printf '%s' "$pane" | tr -s '[:space:]' ' ')"
|
|
101
|
-
if printf '%s' "$pane" | grep -qE "$BUSY_RE"; then
|
|
102
|
-
stable=0 # thinking / tool use / live timer → busy
|
|
103
|
-
elif [ -n "$sig" ] && [ "$sig" = "$prev" ] && printf '%s' "$pane" | grep -q "❯"; then
|
|
104
|
-
stable=$((stable+1)); [ "$stable" -ge "$STABLE_NEEDED" ] && break
|
|
105
|
-
else
|
|
106
|
-
stable=0 # answer still streaming → pane changing
|
|
107
|
-
fi
|
|
108
|
-
prev="$sig"
|
|
109
|
-
sleep 0.5
|
|
110
|
-
done
|
|
111
|
-
sleep 1
|
|
112
|
-
|
|
113
|
-
tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
|
|
114
|
-
echo "captured $(wc -l < "$OUT") lines -> $OUT"
|
|
115
|
-
grep -oE "Done \([^)]*\)|[A-Z][a-z]+ for ([0-9]+m )?[0-9]+s" "$OUT" | tail -1
|
|
116
|
-
grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
|
|
117
|
-
tmux kill-session -t "$SESSION" 2>/dev/null
|
|
118
|
-
|
|
119
|
-
# Clean tool breakdown from the session logs (main + subagents).
|
|
120
|
-
node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
|
|
3
|
+
# agent to finish, then print the tool-call breakdown from the session logs.
|
|
4
|
+
#
|
|
5
|
+
# Why interactive (not `claude -p`): headless print-mode picks the
|
|
6
|
+
# general-purpose subagent, while real interactive sessions delegate to the
|
|
7
|
+
# Explore subagent (or drive homegraph from the main thread). Only the
|
|
8
|
+
# interactive TUI reproduces the behavior users actually see. (Idle-detection
|
|
9
|
+
# technique borrowed from devpit's WaitForIdle.)
|
|
10
|
+
#
|
|
11
|
+
# Usage: itrun.sh <repo-path> <label> "<prompt>"
|
|
12
|
+
# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
|
|
13
|
+
# Requires: tmux 3.0+, a logged-in `claude` CLI, homegraph MCP configured.
|
|
14
|
+
set -uo pipefail
|
|
15
|
+
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
|
16
|
+
SESSION="cgt_${LABEL}"
|
|
17
|
+
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
|
18
|
+
OUT="$OUT_DIR/itrun-${LABEL}.txt"
|
|
19
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
20
|
+
|
|
21
|
+
cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
|
|
22
|
+
|
|
23
|
+
tmux kill-session -t "$SESSION" 2>/dev/null
|
|
24
|
+
|
|
25
|
+
# Wide pane so the TUI doesn't hard-wrap tool lines.
|
|
26
|
+
tmux new-session -d -s "$SESSION" -x 230 -y 60
|
|
27
|
+
tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
|
|
28
|
+
|
|
29
|
+
# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
|
|
30
|
+
# welcome screen seconds before the input actually accepts keystrokes, so this is
|
|
31
|
+
# necessary but NOT sufficient — the type-and-verify loop below is what proves
|
|
32
|
+
# the input is live.
|
|
33
|
+
ready=0
|
|
34
|
+
for _ in $(seq 1 120); do
|
|
35
|
+
cap | grep -q "❯" && { ready=1; break; }
|
|
36
|
+
sleep 0.5
|
|
37
|
+
done
|
|
38
|
+
[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
|
39
|
+
|
|
40
|
+
# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
|
|
41
|
+
# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
|
|
42
|
+
# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
|
|
43
|
+
# cleared before the type-and-verify loop or keystrokes land on the menu.
|
|
44
|
+
for _ in $(seq 1 20); do
|
|
45
|
+
cap | grep -q "trust this folder" || break
|
|
46
|
+
tmux send-keys -t "$SESSION" Enter
|
|
47
|
+
sleep 1
|
|
48
|
+
done
|
|
49
|
+
|
|
50
|
+
# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
|
|
51
|
+
# landed in the input box, retry if it didn't (handles the early-❯ race where
|
|
52
|
+
# the welcome screen shows the prompt glyph but MCP init is still eating keys).
|
|
53
|
+
needle="${PROMPT:0:24}"
|
|
54
|
+
typed=0
|
|
55
|
+
for _ in $(seq 1 30); do
|
|
56
|
+
tmux send-keys -l -t "$SESSION" "$PROMPT"
|
|
57
|
+
sleep 1
|
|
58
|
+
if cap | grep -Fq "$needle"; then typed=1; break; fi
|
|
59
|
+
# Clear whatever partial text may have landed, then retry.
|
|
60
|
+
tmux send-keys -t "$SESSION" C-u
|
|
61
|
+
sleep 1
|
|
62
|
+
done
|
|
63
|
+
[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
|
64
|
+
sleep 0.5
|
|
65
|
+
tmux send-keys -t "$SESSION" Enter
|
|
66
|
+
|
|
67
|
+
# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
|
|
68
|
+
# EVERY working state shows — both the pre-stream thinking phase
|
|
69
|
+
# "(8s · thinking with max effort)" and the streaming phase
|
|
70
|
+
# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
|
|
71
|
+
# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
|
|
72
|
+
# (some TUI versions/states show one but not the others).
|
|
73
|
+
BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
|
|
74
|
+
|
|
75
|
+
# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
|
|
76
|
+
# fail loudly rather than silently reporting an empty run.
|
|
77
|
+
started=0
|
|
78
|
+
for _ in $(seq 1 120); do
|
|
79
|
+
cap | grep -qE "$BUSY_RE" && { started=1; break; }
|
|
80
|
+
sleep 0.5
|
|
81
|
+
done
|
|
82
|
+
[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
|
83
|
+
|
|
84
|
+
# Poll for idle. CRITICAL: Opus 4.8 (extended thinking) renders NO spinner /
|
|
85
|
+
# "esc to interrupt" / timer while it STREAMS its final answer — those appear
|
|
86
|
+
# only during the thinking + tool-use phases ("✻ Marinating… (32s · ↓ 1.3k
|
|
87
|
+
# tokens · thinking with max effort)"). So BUSY_RE reads "not busy" for the whole
|
|
88
|
+
# 10-30s answer stream, and any short not-busy threshold kills the run mid-answer
|
|
89
|
+
# (the truncation bug). We therefore detect "done" by CONTENT STABILITY, not by a
|
|
90
|
+
# spinner string: while the agent streams, the captured pane changes every poll,
|
|
91
|
+
# so stability never accrues; it accrues only once the agent has finished and the
|
|
92
|
+
# static "✻ Brewed for 1m 9s" summary is all that is left. BUSY_RE still hard-
|
|
93
|
+
# resets stability (covers thinking/tool-use/live-timer, where text can briefly
|
|
94
|
+
# sit still). Need STABLE_NEEDED polls (~8s) of zero pane change + ❯ present.
|
|
95
|
+
# Content-stability is model-agnostic — it survives future spinner re-wordings.
|
|
96
|
+
STABLE_NEEDED=16
|
|
97
|
+
prev=""; stable=0
|
|
98
|
+
for _ in $(seq 1 2400); do # up to ~20 min
|
|
99
|
+
pane="$(cap)"
|
|
100
|
+
sig="$(printf '%s' "$pane" | tr -s '[:space:]' ' ')"
|
|
101
|
+
if printf '%s' "$pane" | grep -qE "$BUSY_RE"; then
|
|
102
|
+
stable=0 # thinking / tool use / live timer → busy
|
|
103
|
+
elif [ -n "$sig" ] && [ "$sig" = "$prev" ] && printf '%s' "$pane" | grep -q "❯"; then
|
|
104
|
+
stable=$((stable+1)); [ "$stable" -ge "$STABLE_NEEDED" ] && break
|
|
105
|
+
else
|
|
106
|
+
stable=0 # answer still streaming → pane changing
|
|
107
|
+
fi
|
|
108
|
+
prev="$sig"
|
|
109
|
+
sleep 0.5
|
|
110
|
+
done
|
|
111
|
+
sleep 1
|
|
112
|
+
|
|
113
|
+
tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
|
|
114
|
+
echo "captured $(wc -l < "$OUT") lines -> $OUT"
|
|
115
|
+
grep -oE "Done \([^)]*\)|[A-Z][a-z]+ for ([0-9]+m )?[0-9]+s" "$OUT" | tail -1
|
|
116
|
+
grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
|
|
117
|
+
tmux kill-session -t "$SESSION" 2>/dev/null
|
|
118
|
+
|
|
119
|
+
# Clean tool breakdown from the session logs (main + subagents).
|
|
120
|
+
node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true
|