homegraph 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +243 -243
- package/dist/bin/homegraph.js +371 -9
- package/dist/bin/homegraph.js.map +1 -1
- package/dist/db/migrations.js +18 -18
- package/dist/db/queries.js +140 -140
- package/dist/db/schema.sql +152 -152
- package/dist/directory.js +5 -5
- package/dist/extraction/wasm/tree-sitter-c_sharp.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-pascal.wasm +0 -0
- package/dist/installer/instructions-template.js +9 -9
- package/dist/installer/targets/shared.d.ts +5 -6
- package/dist/installer/targets/shared.d.ts.map +1 -1
- package/dist/installer/targets/shared.js +5 -6
- package/dist/installer/targets/shared.js.map +1 -1
- package/dist/mcp/liveness-watchdog.js +16 -16
- package/dist/mcp/server-instructions.js +69 -69
- package/dist/mcp/tools.d.ts +17 -4
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +192 -22
- package/dist/mcp/tools.js.map +1 -1
- package/dist/reasoning/reasoner.js +32 -32
- package/dist/spec/config.d.ts +43 -0
- package/dist/spec/config.d.ts.map +1 -0
- package/dist/spec/config.js +275 -0
- package/dist/spec/config.js.map +1 -0
- package/dist/spec/db/commit-node.d.ts +23 -0
- package/dist/spec/db/commit-node.d.ts.map +1 -0
- package/dist/spec/db/commit-node.js +62 -0
- package/dist/spec/db/commit-node.js.map +1 -0
- package/dist/spec/db/fragment-node.d.ts +23 -0
- package/dist/spec/db/fragment-node.d.ts.map +1 -0
- package/dist/spec/db/fragment-node.js +120 -0
- package/dist/spec/db/fragment-node.js.map +1 -0
- package/dist/spec/db/fts.d.ts +60 -0
- package/dist/spec/db/fts.d.ts.map +1 -0
- package/dist/spec/db/fts.js +285 -0
- package/dist/spec/db/fts.js.map +1 -0
- package/dist/spec/db/index.d.ts +13 -0
- package/dist/spec/db/index.d.ts.map +1 -0
- package/dist/spec/db/index.js +50 -0
- package/dist/spec/db/index.js.map +1 -0
- package/dist/spec/db/relations.d.ts +55 -0
- package/dist/spec/db/relations.d.ts.map +1 -0
- package/dist/spec/db/relations.js +158 -0
- package/dist/spec/db/relations.js.map +1 -0
- package/dist/spec/db/schema.d.ts +30 -0
- package/dist/spec/db/schema.d.ts.map +1 -0
- package/dist/spec/db/schema.js +87 -0
- package/dist/spec/db/schema.js.map +1 -0
- package/dist/spec/db/schema.sql +107 -0
- package/dist/spec/db/spec-node.d.ts +41 -0
- package/dist/spec/db/spec-node.d.ts.map +1 -0
- package/dist/spec/db/spec-node.js +114 -0
- package/dist/spec/db/spec-node.js.map +1 -0
- package/dist/spec/evolve/impact-locator.d.ts +13 -0
- package/dist/spec/evolve/impact-locator.d.ts.map +1 -0
- package/dist/spec/evolve/impact-locator.js +25 -0
- package/dist/spec/evolve/impact-locator.js.map +1 -0
- package/dist/spec/evolve/llm-client.d.ts +50 -0
- package/dist/spec/evolve/llm-client.d.ts.map +1 -0
- package/dist/spec/evolve/llm-client.js +176 -0
- package/dist/spec/evolve/llm-client.js.map +1 -0
- package/dist/spec/evolve/logic-checker.d.ts +12 -0
- package/dist/spec/evolve/logic-checker.d.ts.map +1 -0
- package/dist/spec/evolve/logic-checker.js +48 -0
- package/dist/spec/evolve/logic-checker.js.map +1 -0
- package/dist/spec/evolve/pipeline.d.ts +40 -0
- package/dist/spec/evolve/pipeline.d.ts.map +1 -0
- package/dist/spec/evolve/pipeline.js +314 -0
- package/dist/spec/evolve/pipeline.js.map +1 -0
- package/dist/spec/evolve/spec-rewriter.d.ts +42 -0
- package/dist/spec/evolve/spec-rewriter.d.ts.map +1 -0
- package/dist/spec/evolve/spec-rewriter.js +254 -0
- package/dist/spec/evolve/spec-rewriter.js.map +1 -0
- package/dist/spec/graph/queries.d.ts +58 -0
- package/dist/spec/graph/queries.d.ts.map +1 -0
- package/dist/spec/graph/queries.js +215 -0
- package/dist/spec/graph/queries.js.map +1 -0
- package/dist/spec/mining/diff-parser.d.ts +33 -0
- package/dist/spec/mining/diff-parser.d.ts.map +1 -0
- package/dist/spec/mining/diff-parser.js +166 -0
- package/dist/spec/mining/diff-parser.js.map +1 -0
- package/dist/spec/mining/git-scanner.d.ts +91 -0
- package/dist/spec/mining/git-scanner.d.ts.map +1 -0
- package/dist/spec/mining/git-scanner.js +266 -0
- package/dist/spec/mining/git-scanner.js.map +1 -0
- package/dist/spec/mining/pipeline.d.ts +53 -0
- package/dist/spec/mining/pipeline.d.ts.map +1 -0
- package/dist/spec/mining/pipeline.js +165 -0
- package/dist/spec/mining/pipeline.js.map +1 -0
- package/dist/spec/mining/scope-resolver.d.ts +45 -0
- package/dist/spec/mining/scope-resolver.d.ts.map +1 -0
- package/dist/spec/mining/scope-resolver.js +103 -0
- package/dist/spec/mining/scope-resolver.js.map +1 -0
- package/dist/spec/mining/spec-extractor.d.ts +69 -0
- package/dist/spec/mining/spec-extractor.d.ts.map +1 -0
- package/dist/spec/mining/spec-extractor.js +369 -0
- package/dist/spec/mining/spec-extractor.js.map +1 -0
- package/dist/spec/types.d.ts +149 -0
- package/dist/spec/types.d.ts.map +1 -0
- package/dist/spec/types.js +15 -0
- package/dist/spec/types.js.map +1 -0
- package/dist/spec/utils.d.ts +166 -0
- package/dist/spec/utils.d.ts.map +1 -0
- package/dist/spec/utils.js +461 -0
- package/dist/spec/utils.js.map +1 -0
- package/package.json +58 -57
- package/scripts/add-lang/bench.sh +60 -60
- package/scripts/add-lang/check-grammar.mjs +75 -75
- package/scripts/add-lang/dump-ast.mjs +103 -103
- package/scripts/add-lang/verify-extraction.mjs +70 -70
- package/scripts/agent-eval/ab-adoption.sh +91 -91
- package/scripts/agent-eval/ab-hook.sh +86 -86
- package/scripts/agent-eval/ab-impl.sh +78 -78
- package/scripts/agent-eval/ab-new-vs-baseline.sh +102 -102
- package/scripts/agent-eval/ab-sufficiency.sh +78 -78
- package/scripts/agent-eval/arms-F.sh +21 -21
- package/scripts/agent-eval/arms-matrix.sh +37 -37
- package/scripts/agent-eval/audit.sh +68 -68
- package/scripts/agent-eval/bench-readme.sh +28 -28
- package/scripts/agent-eval/bench-why-repo.sh +22 -22
- package/scripts/agent-eval/block-read-hook.sh +19 -19
- package/scripts/agent-eval/hook-settings.json +15 -15
- package/scripts/agent-eval/itrun.sh +120 -120
- package/scripts/agent-eval/offload-eval-3arm.sh +72 -72
- package/scripts/agent-eval/offload-eval-cost.mjs +133 -133
- package/scripts/agent-eval/offload-eval-effort.mjs +108 -108
- package/scripts/agent-eval/offload-eval-frontload-matrix.sh +25 -25
- package/scripts/agent-eval/offload-eval-frontload.sh +47 -47
- package/scripts/agent-eval/offload-eval-ground-truth.json +18 -18
- package/scripts/agent-eval/offload-eval-hook.mjs +84 -84
- package/scripts/agent-eval/offload-eval-judge.mjs +103 -103
- package/scripts/agent-eval/offload-eval-matrix.sh +20 -20
- package/scripts/agent-eval/offload-eval-metrics.mjs +94 -94
- package/scripts/agent-eval/offload-eval-refs1.sh +50 -50
- package/scripts/agent-eval/offload-eval-setup.sh +24 -24
- package/scripts/agent-eval/offload-eval-styles.sh +71 -71
- package/scripts/agent-eval/offload-eval-summarize.mjs +68 -68
- package/scripts/agent-eval/offload-eval.md +76 -76
- package/scripts/agent-eval/parse-arms.mjs +116 -116
- package/scripts/agent-eval/parse-bench-readme.mjs +84 -84
- package/scripts/agent-eval/parse-run.mjs +45 -45
- package/scripts/agent-eval/parse-session.mjs +93 -93
- package/scripts/agent-eval/probe-context.mjs +21 -21
- package/scripts/agent-eval/probe-explore.mjs +40 -40
- package/scripts/agent-eval/probe-node.mjs +20 -20
- package/scripts/agent-eval/probe-sweep.mjs +119 -119
- package/scripts/agent-eval/probe-trace.mjs +20 -20
- package/scripts/agent-eval/redirect-read-hook.sh +38 -38
- package/scripts/agent-eval/run-agent.sh +34 -34
- package/scripts/agent-eval/run-all.sh +69 -69
- package/scripts/agent-eval/run-arms.sh +56 -56
- package/scripts/agent-eval/seq-matrix.mjs +137 -137
- package/scripts/build-bundle.sh +118 -118
- package/scripts/extract-release-notes.mjs +130 -130
- package/scripts/local-install.sh +41 -41
- package/scripts/npm-sdk.js +75 -75
- package/scripts/npm-shim.js +246 -246
- package/scripts/pack-npm.sh +119 -119
- package/scripts/prepare-release.mjs +270 -270
- package/scripts/qa_eval/README.md +407 -404
- package/scripts/qa_eval/_test_deveco_probe.py +41 -41
- package/scripts/qa_eval/agent_runner.py +526 -526
- package/scripts/qa_eval/data/.gitignore +4 -4
- package/scripts/qa_eval/data/test-set.jsonl +2 -22
- package/scripts/qa_eval/eval_metrics.py +274 -233
- package/scripts/qa_eval/external_agent.py +976 -671
- package/scripts/qa_eval/llm_config.py +92 -92
- package/scripts/qa_eval/memory_monitor.py +132 -132
- package/scripts/qa_eval/my_answer_accuracy.py +187 -187
- package/scripts/qa_eval/requirements.txt +2 -2
- package/scripts/qa_eval/run_pipeline.py +804 -711
- package/scripts/qa_eval/stats_efficiency.py +279 -279
- package/scripts/qa_eval/stats_scores.py +207 -207
|
@@ -1,70 +1,70 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// Sanity-check that homegraph extracted REAL symbols (not just file/import nodes)
|
|
3
|
-
// from a repo for a given language. Exits non-zero on a critical failure so it
|
|
4
|
-
// can drive a write-extractor -> build -> re-check loop.
|
|
5
|
-
//
|
|
6
|
-
// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
|
|
7
|
-
// Reads `homegraph status <repo> --json` using whatever homegraph is on PATH,
|
|
8
|
-
// so it reflects the binary that built the index.
|
|
9
|
-
//
|
|
10
|
-
// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
|
|
11
|
-
|
|
12
|
-
import { execFileSync } from 'node:child_process';
|
|
13
|
-
|
|
14
|
-
const [repo, lang] = process.argv.slice(2);
|
|
15
|
-
if (!repo || !lang) {
|
|
16
|
-
console.error('usage: verify-extraction.mjs <repo-path> <lang>');
|
|
17
|
-
process.exit(2);
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
let status;
|
|
21
|
-
try {
|
|
22
|
-
const out = execFileSync('homegraph', ['status', repo, '--json'], { encoding: 'utf8' });
|
|
23
|
-
status = JSON.parse(out);
|
|
24
|
-
} catch (e) {
|
|
25
|
-
console.error(`[verify] could not read homegraph status for ${repo}: ${e.message}`);
|
|
26
|
-
process.exit(2);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
// Kinds that prove the extractor mapped AST node types (everything except
|
|
30
|
-
// 'file' and 'import', which homegraph creates structurally for any language).
|
|
31
|
-
const SYMBOL_KINDS = new Set([
|
|
32
|
-
'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
|
|
33
|
-
'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
|
|
34
|
-
'type_alias', 'namespace', 'route', 'component',
|
|
35
|
-
]);
|
|
36
|
-
|
|
37
|
-
const byKind = status.nodesByKind || {};
|
|
38
|
-
const langs = status.languages || [];
|
|
39
|
-
const files = status.fileCount || 0;
|
|
40
|
-
const edges = status.edgeCount || 0;
|
|
41
|
-
const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
|
|
42
|
-
const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
|
|
43
|
-
|
|
44
|
-
const checks = [];
|
|
45
|
-
const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
|
|
46
|
-
|
|
47
|
-
add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
|
|
48
|
-
add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
|
|
49
|
-
add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
|
|
50
|
-
add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
|
|
51
|
-
add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
|
|
52
|
-
|
|
53
|
-
console.log(`\n# Extraction check — ${repo} (lang=${lang}, backend=${status.backend})`);
|
|
54
|
-
console.log(` files=${files} nodes=${status.nodeCount} edges=${edges}`);
|
|
55
|
-
console.log(` nodesByKind: ${JSON.stringify(byKind)}\n`);
|
|
56
|
-
for (const c of checks) console.log(` ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
|
|
57
|
-
|
|
58
|
-
const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
|
|
59
|
-
const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
|
|
60
|
-
console.log();
|
|
61
|
-
if (critical.length) {
|
|
62
|
-
console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
|
|
63
|
-
process.exit(1);
|
|
64
|
-
}
|
|
65
|
-
if (soft.length) {
|
|
66
|
-
console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
|
|
67
|
-
process.exit(0);
|
|
68
|
-
}
|
|
69
|
-
console.log('RESULT: PASS — extraction looks healthy.');
|
|
70
|
-
process.exit(0);
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Sanity-check that homegraph extracted REAL symbols (not just file/import nodes)
|
|
3
|
+
// from a repo for a given language. Exits non-zero on a critical failure so it
|
|
4
|
+
// can drive a write-extractor -> build -> re-check loop.
|
|
5
|
+
//
|
|
6
|
+
// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
|
|
7
|
+
// Reads `homegraph status <repo> --json` using whatever homegraph is on PATH,
|
|
8
|
+
// so it reflects the binary that built the index.
|
|
9
|
+
//
|
|
10
|
+
// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
|
|
11
|
+
|
|
12
|
+
import { execFileSync } from 'node:child_process';
|
|
13
|
+
|
|
14
|
+
const [repo, lang] = process.argv.slice(2);
|
|
15
|
+
if (!repo || !lang) {
|
|
16
|
+
console.error('usage: verify-extraction.mjs <repo-path> <lang>');
|
|
17
|
+
process.exit(2);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
let status;
|
|
21
|
+
try {
|
|
22
|
+
const out = execFileSync('homegraph', ['status', repo, '--json'], { encoding: 'utf8' });
|
|
23
|
+
status = JSON.parse(out);
|
|
24
|
+
} catch (e) {
|
|
25
|
+
console.error(`[verify] could not read homegraph status for ${repo}: ${e.message}`);
|
|
26
|
+
process.exit(2);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Kinds that prove the extractor mapped AST node types (everything except
|
|
30
|
+
// 'file' and 'import', which homegraph creates structurally for any language).
|
|
31
|
+
const SYMBOL_KINDS = new Set([
|
|
32
|
+
'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
|
|
33
|
+
'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
|
|
34
|
+
'type_alias', 'namespace', 'route', 'component',
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
const byKind = status.nodesByKind || {};
|
|
38
|
+
const langs = status.languages || [];
|
|
39
|
+
const files = status.fileCount || 0;
|
|
40
|
+
const edges = status.edgeCount || 0;
|
|
41
|
+
const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
|
|
42
|
+
const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
|
|
43
|
+
|
|
44
|
+
const checks = [];
|
|
45
|
+
const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
|
|
46
|
+
|
|
47
|
+
add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
|
|
48
|
+
add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
|
|
49
|
+
add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
|
|
50
|
+
add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
|
|
51
|
+
add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
|
|
52
|
+
|
|
53
|
+
console.log(`\n# Extraction check — ${repo} (lang=${lang}, backend=${status.backend})`);
|
|
54
|
+
console.log(` files=${files} nodes=${status.nodeCount} edges=${edges}`);
|
|
55
|
+
console.log(` nodesByKind: ${JSON.stringify(byKind)}\n`);
|
|
56
|
+
for (const c of checks) console.log(` ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
|
|
57
|
+
|
|
58
|
+
const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
|
|
59
|
+
const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
|
|
60
|
+
console.log();
|
|
61
|
+
if (critical.length) {
|
|
62
|
+
console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
if (soft.length) {
|
|
66
|
+
console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
|
|
67
|
+
process.exit(0);
|
|
68
|
+
}
|
|
69
|
+
console.log('RESULT: PASS — extraction looks healthy.');
|
|
70
|
+
process.exit(0);
|
|
@@ -1,91 +1,91 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Does the agent PICK homegraph_node to read a file, vs the built-in Read tool?
|
|
3
|
-
# Build A/B: NEW build (HEAD, homegraph_node has Read parity) vs BASELINE build
|
|
4
|
-
# (a ref where it doesn't), BOTH homegraph-attached + pre-warmed, same task. The
|
|
5
|
-
# metric is tool CHOICE: Read calls vs homegraph_node[file] calls per run.
|
|
6
|
-
#
|
|
7
|
-
# Usage: ab-adoption.sh <indexed-repo> "<task>" [runs-per-arm] [baseline-ref]
|
|
8
|
-
# Env: AGENT_EVAL_OUT (default: /tmp/ab-adoption)
|
|
9
|
-
set -uo pipefail
|
|
10
|
-
TARGET="${1:?usage: ab-adoption.sh <indexed-repo> \"<task>\" [runs] [baseline-ref]}"
|
|
11
|
-
TASK="${2:?task required}"
|
|
12
|
-
RUNS="${3:-2}"
|
|
13
|
-
BASE_REF="${4:-HEAD~1}"
|
|
14
|
-
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
15
|
-
BIN="$ENGINE/dist/bin/homegraph.js"
|
|
16
|
-
OUT="${AGENT_EVAL_OUT:-/tmp/ab-adoption}"
|
|
17
|
-
|
|
18
|
-
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
|
19
|
-
[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
|
|
20
|
-
git -C "$ENGINE" diff --quiet && git -C "$ENGINE" diff --cached --quiet || { echo "engine has uncommitted changes — commit/stash first"; exit 1; }
|
|
21
|
-
CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
|
|
22
|
-
[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD"; exit 1; }
|
|
23
|
-
|
|
24
|
-
cleanup() {
|
|
25
|
-
pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
|
|
26
|
-
git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
|
|
27
|
-
( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
|
|
28
|
-
}
|
|
29
|
-
trap cleanup EXIT
|
|
30
|
-
mkdir -p "$OUT"
|
|
31
|
-
echo "###### target=$TARGET runs/arm=$RUNS baseline=$BASE_REF"
|
|
32
|
-
echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
|
|
33
|
-
echo "###### task=$TASK"; echo
|
|
34
|
-
|
|
35
|
-
prewarm() {
|
|
36
|
-
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
|
37
|
-
HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
|
38
|
-
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
# Per-run tool-choice counts: Read vs homegraph_node[file] vs [symbol].
|
|
42
|
-
count() {
|
|
43
|
-
node -e '
|
|
44
|
-
const fs=require("fs");
|
|
45
|
-
const lines=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
|
46
|
-
let read=0,cgFile=0,cgSym=0,cgOther=0,exposed="?";
|
|
47
|
-
for(const l of lines){try{const o=JSON.parse(l);
|
|
48
|
-
if(o.type==="system"&&o.subtype==="init"){exposed=(o.tools||[]).filter(t=>/homegraph/.test(t)).length;}
|
|
49
|
-
const blocks=o.message?.content||[];
|
|
50
|
-
for(const b of (Array.isArray(blocks)?blocks:[])){
|
|
51
|
-
if(b.type!=="tool_use")continue;
|
|
52
|
-
if(b.name==="Read")read++;
|
|
53
|
-
else if(b.name==="mcp__homegraph__homegraph_node"){ if(b.input&&b.input.symbol)cgSym++; else cgFile++; }
|
|
54
|
-
else if(/mcp__homegraph__/.test(b.name))cgOther++;
|
|
55
|
-
}
|
|
56
|
-
}catch{}}
|
|
57
|
-
console.log(` Read=${read} homegraph_node[file]=${cgFile} homegraph_node[symbol]=${cgSym} other_cg=${cgOther} (cg exposed=${exposed})`);
|
|
58
|
-
' "$1"
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
run_arm() { # label, N
|
|
62
|
-
local label="$1" n="$2"
|
|
63
|
-
local c="$OUT/mcp-$label.json"
|
|
64
|
-
for i in $(seq 1 "$n"); do
|
|
65
|
-
local tgt="$OUT/t-$label-$i"
|
|
66
|
-
rm -rf "$tgt"
|
|
67
|
-
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$tgt/"
|
|
68
|
-
node "$BIN" init "$tgt" >/dev/null 2>&1
|
|
69
|
-
printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
|
70
|
-
prewarm "$tgt"
|
|
71
|
-
echo "----- [$label] run $i -----"
|
|
72
|
-
( cd "$tgt" && claude -p "$TASK" \
|
|
73
|
-
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
|
74
|
-
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
|
75
|
-
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
|
|
76
|
-
count "$OUT/run-$label-$i.jsonl"
|
|
77
|
-
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
|
78
|
-
done
|
|
79
|
-
echo
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
echo "== NEW build (HEAD: homegraph_node has Read parity) =="
|
|
83
|
-
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
|
84
|
-
run_arm new "$RUNS"
|
|
85
|
-
|
|
86
|
-
echo "== BASELINE build ($BASE_REF) =="
|
|
87
|
-
git -C "$ENGINE" checkout "$BASE_REF" -- $CHANGED
|
|
88
|
-
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
|
89
|
-
run_arm baseline "$RUNS"
|
|
90
|
-
|
|
91
|
-
echo "###### DONE — compare [new] vs [baseline]: does homegraph_node[file] rise / Read fall? Logs: $OUT"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Does the agent PICK homegraph_node to read a file, vs the built-in Read tool?
|
|
3
|
+
# Build A/B: NEW build (HEAD, homegraph_node has Read parity) vs BASELINE build
|
|
4
|
+
# (a ref where it doesn't), BOTH homegraph-attached + pre-warmed, same task. The
|
|
5
|
+
# metric is tool CHOICE: Read calls vs homegraph_node[file] calls per run.
|
|
6
|
+
#
|
|
7
|
+
# Usage: ab-adoption.sh <indexed-repo> "<task>" [runs-per-arm] [baseline-ref]
|
|
8
|
+
# Env: AGENT_EVAL_OUT (default: /tmp/ab-adoption)
|
|
9
|
+
set -uo pipefail
|
|
10
|
+
TARGET="${1:?usage: ab-adoption.sh <indexed-repo> \"<task>\" [runs] [baseline-ref]}"
|
|
11
|
+
TASK="${2:?task required}"
|
|
12
|
+
RUNS="${3:-2}"
|
|
13
|
+
BASE_REF="${4:-HEAD~1}"
|
|
14
|
+
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
15
|
+
BIN="$ENGINE/dist/bin/homegraph.js"
|
|
16
|
+
OUT="${AGENT_EVAL_OUT:-/tmp/ab-adoption}"
|
|
17
|
+
|
|
18
|
+
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
|
19
|
+
[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
|
|
20
|
+
git -C "$ENGINE" diff --quiet && git -C "$ENGINE" diff --cached --quiet || { echo "engine has uncommitted changes — commit/stash first"; exit 1; }
|
|
21
|
+
CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
|
|
22
|
+
[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD"; exit 1; }
|
|
23
|
+
|
|
24
|
+
cleanup() {
|
|
25
|
+
pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
|
|
26
|
+
git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
|
|
27
|
+
( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
|
|
28
|
+
}
|
|
29
|
+
trap cleanup EXIT
|
|
30
|
+
mkdir -p "$OUT"
|
|
31
|
+
echo "###### target=$TARGET runs/arm=$RUNS baseline=$BASE_REF"
|
|
32
|
+
echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
|
|
33
|
+
echo "###### task=$TASK"; echo
|
|
34
|
+
|
|
35
|
+
prewarm() {
|
|
36
|
+
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
|
37
|
+
HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
|
38
|
+
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
# Per-run tool-choice counts: Read vs homegraph_node[file] vs [symbol].
|
|
42
|
+
count() {
|
|
43
|
+
node -e '
|
|
44
|
+
const fs=require("fs");
|
|
45
|
+
const lines=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
|
46
|
+
let read=0,cgFile=0,cgSym=0,cgOther=0,exposed="?";
|
|
47
|
+
for(const l of lines){try{const o=JSON.parse(l);
|
|
48
|
+
if(o.type==="system"&&o.subtype==="init"){exposed=(o.tools||[]).filter(t=>/homegraph/.test(t)).length;}
|
|
49
|
+
const blocks=o.message?.content||[];
|
|
50
|
+
for(const b of (Array.isArray(blocks)?blocks:[])){
|
|
51
|
+
if(b.type!=="tool_use")continue;
|
|
52
|
+
if(b.name==="Read")read++;
|
|
53
|
+
else if(b.name==="mcp__homegraph__homegraph_node"){ if(b.input&&b.input.symbol)cgSym++; else cgFile++; }
|
|
54
|
+
else if(/mcp__homegraph__/.test(b.name))cgOther++;
|
|
55
|
+
}
|
|
56
|
+
}catch{}}
|
|
57
|
+
console.log(` Read=${read} homegraph_node[file]=${cgFile} homegraph_node[symbol]=${cgSym} other_cg=${cgOther} (cg exposed=${exposed})`);
|
|
58
|
+
' "$1"
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
run_arm() { # label, N
|
|
62
|
+
local label="$1" n="$2"
|
|
63
|
+
local c="$OUT/mcp-$label.json"
|
|
64
|
+
for i in $(seq 1 "$n"); do
|
|
65
|
+
local tgt="$OUT/t-$label-$i"
|
|
66
|
+
rm -rf "$tgt"
|
|
67
|
+
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$tgt/"
|
|
68
|
+
node "$BIN" init "$tgt" >/dev/null 2>&1
|
|
69
|
+
printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
|
70
|
+
prewarm "$tgt"
|
|
71
|
+
echo "----- [$label] run $i -----"
|
|
72
|
+
( cd "$tgt" && claude -p "$TASK" \
|
|
73
|
+
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
|
74
|
+
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
|
75
|
+
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
|
|
76
|
+
count "$OUT/run-$label-$i.jsonl"
|
|
77
|
+
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
|
78
|
+
done
|
|
79
|
+
echo
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
echo "== NEW build (HEAD: homegraph_node has Read parity) =="
|
|
83
|
+
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
|
84
|
+
run_arm new "$RUNS"
|
|
85
|
+
|
|
86
|
+
echo "== BASELINE build ($BASE_REF) =="
|
|
87
|
+
git -C "$ENGINE" checkout "$BASE_REF" -- $CHANGED
|
|
88
|
+
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
|
89
|
+
run_arm baseline "$RUNS"
|
|
90
|
+
|
|
91
|
+
echo "###### DONE — compare [new] vs [baseline]: does homegraph_node[file] rise / Read fall? Logs: $OUT"
|
|
@@ -1,86 +1,86 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# A/B the PreToolUse(Read) REDIRECT hook (P1): does steering Read → homegraph_node
|
|
3
|
-
# file-view actually move the agent off Read during implementation? BOTH arms use
|
|
4
|
-
# the CURRENT build with homegraph attached and pre-warmed; the only difference is
|
|
5
|
-
# the hook. Isolates the hook's behavioral effect from the build/file-view change
|
|
6
|
-
# (use ab-new-vs-baseline.sh for the build A/B).
|
|
7
|
-
#
|
|
8
|
-
# arm [nohook] — homegraph on, no hook (does the better file-view get picked on its own?)
|
|
9
|
-
# arm [hook] — homegraph on, + redirect hook (does routing close it?)
|
|
10
|
-
#
|
|
11
|
-
# Reliable attach (works nested): each arm pre-warms a persistent daemon and skips
|
|
12
|
-
# the startup re-exec (HOMEGRAPH_WASM_RELAUNCHED=1), so claude connects before the
|
|
13
|
-
# agent's first turn. Judge by ACTUAL homegraph usage in parse-run.mjs's "by type",
|
|
14
|
-
# not claude's init snapshot (which can read pending even when it then connects).
|
|
15
|
-
#
|
|
16
|
-
# Usage: ab-hook.sh <indexed-repo> "<implementation task>" [runs-per-arm]
|
|
17
|
-
# <indexed-repo> a repo with a .homegraph index (copied per arm; never mutated)
|
|
18
|
-
# "<task>" a GENUINELY-NEW implementation task (verify it isn't already done)
|
|
19
|
-
# [runs-per-arm] default 2 (n=1 is noisy — the doctrine says >=2)
|
|
20
|
-
# Env: AGENT_EVAL_OUT (default: /tmp/ab-hook)
|
|
21
|
-
set -uo pipefail
|
|
22
|
-
|
|
23
|
-
TARGET="${1:?usage: ab-hook.sh <indexed-repo> \"<task>\" [runs-per-arm]}"
|
|
24
|
-
TASK="${2:?task required}"
|
|
25
|
-
RUNS="${3:-2}"
|
|
26
|
-
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
27
|
-
BIN="$ENGINE/dist/bin/homegraph.js"
|
|
28
|
-
HOOK="$ENGINE/scripts/agent-eval/redirect-read-hook.sh"
|
|
29
|
-
OUT="${AGENT_EVAL_OUT:-/tmp/ab-hook}"
|
|
30
|
-
PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
|
|
31
|
-
|
|
32
|
-
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
|
33
|
-
command -v jq >/dev/null || { echo "jq not on PATH (the hook needs it)"; exit 1; }
|
|
34
|
-
[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
|
|
35
|
-
chmod +x "$HOOK"
|
|
36
|
-
|
|
37
|
-
cleanup() { pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
|
|
38
|
-
trap cleanup EXIT
|
|
39
|
-
|
|
40
|
-
mkdir -p "$OUT"
|
|
41
|
-
echo "###### engine=$ENGINE"
|
|
42
|
-
echo "###### target=$TARGET runs/arm=$RUNS"
|
|
43
|
-
echo "###### task=$TASK"
|
|
44
|
-
echo
|
|
45
|
-
|
|
46
|
-
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
|
47
|
-
|
|
48
|
-
# A settings file carrying ONLY the PreToolUse(Read) redirect hook.
|
|
49
|
-
HOOK_SETTINGS="$OUT/hook-settings.json"
|
|
50
|
-
jq -n --arg cmd "bash $HOOK" \
|
|
51
|
-
'{hooks:{PreToolUse:[{matcher:"Read",hooks:[{type:"command",command:$cmd}]}]}}' > "$HOOK_SETTINGS"
|
|
52
|
-
|
|
53
|
-
prewarm() { # target — spawn a persistent daemon and wait for its socket
|
|
54
|
-
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
|
55
|
-
HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
|
56
|
-
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
|
57
|
-
&& echo " daemon warm: $1" || echo " WARN: daemon never bound for $1"
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
run_one() { # arm-label, run-index, use-hook(0|1)
|
|
61
|
-
local label="$1" idx="$2" hook="$3"
|
|
62
|
-
local tgt="$OUT/t-$label-$idx" c="$OUT/mcp-$label.json"
|
|
63
|
-
rm -rf "$tgt"
|
|
64
|
-
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$tgt/"
|
|
65
|
-
node "$BIN" init "$tgt" >/dev/null 2>&1
|
|
66
|
-
printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
|
67
|
-
prewarm "$tgt"
|
|
68
|
-
local extra=()
|
|
69
|
-
[ "$hook" = "1" ] && extra=(--settings "$HOOK_SETTINGS")
|
|
70
|
-
echo "----- [$label] run $idx -----"
|
|
71
|
-
# ${extra[@]+...} guard: bash 3.2 (macOS) under `set -u` errors on an empty
|
|
72
|
-
# array expansion otherwise, which would skip the no-hook arm's claude run.
|
|
73
|
-
( cd "$tgt" && claude -p "$TASK" \
|
|
74
|
-
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
|
75
|
-
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
|
|
76
|
-
</dev/null > "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" )
|
|
77
|
-
node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$idx.jsonl)"
|
|
78
|
-
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
|
79
|
-
echo
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
for i in $(seq 1 "$RUNS"); do run_one nohook "$i" 0; done
|
|
83
|
-
for i in $(seq 1 "$RUNS"); do run_one hook "$i" 1; done
|
|
84
|
-
|
|
85
|
-
echo "###### DONE. Compare [nohook] vs [hook] 'by type' — Read should fall and"
|
|
86
|
-
echo "###### mcp__homegraph__homegraph_node should rise in the [hook] arm. Logs: $OUT"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# A/B the PreToolUse(Read) REDIRECT hook (P1): does steering Read → homegraph_node
|
|
3
|
+
# file-view actually move the agent off Read during implementation? BOTH arms use
|
|
4
|
+
# the CURRENT build with homegraph attached and pre-warmed; the only difference is
|
|
5
|
+
# the hook. Isolates the hook's behavioral effect from the build/file-view change
|
|
6
|
+
# (use ab-new-vs-baseline.sh for the build A/B).
|
|
7
|
+
#
|
|
8
|
+
# arm [nohook] — homegraph on, no hook (does the better file-view get picked on its own?)
|
|
9
|
+
# arm [hook] — homegraph on, + redirect hook (does routing close it?)
|
|
10
|
+
#
|
|
11
|
+
# Reliable attach (works nested): each arm pre-warms a persistent daemon and skips
|
|
12
|
+
# the startup re-exec (HOMEGRAPH_WASM_RELAUNCHED=1), so claude connects before the
|
|
13
|
+
# agent's first turn. Judge by ACTUAL homegraph usage in parse-run.mjs's "by type",
|
|
14
|
+
# not claude's init snapshot (which can read pending even when it then connects).
|
|
15
|
+
#
|
|
16
|
+
# Usage: ab-hook.sh <indexed-repo> "<implementation task>" [runs-per-arm]
|
|
17
|
+
# <indexed-repo> a repo with a .homegraph index (copied per arm; never mutated)
|
|
18
|
+
# "<task>" a GENUINELY-NEW implementation task (verify it isn't already done)
|
|
19
|
+
# [runs-per-arm] default 2 (n=1 is noisy — the doctrine says >=2)
|
|
20
|
+
# Env: AGENT_EVAL_OUT (default: /tmp/ab-hook)
|
|
21
|
+
set -uo pipefail
|
|
22
|
+
|
|
23
|
+
TARGET="${1:?usage: ab-hook.sh <indexed-repo> \"<task>\" [runs-per-arm]}"
|
|
24
|
+
TASK="${2:?task required}"
|
|
25
|
+
RUNS="${3:-2}"
|
|
26
|
+
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
27
|
+
BIN="$ENGINE/dist/bin/homegraph.js"
|
|
28
|
+
HOOK="$ENGINE/scripts/agent-eval/redirect-read-hook.sh"
|
|
29
|
+
OUT="${AGENT_EVAL_OUT:-/tmp/ab-hook}"
|
|
30
|
+
PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
|
|
31
|
+
|
|
32
|
+
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
|
33
|
+
command -v jq >/dev/null || { echo "jq not on PATH (the hook needs it)"; exit 1; }
|
|
34
|
+
[ -d "$TARGET/.homegraph" ] || { echo "target not indexed: run 'homegraph init $TARGET' first"; exit 1; }
|
|
35
|
+
chmod +x "$HOOK"
|
|
36
|
+
|
|
37
|
+
cleanup() { pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
|
|
38
|
+
trap cleanup EXIT
|
|
39
|
+
|
|
40
|
+
mkdir -p "$OUT"
|
|
41
|
+
echo "###### engine=$ENGINE"
|
|
42
|
+
echo "###### target=$TARGET runs/arm=$RUNS"
|
|
43
|
+
echo "###### task=$TASK"
|
|
44
|
+
echo
|
|
45
|
+
|
|
46
|
+
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
|
47
|
+
|
|
48
|
+
# A settings file carrying ONLY the PreToolUse(Read) redirect hook.
|
|
49
|
+
HOOK_SETTINGS="$OUT/hook-settings.json"
|
|
50
|
+
jq -n --arg cmd "bash $HOOK" \
|
|
51
|
+
'{hooks:{PreToolUse:[{matcher:"Read",hooks:[{type:"command",command:$cmd}]}]}}' > "$HOOK_SETTINGS"
|
|
52
|
+
|
|
53
|
+
prewarm() { # target — spawn a persistent daemon and wait for its socket
|
|
54
|
+
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
|
55
|
+
HOMEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
|
56
|
+
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.homegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
|
57
|
+
&& echo " daemon warm: $1" || echo " WARN: daemon never bound for $1"
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
run_one() { # arm-label, run-index, use-hook(0|1)
|
|
61
|
+
local label="$1" idx="$2" hook="$3"
|
|
62
|
+
local tgt="$OUT/t-$label-$idx" c="$OUT/mcp-$label.json"
|
|
63
|
+
rm -rf "$tgt"
|
|
64
|
+
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .homegraph "$TARGET/" "$tgt/"
|
|
65
|
+
node "$BIN" init "$tgt" >/dev/null 2>&1
|
|
66
|
+
printf '{"mcpServers":{"homegraph":{"command":"env","args":["HOMEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
|
67
|
+
prewarm "$tgt"
|
|
68
|
+
local extra=()
|
|
69
|
+
[ "$hook" = "1" ] && extra=(--settings "$HOOK_SETTINGS")
|
|
70
|
+
echo "----- [$label] run $idx -----"
|
|
71
|
+
# ${extra[@]+...} guard: bash 3.2 (macOS) under `set -u` errors on an empty
|
|
72
|
+
# array expansion otherwise, which would skip the no-hook arm's claude run.
|
|
73
|
+
( cd "$tgt" && claude -p "$TASK" \
|
|
74
|
+
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
|
75
|
+
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
|
|
76
|
+
</dev/null > "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" )
|
|
77
|
+
node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$idx.jsonl)"
|
|
78
|
+
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
|
79
|
+
echo
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
for i in $(seq 1 "$RUNS"); do run_one nohook "$i" 0; done
|
|
83
|
+
for i in $(seq 1 "$RUNS"); do run_one hook "$i" 1; done
|
|
84
|
+
|
|
85
|
+
echo "###### DONE. Compare [nohook] vs [hook] 'by type' — Read should fall and"
|
|
86
|
+
echo "###### mcp__homegraph__homegraph_node should rise in the [hook] arm. Logs: $OUT"
|