homegraph 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +243 -243
- package/dist/bin/homegraph.js +371 -9
- package/dist/bin/homegraph.js.map +1 -1
- package/dist/db/migrations.js +18 -18
- package/dist/db/queries.js +140 -140
- package/dist/db/schema.sql +152 -152
- package/dist/directory.js +5 -5
- package/dist/extraction/wasm/tree-sitter-c_sharp.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-pascal.wasm +0 -0
- package/dist/installer/instructions-template.js +9 -9
- package/dist/installer/targets/shared.d.ts +5 -6
- package/dist/installer/targets/shared.d.ts.map +1 -1
- package/dist/installer/targets/shared.js +5 -6
- package/dist/installer/targets/shared.js.map +1 -1
- package/dist/mcp/liveness-watchdog.js +16 -16
- package/dist/mcp/server-instructions.js +69 -69
- package/dist/mcp/tools.d.ts +17 -4
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +192 -22
- package/dist/mcp/tools.js.map +1 -1
- package/dist/reasoning/reasoner.js +32 -32
- package/dist/spec/config.d.ts +43 -0
- package/dist/spec/config.d.ts.map +1 -0
- package/dist/spec/config.js +275 -0
- package/dist/spec/config.js.map +1 -0
- package/dist/spec/db/commit-node.d.ts +23 -0
- package/dist/spec/db/commit-node.d.ts.map +1 -0
- package/dist/spec/db/commit-node.js +62 -0
- package/dist/spec/db/commit-node.js.map +1 -0
- package/dist/spec/db/fragment-node.d.ts +23 -0
- package/dist/spec/db/fragment-node.d.ts.map +1 -0
- package/dist/spec/db/fragment-node.js +120 -0
- package/dist/spec/db/fragment-node.js.map +1 -0
- package/dist/spec/db/fts.d.ts +60 -0
- package/dist/spec/db/fts.d.ts.map +1 -0
- package/dist/spec/db/fts.js +285 -0
- package/dist/spec/db/fts.js.map +1 -0
- package/dist/spec/db/index.d.ts +13 -0
- package/dist/spec/db/index.d.ts.map +1 -0
- package/dist/spec/db/index.js +50 -0
- package/dist/spec/db/index.js.map +1 -0
- package/dist/spec/db/relations.d.ts +55 -0
- package/dist/spec/db/relations.d.ts.map +1 -0
- package/dist/spec/db/relations.js +158 -0
- package/dist/spec/db/relations.js.map +1 -0
- package/dist/spec/db/schema.d.ts +30 -0
- package/dist/spec/db/schema.d.ts.map +1 -0
- package/dist/spec/db/schema.js +87 -0
- package/dist/spec/db/schema.js.map +1 -0
- package/dist/spec/db/schema.sql +107 -0
- package/dist/spec/db/spec-node.d.ts +41 -0
- package/dist/spec/db/spec-node.d.ts.map +1 -0
- package/dist/spec/db/spec-node.js +114 -0
- package/dist/spec/db/spec-node.js.map +1 -0
- package/dist/spec/evolve/impact-locator.d.ts +13 -0
- package/dist/spec/evolve/impact-locator.d.ts.map +1 -0
- package/dist/spec/evolve/impact-locator.js +25 -0
- package/dist/spec/evolve/impact-locator.js.map +1 -0
- package/dist/spec/evolve/llm-client.d.ts +50 -0
- package/dist/spec/evolve/llm-client.d.ts.map +1 -0
- package/dist/spec/evolve/llm-client.js +176 -0
- package/dist/spec/evolve/llm-client.js.map +1 -0
- package/dist/spec/evolve/logic-checker.d.ts +12 -0
- package/dist/spec/evolve/logic-checker.d.ts.map +1 -0
- package/dist/spec/evolve/logic-checker.js +48 -0
- package/dist/spec/evolve/logic-checker.js.map +1 -0
- package/dist/spec/evolve/pipeline.d.ts +40 -0
- package/dist/spec/evolve/pipeline.d.ts.map +1 -0
- package/dist/spec/evolve/pipeline.js +314 -0
- package/dist/spec/evolve/pipeline.js.map +1 -0
- package/dist/spec/evolve/spec-rewriter.d.ts +42 -0
- package/dist/spec/evolve/spec-rewriter.d.ts.map +1 -0
- package/dist/spec/evolve/spec-rewriter.js +254 -0
- package/dist/spec/evolve/spec-rewriter.js.map +1 -0
- package/dist/spec/graph/queries.d.ts +58 -0
- package/dist/spec/graph/queries.d.ts.map +1 -0
- package/dist/spec/graph/queries.js +215 -0
- package/dist/spec/graph/queries.js.map +1 -0
- package/dist/spec/mining/diff-parser.d.ts +33 -0
- package/dist/spec/mining/diff-parser.d.ts.map +1 -0
- package/dist/spec/mining/diff-parser.js +166 -0
- package/dist/spec/mining/diff-parser.js.map +1 -0
- package/dist/spec/mining/git-scanner.d.ts +91 -0
- package/dist/spec/mining/git-scanner.d.ts.map +1 -0
- package/dist/spec/mining/git-scanner.js +266 -0
- package/dist/spec/mining/git-scanner.js.map +1 -0
- package/dist/spec/mining/pipeline.d.ts +53 -0
- package/dist/spec/mining/pipeline.d.ts.map +1 -0
- package/dist/spec/mining/pipeline.js +165 -0
- package/dist/spec/mining/pipeline.js.map +1 -0
- package/dist/spec/mining/scope-resolver.d.ts +45 -0
- package/dist/spec/mining/scope-resolver.d.ts.map +1 -0
- package/dist/spec/mining/scope-resolver.js +103 -0
- package/dist/spec/mining/scope-resolver.js.map +1 -0
- package/dist/spec/mining/spec-extractor.d.ts +69 -0
- package/dist/spec/mining/spec-extractor.d.ts.map +1 -0
- package/dist/spec/mining/spec-extractor.js +369 -0
- package/dist/spec/mining/spec-extractor.js.map +1 -0
- package/dist/spec/types.d.ts +149 -0
- package/dist/spec/types.d.ts.map +1 -0
- package/dist/spec/types.js +15 -0
- package/dist/spec/types.js.map +1 -0
- package/dist/spec/utils.d.ts +166 -0
- package/dist/spec/utils.d.ts.map +1 -0
- package/dist/spec/utils.js +461 -0
- package/dist/spec/utils.js.map +1 -0
- package/package.json +58 -57
- package/scripts/add-lang/bench.sh +60 -60
- package/scripts/add-lang/check-grammar.mjs +75 -75
- package/scripts/add-lang/dump-ast.mjs +103 -103
- package/scripts/add-lang/verify-extraction.mjs +70 -70
- package/scripts/agent-eval/ab-adoption.sh +91 -91
- package/scripts/agent-eval/ab-hook.sh +86 -86
- package/scripts/agent-eval/ab-impl.sh +78 -78
- package/scripts/agent-eval/ab-new-vs-baseline.sh +102 -102
- package/scripts/agent-eval/ab-sufficiency.sh +78 -78
- package/scripts/agent-eval/arms-F.sh +21 -21
- package/scripts/agent-eval/arms-matrix.sh +37 -37
- package/scripts/agent-eval/audit.sh +68 -68
- package/scripts/agent-eval/bench-readme.sh +28 -28
- package/scripts/agent-eval/bench-why-repo.sh +22 -22
- package/scripts/agent-eval/block-read-hook.sh +19 -19
- package/scripts/agent-eval/hook-settings.json +15 -15
- package/scripts/agent-eval/itrun.sh +120 -120
- package/scripts/agent-eval/offload-eval-3arm.sh +72 -72
- package/scripts/agent-eval/offload-eval-cost.mjs +133 -133
- package/scripts/agent-eval/offload-eval-effort.mjs +108 -108
- package/scripts/agent-eval/offload-eval-frontload-matrix.sh +25 -25
- package/scripts/agent-eval/offload-eval-frontload.sh +47 -47
- package/scripts/agent-eval/offload-eval-ground-truth.json +18 -18
- package/scripts/agent-eval/offload-eval-hook.mjs +84 -84
- package/scripts/agent-eval/offload-eval-judge.mjs +103 -103
- package/scripts/agent-eval/offload-eval-matrix.sh +20 -20
- package/scripts/agent-eval/offload-eval-metrics.mjs +94 -94
- package/scripts/agent-eval/offload-eval-refs1.sh +50 -50
- package/scripts/agent-eval/offload-eval-setup.sh +24 -24
- package/scripts/agent-eval/offload-eval-styles.sh +71 -71
- package/scripts/agent-eval/offload-eval-summarize.mjs +68 -68
- package/scripts/agent-eval/offload-eval.md +76 -76
- package/scripts/agent-eval/parse-arms.mjs +116 -116
- package/scripts/agent-eval/parse-bench-readme.mjs +84 -84
- package/scripts/agent-eval/parse-run.mjs +45 -45
- package/scripts/agent-eval/parse-session.mjs +93 -93
- package/scripts/agent-eval/probe-context.mjs +21 -21
- package/scripts/agent-eval/probe-explore.mjs +40 -40
- package/scripts/agent-eval/probe-node.mjs +20 -20
- package/scripts/agent-eval/probe-sweep.mjs +119 -119
- package/scripts/agent-eval/probe-trace.mjs +20 -20
- package/scripts/agent-eval/redirect-read-hook.sh +38 -38
- package/scripts/agent-eval/run-agent.sh +34 -34
- package/scripts/agent-eval/run-all.sh +69 -69
- package/scripts/agent-eval/run-arms.sh +56 -56
- package/scripts/agent-eval/seq-matrix.mjs +137 -137
- package/scripts/build-bundle.sh +118 -118
- package/scripts/extract-release-notes.mjs +130 -130
- package/scripts/local-install.sh +41 -41
- package/scripts/npm-sdk.js +75 -75
- package/scripts/npm-shim.js +246 -246
- package/scripts/pack-npm.sh +119 -119
- package/scripts/prepare-release.mjs +270 -270
- package/scripts/qa_eval/README.md +407 -404
- package/scripts/qa_eval/_test_deveco_probe.py +41 -41
- package/scripts/qa_eval/agent_runner.py +526 -526
- package/scripts/qa_eval/data/.gitignore +4 -4
- package/scripts/qa_eval/data/test-set.jsonl +2 -22
- package/scripts/qa_eval/eval_metrics.py +274 -233
- package/scripts/qa_eval/external_agent.py +976 -671
- package/scripts/qa_eval/llm_config.py +92 -92
- package/scripts/qa_eval/memory_monitor.py +132 -132
- package/scripts/qa_eval/my_answer_accuracy.py +187 -187
- package/scripts/qa_eval/requirements.txt +2 -2
- package/scripts/qa_eval/run_pipeline.py +804 -711
- package/scripts/qa_eval/stats_efficiency.py +279 -279
- package/scripts/qa_eval/stats_scores.py +207 -207
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// Probe homegraph_trace against an index using the built dist.
|
|
3
|
-
// Usage: node probe-trace.mjs <repo-with-.homegraph> <from> <to>
|
|
4
|
-
import { pathToFileURL } from 'node:url';
|
|
5
|
-
import { resolve } from 'node:path';
|
|
6
|
-
|
|
7
|
-
const [, , repo, from, to] = process.argv;
|
|
8
|
-
if (!repo || !from || !to) { console.error('usage: probe-trace.mjs <repo> <from> <to>'); process.exit(1); }
|
|
9
|
-
|
|
10
|
-
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
11
|
-
const idx = await load('dist/index.js');
|
|
12
|
-
const tools = await load('dist/mcp/tools.js');
|
|
13
|
-
const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
|
|
14
|
-
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
15
|
-
|
|
16
|
-
const cg = HomeGraph.openSync(repo);
|
|
17
|
-
const h = new ToolHandler(cg);
|
|
18
|
-
const res = await h.execute('homegraph_trace', { from, to });
|
|
19
|
-
console.log(res.content?.[0]?.text ?? '(no text)');
|
|
20
|
-
try { cg.close?.(); } catch {}
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Probe homegraph_trace against an index using the built dist.
|
|
3
|
+
// Usage: node probe-trace.mjs <repo-with-.homegraph> <from> <to>
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { resolve } from 'node:path';
|
|
6
|
+
|
|
7
|
+
const [, , repo, from, to] = process.argv;
|
|
8
|
+
if (!repo || !from || !to) { console.error('usage: probe-trace.mjs <repo> <from> <to>'); process.exit(1); }
|
|
9
|
+
|
|
10
|
+
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
11
|
+
const idx = await load('dist/index.js');
|
|
12
|
+
const tools = await load('dist/mcp/tools.js');
|
|
13
|
+
const HomeGraph = idx.default?.default ?? idx.default ?? idx.HomeGraph;
|
|
14
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
15
|
+
|
|
16
|
+
const cg = HomeGraph.openSync(repo);
|
|
17
|
+
const h = new ToolHandler(cg);
|
|
18
|
+
const res = await h.execute('homegraph_trace', { from, to });
|
|
19
|
+
console.log(res.content?.[0]?.text ?? '(no text)');
|
|
20
|
+
try { cg.close?.(); } catch {}
|
|
@@ -1,38 +1,38 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# PreToolUse(Read) REDIRECT hook — prototype for A/B (P1: get agents off Read and
|
|
3
|
-
# onto homegraph_node during implementation, not just for Q&A).
|
|
4
|
-
#
|
|
5
|
-
# When the agent Reads a SOURCE file, deny it and steer to homegraph_node's
|
|
6
|
-
# file-view, which (as of the Lever-1 change) returns the WHOLE file verbatim
|
|
7
|
-
# WITH line numbers — imports, top-level code, comments and all — PLUS the file's
|
|
8
|
-
# blast radius, in one call. That output is a strict superset of Read, so the
|
|
9
|
-
# redirect is lossless: the agent loses nothing by taking it, and gains who-
|
|
10
|
-
# depends-on-this for the edit it's about to make.
|
|
11
|
-
#
|
|
12
|
-
# Differs from block-read-hook.sh (which steers to explore/node-by-symbol): this
|
|
13
|
-
# names the FILE-VIEW path explicitly (file:"<base>" + includeCode:true), the
|
|
14
|
-
# 1:1 Read replacement we're trying to get picked during implementation.
|
|
15
|
-
#
|
|
16
|
-
# Non-source files (configs, docs, lockfiles, .env) pass through to a real Read.
|
|
17
|
-
# A redirect to a file homegraph hasn't indexed SELF-CORRECTS: the file-view
|
|
18
|
-
# replies "No indexed file matches … Read it directly", so a just-created file
|
|
19
|
-
# never dead-ends — the agent Reads it on the next turn.
|
|
20
|
-
#
|
|
21
|
-
# Wire via: claude ... --settings <settings-with-this-as-PreToolUse(Read)>
|
|
22
|
-
# Eval artifact only. The production version is an indexed-aware `homegraph`
|
|
23
|
-
# subcommand (cross-platform — no bash/jq — and queries the index so it never
|
|
24
|
-
# bounces a new/un-indexed file), wired opt-in by the installer.
|
|
25
|
-
set -uo pipefail
|
|
26
|
-
input="$(cat)"
|
|
27
|
-
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
|
28
|
-
[ -n "$fp" ] || exit 0
|
|
29
|
-
base="$(basename "$fp")"
|
|
30
|
-
|
|
31
|
-
case "$fp" in
|
|
32
|
-
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.scala|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte|*.m|*.mm)
|
|
33
|
-
msg="homegraph has this file indexed (kept in sync on every edit). Call homegraph_node with file:\"$base\" and includeCode:true instead of Read — it returns the WHOLE file verbatim WITH line numbers (imports, top-level code and all — safe to base an Edit on) PLUS which files depend on it, in one call. Treat its output as already-Read; do not Read this file. (If it answers that the file isn't indexed — e.g. you just created it — then Read it directly.)"
|
|
34
|
-
jq -n --arg m "$msg" '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
|
35
|
-
exit 0
|
|
36
|
-
;;
|
|
37
|
-
esac
|
|
38
|
-
exit 0
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# PreToolUse(Read) REDIRECT hook — prototype for A/B (P1: get agents off Read and
|
|
3
|
+
# onto homegraph_node during implementation, not just for Q&A).
|
|
4
|
+
#
|
|
5
|
+
# When the agent Reads a SOURCE file, deny it and steer to homegraph_node's
|
|
6
|
+
# file-view, which (as of the Lever-1 change) returns the WHOLE file verbatim
|
|
7
|
+
# WITH line numbers — imports, top-level code, comments and all — PLUS the file's
|
|
8
|
+
# blast radius, in one call. That output is a strict superset of Read, so the
|
|
9
|
+
# redirect is lossless: the agent loses nothing by taking it, and gains who-
|
|
10
|
+
# depends-on-this for the edit it's about to make.
|
|
11
|
+
#
|
|
12
|
+
# Differs from block-read-hook.sh (which steers to explore/node-by-symbol): this
|
|
13
|
+
# names the FILE-VIEW path explicitly (file:"<base>" + includeCode:true), the
|
|
14
|
+
# 1:1 Read replacement we're trying to get picked during implementation.
|
|
15
|
+
#
|
|
16
|
+
# Non-source files (configs, docs, lockfiles, .env) pass through to a real Read.
|
|
17
|
+
# A redirect to a file homegraph hasn't indexed SELF-CORRECTS: the file-view
|
|
18
|
+
# replies "No indexed file matches … Read it directly", so a just-created file
|
|
19
|
+
# never dead-ends — the agent Reads it on the next turn.
|
|
20
|
+
#
|
|
21
|
+
# Wire via: claude ... --settings <settings-with-this-as-PreToolUse(Read)>
|
|
22
|
+
# Eval artifact only. The production version is an indexed-aware `homegraph`
|
|
23
|
+
# subcommand (cross-platform — no bash/jq — and queries the index so it never
|
|
24
|
+
# bounces a new/un-indexed file), wired opt-in by the installer.
|
|
25
|
+
set -uo pipefail
|
|
26
|
+
input="$(cat)"
|
|
27
|
+
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
|
28
|
+
[ -n "$fp" ] || exit 0
|
|
29
|
+
base="$(basename "$fp")"
|
|
30
|
+
|
|
31
|
+
case "$fp" in
|
|
32
|
+
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.scala|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte|*.m|*.mm)
|
|
33
|
+
msg="homegraph has this file indexed (kept in sync on every edit). Call homegraph_node with file:\"$base\" and includeCode:true instead of Read — it returns the WHOLE file verbatim WITH line numbers (imports, top-level code and all — safe to base an Edit on) PLUS which files depend on it, in one call. Treat its output as already-Read; do not Read this file. (If it answers that the file isn't indexed — e.g. you just created it — then Read it directly.)"
|
|
34
|
+
jq -n --arg m "$msg" '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
|
35
|
+
exit 0
|
|
36
|
+
;;
|
|
37
|
+
esac
|
|
38
|
+
exit 0
|
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Headless Claude Code run against a repo with homegraph MCP, capturing the
|
|
3
|
-
# full stream-json so we can see tool calls + token usage. Complements the
|
|
4
|
-
# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
|
|
5
|
-
# tokens/cost, but defaults to the general-purpose subagent (not Explore).
|
|
6
|
-
# To force the Explore path, ask for it in the prompt.
|
|
7
|
-
#
|
|
8
|
-
# Usage: run-agent.sh <repo-path> <label> "<prompt>"
|
|
9
|
-
# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (homegraph dist binary)
|
|
10
|
-
set -uo pipefail
|
|
11
|
-
|
|
12
|
-
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
|
13
|
-
CG_BIN="${CG_BIN:-$(command -v homegraph || echo /usr/local/bin/homegraph)}"
|
|
14
|
-
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
|
15
|
-
OUT="$OUT_DIR/run-${LABEL}.jsonl"
|
|
16
|
-
|
|
17
|
-
MCP_CONFIG=$(cat <<JSON
|
|
18
|
-
{"mcpServers":{"homegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
|
|
19
|
-
JSON
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
echo "→ running [$LABEL] in $REPO"
|
|
23
|
-
cd "$REPO" || exit 1
|
|
24
|
-
|
|
25
|
-
claude -p "$PROMPT" \
|
|
26
|
-
--output-format stream-json --verbose \
|
|
27
|
-
--permission-mode bypassPermissions \
|
|
28
|
-
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
|
29
|
-
--max-budget-usd 2 \
|
|
30
|
-
--strict-mcp-config --mcp-config "$MCP_CONFIG" \
|
|
31
|
-
> "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
|
|
32
|
-
|
|
33
|
-
echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
|
|
34
|
-
node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Headless Claude Code run against a repo with homegraph MCP, capturing the
|
|
3
|
+
# full stream-json so we can see tool calls + token usage. Complements the
|
|
4
|
+
# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
|
|
5
|
+
# tokens/cost, but defaults to the general-purpose subagent (not Explore).
|
|
6
|
+
# To force the Explore path, ask for it in the prompt.
|
|
7
|
+
#
|
|
8
|
+
# Usage: run-agent.sh <repo-path> <label> "<prompt>"
|
|
9
|
+
# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (homegraph dist binary)
|
|
10
|
+
set -uo pipefail
|
|
11
|
+
|
|
12
|
+
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
|
13
|
+
CG_BIN="${CG_BIN:-$(command -v homegraph || echo /usr/local/bin/homegraph)}"
|
|
14
|
+
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
|
15
|
+
OUT="$OUT_DIR/run-${LABEL}.jsonl"
|
|
16
|
+
|
|
17
|
+
MCP_CONFIG=$(cat <<JSON
|
|
18
|
+
{"mcpServers":{"homegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
|
|
19
|
+
JSON
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
echo "→ running [$LABEL] in $REPO"
|
|
23
|
+
cd "$REPO" || exit 1
|
|
24
|
+
|
|
25
|
+
claude -p "$PROMPT" \
|
|
26
|
+
--output-format stream-json --verbose \
|
|
27
|
+
--permission-mode bypassPermissions \
|
|
28
|
+
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
|
29
|
+
--max-budget-usd 2 \
|
|
30
|
+
--strict-mcp-config --mcp-config "$MCP_CONFIG" \
|
|
31
|
+
> "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
|
|
32
|
+
|
|
33
|
+
echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
|
|
34
|
+
node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true
|
|
@@ -1,69 +1,69 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# With/without A/B (and optional interactive) eval for a homegraph version on a
|
|
3
|
-
# repo. Codegraph is the ONLY variable: both arms launch claude with
|
|
4
|
-
# --strict-mcp-config — with = homegraph-only MCP (pointed at $CG_BIN),
|
|
5
|
-
# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
|
|
6
|
-
#
|
|
7
|
-
# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
|
|
8
|
-
# Env: CG_BIN homegraph binary (default: command -v homegraph)
|
|
9
|
-
# AGENT_EVAL_OUT output dir (default: /tmp/agent-eval)
|
|
10
|
-
# MODEL / EFFORT claude model/effort (default: sonnet / high — the
|
|
11
|
-
# standing A/B policy; see CLAUDE.md, don't raise)
|
|
12
|
-
set -uo pipefail
|
|
13
|
-
|
|
14
|
-
REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
|
|
15
|
-
Q="${2:?question required}"
|
|
16
|
-
MODE="${3:-headless}"
|
|
17
|
-
CG_BIN="${CG_BIN:-$(command -v homegraph)}"
|
|
18
|
-
OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
|
|
19
|
-
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
|
20
|
-
mkdir -p "$OUT"
|
|
21
|
-
|
|
22
|
-
[ -n "$CG_BIN" ] || { echo "no homegraph binary on PATH (set CG_BIN)"; exit 1; }
|
|
23
|
-
[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO — index it first"; exit 1; }
|
|
24
|
-
case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
|
|
25
|
-
|
|
26
|
-
# MCP config files (path form avoids inline-JSON quoting through tmux).
|
|
27
|
-
cat > "$OUT/mcp-homegraph.json" <<JSON
|
|
28
|
-
{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
|
29
|
-
JSON
|
|
30
|
-
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
|
|
31
|
-
|
|
32
|
-
echo "###### homegraph: $CG_BIN"
|
|
33
|
-
echo "###### repo: $REPO"
|
|
34
|
-
echo "###### question: $Q"
|
|
35
|
-
echo
|
|
36
|
-
|
|
37
|
-
# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
|
|
38
|
-
headless() {
|
|
39
|
-
local label="$1" cfg="$2"
|
|
40
|
-
echo "############################## HEADLESS [$label] ##############################"
|
|
41
|
-
( cd "$REPO" && claude -p "$Q" \
|
|
42
|
-
--output-format stream-json --verbose \
|
|
43
|
-
--permission-mode bypassPermissions \
|
|
44
|
-
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
|
45
|
-
--max-budget-usd 4 \
|
|
46
|
-
--strict-mcp-config --mcp-config "$cfg" \
|
|
47
|
-
> "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
|
|
48
|
-
echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
|
|
49
|
-
tail -2 "$OUT/run-$label.err" 2>/dev/null
|
|
50
|
-
node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
|
|
51
|
-
echo
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
|
|
55
|
-
headless "headless-with" "$OUT/mcp-homegraph.json"
|
|
56
|
-
headless "headless-without" "$OUT/mcp-empty.json"
|
|
57
|
-
fi
|
|
58
|
-
|
|
59
|
-
if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
|
|
60
|
-
echo "############################## INTERACTIVE [with] ##############################"
|
|
61
|
-
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-homegraph.json" \
|
|
62
|
-
bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
|
|
63
|
-
echo
|
|
64
|
-
echo "############################## INTERACTIVE [without] ##############################"
|
|
65
|
-
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
|
|
66
|
-
bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
|
|
67
|
-
echo
|
|
68
|
-
fi
|
|
69
|
-
echo "############################## RUN-ALL COMPLETE ##############################"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# With/without A/B (and optional interactive) eval for a homegraph version on a
|
|
3
|
+
# repo. Codegraph is the ONLY variable: both arms launch claude with
|
|
4
|
+
# --strict-mcp-config — with = homegraph-only MCP (pointed at $CG_BIN),
|
|
5
|
+
# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
|
|
6
|
+
#
|
|
7
|
+
# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
|
|
8
|
+
# Env: CG_BIN homegraph binary (default: command -v homegraph)
|
|
9
|
+
# AGENT_EVAL_OUT output dir (default: /tmp/agent-eval)
|
|
10
|
+
# MODEL / EFFORT claude model/effort (default: sonnet / high — the
|
|
11
|
+
# standing A/B policy; see CLAUDE.md, don't raise)
|
|
12
|
+
set -uo pipefail
|
|
13
|
+
|
|
14
|
+
REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
|
|
15
|
+
Q="${2:?question required}"
|
|
16
|
+
MODE="${3:-headless}"
|
|
17
|
+
CG_BIN="${CG_BIN:-$(command -v homegraph)}"
|
|
18
|
+
OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
|
|
19
|
+
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
|
20
|
+
mkdir -p "$OUT"
|
|
21
|
+
|
|
22
|
+
[ -n "$CG_BIN" ] || { echo "no homegraph binary on PATH (set CG_BIN)"; exit 1; }
|
|
23
|
+
[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO — index it first"; exit 1; }
|
|
24
|
+
case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
|
|
25
|
+
|
|
26
|
+
# MCP config files (path form avoids inline-JSON quoting through tmux).
|
|
27
|
+
cat > "$OUT/mcp-homegraph.json" <<JSON
|
|
28
|
+
{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
|
29
|
+
JSON
|
|
30
|
+
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
|
|
31
|
+
|
|
32
|
+
echo "###### homegraph: $CG_BIN"
|
|
33
|
+
echo "###### repo: $REPO"
|
|
34
|
+
echo "###### question: $Q"
|
|
35
|
+
echo
|
|
36
|
+
|
|
37
|
+
# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
|
|
38
|
+
headless() {
|
|
39
|
+
local label="$1" cfg="$2"
|
|
40
|
+
echo "############################## HEADLESS [$label] ##############################"
|
|
41
|
+
( cd "$REPO" && claude -p "$Q" \
|
|
42
|
+
--output-format stream-json --verbose \
|
|
43
|
+
--permission-mode bypassPermissions \
|
|
44
|
+
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
|
45
|
+
--max-budget-usd 4 \
|
|
46
|
+
--strict-mcp-config --mcp-config "$cfg" \
|
|
47
|
+
> "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
|
|
48
|
+
echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
|
|
49
|
+
tail -2 "$OUT/run-$label.err" 2>/dev/null
|
|
50
|
+
node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
|
|
51
|
+
echo
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
|
|
55
|
+
headless "headless-with" "$OUT/mcp-homegraph.json"
|
|
56
|
+
headless "headless-without" "$OUT/mcp-empty.json"
|
|
57
|
+
fi
|
|
58
|
+
|
|
59
|
+
if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
|
|
60
|
+
echo "############################## INTERACTIVE [with] ##############################"
|
|
61
|
+
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-homegraph.json" \
|
|
62
|
+
bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
|
|
63
|
+
echo
|
|
64
|
+
echo "############################## INTERACTIVE [without] ##############################"
|
|
65
|
+
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
|
|
66
|
+
bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
|
|
67
|
+
echo
|
|
68
|
+
fi
|
|
69
|
+
echo "############################## RUN-ALL COMPLETE ##############################"
|
|
@@ -1,56 +1,56 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Tool-surface ablation — run ONE repo+question under ONE arm.
|
|
3
|
-
#
|
|
4
|
-
# Arms vary (exposed homegraph tools, trace-first steering). Tools are trimmed
|
|
5
|
-
# SERVER-SIDE via HOMEGRAPH_MCP_TOOLS in the MCP config's `env` block, so an
|
|
6
|
-
# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
|
|
7
|
-
# denied-call confound (which --disallowedTools would introduce). Steering is
|
|
8
|
-
# injected with --append-system-prompt, so no rebuild of the shipped
|
|
9
|
-
# server-instructions is needed to A/B it.
|
|
10
|
-
#
|
|
11
|
-
# A control all tools no steering
|
|
12
|
-
# B steer all tools trace-first
|
|
13
|
-
# C no-explore hide explore trace-first
|
|
14
|
-
# D trace-centric hide explore+context trace-first
|
|
15
|
-
# E control-probe hide explore+context trace-first (caller passes a NON-flow Q)
|
|
16
|
-
#
|
|
17
|
-
# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
|
|
18
|
-
set -uo pipefail
|
|
19
|
-
REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
|
|
20
|
-
CG_BIN="${CG_BIN:-$(command -v homegraph)}"
|
|
21
|
-
OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
|
|
22
|
-
mkdir -p "$OUT"
|
|
23
|
-
[ -n "$CG_BIN" ] || { echo "no homegraph binary (set CG_BIN)"; exit 1; }
|
|
24
|
-
[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO"; exit 1; }
|
|
25
|
-
|
|
26
|
-
STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call homegraph_trace(from,to) FIRST — one call returns the whole path. Use homegraph_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
|
|
27
|
-
KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
|
|
28
|
-
KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
|
|
29
|
-
|
|
30
|
-
case "$ARM" in
|
|
31
|
-
A|G|H|I) TOOLS=""; STEERING="" ;; # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
|
|
32
|
-
B|F) TOOLS=""; STEERING="$STEER" ;; # F = B's surface, run on the body-inlining trace build
|
|
33
|
-
C) TOOLS="$KEEP_NO_EXPLORE"; STEERING="$STEER" ;;
|
|
34
|
-
D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
|
|
35
|
-
*) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
|
|
36
|
-
esac
|
|
37
|
-
|
|
38
|
-
CFG="$OUT/mcp-$ARM.json"
|
|
39
|
-
if [ -n "$TOOLS" ]; then
|
|
40
|
-
cat > "$CFG" <<JSON
|
|
41
|
-
{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"HOMEGRAPH_MCP_TOOLS":"$TOOLS"}}}}
|
|
42
|
-
JSON
|
|
43
|
-
else
|
|
44
|
-
cat > "$CFG" <<JSON
|
|
45
|
-
{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
|
46
|
-
JSON
|
|
47
|
-
fi
|
|
48
|
-
|
|
49
|
-
LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
|
|
50
|
-
ARGS=( -p "$Q" --output-format stream-json --verbose
|
|
51
|
-
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
|
|
52
|
-
--strict-mcp-config --mcp-config "$CFG" )
|
|
53
|
-
[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
|
|
54
|
-
|
|
55
|
-
( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
|
|
56
|
-
echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Tool-surface ablation — run ONE repo+question under ONE arm.
|
|
3
|
+
#
|
|
4
|
+
# Arms vary (exposed homegraph tools, trace-first steering). Tools are trimmed
|
|
5
|
+
# SERVER-SIDE via HOMEGRAPH_MCP_TOOLS in the MCP config's `env` block, so an
|
|
6
|
+
# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
|
|
7
|
+
# denied-call confound (which --disallowedTools would introduce). Steering is
|
|
8
|
+
# injected with --append-system-prompt, so no rebuild of the shipped
|
|
9
|
+
# server-instructions is needed to A/B it.
|
|
10
|
+
#
|
|
11
|
+
# A control all tools no steering
|
|
12
|
+
# B steer all tools trace-first
|
|
13
|
+
# C no-explore hide explore trace-first
|
|
14
|
+
# D trace-centric hide explore+context trace-first
|
|
15
|
+
# E control-probe hide explore+context trace-first (caller passes a NON-flow Q)
|
|
16
|
+
#
|
|
17
|
+
# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
|
|
18
|
+
set -uo pipefail
|
|
19
|
+
REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
|
|
20
|
+
CG_BIN="${CG_BIN:-$(command -v homegraph)}"
|
|
21
|
+
OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
|
|
22
|
+
mkdir -p "$OUT"
|
|
23
|
+
[ -n "$CG_BIN" ] || { echo "no homegraph binary (set CG_BIN)"; exit 1; }
|
|
24
|
+
[ -d "$REPO/.homegraph" ] || { echo "no .homegraph index at $REPO"; exit 1; }
|
|
25
|
+
|
|
26
|
+
STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call homegraph_trace(from,to) FIRST — one call returns the whole path. Use homegraph_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
|
|
27
|
+
KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
|
|
28
|
+
KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
|
|
29
|
+
|
|
30
|
+
case "$ARM" in
|
|
31
|
+
A|G|H|I) TOOLS=""; STEERING="" ;; # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
|
|
32
|
+
B|F) TOOLS=""; STEERING="$STEER" ;; # F = B's surface, run on the body-inlining trace build
|
|
33
|
+
C) TOOLS="$KEEP_NO_EXPLORE"; STEERING="$STEER" ;;
|
|
34
|
+
D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
|
|
35
|
+
*) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
|
|
36
|
+
esac
|
|
37
|
+
|
|
38
|
+
CFG="$OUT/mcp-$ARM.json"
|
|
39
|
+
if [ -n "$TOOLS" ]; then
|
|
40
|
+
cat > "$CFG" <<JSON
|
|
41
|
+
{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"HOMEGRAPH_MCP_TOOLS":"$TOOLS"}}}}
|
|
42
|
+
JSON
|
|
43
|
+
else
|
|
44
|
+
cat > "$CFG" <<JSON
|
|
45
|
+
{"mcpServers":{"homegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
|
46
|
+
JSON
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
|
|
50
|
+
ARGS=( -p "$Q" --output-format stream-json --verbose
|
|
51
|
+
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
|
|
52
|
+
--strict-mcp-config --mcp-config "$CFG" )
|
|
53
|
+
[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
|
|
54
|
+
|
|
55
|
+
( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
|
|
56
|
+
echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"
|