@saluzi/codegraph 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +885 -0
- package/package.json +42 -10
- package/scripts/add-lang/bench.sh +60 -0
- package/scripts/add-lang/check-grammar.mjs +75 -0
- package/scripts/add-lang/dump-ast.mjs +103 -0
- package/scripts/add-lang/verify-extraction.mjs +70 -0
- package/scripts/agent-eval/ab-adoption.sh +91 -0
- package/scripts/agent-eval/ab-hook.sh +86 -0
- package/scripts/agent-eval/ab-impl.sh +78 -0
- package/scripts/agent-eval/ab-new-vs-baseline.sh +102 -0
- package/scripts/agent-eval/ab-sufficiency.sh +78 -0
- package/scripts/agent-eval/arms-F.sh +21 -0
- package/scripts/agent-eval/arms-matrix.sh +37 -0
- package/scripts/agent-eval/audit.sh +68 -0
- package/scripts/agent-eval/bench-readme.sh +28 -0
- package/scripts/agent-eval/bench-why-repo.sh +22 -0
- package/scripts/agent-eval/block-read-hook.sh +19 -0
- package/scripts/agent-eval/hook-settings.json +15 -0
- package/scripts/agent-eval/itrun.sh +120 -0
- package/scripts/agent-eval/offload-eval-3arm.sh +72 -0
- package/scripts/agent-eval/offload-eval-cost.mjs +133 -0
- package/scripts/agent-eval/offload-eval-effort.mjs +108 -0
- package/scripts/agent-eval/offload-eval-frontload-matrix.sh +25 -0
- package/scripts/agent-eval/offload-eval-frontload.sh +47 -0
- package/scripts/agent-eval/offload-eval-ground-truth.json +18 -0
- package/scripts/agent-eval/offload-eval-hook.mjs +84 -0
- package/scripts/agent-eval/offload-eval-judge.mjs +103 -0
- package/scripts/agent-eval/offload-eval-matrix.sh +20 -0
- package/scripts/agent-eval/offload-eval-metrics.mjs +94 -0
- package/scripts/agent-eval/offload-eval-refs1.sh +50 -0
- package/scripts/agent-eval/offload-eval-setup.sh +24 -0
- package/scripts/agent-eval/offload-eval-styles.sh +72 -0
- package/scripts/agent-eval/offload-eval-summarize.mjs +68 -0
- package/scripts/agent-eval/offload-eval.md +76 -0
- package/scripts/agent-eval/parse-arms.mjs +116 -0
- package/scripts/agent-eval/parse-bench-readme.mjs +84 -0
- package/scripts/agent-eval/parse-run.mjs +45 -0
- package/scripts/agent-eval/parse-session.mjs +93 -0
- package/scripts/agent-eval/probe-context.mjs +21 -0
- package/scripts/agent-eval/probe-explore.mjs +40 -0
- package/scripts/agent-eval/probe-node.mjs +20 -0
- package/scripts/agent-eval/probe-sweep.mjs +119 -0
- package/scripts/agent-eval/probe-trace.mjs +20 -0
- package/scripts/agent-eval/redirect-read-hook.sh +38 -0
- package/scripts/agent-eval/repro-concurrent-explore.mjs +119 -0
- package/scripts/agent-eval/repro-daemon-clients.mjs +125 -0
- package/scripts/agent-eval/run-agent.sh +34 -0
- package/scripts/agent-eval/run-all.sh +75 -0
- package/scripts/agent-eval/run-arms.sh +56 -0
- package/scripts/agent-eval/seq-matrix.mjs +137 -0
- package/scripts/build-bundle.sh +123 -0
- package/scripts/extract-release-notes.mjs +130 -0
- package/scripts/local-install.sh +41 -0
- package/scripts/npm-sdk.js +75 -0
- package/scripts/npm-shim.js +275 -0
- package/scripts/pack-npm.sh +119 -0
- package/scripts/prepare-release.mjs +270 -0
- package/dist/bin/codegraph.d.ts +0 -25
- package/dist/bin/node-version-check.d.ts +0 -37
- package/dist/bin/uninstall.d.ts +0 -14
- package/dist/context/formatter.d.ts +0 -33
- package/dist/context/formatter.js +0 -244
- package/dist/context/index.d.ts +0 -117
- package/dist/context/index.js +0 -1050
- package/dist/db/index.d.ts +0 -101
- package/dist/db/index.js +0 -250
- package/dist/db/migrations.d.ts +0 -47
- package/dist/db/migrations.js +0 -131
- package/dist/db/queries.d.ts +0 -291
- package/dist/db/queries.js +0 -1349
- package/dist/db/schema.sql +0 -151
- package/dist/db/sqlite-adapter.d.ts +0 -49
- package/dist/db/sqlite-adapter.js +0 -141
- package/dist/directory.d.ts +0 -62
- package/dist/directory.js +0 -264
- package/dist/errors.d.ts +0 -149
- package/dist/errors.js +0 -219
- package/dist/extraction/dfm-extractor.d.ts +0 -31
- package/dist/extraction/dfm-extractor.js +0 -151
- package/dist/extraction/grammars.d.ts +0 -94
- package/dist/extraction/grammars.js +0 -357
- package/dist/extraction/index.d.ts +0 -148
- package/dist/extraction/index.js +0 -1286
- package/dist/extraction/languages/c-cpp.d.ts +0 -4
- package/dist/extraction/languages/c-cpp.js +0 -126
- package/dist/extraction/languages/csharp.d.ts +0 -3
- package/dist/extraction/languages/csharp.js +0 -72
- package/dist/extraction/languages/dart.d.ts +0 -3
- package/dist/extraction/languages/dart.js +0 -192
- package/dist/extraction/languages/go.d.ts +0 -3
- package/dist/extraction/languages/go.js +0 -58
- package/dist/extraction/languages/index.d.ts +0 -10
- package/dist/extraction/languages/index.js +0 -49
- package/dist/extraction/languages/java.d.ts +0 -3
- package/dist/extraction/languages/java.js +0 -64
- package/dist/extraction/languages/javascript.d.ts +0 -3
- package/dist/extraction/languages/javascript.js +0 -90
- package/dist/extraction/languages/kotlin.d.ts +0 -3
- package/dist/extraction/languages/kotlin.js +0 -253
- package/dist/extraction/languages/lua.d.ts +0 -3
- package/dist/extraction/languages/lua.js +0 -150
- package/dist/extraction/languages/luau.d.ts +0 -3
- package/dist/extraction/languages/luau.js +0 -37
- package/dist/extraction/languages/pascal.d.ts +0 -3
- package/dist/extraction/languages/pascal.js +0 -66
- package/dist/extraction/languages/php.d.ts +0 -3
- package/dist/extraction/languages/php.js +0 -107
- package/dist/extraction/languages/python.d.ts +0 -3
- package/dist/extraction/languages/python.js +0 -56
- package/dist/extraction/languages/ruby.d.ts +0 -3
- package/dist/extraction/languages/ruby.js +0 -114
- package/dist/extraction/languages/rust.d.ts +0 -3
- package/dist/extraction/languages/rust.js +0 -109
- package/dist/extraction/languages/scala.d.ts +0 -3
- package/dist/extraction/languages/scala.js +0 -139
- package/dist/extraction/languages/swift.d.ts +0 -3
- package/dist/extraction/languages/swift.js +0 -91
- package/dist/extraction/languages/typescript.d.ts +0 -3
- package/dist/extraction/languages/typescript.js +0 -129
- package/dist/extraction/liquid-extractor.d.ts +0 -52
- package/dist/extraction/liquid-extractor.js +0 -313
- package/dist/extraction/parse-worker.d.ts +0 -8
- package/dist/extraction/parse-worker.js +0 -94
- package/dist/extraction/svelte-extractor.d.ts +0 -56
- package/dist/extraction/svelte-extractor.js +0 -272
- package/dist/extraction/tree-sitter-helpers.d.ts +0 -39
- package/dist/extraction/tree-sitter-helpers.js +0 -103
- package/dist/extraction/tree-sitter-types.d.ts +0 -191
- package/dist/extraction/tree-sitter-types.js +0 -10
- package/dist/extraction/tree-sitter.d.ts +0 -238
- package/dist/extraction/tree-sitter.js +0 -2430
- package/dist/extraction/vue-extractor.d.ts +0 -36
- package/dist/extraction/vue-extractor.js +0 -163
- package/dist/extraction/wasm/tree-sitter-c.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-c_sharp.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-cpp.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-dart.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-go.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-java.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-javascript.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-kotlin.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-lua.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-luau.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-pascal.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-php.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-python.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-ruby.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-rust.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-scala.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-swift.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-tsx.wasm +0 -0
- package/dist/extraction/wasm/tree-sitter-typescript.wasm +0 -0
- package/dist/extraction/wasm-runtime-flags.d.ts +0 -46
- package/dist/extraction/wasm-runtime-flags.js +0 -105
- package/dist/graph/index.d.ts +0 -8
- package/dist/graph/index.js +0 -13
- package/dist/graph/queries.d.ts +0 -109
- package/dist/graph/queries.js +0 -366
- package/dist/graph/traversal.d.ts +0 -137
- package/dist/graph/traversal.js +0 -528
- package/dist/index.d.ts +0 -509
- package/dist/index.js +0 -800
- package/dist/installer/claude-md-template.d.ts +0 -19
- package/dist/installer/config-writer.d.ts +0 -29
- package/dist/installer/index.d.ts +0 -140
- package/dist/installer/instructions-template.d.ts +0 -30
- package/dist/installer/targets/claude.d.ts +0 -55
- package/dist/installer/targets/codex.d.ts +0 -18
- package/dist/installer/targets/cursor.d.ts +0 -35
- package/dist/installer/targets/hermes.d.ts +0 -18
- package/dist/installer/targets/opencode.d.ts +0 -30
- package/dist/installer/targets/registry.d.ts +0 -38
- package/dist/installer/targets/shared.d.ts +0 -92
- package/dist/installer/targets/toml.d.ts +0 -64
- package/dist/installer/targets/types.d.ts +0 -122
- package/dist/mcp/index.d.ts +0 -98
- package/dist/mcp/server-instructions.d.ts +0 -20
- package/dist/mcp/tools.d.ts +0 -269
- package/dist/mcp/transport.d.ts +0 -117
- package/dist/resolution/frameworks/cargo-workspace.d.ts +0 -20
- package/dist/resolution/frameworks/cargo-workspace.js +0 -225
- package/dist/resolution/frameworks/csharp.d.ts +0 -8
- package/dist/resolution/frameworks/csharp.js +0 -213
- package/dist/resolution/frameworks/drupal.d.ts +0 -51
- package/dist/resolution/frameworks/drupal.js +0 -335
- package/dist/resolution/frameworks/express.d.ts +0 -8
- package/dist/resolution/frameworks/express.js +0 -225
- package/dist/resolution/frameworks/go.d.ts +0 -8
- package/dist/resolution/frameworks/go.js +0 -158
- package/dist/resolution/frameworks/index.d.ts +0 -52
- package/dist/resolution/frameworks/index.js +0 -137
- package/dist/resolution/frameworks/java.d.ts +0 -8
- package/dist/resolution/frameworks/java.js +0 -177
- package/dist/resolution/frameworks/laravel.d.ts +0 -13
- package/dist/resolution/frameworks/laravel.js +0 -248
- package/dist/resolution/frameworks/nestjs.d.ts +0 -26
- package/dist/resolution/frameworks/nestjs.js +0 -374
- package/dist/resolution/frameworks/python.d.ts +0 -10
- package/dist/resolution/frameworks/python.js +0 -278
- package/dist/resolution/frameworks/react.d.ts +0 -8
- package/dist/resolution/frameworks/react.js +0 -272
- package/dist/resolution/frameworks/ruby.d.ts +0 -8
- package/dist/resolution/frameworks/ruby.js +0 -198
- package/dist/resolution/frameworks/rust.d.ts +0 -8
- package/dist/resolution/frameworks/rust.js +0 -207
- package/dist/resolution/frameworks/svelte.d.ts +0 -9
- package/dist/resolution/frameworks/svelte.js +0 -249
- package/dist/resolution/frameworks/swift.d.ts +0 -10
- package/dist/resolution/frameworks/swift.js +0 -376
- package/dist/resolution/frameworks/vue.d.ts +0 -9
- package/dist/resolution/frameworks/vue.js +0 -306
- package/dist/resolution/import-resolver.d.ts +0 -61
- package/dist/resolution/import-resolver.js +0 -663
- package/dist/resolution/index.d.ts +0 -118
- package/dist/resolution/index.js +0 -744
- package/dist/resolution/lru-cache.d.ts +0 -24
- package/dist/resolution/lru-cache.js +0 -62
- package/dist/resolution/name-matcher.d.ts +0 -50
- package/dist/resolution/name-matcher.js +0 -384
- package/dist/resolution/path-aliases.d.ts +0 -72
- package/dist/resolution/path-aliases.js +0 -238
- package/dist/resolution/strip-comments.d.ts +0 -40
- package/dist/resolution/strip-comments.js +0 -441
- package/dist/resolution/types.d.ts +0 -181
- package/dist/resolution/types.js +0 -8
- package/dist/search/query-parser.d.ts +0 -61
- package/dist/search/query-parser.js +0 -177
- package/dist/search/query-utils.d.ts +0 -59
- package/dist/search/query-utils.js +0 -383
- package/dist/sync/git-hooks.d.ts +0 -54
- package/dist/sync/git-hooks.js +0 -223
- package/dist/sync/index.d.ts +0 -25
- package/dist/sync/index.js +0 -28
- package/dist/sync/watch-policy.d.ts +0 -51
- package/dist/sync/watch-policy.js +0 -124
- package/dist/sync/watcher.d.ts +0 -83
- package/dist/sync/watcher.js +0 -192
- package/dist/types.d.ts +0 -433
- package/dist/types.js +0 -75
- package/dist/ui/glyphs.d.ts +0 -42
- package/dist/ui/shimmer-progress.d.ts +0 -11
- package/dist/ui/shimmer-worker.d.ts +0 -2
- package/dist/ui/types.d.ts +0 -20
- package/dist/utils.d.ts +0 -231
- package/dist/utils.js +0 -549
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Parse the newest Claude Code session log for a project + its subagent logs,
|
|
3
|
+
// and report the tool-call breakdown (main + subagents). Works for interactive
|
|
4
|
+
// runs (driven via itrun.sh) — Claude Code writes full transcripts to
|
|
5
|
+
// ~/.claude/projects/<escaped-cwd>/<session>.jsonl with subagents/ alongside.
|
|
6
|
+
import { readFileSync, readdirSync, statSync, existsSync, realpathSync } from 'fs';
|
|
7
|
+
import { join } from 'path';
|
|
8
|
+
import { homedir } from 'os';
|
|
9
|
+
|
|
10
|
+
const projectArg = process.argv[2];
|
|
11
|
+
if (!projectArg) { console.error('usage: parse-session.mjs <project-dir>'); process.exit(1); }
|
|
12
|
+
|
|
13
|
+
// Claude Code escapes the (real) cwd by replacing every "/" with "-".
|
|
14
|
+
const real = realpathSync(projectArg);
|
|
15
|
+
const escaped = real.replace(/\//g, '-');
|
|
16
|
+
const projDir = join(homedir(), '.claude', 'projects', escaped);
|
|
17
|
+
if (!existsSync(projDir)) { console.error('no session logs at', projDir); process.exit(1); }
|
|
18
|
+
|
|
19
|
+
// Newest top-level session .jsonl
|
|
20
|
+
const sessions = readdirSync(projDir)
|
|
21
|
+
.filter(f => f.endsWith('.jsonl'))
|
|
22
|
+
.map(f => ({ f, m: statSync(join(projDir, f)).mtimeMs }))
|
|
23
|
+
.sort((a, b) => b.m - a.m);
|
|
24
|
+
if (sessions.length === 0) { console.error('no .jsonl sessions in', projDir); process.exit(1); }
|
|
25
|
+
const sessionId = sessions[0].f.replace('.jsonl', '');
|
|
26
|
+
|
|
27
|
+
function tally(file) {
|
|
28
|
+
const counts = {};
|
|
29
|
+
for (const line of readFileSync(file, 'utf8').split('\n')) {
|
|
30
|
+
if (!line) continue;
|
|
31
|
+
let ev; try { ev = JSON.parse(line); } catch { continue; }
|
|
32
|
+
const content = ev.message?.content;
|
|
33
|
+
if (!Array.isArray(content)) continue;
|
|
34
|
+
for (const b of content) {
|
|
35
|
+
if (b.type === 'tool_use') counts[b.name] = (counts[b.name] || 0) + 1;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return counts;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Sum token usage from a transcript. The TUI's "Done (…Xk tokens…)" line only
|
|
42
|
+
// covers a subagent's throughput; this works for main-thread runs too and is
|
|
43
|
+
// consistent across both paths. `gen` = output, `fresh` = uncached input
|
|
44
|
+
// (input + cache_creation), `cached` = cache reads (≈free), `total` = all.
|
|
45
|
+
function sumTokens(file) {
|
|
46
|
+
const t = { gen: 0, fresh: 0, cached: 0 };
|
|
47
|
+
for (const line of readFileSync(file, 'utf8').split('\n')) {
|
|
48
|
+
if (!line) continue;
|
|
49
|
+
let ev; try { ev = JSON.parse(line); } catch { continue; }
|
|
50
|
+
const u = ev.message?.usage;
|
|
51
|
+
if (!u) continue;
|
|
52
|
+
t.gen += u.output_tokens || 0;
|
|
53
|
+
t.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
|
|
54
|
+
t.cached += u.cache_read_input_tokens || 0;
|
|
55
|
+
}
|
|
56
|
+
return t;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const mainCounts = tally(join(projDir, sessionId + '.jsonl'));
|
|
60
|
+
|
|
61
|
+
// Subagent transcripts live under <session>/subagents/*.jsonl
|
|
62
|
+
const subDir = join(projDir, sessionId, 'subagents');
|
|
63
|
+
const subCounts = {};
|
|
64
|
+
let subAgentFiles = 0;
|
|
65
|
+
if (existsSync(subDir)) {
|
|
66
|
+
for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) {
|
|
67
|
+
subAgentFiles++;
|
|
68
|
+
const c = tally(join(subDir, f));
|
|
69
|
+
for (const [k, v] of Object.entries(c)) subCounts[k] = (subCounts[k] || 0) + v;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const fmt = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])
|
|
74
|
+
.map(([k, v]) => ` ${String(v).padStart(3)} ${k}`).join('\n') || ' (none)';
|
|
75
|
+
|
|
76
|
+
console.log(`session: ${sessionId}`);
|
|
77
|
+
console.log(`\nMAIN thread tools:\n${fmt(mainCounts)}`);
|
|
78
|
+
console.log(`\nSUBAGENT tools (${subAgentFiles} subagent transcript${subAgentFiles === 1 ? '' : 's'}):\n${fmt(subCounts)}`);
|
|
79
|
+
|
|
80
|
+
const explore = subCounts['mcp__codegraph__codegraph_explore'] || mainCounts['mcp__codegraph__codegraph_explore'] || 0;
|
|
81
|
+
const reads = (subCounts['Read'] || 0) + (mainCounts['Read'] || 0);
|
|
82
|
+
const greps = (subCounts['Grep'] || 0) + (mainCounts['Grep'] || 0) + (subCounts['Bash'] || 0) + (mainCounts['Bash'] || 0);
|
|
83
|
+
console.log(`\nVERDICT: codegraph_explore used ${explore}x | Read ${reads} | Grep/Bash ${greps}`);
|
|
84
|
+
|
|
85
|
+
// Token totals (main + subagents), consistent across main-thread and subagent runs.
|
|
86
|
+
const tok = { gen: 0, fresh: 0, cached: 0 };
|
|
87
|
+
const addTok = (t) => { tok.gen += t.gen; tok.fresh += t.fresh; tok.cached += t.cached; };
|
|
88
|
+
addTok(sumTokens(join(projDir, sessionId + '.jsonl')));
|
|
89
|
+
if (existsSync(subDir)) {
|
|
90
|
+
for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) addTok(sumTokens(join(subDir, f)));
|
|
91
|
+
}
|
|
92
|
+
const k = (n) => (n / 1000).toFixed(1) + 'k';
|
|
93
|
+
console.log(`TOKENS: gen ${k(tok.gen)} | fresh-in ${k(tok.fresh)} | cached-in ${k(tok.cached)} | billable≈ ${k(tok.gen + tok.fresh)}`);
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Probe codegraph_context (with call-paths) against an index using the built dist.
|
|
3
|
+
// Usage: node probe-context.mjs <repo-with-.codegraph> <task words...>
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { resolve } from 'node:path';
|
|
6
|
+
|
|
7
|
+
const [, , repo, ...taskParts] = process.argv;
|
|
8
|
+
const task = taskParts.join(' ');
|
|
9
|
+
if (!repo || !task) { console.error('usage: probe-context.mjs <repo> <task...>'); process.exit(1); }
|
|
10
|
+
|
|
11
|
+
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
12
|
+
const idx = await load('dist/index.js');
|
|
13
|
+
const tools = await load('dist/mcp/tools.js');
|
|
14
|
+
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
15
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
16
|
+
|
|
17
|
+
const cg = CodeGraph.openSync(repo);
|
|
18
|
+
const h = new ToolHandler(cg);
|
|
19
|
+
const res = await h.execute('codegraph_context', { task });
|
|
20
|
+
console.log(res.content?.[0]?.text ?? '(no text)');
|
|
21
|
+
try { cg.close?.(); } catch {}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// One-shot probe: run handleExplore against an existing index using the built
|
|
3
|
+
// dist, print the output + a few stats. Lets us verify explore's coverage fix
|
|
4
|
+
// without a full agent run. Usage: node probe-explore.mjs <repo-with-.codegraph> "<query>"
|
|
5
|
+
import { pathToFileURL } from 'node:url';
|
|
6
|
+
import { resolve } from 'node:path';
|
|
7
|
+
|
|
8
|
+
const [, , repo, query] = process.argv;
|
|
9
|
+
if (!repo || !query) {
|
|
10
|
+
console.error('usage: probe-explore.mjs <repo> "<query>"');
|
|
11
|
+
process.exit(1);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
15
|
+
const idx = await load('dist/index.js');
|
|
16
|
+
const tools = await load('dist/mcp/tools.js');
|
|
17
|
+
|
|
18
|
+
// esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
|
|
19
|
+
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
20
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
21
|
+
|
|
22
|
+
if (typeof CodeGraph?.openSync !== 'function') {
|
|
23
|
+
console.error('could not resolve CodeGraph.openSync; index keys:', Object.keys(idx), 'default keys:', idx.default && Object.keys(idx.default));
|
|
24
|
+
process.exit(2);
|
|
25
|
+
}
|
|
26
|
+
if (typeof ToolHandler !== 'function') {
|
|
27
|
+
console.error('could not resolve ToolHandler; tools keys:', Object.keys(tools));
|
|
28
|
+
process.exit(2);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const cg = CodeGraph.openSync(repo);
|
|
32
|
+
const h = new ToolHandler(cg);
|
|
33
|
+
const res = await h.execute('codegraph_explore', { query });
|
|
34
|
+
const text = res.content?.[0]?.text ?? '(no text)';
|
|
35
|
+
console.log(text);
|
|
36
|
+
console.error('\n--- PROBE STATS ---');
|
|
37
|
+
console.error('output chars:', text.length);
|
|
38
|
+
console.error('triggerRender body present (-> setState({})):', /triggerRender[\s\S]{0,400}setState\(\{\}\)/.test(text));
|
|
39
|
+
console.error('App.tsx in source section:', /\*\*`.*App\.tsx`\*\* —/.test(text));
|
|
40
|
+
try { cg.close?.(); } catch {}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Probe codegraph_node (with trail) against an index using the built dist.
|
|
3
|
+
// Usage: node probe-node.mjs <repo-with-.codegraph> <symbol> [code]
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { resolve } from 'node:path';
|
|
6
|
+
|
|
7
|
+
const [, , repo, symbol, code] = process.argv;
|
|
8
|
+
if (!repo || !symbol) { console.error('usage: probe-node.mjs <repo> <symbol> [code]'); process.exit(1); }
|
|
9
|
+
|
|
10
|
+
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
11
|
+
const idx = await load('dist/index.js');
|
|
12
|
+
const tools = await load('dist/mcp/tools.js');
|
|
13
|
+
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
14
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
15
|
+
|
|
16
|
+
const cg = CodeGraph.openSync(repo);
|
|
17
|
+
const h = new ToolHandler(cg);
|
|
18
|
+
const res = await h.execute('codegraph_node', { symbol, includeCode: code === 'code' });
|
|
19
|
+
console.log(res.content?.[0]?.text ?? '(no text)');
|
|
20
|
+
try { cg.close?.(); } catch {}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// probe-sweep — direct MCP test across N repos × N tools, no claude needed.
|
|
3
|
+
//
|
|
4
|
+
// Measures response characteristics (size, sections present, signals fired)
|
|
5
|
+
// for each (repo, query) pair against the built dist/. Sub-second per probe;
|
|
6
|
+
// the full sweep below runs in ~10-30s vs hours for a real claude audit.
|
|
7
|
+
//
|
|
8
|
+
// Use this to iterate on backend changes rapidly: change tools.ts /
|
|
9
|
+
// context-builder, npm run build, re-run probe-sweep, compare. Once a
|
|
10
|
+
// change looks good on probe metrics, run a focused claude audit for the
|
|
11
|
+
// few repos that matter to confirm end-to-end cost behavior.
|
|
12
|
+
//
|
|
13
|
+
// Usage: node scripts/agent-eval/probe-sweep.mjs [--tool=context|explore|trace] [--repos=a,b,c]
|
|
14
|
+
import { pathToFileURL } from 'node:url';
|
|
15
|
+
import { resolve } from 'node:path';
|
|
16
|
+
|
|
17
|
+
const args = Object.fromEntries(
|
|
18
|
+
process.argv.slice(2).map(a => a.startsWith('--') ? a.slice(2).split('=') : [a, true])
|
|
19
|
+
);
|
|
20
|
+
const TOOL = args.tool ?? 'context';
|
|
21
|
+
|
|
22
|
+
const load = (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
23
|
+
const idx = await load('dist/index.js');
|
|
24
|
+
const tools = await load('dist/mcp/tools.js');
|
|
25
|
+
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
26
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
27
|
+
|
|
28
|
+
// Each entry: repo, query, optional 2nd arg for trace (from, to).
|
|
29
|
+
// The query is the same prompt used in the real claude audits, so probe
|
|
30
|
+
// output is directly comparable to the agent's would-be input.
|
|
31
|
+
const SWEEP = [
|
|
32
|
+
// Small realworld template repos (the loss cases from the cross-language sweep)
|
|
33
|
+
{ id: 'gin-rw', repo: '/tmp/codegraph-corpus/gin-realworld', q: 'How does this Gin app route a request through its middleware chain to a handler?' },
|
|
34
|
+
{ id: 'go-mux', repo: '/tmp/codegraph-corpus/go-mux', q: 'How does this gorilla/mux app route a request to its handler?' },
|
|
35
|
+
{ id: 'fastapi-rw', repo: '/tmp/codegraph-corpus/fastapi-realworld', q: 'How does FastAPI route a request through its dependencies to a handler?' },
|
|
36
|
+
{ id: 'spring-pc', repo: '/tmp/codegraph-corpus/spring-petclinic', q: 'How does Spring route an HTTP request to a controller method?' },
|
|
37
|
+
{ id: 'axum-rw', repo: '/tmp/codegraph-corpus/rust-axum-realworld', q: 'How does Axum route a request to its handler in this app?' },
|
|
38
|
+
{ id: 'express-rw', repo: '/tmp/codegraph-corpus/express-realworld', q: 'How does this Express app route a request through middleware to a handler?' },
|
|
39
|
+
{ id: 'kotlin-pc', repo: '/tmp/codegraph-corpus/kotlin-petclinic', q: 'How does the Kotlin Spring app route an HTTP request to its handler?' },
|
|
40
|
+
{ id: 'flask-mb', repo: '/tmp/codegraph-corpus/flask-microblog', q: 'How does this Flask app route a request to a view function?' },
|
|
41
|
+
{ id: 'vapor-tpl', repo: '/tmp/codegraph-corpus/vapor-template', q: 'How does Vapor route an HTTP request to its handler?' },
|
|
42
|
+
{ id: 'cpp-leveldb', repo: '/tmp/codegraph-corpus/cpp-leveldb', q: 'How does LevelDB handle a Put operation through to disk?' },
|
|
43
|
+
{ id: 'lualine', repo: '/tmp/codegraph-corpus/lualine.nvim', q: 'How does lualine assemble and render the statusline?' },
|
|
44
|
+
{ id: 'drupal-admin', repo: '/tmp/codegraph-corpus/drupal-admintoolbar', q: 'How does the Drupal admin toolbar module render its toolbar?' },
|
|
45
|
+
{ id: 'svelte-rw', repo: '/tmp/codegraph-corpus/svelte-realworld', q: 'How does this SvelteKit app route a request to a handler?' },
|
|
46
|
+
{ id: 'react-rw', repo: '/tmp/codegraph-corpus/react-realworld', q: 'How does this React app fetch and display articles?' },
|
|
47
|
+
{ id: 'rails-rw', repo: '/tmp/codegraph-corpus/rails-realworld', q: 'How does Rails route a request to a controller action?' },
|
|
48
|
+
{ id: 'flask-rest', repo: '/tmp/codegraph-corpus/flask-restful-realworld', q: 'How does Flask-RESTful route a request to a resource method?' },
|
|
49
|
+
{ id: 'laravel-rw', repo: '/tmp/codegraph-corpus/laravel-realworld', q: 'How does Laravel route a request to the controller method?' },
|
|
50
|
+
{ id: 'aspnet-rw', repo: '/tmp/codegraph-corpus/aspnet-realworld', q: 'How does ASP.NET route a request to the controller action?' },
|
|
51
|
+
// The iter7 wins/ties (to make sure we don't regress)
|
|
52
|
+
{ id: 'cobra', repo: '/tmp/codegraph-corpus/cobra', q: 'How does cobra parse commands and flags?' },
|
|
53
|
+
{ id: 'sinatra', repo: '/tmp/codegraph-corpus/sinatra', q: 'How does sinatra route a request to its handler?' },
|
|
54
|
+
{ id: 'slim', repo: '/tmp/codegraph-corpus/slim', q: 'How does slim route a request and apply middleware?' },
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
// Detect signals in response text — these are the levers we've added that
|
|
58
|
+
// otherwise only show up via "agent ran X more tool calls" downstream.
|
|
59
|
+
const detect = (text) => ({
|
|
60
|
+
hasEntryPoints: /^### Entry Points/m.test(text),
|
|
61
|
+
hasRelatedSymbols: /^### Related Symbols/m.test(text),
|
|
62
|
+
hasFlowTrace: /^## Inline flow trace/m.test(text),
|
|
63
|
+
hasRouteManifest: /^## Routing manifest/m.test(text),
|
|
64
|
+
hasTopHandler: /^### Top handler file/m.test(text),
|
|
65
|
+
hasSmallRepoTail: /This project is small/.test(text),
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
const filterRepos = args.repos ? new Set(String(args.repos).split(',')) : null;
|
|
69
|
+
const subjects = SWEEP.filter(s => !filterRepos || filterRepos.has(s.id));
|
|
70
|
+
|
|
71
|
+
const t0 = Date.now();
|
|
72
|
+
const rows = [];
|
|
73
|
+
for (const s of subjects) {
|
|
74
|
+
try {
|
|
75
|
+
const cg = CodeGraph.openSync(s.repo);
|
|
76
|
+
const handler = new ToolHandler(cg);
|
|
77
|
+
const t1 = Date.now();
|
|
78
|
+
const res = await handler.execute('codegraph_' + TOOL,
|
|
79
|
+
TOOL === 'context' ? { task: s.q } :
|
|
80
|
+
TOOL === 'explore' ? { query: s.q } : { from: 'main', to: 'main' });
|
|
81
|
+
const text = res.content?.[0]?.text ?? '';
|
|
82
|
+
const signals = detect(text);
|
|
83
|
+
rows.push({
|
|
84
|
+
id: s.id,
|
|
85
|
+
ms: Date.now() - t1,
|
|
86
|
+
chars: text.length,
|
|
87
|
+
lines: text.split('\n').length,
|
|
88
|
+
...signals,
|
|
89
|
+
});
|
|
90
|
+
try { cg.close?.(); } catch {}
|
|
91
|
+
} catch (e) {
|
|
92
|
+
rows.push({ id: s.id, error: String(e).slice(0, 80) });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Pretty-print as a compact table.
|
|
97
|
+
const fmt = (r) =>
|
|
98
|
+
r.error
|
|
99
|
+
? ` ${r.id.padEnd(13)} ERROR: ${r.error}`
|
|
100
|
+
: ` ${r.id.padEnd(13)} ${String(r.chars).padStart(6)}c ${String(r.lines).padStart(4)}L ${String(r.ms).padStart(4)}ms` +
|
|
101
|
+
` ${r.hasEntryPoints ? 'EP ' : ' '}` +
|
|
102
|
+
`${r.hasFlowTrace ? 'TRC ' : ' '}` +
|
|
103
|
+
`${r.hasRouteManifest ? 'MAN ' : ' '}` +
|
|
104
|
+
`${r.hasTopHandler ? 'HND ' : ' '}` +
|
|
105
|
+
`${r.hasSmallRepoTail ? 'TAIL' : ' '}`;
|
|
106
|
+
console.log(`=== probe-sweep tool=${TOOL} n=${subjects.length} (${Date.now() - t0}ms total) ===`);
|
|
107
|
+
console.log(' id chars lines ms signals');
|
|
108
|
+
console.log(' ' + '-'.repeat(56));
|
|
109
|
+
for (const r of rows) console.log(fmt(r));
|
|
110
|
+
|
|
111
|
+
// Sum + medians for the size pillar
|
|
112
|
+
const sizes = rows.filter(r => !r.error).map(r => r.chars);
|
|
113
|
+
sizes.sort((a, b) => a - b);
|
|
114
|
+
const median = sizes[Math.floor(sizes.length / 2)];
|
|
115
|
+
const sum = sizes.reduce((a, b) => a + b, 0);
|
|
116
|
+
console.log(` ${'-'.repeat(64)}`);
|
|
117
|
+
console.log(` median=${median}c total=${sum}c ` +
|
|
118
|
+
`manifest=${rows.filter(r => r.hasRouteManifest).length}/${rows.filter(r => !r.error).length} ` +
|
|
119
|
+
`top-handler=${rows.filter(r => r.hasTopHandler).length}/${rows.filter(r => !r.error).length}`);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Probe codegraph_trace against an index using the built dist.
|
|
3
|
+
// Usage: node probe-trace.mjs <repo-with-.codegraph> <from> <to>
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { resolve } from 'node:path';
|
|
6
|
+
|
|
7
|
+
const [, , repo, from, to] = process.argv;
|
|
8
|
+
if (!repo || !from || !to) { console.error('usage: probe-trace.mjs <repo> <from> <to>'); process.exit(1); }
|
|
9
|
+
|
|
10
|
+
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
11
|
+
const idx = await load('dist/index.js');
|
|
12
|
+
const tools = await load('dist/mcp/tools.js');
|
|
13
|
+
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
14
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
15
|
+
|
|
16
|
+
const cg = CodeGraph.openSync(repo);
|
|
17
|
+
const h = new ToolHandler(cg);
|
|
18
|
+
const res = await h.execute('codegraph_trace', { from, to });
|
|
19
|
+
console.log(res.content?.[0]?.text ?? '(no text)');
|
|
20
|
+
try { cg.close?.(); } catch {}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# PreToolUse(Read) REDIRECT hook — prototype for A/B (P1: get agents off Read and
|
|
3
|
+
# onto codegraph_node during implementation, not just for Q&A).
|
|
4
|
+
#
|
|
5
|
+
# When the agent Reads a SOURCE file, deny it and steer to codegraph_node's
|
|
6
|
+
# file-view, which (as of the Lever-1 change) returns the WHOLE file verbatim
|
|
7
|
+
# WITH line numbers — imports, top-level code, comments and all — PLUS the file's
|
|
8
|
+
# blast radius, in one call. That output is a strict superset of Read, so the
|
|
9
|
+
# redirect is lossless: the agent loses nothing by taking it, and gains who-
|
|
10
|
+
# depends-on-this for the edit it's about to make.
|
|
11
|
+
#
|
|
12
|
+
# Differs from block-read-hook.sh (which steers to explore/node-by-symbol): this
|
|
13
|
+
# names the FILE-VIEW path explicitly (file:"<base>" + includeCode:true), the
|
|
14
|
+
# 1:1 Read replacement we're trying to get picked during implementation.
|
|
15
|
+
#
|
|
16
|
+
# Non-source files (configs, docs, lockfiles, .env) pass through to a real Read.
|
|
17
|
+
# A redirect to a file codegraph hasn't indexed SELF-CORRECTS: the file-view
|
|
18
|
+
# replies "No indexed file matches … Read it directly", so a just-created file
|
|
19
|
+
# never dead-ends — the agent Reads it on the next turn.
|
|
20
|
+
#
|
|
21
|
+
# Wire via: claude ... --settings <settings-with-this-as-PreToolUse(Read)>
|
|
22
|
+
# Eval artifact only. The production version is an indexed-aware `codegraph`
|
|
23
|
+
# subcommand (cross-platform — no bash/jq — and queries the index so it never
|
|
24
|
+
# bounces a new/un-indexed file), wired opt-in by the installer.
|
|
25
|
+
set -uo pipefail
|
|
26
|
+
input="$(cat)"
|
|
27
|
+
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
|
28
|
+
[ -n "$fp" ] || exit 0
|
|
29
|
+
base="$(basename "$fp")"
|
|
30
|
+
|
|
31
|
+
case "$fp" in
|
|
32
|
+
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.scala|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte|*.m|*.mm)
|
|
33
|
+
msg="codegraph has this file indexed (kept in sync on every edit). Call codegraph_node with file:\"$base\" and includeCode:true instead of Read — it returns the WHOLE file verbatim WITH line numbers (imports, top-level code and all — safe to base an Edit on) PLUS which files depend on it, in one call. Treat its output as already-Read; do not Read this file. (If it answers that the file isn't indexed — e.g. you just created it — then Read it directly.)"
|
|
34
|
+
jq -n --arg m "$msg" '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
|
35
|
+
exit 0
|
|
36
|
+
;;
|
|
37
|
+
esac
|
|
38
|
+
exit 0
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Reproduction harness A — does the shared daemon serialize concurrent explore?
|
|
3
|
+
//
|
|
4
|
+
// Mirrors the daemon's reality: ONE CodeGraph + ONE ToolHandler (as MCPEngine
|
|
5
|
+
// shares across all sessions), then fires N concurrent codegraph_explore calls
|
|
6
|
+
// and measures:
|
|
7
|
+
// - each call's wall-clock latency + completion order
|
|
8
|
+
// - an event-loop HEARTBEAT (setInterval 50ms): the max gap between ticks is a
|
|
9
|
+
// direct measure of how long synchronous compute blocked the loop. In the
|
|
10
|
+
// real daemon a blocked loop can't flush a finished response or read the
|
|
11
|
+
// next request, so this gap is what starves the MCP transport.
|
|
12
|
+
//
|
|
13
|
+
// Usage: node repro-concurrent-explore.mjs <repo-with-.codegraph> <N> [timeoutMs]
|
|
14
|
+
import { pathToFileURL } from 'node:url';
|
|
15
|
+
import { resolve } from 'node:path';
|
|
16
|
+
import { performance } from 'node:perf_hooks';
|
|
17
|
+
|
|
18
|
+
const [, , repo, nRaw, timeoutRaw] = process.argv;
|
|
19
|
+
if (!repo) {
|
|
20
|
+
console.error('usage: repro-concurrent-explore.mjs <repo> <N=10> [timeoutMs=60000]');
|
|
21
|
+
process.exit(1);
|
|
22
|
+
}
|
|
23
|
+
const N = Number(nRaw) || 10;
|
|
24
|
+
const TIMEOUT_MS = Number(timeoutRaw) || 60000; // ~ MCP SDK default request timeout
|
|
25
|
+
|
|
26
|
+
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
|
27
|
+
const idx = await load('dist/index.js');
|
|
28
|
+
const tools = await load('dist/mcp/tools.js');
|
|
29
|
+
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
30
|
+
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
31
|
+
|
|
32
|
+
// Distinct queries so no two calls are trivially identical. Mix of NL questions
|
|
33
|
+
// (exercise FTS + RWR over the whole graph) — the expensive explore path.
|
|
34
|
+
const QUERIES = [
|
|
35
|
+
'how does the text model handle edits and undo',
|
|
36
|
+
'how does the file service watch for changes on disk',
|
|
37
|
+
'how does the keybinding service resolve a chord to a command',
|
|
38
|
+
'how does the extension host activate an extension',
|
|
39
|
+
'how does the editor render decorations in the viewport',
|
|
40
|
+
'how does the search service stream results to the UI',
|
|
41
|
+
'how does the terminal process manager spawn a shell',
|
|
42
|
+
'how does the configuration service merge user and workspace settings',
|
|
43
|
+
'how does the debug adapter forward breakpoints to the runtime',
|
|
44
|
+
'how does the quick input widget filter its items',
|
|
45
|
+
'how does the notification service queue and show toasts',
|
|
46
|
+
'how does the git extension compute the diff for a file',
|
|
47
|
+
'how does the language features registry dispatch a hover request',
|
|
48
|
+
'how does the workbench layout restore editor groups on reload',
|
|
49
|
+
'how does the storage service persist state between sessions',
|
|
50
|
+
'how does the menu service build a context menu from contributions',
|
|
51
|
+
];
|
|
52
|
+
|
|
53
|
+
const cg = CodeGraph.openSync(repo);
|
|
54
|
+
let fileCount = 0;
|
|
55
|
+
try { fileCount = cg.getStats().fileCount; } catch {}
|
|
56
|
+
const handler = new ToolHandler(cg);
|
|
57
|
+
|
|
58
|
+
// --- event-loop heartbeat ---
|
|
59
|
+
let lastTick = performance.now();
|
|
60
|
+
let maxGap = 0;
|
|
61
|
+
const gaps = [];
|
|
62
|
+
const hb = setInterval(() => {
|
|
63
|
+
const now = performance.now();
|
|
64
|
+
const gap = now - lastTick;
|
|
65
|
+
lastTick = now;
|
|
66
|
+
if (gap > 60) gaps.push(Math.round(gap)); // expected ~50ms; record stalls
|
|
67
|
+
if (gap > maxGap) maxGap = gap;
|
|
68
|
+
}, 50);
|
|
69
|
+
|
|
70
|
+
function runOne(i) {
|
|
71
|
+
const q = QUERIES[i % QUERIES.length];
|
|
72
|
+
const startedAt = performance.now();
|
|
73
|
+
let timer;
|
|
74
|
+
const timeout = new Promise((res) => {
|
|
75
|
+
timer = setTimeout(() => res({ timedOut: true }), TIMEOUT_MS);
|
|
76
|
+
});
|
|
77
|
+
const work = handler
|
|
78
|
+
.execute('codegraph_explore', { query: q })
|
|
79
|
+
.then((r) => ({ ok: !r.isError, chars: r.content?.[0]?.text?.length ?? 0 }))
|
|
80
|
+
.catch((e) => ({ ok: false, err: String(e?.message ?? e) }));
|
|
81
|
+
return Promise.race([work, timeout]).then((r) => {
|
|
82
|
+
clearTimeout(timer);
|
|
83
|
+
return { i, q, ms: Math.round(performance.now() - startedAt), ...r };
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Baseline: one warm single call (so the first-call cold paths don't skew N).
|
|
88
|
+
const warmStart = performance.now();
|
|
89
|
+
await runOne(0);
|
|
90
|
+
const warmMs = Math.round(performance.now() - warmStart);
|
|
91
|
+
|
|
92
|
+
// Reset heartbeat stats for the concurrent run.
|
|
93
|
+
gaps.length = 0; maxGap = 0; lastTick = performance.now();
|
|
94
|
+
|
|
95
|
+
const batchStart = performance.now();
|
|
96
|
+
const results = await Promise.all(Array.from({ length: N }, (_, i) => runOne(i)));
|
|
97
|
+
const batchMs = Math.round(performance.now() - batchStart);
|
|
98
|
+
clearInterval(hb);
|
|
99
|
+
|
|
100
|
+
const lat = results.map((r) => r.ms).sort((a, b) => a - b);
|
|
101
|
+
const timeouts = results.filter((r) => r.timedOut).length;
|
|
102
|
+
const p = (q) => lat[Math.min(lat.length - 1, Math.floor(q * lat.length))];
|
|
103
|
+
|
|
104
|
+
console.log('='.repeat(64));
|
|
105
|
+
console.log(`repo=${repo}`);
|
|
106
|
+
console.log(`fileCount=${fileCount} N=${N} perCallTimeout=${TIMEOUT_MS}ms`);
|
|
107
|
+
console.log(`single warm explore: ${warmMs}ms`);
|
|
108
|
+
console.log('-'.repeat(64));
|
|
109
|
+
console.log(`concurrent batch wall-clock: ${batchMs}ms`);
|
|
110
|
+
console.log(`per-call latency min=${lat[0]} p50=${p(0.5)} p90=${p(0.9)} max=${lat[lat.length - 1]} (ms)`);
|
|
111
|
+
console.log(`TIMEOUTS (>${TIMEOUT_MS}ms): ${timeouts} / ${N}`);
|
|
112
|
+
console.log(`event-loop max stall: ${Math.round(maxGap)}ms stalls>60ms: ${gaps.length}`);
|
|
113
|
+
console.log(` sum of stalls: ${gaps.reduce((a, b) => a + b, 0)}ms biggest 5: ${gaps.sort((a,b)=>b-a).slice(0,5).join(', ')}`);
|
|
114
|
+
console.log('-'.repeat(64));
|
|
115
|
+
console.log('SERIALIZATION CHECK:');
|
|
116
|
+
console.log(` if serialized, batch ≈ N×single = ~${N * warmMs}ms; actual=${batchMs}ms (ratio ${(batchMs / (N * warmMs)).toFixed(2)})`);
|
|
117
|
+
console.log(` max latency / single = ${(lat[lat.length - 1] / warmMs).toFixed(1)}× (≈N means last call waited for all others)`);
|
|
118
|
+
console.log('='.repeat(64));
|
|
119
|
+
try { cg.close?.(); } catch {}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Reproduction harness B — the FAITHFUL opencode scenario.
|
|
3
|
+
//
|
|
4
|
+
// Spawns N real `codegraph serve --mcp --path <repo>` processes (each becomes a
|
|
5
|
+
// proxy that attaches to ONE shared daemon — exactly what opencode does with N
|
|
6
|
+
// subagents), drives clean MCP JSON-RPC over each child's stdio, then fires ONE
|
|
7
|
+
// concurrent wave of codegraph_explore tools/call across all N and measures
|
|
8
|
+
// end-to-end latency + timeouts. This captures transport-flush starvation: a
|
|
9
|
+
// daemon event-loop blocked in synchronous explore compute can neither read the
|
|
10
|
+
// next request nor flush a finished response.
|
|
11
|
+
//
|
|
12
|
+
// Usage: node repro-daemon-clients.mjs <repo> <N=10> [perCallTimeoutMs=60000] [warm=1]
|
|
13
|
+
import { spawn } from 'node:child_process';
|
|
14
|
+
import { performance } from 'node:perf_hooks';
|
|
15
|
+
import { resolve } from 'node:path';
|
|
16
|
+
|
|
17
|
+
const [, , repoRaw, nRaw, timeoutRaw, warmRaw] = process.argv;
|
|
18
|
+
const repo = resolve(repoRaw || '.');
|
|
19
|
+
const N = Number(nRaw) || 10;
|
|
20
|
+
const TIMEOUT_MS = Number(timeoutRaw) || 60000;
|
|
21
|
+
const WARM = warmRaw === undefined ? true : warmRaw !== '0';
|
|
22
|
+
const CLI = resolve('dist/bin/codegraph.js');
|
|
23
|
+
|
|
24
|
+
const QUERIES = [
|
|
25
|
+
'how does the text model handle edits and undo',
|
|
26
|
+
'how does the file service watch for changes on disk',
|
|
27
|
+
'how does the keybinding service resolve a chord to a command',
|
|
28
|
+
'how does the extension host activate an extension',
|
|
29
|
+
'how does the editor render decorations in the viewport',
|
|
30
|
+
'how does the search service stream results to the UI',
|
|
31
|
+
'how does the terminal process manager spawn a shell',
|
|
32
|
+
'how does the configuration service merge user and workspace settings',
|
|
33
|
+
'how does the debug adapter forward breakpoints to the runtime',
|
|
34
|
+
'how does the quick input widget filter its items',
|
|
35
|
+
'how does the notification service queue and show toasts',
|
|
36
|
+
'how does the git extension compute the diff for a file',
|
|
37
|
+
];
|
|
38
|
+
|
|
39
|
+
function makeClient(id) {
|
|
40
|
+
const child = spawn('node', [CLI, 'serve', '--mcp', '--path', repo], {
|
|
41
|
+
env: { ...process.env, CODEGRAPH_TELEMETRY: '0', DO_NOT_TRACK: '1', CODEGRAPH_MCP_LOG_ATTACH: '0' },
|
|
42
|
+
stdio: ['pipe', 'pipe', 'inherit'],
|
|
43
|
+
});
|
|
44
|
+
let buf = '';
|
|
45
|
+
const waiters = new Map(); // id -> resolve
|
|
46
|
+
child.stdout.setEncoding('utf8');
|
|
47
|
+
child.stdout.on('data', (chunk) => {
|
|
48
|
+
buf += chunk;
|
|
49
|
+
let idx;
|
|
50
|
+
while ((idx = buf.indexOf('\n')) !== -1) {
|
|
51
|
+
const line = buf.slice(0, idx).trim();
|
|
52
|
+
buf = buf.slice(idx + 1);
|
|
53
|
+
if (!line) continue;
|
|
54
|
+
let msg; try { msg = JSON.parse(line); } catch { continue; }
|
|
55
|
+
if (msg.id !== undefined && waiters.has(msg.id)) {
|
|
56
|
+
waiters.get(msg.id)(msg);
|
|
57
|
+
waiters.delete(msg.id);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
const send = (obj) => child.stdin.write(JSON.stringify(obj) + '\n');
|
|
62
|
+
const request = (method, params, rpcId, timeoutMs) =>
|
|
63
|
+
new Promise((res) => {
|
|
64
|
+
let timer;
|
|
65
|
+
if (timeoutMs) timer = setTimeout(() => { waiters.delete(rpcId); res({ __timeout: true }); }, timeoutMs);
|
|
66
|
+
waiters.set(rpcId, (m) => { if (timer) clearTimeout(timer); res(m); });
|
|
67
|
+
send({ jsonrpc: '2.0', id: rpcId, method, params });
|
|
68
|
+
});
|
|
69
|
+
return { id, child, send, request };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
73
|
+
|
|
74
|
+
const clients = Array.from({ length: N }, (_, i) => makeClient(i));
|
|
75
|
+
|
|
76
|
+
// Initialize every client (handshake is answered locally by each proxy, instant).
|
|
77
|
+
await Promise.all(clients.map((c) =>
|
|
78
|
+
c.request('initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'repro', version: '1' } }, `init-${c.id}`, 10000)
|
|
79
|
+
.then(() => c.send({ jsonrpc: '2.0', method: 'initialized' }))
|
|
80
|
+
));
|
|
81
|
+
|
|
82
|
+
// Warm the daemon: one explore through client 0 forces daemon spawn + project
|
|
83
|
+
// open + catch-up gate to complete, so the concurrent wave measures the STEADY
|
|
84
|
+
// state (the user's real scenario after the first call), not cold start.
|
|
85
|
+
if (WARM) {
|
|
86
|
+
process.stderr.write('[repro] warming daemon (first explore triggers spawn+open+catchup)...\n');
|
|
87
|
+
const t0 = performance.now();
|
|
88
|
+
const r = await clients[0].request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[0] } }, 'warm-0', 120000);
|
|
89
|
+
process.stderr.write(`[repro] warm explore took ${Math.round(performance.now() - t0)}ms (timeout=${!!r.__timeout})\n`);
|
|
90
|
+
await sleep(500);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// THE WAVE: fire one explore on every client as simultaneously as possible.
|
|
94
|
+
process.stderr.write(`[repro] firing ${N} concurrent explores...\n`);
|
|
95
|
+
const waveStart = performance.now();
|
|
96
|
+
const results = await Promise.all(clients.map((c, i) => {
|
|
97
|
+
const started = performance.now();
|
|
98
|
+
return c.request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[i % QUERIES.length] } }, `call-${c.id}`, TIMEOUT_MS)
|
|
99
|
+
.then((m) => ({
|
|
100
|
+
id: c.id,
|
|
101
|
+
ms: Math.round(performance.now() - started),
|
|
102
|
+
timedOut: !!m.__timeout,
|
|
103
|
+
ok: !!m.result && !m.result.isError,
|
|
104
|
+
chars: m.result?.content?.[0]?.text?.length ?? 0,
|
|
105
|
+
}));
|
|
106
|
+
}));
|
|
107
|
+
const waveMs = Math.round(performance.now() - waveStart);
|
|
108
|
+
|
|
109
|
+
const lat = results.map((r) => r.ms).sort((a, b) => a - b);
|
|
110
|
+
const timeouts = results.filter((r) => r.timedOut).length;
|
|
111
|
+
const p = (q) => lat[Math.min(lat.length - 1, Math.floor(q * lat.length))];
|
|
112
|
+
|
|
113
|
+
console.log('='.repeat(64));
|
|
114
|
+
console.log(`HARNESS B (real daemon + ${N} proxies) repo=${repo}`);
|
|
115
|
+
console.log(`warm=${WARM} perCallTimeout=${TIMEOUT_MS}ms`);
|
|
116
|
+
console.log('-'.repeat(64));
|
|
117
|
+
console.log(`wave wall-clock: ${waveMs}ms`);
|
|
118
|
+
console.log(`per-call latency min=${lat[0]} p50=${p(0.5)} p90=${p(0.9)} max=${lat[lat.length - 1]} (ms)`);
|
|
119
|
+
console.log(`TIMEOUTS (>${TIMEOUT_MS}ms): ${timeouts} / ${N}`);
|
|
120
|
+
console.log(`completion order (id:ms): ${results.slice().sort((a,b)=>a.ms-b.ms).map(r=>`${r.id}:${r.ms}`).join(' ')}`);
|
|
121
|
+
console.log('='.repeat(64));
|
|
122
|
+
|
|
123
|
+
for (const c of clients) { try { c.child.stdin.end(); c.child.kill('SIGTERM'); } catch {} }
|
|
124
|
+
await sleep(300);
|
|
125
|
+
process.exit(0);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Headless Claude Code run against a repo with codegraph MCP, capturing the
|
|
3
|
+
# full stream-json so we can see tool calls + token usage. Complements the
|
|
4
|
+
# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
|
|
5
|
+
# tokens/cost, but defaults to the general-purpose subagent (not Explore).
|
|
6
|
+
# To force the Explore path, ask for it in the prompt.
|
|
7
|
+
#
|
|
8
|
+
# Usage: run-agent.sh <repo-path> <label> "<prompt>"
|
|
9
|
+
# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (codegraph dist binary)
|
|
10
|
+
set -uo pipefail
|
|
11
|
+
|
|
12
|
+
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
|
13
|
+
CG_BIN="${CG_BIN:-$(command -v codegraph || echo /usr/local/bin/codegraph)}"
|
|
14
|
+
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
|
15
|
+
OUT="$OUT_DIR/run-${LABEL}.jsonl"
|
|
16
|
+
|
|
17
|
+
MCP_CONFIG=$(cat <<JSON
|
|
18
|
+
{"mcpServers":{"codegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
|
|
19
|
+
JSON
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
echo "→ running [$LABEL] in $REPO"
|
|
23
|
+
cd "$REPO" || exit 1
|
|
24
|
+
|
|
25
|
+
claude -p "$PROMPT" \
|
|
26
|
+
--output-format stream-json --verbose \
|
|
27
|
+
--permission-mode bypassPermissions \
|
|
28
|
+
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
|
29
|
+
--max-budget-usd 2 \
|
|
30
|
+
--strict-mcp-config --mcp-config "$MCP_CONFIG" \
|
|
31
|
+
> "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
|
|
32
|
+
|
|
33
|
+
echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
|
|
34
|
+
node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true
|