@iderouter/index-mcp 0.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/package.json +26 -0
- package/scripts/benchmark-all.mjs +177 -0
- package/scripts/benchmark-auto-continuation.mjs +188 -0
- package/scripts/benchmark-background-fine-resume.mjs +245 -0
- package/scripts/benchmark-background-fine-wait.mjs +76 -0
- package/scripts/benchmark-background-fine.mjs +132 -0
- package/scripts/benchmark-clean-snapshot.mjs +83 -0
- package/scripts/benchmark-coarse-ready-search.mjs +161 -0
- package/scripts/benchmark-deferred.mjs +62 -0
- package/scripts/benchmark-first-semantic-visible.mjs +151 -0
- package/scripts/benchmark-gate.mjs +107 -0
- package/scripts/benchmark-generic-resumed-single-chunk-embed.mjs +104 -0
- package/scripts/benchmark-noop.mjs +24 -0
- package/scripts/benchmark-priority-ready-search.mjs +165 -0
- package/scripts/benchmark-repeat-search.mjs +148 -0
- package/scripts/benchmark-resumed-retry-burst.mjs +187 -0
- package/scripts/benchmark-resumed-single-chunk-success.mjs +154 -0
- package/scripts/benchmark-resumed-single-chunk.mjs +146 -0
- package/scripts/benchmark-single-priority-chunk-embed.mjs +145 -0
- package/scripts/benchmark-small-change.mjs +146 -0
- package/scripts/benchmark-stage-summary.mjs +88 -0
- package/scripts/lib/auto-continuation-state.mjs +34 -0
- package/scripts/lib/benchmark-query-packs.mjs +123 -0
- package/scripts/lib/benchmark-snapshot.mjs +109 -0
- package/scripts/lib/mcp-bench.mjs +455 -0
- package/src/architecture-query-fallback.js +50 -0
- package/src/background-definition-chunks.js +199 -0
- package/src/background-embedding-profile.js +64 -0
- package/src/background-fine-budget.js +18 -0
- package/src/background-fine-runtime.js +179 -0
- package/src/background-fine-selection.js +332 -0
- package/src/checkpoint-policy.js +16 -0
- package/src/conflict-policy.js +17 -0
- package/src/deferred-retry-delay.js +14 -0
- package/src/deferred-retry-status.js +10 -0
- package/src/embedding-attempt-ordinal.js +17 -0
- package/src/embedding-failure-penalty.js +60 -0
- package/src/embedding-failure-policy.js +52 -0
- package/src/embedding-flush-timeout.js +33 -0
- package/src/embedding-inflight-status.js +18 -0
- package/src/embedding-model-policy.js +44 -0
- package/src/embedding-next-switch.js +18 -0
- package/src/embedding-request-status-detail.js +25 -0
- package/src/embedding-request-status.js +22 -0
- package/src/embedding-selection-order.js +23 -0
- package/src/fine-run-queue.js +14 -0
- package/src/index.js +7970 -0
- package/src/job-supersession.js +25 -0
- package/src/priority-progress.js +20 -0
- package/src/priority-ready-anchor-coverage-normalize.js +18 -0
- package/src/priority-ready-anchor-coverage.js +23 -0
- package/src/priority-ready-hotspots.js +344 -0
- package/src/priority-ready-status.js +30 -0
- package/src/priority-ready-targets.js +45 -0
- package/src/priority-usable-attempt-plan.js +44 -0
- package/src/priority-usable-attempt-timeout.js +18 -0
- package/src/priority-usable-fast-path.js +11 -0
- package/src/priority-usable-probe-order.js +34 -0
- package/src/remote-strategy-failure-cache.js +55 -0
- package/src/resume-seed.js +9 -0
- package/src/semantic-first-checkpoint.js +8 -0
- package/src/semantic-slow-path.js +10 -0
- package/src/single-chunk-attempt-timeout.js +13 -0
- package/src/single-chunk-embedding-content.js +26 -0
- package/src/single-chunk-embedding-policy.js +18 -0
- package/src/single-chunk-provider-order.js +12 -0
- package/src/single-chunk-provider-policy.js +63 -0
- package/src/worker-lock-retry.js +24 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { parseDiagnostics, sleep, spawnMcpClient } from "./lib/mcp-bench.mjs";
|
|
6
|
+
|
|
7
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
8
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
9
|
+
const RECHECK_MS = Number(process.env.IDEROUTER_BENCH_RECHECK_MS || 1500);
|
|
10
|
+
|
|
11
|
+
function extractSummary(text) {
|
|
12
|
+
const diagnostics = parseDiagnostics(text);
|
|
13
|
+
return {
|
|
14
|
+
status: text.split("\n")[0],
|
|
15
|
+
deferred_ready: /^Priority semantic search is ready\b/.test(text) && /background fine rebuild .*deferred/i.test(text),
|
|
16
|
+
failed: /^Last index job .* failed\b/.test(text),
|
|
17
|
+
diagnostics,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function main() {
|
|
22
|
+
const { child, client } = spawnMcpClient(process.env);
|
|
23
|
+
try {
|
|
24
|
+
await client.initialize();
|
|
25
|
+
const startedAt = Date.now();
|
|
26
|
+
const firstText = await client.callTool("get_indexing_status", { path: TARGET_PATH });
|
|
27
|
+
await sleep(Math.max(POLL_MS, RECHECK_MS));
|
|
28
|
+
const secondText = await client.callTool("get_indexing_status", { path: TARGET_PATH });
|
|
29
|
+
const wallMs = Date.now() - startedAt;
|
|
30
|
+
|
|
31
|
+
const first = extractSummary(firstText);
|
|
32
|
+
const second = extractSummary(secondText);
|
|
33
|
+
const output = {
|
|
34
|
+
path: TARGET_PATH,
|
|
35
|
+
wall_ms: wallMs,
|
|
36
|
+
first_status: first.status,
|
|
37
|
+
second_status: second.status,
|
|
38
|
+
first_deferred_ready: first.deferred_ready,
|
|
39
|
+
second_deferred_ready: second.deferred_ready,
|
|
40
|
+
first_failed: first.failed,
|
|
41
|
+
second_failed: second.failed,
|
|
42
|
+
stable_status: first.status === second.status,
|
|
43
|
+
resumed_unexpectedly: first.deferred_ready && !second.deferred_ready,
|
|
44
|
+
scan_ms: Number(second.diagnostics.scan_ms || first.diagnostics.scan_ms || 0),
|
|
45
|
+
total_ms: Number(second.diagnostics.total_ms || first.diagnostics.total_ms || 0),
|
|
46
|
+
strategy_version: second.diagnostics.strategy_version || first.diagnostics.strategy_version || "",
|
|
47
|
+
strategy_source: second.diagnostics.strategy_source || first.diagnostics.strategy_source || "",
|
|
48
|
+
embedding_model_source: second.diagnostics.embedding_model_source || first.diagnostics.embedding_model_source || "",
|
|
49
|
+
background_fine_target_files: Number(second.diagnostics.background_fine_target_files || first.diagnostics.background_fine_target_files || 0),
|
|
50
|
+
background_fine_target_chunks: Number(second.diagnostics.background_fine_target_chunks || first.diagnostics.background_fine_target_chunks || 0),
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
|
|
54
|
+
} finally {
|
|
55
|
+
child.kill("SIGTERM");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
main().catch((error) => {
|
|
60
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
61
|
+
process.exitCode = 1;
|
|
62
|
+
});
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
cleanupBenchmarkRun,
|
|
10
|
+
installBenchmarkSignalCleanup,
|
|
11
|
+
readIndexMeta,
|
|
12
|
+
readJobStatus,
|
|
13
|
+
sleep,
|
|
14
|
+
spawnMcpClient,
|
|
15
|
+
} from "./lib/mcp-bench.mjs";
|
|
16
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
17
|
+
import { isCoarseDeferredReady } from "./lib/auto-continuation-state.mjs";
|
|
18
|
+
|
|
19
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
20
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 1000);
|
|
21
|
+
const FIRST_PHASE_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_AUTO_CONTINUE_FIRST_PHASE_TIMEOUT_MS || 180000);
|
|
22
|
+
const SEMANTIC_VISIBLE_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_FIRST_SEMANTIC_VISIBLE_TIMEOUT_MS || 180000);
|
|
23
|
+
|
|
24
|
+
function snapshotState(meta, job, statusText) {
|
|
25
|
+
return {
|
|
26
|
+
status_text: statusText.split("\n")[0],
|
|
27
|
+
job_id: job?.id || "",
|
|
28
|
+
job_status: job?.status || "",
|
|
29
|
+
current_step: job?.currentStep || "",
|
|
30
|
+
progress: Number(job?.progress || 0),
|
|
31
|
+
embedded_count: Number(job?.embeddedCount || 0),
|
|
32
|
+
total_chunks: Number(job?.totalChunks || 0),
|
|
33
|
+
background_fine_deferred: Boolean(job?.backgroundFineDeferred),
|
|
34
|
+
semantic_retry_deferred: Boolean(job?.semanticRetryDeferred),
|
|
35
|
+
searchable_modes: job?.searchableModes || meta?.searchableModes || "",
|
|
36
|
+
fine_semantic_files: Number(meta?.fineSemanticFileCount || 0),
|
|
37
|
+
fine_semantic_chunks: Number(meta?.fineSemanticChunkCount || 0),
|
|
38
|
+
updated_at: job?.updatedAt || meta?.updatedAt || "",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function waitForCoarseDeferredReady(client, snapshotDir, indexHome) {
|
|
43
|
+
const startedAt = Date.now();
|
|
44
|
+
let lastText = "";
|
|
45
|
+
while (Date.now() - startedAt < FIRST_PHASE_TIMEOUT_MS) {
|
|
46
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
47
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
48
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
49
|
+
const state = snapshotState(meta, job, lastText);
|
|
50
|
+
if (
|
|
51
|
+
Boolean(job?.backgroundFineDeferred) &&
|
|
52
|
+
!Boolean(job?.priorityFineReady) &&
|
|
53
|
+
isCoarseDeferredReady(state)
|
|
54
|
+
) {
|
|
55
|
+
return {
|
|
56
|
+
settled: true,
|
|
57
|
+
wall_ms: Date.now() - startedAt,
|
|
58
|
+
state,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
await sleep(POLL_MS);
|
|
62
|
+
}
|
|
63
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
64
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
65
|
+
return {
|
|
66
|
+
settled: false,
|
|
67
|
+
wall_ms: Date.now() - startedAt,
|
|
68
|
+
state: snapshotState(meta, job, lastText),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function waitForFirstSemanticVisible(client, snapshotDir, indexHome) {
|
|
73
|
+
const startedAt = Date.now();
|
|
74
|
+
let lastText = "";
|
|
75
|
+
while (Date.now() - startedAt < SEMANTIC_VISIBLE_TIMEOUT_MS) {
|
|
76
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
77
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
78
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
79
|
+
const state = snapshotState(meta, job, lastText);
|
|
80
|
+
if (state.fine_semantic_chunks > 0 || state.fine_semantic_files > 0) {
|
|
81
|
+
return {
|
|
82
|
+
settled: true,
|
|
83
|
+
wall_ms: Date.now() - startedAt,
|
|
84
|
+
state,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
await sleep(POLL_MS);
|
|
88
|
+
}
|
|
89
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
90
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
91
|
+
return {
|
|
92
|
+
settled: false,
|
|
93
|
+
wall_ms: Date.now() - startedAt,
|
|
94
|
+
state: snapshotState(meta, job, lastText),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function main() {
|
|
99
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
100
|
+
const snapshotDir = snapshot.snapshotDir;
|
|
101
|
+
const isolatedIndexHome = await fs.mkdtemp(path.join(os.tmpdir(), "iderouter-bench-first-semantic-"));
|
|
102
|
+
const env = {
|
|
103
|
+
...process.env,
|
|
104
|
+
IDEROUTER_INDEX_HOME: isolatedIndexHome,
|
|
105
|
+
};
|
|
106
|
+
const { child, client } = spawnMcpClient(env);
|
|
107
|
+
const uninstallSignalCleanup = installBenchmarkSignalCleanup(() => ({
|
|
108
|
+
child,
|
|
109
|
+
indexHome: isolatedIndexHome,
|
|
110
|
+
targetPath: snapshotDir,
|
|
111
|
+
snapshotDir,
|
|
112
|
+
}));
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
await client.initialize();
|
|
116
|
+
const startText = await client.callTool("index_codebase", { path: snapshotDir, wait: false, cloud: false });
|
|
117
|
+
const firstPhase = await waitForCoarseDeferredReady(client, snapshotDir, isolatedIndexHome);
|
|
118
|
+
const firstSemanticVisible = firstPhase.settled
|
|
119
|
+
? await waitForFirstSemanticVisible(client, snapshotDir, isolatedIndexHome)
|
|
120
|
+
: { settled: false, wall_ms: 0, state: firstPhase.state };
|
|
121
|
+
|
|
122
|
+
process.stdout.write(
|
|
123
|
+
`${JSON.stringify(
|
|
124
|
+
{
|
|
125
|
+
path: TARGET_PATH,
|
|
126
|
+
snapshot_path: snapshotDir,
|
|
127
|
+
snapshot_mode: snapshot.mode,
|
|
128
|
+
isolated_index_home: isolatedIndexHome,
|
|
129
|
+
start_status: startText.split("\n")[0],
|
|
130
|
+
first_phase: firstPhase,
|
|
131
|
+
first_semantic_visible: firstSemanticVisible,
|
|
132
|
+
},
|
|
133
|
+
null,
|
|
134
|
+
2,
|
|
135
|
+
)}\n`,
|
|
136
|
+
);
|
|
137
|
+
} finally {
|
|
138
|
+
uninstallSignalCleanup();
|
|
139
|
+
await cleanupBenchmarkRun({
|
|
140
|
+
child,
|
|
141
|
+
indexHome: isolatedIndexHome,
|
|
142
|
+
targetPath: snapshotDir,
|
|
143
|
+
snapshotDir,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
main().catch((error) => {
|
|
149
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
150
|
+
process.exitCode = 1;
|
|
151
|
+
});
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
10
|
+
const SCRIPT_DIR = path.resolve("scripts");
|
|
11
|
+
const PRIORITY_READY_MAX_MS = Number(process.env.IDEROUTER_BENCH_GATE_PRIORITY_READY_MAX_MS || 20000);
|
|
12
|
+
const PRIORITY_SEARCH_MAX_MS = Number(process.env.IDEROUTER_BENCH_GATE_PRIORITY_SEARCH_MAX_MS || 300);
|
|
13
|
+
const REPEAT_SPEEDUP_MIN = Number(process.env.IDEROUTER_BENCH_GATE_REPEAT_SPEEDUP_MIN || 5);
|
|
14
|
+
|
|
15
|
+
async function runScript(scriptName) {
|
|
16
|
+
const scriptPath = path.join(SCRIPT_DIR, scriptName);
|
|
17
|
+
const { stdout } = await execFileAsync(process.execPath, [scriptPath, TARGET_PATH], {
|
|
18
|
+
cwd: process.cwd(),
|
|
19
|
+
env: process.env,
|
|
20
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
21
|
+
});
|
|
22
|
+
return JSON.parse(String(stdout || "").trim());
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function fail(message, details) {
|
|
26
|
+
const output = {
|
|
27
|
+
ok: false,
|
|
28
|
+
message,
|
|
29
|
+
details,
|
|
30
|
+
};
|
|
31
|
+
process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
|
|
32
|
+
process.exitCode = 1;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function main() {
|
|
36
|
+
const priority = await runScript("benchmark-priority-ready-search.mjs");
|
|
37
|
+
const repeat = await runScript("benchmark-repeat-search.mjs");
|
|
38
|
+
|
|
39
|
+
const priorityReadyKind = String(priority?.priority_ready_kind || "not_ready");
|
|
40
|
+
const repeatReadyKind = String(repeat?.priority_ready_kind || "not_ready");
|
|
41
|
+
const priorityReadyMs = Number(priority?.priority_ready?.parsed?.priority_ready_ms || priority?.priority_ready?.wallMs || 0);
|
|
42
|
+
const priorityAnchorCompleted = Number(priority?.priority_ready_anchor_coverage?.completed || 0);
|
|
43
|
+
const priorityAnchorTotal = Number(priority?.priority_ready_anchor_coverage?.total || 0);
|
|
44
|
+
const repeatAnchorCompleted = Number(repeat?.priority_ready_anchor_coverage?.completed || 0);
|
|
45
|
+
const repeatAnchorTotal = Number(repeat?.priority_ready_anchor_coverage?.total || 0);
|
|
46
|
+
const allTop1Hit = Array.isArray(priority?.searches) && priority.searches.every((item) => item?.top1_hit === true);
|
|
47
|
+
const maxSearchMs = Array.isArray(priority?.searches)
|
|
48
|
+
? Math.max(...priority.searches.map((item) => Number(item?.wall_ms || 0)), 0)
|
|
49
|
+
: 0;
|
|
50
|
+
const repeatSpeedup = Number(repeat?.speedup_ratio || 0);
|
|
51
|
+
|
|
52
|
+
const failedChecks = [];
|
|
53
|
+
if (!["priority_semantic", "full_ready"].includes(priorityReadyKind)) {
|
|
54
|
+
failedChecks.push(`priority_ready_kind=${priorityReadyKind} is not semantic-ready`);
|
|
55
|
+
}
|
|
56
|
+
if (!["priority_semantic", "full_ready"].includes(repeatReadyKind)) {
|
|
57
|
+
failedChecks.push(`repeat_ready_kind=${repeatReadyKind} is not semantic-ready`);
|
|
58
|
+
}
|
|
59
|
+
if (priorityAnchorTotal <= 0 || priorityAnchorCompleted < priorityAnchorTotal) {
|
|
60
|
+
failedChecks.push(`priority_ready_anchor_coverage=${priorityAnchorCompleted}/${priorityAnchorTotal} is incomplete`);
|
|
61
|
+
}
|
|
62
|
+
if (repeatAnchorTotal <= 0 || repeatAnchorCompleted < repeatAnchorTotal) {
|
|
63
|
+
failedChecks.push(`repeat_ready_anchor_coverage=${repeatAnchorCompleted}/${repeatAnchorTotal} is incomplete`);
|
|
64
|
+
}
|
|
65
|
+
if (!allTop1Hit) failedChecks.push("priority_ready_all_top1_hit=false");
|
|
66
|
+
if (priorityReadyMs <= 0 || priorityReadyMs > PRIORITY_READY_MAX_MS) {
|
|
67
|
+
failedChecks.push(`priority_ready_ms=${priorityReadyMs} exceeds ${PRIORITY_READY_MAX_MS}`);
|
|
68
|
+
}
|
|
69
|
+
if (maxSearchMs <= 0 || maxSearchMs > PRIORITY_SEARCH_MAX_MS) {
|
|
70
|
+
failedChecks.push(`priority_ready_max_search_ms=${maxSearchMs} exceeds ${PRIORITY_SEARCH_MAX_MS}`);
|
|
71
|
+
}
|
|
72
|
+
if (repeatSpeedup <= 0 || repeatSpeedup < REPEAT_SPEEDUP_MIN) {
|
|
73
|
+
failedChecks.push(`repeat_search_speedup_ratio=${repeatSpeedup} below ${REPEAT_SPEEDUP_MIN}`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const output = {
|
|
77
|
+
ok: failedChecks.length === 0,
|
|
78
|
+
thresholds: {
|
|
79
|
+
priority_ready_max_ms: PRIORITY_READY_MAX_MS,
|
|
80
|
+
priority_search_max_ms: PRIORITY_SEARCH_MAX_MS,
|
|
81
|
+
repeat_speedup_min: REPEAT_SPEEDUP_MIN,
|
|
82
|
+
},
|
|
83
|
+
summary: {
|
|
84
|
+
priority_ready_kind: priorityReadyKind,
|
|
85
|
+
priority_ready_ms: priorityReadyMs,
|
|
86
|
+
priority_ready_anchor_completed: priorityAnchorCompleted,
|
|
87
|
+
priority_ready_anchor_total: priorityAnchorTotal,
|
|
88
|
+
priority_ready_all_top1_hit: allTop1Hit,
|
|
89
|
+
priority_ready_max_search_ms: maxSearchMs,
|
|
90
|
+
repeat_ready_kind: repeatReadyKind,
|
|
91
|
+
repeat_ready_anchor_completed: repeatAnchorCompleted,
|
|
92
|
+
repeat_ready_anchor_total: repeatAnchorTotal,
|
|
93
|
+
repeat_search_speedup_ratio: repeatSpeedup,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
if (failedChecks.length > 0) {
|
|
98
|
+
fail("benchmark gate failed", { ...output, failed_checks: failedChecks });
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
main().catch((error) => {
|
|
106
|
+
fail(error instanceof Error ? error.message : String(error), {});
|
|
107
|
+
});
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import fs from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
|
|
9
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
10
|
+
import { extractPriorityReadyHotspotChunks } from "../src/priority-ready-hotspots.js";
|
|
11
|
+
import { compactEmbeddingContentForSingleResumeChunk } from "../src/single-chunk-embedding-content.js";
|
|
12
|
+
|
|
13
|
+
const execFileAsync = promisify(execFile);
|
|
14
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
15
|
+
const MODEL = String(process.env.IDEROUTER_BENCH_SINGLE_CHUNK_MODEL || "openai/text-embedding-3-small").trim();
|
|
16
|
+
const TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_SINGLE_CHUNK_TIMEOUT_MS || 15000);
|
|
17
|
+
const RELATIVE_PATH = "src/index.js";
|
|
18
|
+
|
|
19
|
+
async function postEmbeddingWithCurl({ endpoint, apiKey, model, input, timeoutMs }) {
|
|
20
|
+
const timeoutSeconds = Math.max(1, Math.ceil(timeoutMs / 1000));
|
|
21
|
+
const marker = "\n__IDEROUTER_HTTP_STATUS__:";
|
|
22
|
+
const payload = JSON.stringify({
|
|
23
|
+
model,
|
|
24
|
+
input: [input],
|
|
25
|
+
});
|
|
26
|
+
const args = [
|
|
27
|
+
"-sS",
|
|
28
|
+
"-X", "POST",
|
|
29
|
+
"--connect-timeout", String(Math.min(15, timeoutSeconds)),
|
|
30
|
+
"--max-time", String(timeoutSeconds),
|
|
31
|
+
"-H", "Content-Type: application/json",
|
|
32
|
+
"-H", `Authorization: Bearer ${apiKey}`,
|
|
33
|
+
"--data-binary", "@-",
|
|
34
|
+
"-w", `${marker}%{http_code}`,
|
|
35
|
+
endpoint,
|
|
36
|
+
];
|
|
37
|
+
const startedAt = Date.now();
|
|
38
|
+
const { stdout } = await execFileAsync("curl", args, {
|
|
39
|
+
input: payload,
|
|
40
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
41
|
+
});
|
|
42
|
+
const wallMs = Date.now() - startedAt;
|
|
43
|
+
const output = String(stdout || "");
|
|
44
|
+
const markerIndex = output.lastIndexOf(marker);
|
|
45
|
+
if (markerIndex < 0) {
|
|
46
|
+
throw new Error("curl did not return HTTP status marker");
|
|
47
|
+
}
|
|
48
|
+
const responseBody = output.slice(0, markerIndex);
|
|
49
|
+
const status = Number(output.slice(markerIndex + marker.length).trim());
|
|
50
|
+
return {
|
|
51
|
+
status,
|
|
52
|
+
wall_ms: wallMs,
|
|
53
|
+
response_preview: responseBody.slice(0, 500),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function main() {
|
|
58
|
+
const apiKey = process.env.IDEROUTER_API_KEY;
|
|
59
|
+
if (!apiKey) throw new Error("IDEROUTER_API_KEY is required");
|
|
60
|
+
const baseURL = process.env.IDEROUTER_BASE_URL || "https://cursor.iderouter.com/v1";
|
|
61
|
+
const endpoint = `${baseURL.replace(/\/+$/, "")}/embeddings`;
|
|
62
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
63
|
+
const filePath = path.join(snapshot.snapshotDir, RELATIVE_PATH);
|
|
64
|
+
try {
|
|
65
|
+
const content = await fs.readFile(filePath, "utf8");
|
|
66
|
+
const chunks = extractPriorityReadyHotspotChunks(content, {
|
|
67
|
+
relativePath: RELATIVE_PATH,
|
|
68
|
+
extension: ".js",
|
|
69
|
+
}, {
|
|
70
|
+
chunkSize: 600,
|
|
71
|
+
chunkCap: 3,
|
|
72
|
+
primaryOnly: true,
|
|
73
|
+
});
|
|
74
|
+
if (!chunks[0]) {
|
|
75
|
+
throw new Error(`generic resumed single chunk not found in ${RELATIVE_PATH}`);
|
|
76
|
+
}
|
|
77
|
+
const payload = compactEmbeddingContentForSingleResumeChunk(chunks[0]);
|
|
78
|
+
const result = await postEmbeddingWithCurl({
|
|
79
|
+
endpoint,
|
|
80
|
+
apiKey,
|
|
81
|
+
model: MODEL,
|
|
82
|
+
input: payload,
|
|
83
|
+
timeoutMs: TIMEOUT_MS,
|
|
84
|
+
});
|
|
85
|
+
process.stdout.write(`${JSON.stringify({
|
|
86
|
+
path: TARGET_PATH,
|
|
87
|
+
snapshot_path: snapshot.snapshotDir,
|
|
88
|
+
model: MODEL,
|
|
89
|
+
timeout_ms: TIMEOUT_MS,
|
|
90
|
+
relative_path: RELATIVE_PATH,
|
|
91
|
+
chunk_preview: chunks[0].content.slice(0, 160),
|
|
92
|
+
payload,
|
|
93
|
+
payload_length: payload.length,
|
|
94
|
+
result,
|
|
95
|
+
}, null, 2)}\n`);
|
|
96
|
+
} finally {
|
|
97
|
+
await fs.rm(snapshot.snapshotDir, { recursive: true, force: true });
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
main().catch((error) => {
|
|
102
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
103
|
+
process.exitCode = 1;
|
|
104
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { runIndexBenchmark } from "./lib/mcp-bench.mjs";
|
|
6
|
+
|
|
7
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
8
|
+
const TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_TIMEOUT_MS || 30000);
|
|
9
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
10
|
+
|
|
11
|
+
async function main() {
|
|
12
|
+
const result = await runIndexBenchmark({
|
|
13
|
+
targetPath: TARGET_PATH,
|
|
14
|
+
timeoutMs: TIMEOUT_MS,
|
|
15
|
+
pollMs: POLL_MS,
|
|
16
|
+
startArgs: { path: TARGET_PATH, wait: false, cloud: false },
|
|
17
|
+
});
|
|
18
|
+
process.stdout.write(`${JSON.stringify(result.parsed, null, 2)}\n`);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
main().catch((error) => {
|
|
22
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
23
|
+
process.exitCode = 1;
|
|
24
|
+
});
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
import { cleanupBenchmarkRun, installBenchmarkSignalCleanup, normalizeBenchmarkResult, readIndexMeta, sleep, spawnMcpClient } from "./lib/mcp-bench.mjs";
|
|
8
|
+
import { getBenchmarkQueryPack } from "./lib/benchmark-query-packs.mjs";
|
|
9
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
10
|
+
|
|
11
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
12
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
13
|
+
const TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_TIMEOUT_MS || 30000);
|
|
14
|
+
const QUERY_PACK = String(process.env.IDEROUTER_BENCH_QUERY_PACK || "core").trim().toLowerCase();
|
|
15
|
+
const QUERIES = getBenchmarkQueryPack(QUERY_PACK);
|
|
16
|
+
|
|
17
|
+
function extractResultPaths(text) {
|
|
18
|
+
return String(text || "")
|
|
19
|
+
.split("\n")
|
|
20
|
+
.map((line) => line.trim())
|
|
21
|
+
.filter((line) => /^\d+\.\s+/.test(line))
|
|
22
|
+
.map((line) => {
|
|
23
|
+
const match = line.match(/^\d+\.\s+(.+?):\d+-\d+\s+score=/);
|
|
24
|
+
return match ? match[1] : null;
|
|
25
|
+
})
|
|
26
|
+
.filter(Boolean);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function evaluatePaths(paths, expected) {
|
|
30
|
+
const top1 = paths[0] || "";
|
|
31
|
+
const top3 = paths.slice(0, 3);
|
|
32
|
+
return {
|
|
33
|
+
top1,
|
|
34
|
+
top3,
|
|
35
|
+
top1_hit: expected.includes(top1),
|
|
36
|
+
top3_hit: top3.some((item) => expected.includes(item)),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function waitUntilPriorityReady(client, targetPath, timeoutMs, pollMs, indexHome) {
|
|
41
|
+
const startedAt = Date.now();
|
|
42
|
+
const deadline = startedAt + timeoutMs;
|
|
43
|
+
let finalText = "";
|
|
44
|
+
while (Date.now() < deadline) {
|
|
45
|
+
const text = await client.callTool("get_indexing_status", { path: targetPath });
|
|
46
|
+
finalText = text;
|
|
47
|
+
const meta = indexHome ? await readIndexMeta(indexHome, targetPath) : null;
|
|
48
|
+
const priorityReadyFromMeta = Boolean(meta?.priorityFineReady) && Number(meta?.fineSemanticChunkCount || 0) > 0;
|
|
49
|
+
const fullReadyFromMeta = Boolean(meta?.complete);
|
|
50
|
+
if (/^Priority semantic search is ready\b/.test(text) || /^Indexed\b/.test(text) || priorityReadyFromMeta || fullReadyFromMeta) {
|
|
51
|
+
return {
|
|
52
|
+
settled: true,
|
|
53
|
+
wallMs: Date.now() - startedAt,
|
|
54
|
+
finalText,
|
|
55
|
+
meta,
|
|
56
|
+
parsed: normalizeBenchmarkResult(targetPath, finalText, Date.now() - startedAt),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
await sleep(pollMs);
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
settled: false,
|
|
63
|
+
wallMs: Date.now() - startedAt,
|
|
64
|
+
finalText,
|
|
65
|
+
meta: indexHome ? await readIndexMeta(indexHome, targetPath) : null,
|
|
66
|
+
parsed: normalizeBenchmarkResult(targetPath, finalText, Date.now() - startedAt),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function anchorCoverageFromMeta(meta) {
|
|
71
|
+
const coverage = meta?.priorityReadyAnchorCoverage || {};
|
|
72
|
+
return {
|
|
73
|
+
total: Number(coverage.total || 0),
|
|
74
|
+
completed: Number(coverage.completed || 0),
|
|
75
|
+
completed_paths: Array.isArray(coverage.completed_paths) ? coverage.completed_paths : [],
|
|
76
|
+
pending_paths: Array.isArray(coverage.pending_paths) ? coverage.pending_paths : [],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function main() {
|
|
81
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
82
|
+
const snapshotDir = snapshot.snapshotDir;
|
|
83
|
+
const isolatedIndexHome = await fs.mkdtemp(path.join(os.tmpdir(), "iderouter-bench-home-"));
|
|
84
|
+
const env = {
|
|
85
|
+
...process.env,
|
|
86
|
+
IDEROUTER_INDEX_HOME: isolatedIndexHome,
|
|
87
|
+
};
|
|
88
|
+
const { child, client } = spawnMcpClient(env);
|
|
89
|
+
const uninstallSignalCleanup = installBenchmarkSignalCleanup(() => ({
|
|
90
|
+
child,
|
|
91
|
+
indexHome: isolatedIndexHome,
|
|
92
|
+
targetPath: snapshotDir,
|
|
93
|
+
snapshotDir,
|
|
94
|
+
}));
|
|
95
|
+
|
|
96
|
+
try {
|
|
97
|
+
await client.initialize();
|
|
98
|
+
const startText = await client.callTool("index_codebase", { path: snapshotDir, wait: false, cloud: false });
|
|
99
|
+
const ready = await waitUntilPriorityReady(
|
|
100
|
+
client,
|
|
101
|
+
snapshotDir,
|
|
102
|
+
Math.max(TIMEOUT_MS, 180000),
|
|
103
|
+
POLL_MS,
|
|
104
|
+
isolatedIndexHome,
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
const searches = [];
|
|
108
|
+
if (ready.settled) {
|
|
109
|
+
for (const item of QUERIES) {
|
|
110
|
+
const startedAt = Date.now();
|
|
111
|
+
const text = await client.callTool("search_code", {
|
|
112
|
+
path: snapshotDir,
|
|
113
|
+
query: item.query,
|
|
114
|
+
limit: 5,
|
|
115
|
+
});
|
|
116
|
+
const wallMs = Date.now() - startedAt;
|
|
117
|
+
const resultPaths = extractResultPaths(text);
|
|
118
|
+
searches.push({
|
|
119
|
+
name: item.name,
|
|
120
|
+
query: item.query,
|
|
121
|
+
wall_ms: wallMs,
|
|
122
|
+
result_paths: resultPaths,
|
|
123
|
+
...evaluatePaths(resultPaths, item.expect_any),
|
|
124
|
+
raw_status: text.split("\n")[0],
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
process.stdout.write(
|
|
130
|
+
`${JSON.stringify(
|
|
131
|
+
{
|
|
132
|
+
path: TARGET_PATH,
|
|
133
|
+
snapshot_path: snapshotDir,
|
|
134
|
+
snapshot_mode: snapshot.mode,
|
|
135
|
+
isolated_index_home: isolatedIndexHome,
|
|
136
|
+
start_status: startText.split("\n")[0],
|
|
137
|
+
query_pack: QUERY_PACK,
|
|
138
|
+
priority_ready: ready,
|
|
139
|
+
priority_ready_anchor_coverage: anchorCoverageFromMeta(ready.meta),
|
|
140
|
+
priority_ready_kind: /^Indexed\b/.test(ready.finalText || "")
|
|
141
|
+
? "full_ready"
|
|
142
|
+
: ((/^Priority semantic search is ready\b/.test(ready.finalText || "") && Number(ready?.meta?.fineSemanticChunkCount || 0) > 0) || ready?.settled)
|
|
143
|
+
? "priority_semantic"
|
|
144
|
+
: "not_ready",
|
|
145
|
+
searches,
|
|
146
|
+
},
|
|
147
|
+
null,
|
|
148
|
+
2,
|
|
149
|
+
)}\n`,
|
|
150
|
+
);
|
|
151
|
+
} finally {
|
|
152
|
+
uninstallSignalCleanup();
|
|
153
|
+
await cleanupBenchmarkRun({
|
|
154
|
+
child,
|
|
155
|
+
indexHome: isolatedIndexHome,
|
|
156
|
+
targetPath: snapshotDir,
|
|
157
|
+
snapshotDir,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
main().catch((error) => {
|
|
163
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
164
|
+
process.exitCode = 1;
|
|
165
|
+
});
|