@iderouter/index-mcp 0.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/package.json +26 -0
- package/scripts/benchmark-all.mjs +177 -0
- package/scripts/benchmark-auto-continuation.mjs +188 -0
- package/scripts/benchmark-background-fine-resume.mjs +245 -0
- package/scripts/benchmark-background-fine-wait.mjs +76 -0
- package/scripts/benchmark-background-fine.mjs +132 -0
- package/scripts/benchmark-clean-snapshot.mjs +83 -0
- package/scripts/benchmark-coarse-ready-search.mjs +161 -0
- package/scripts/benchmark-deferred.mjs +62 -0
- package/scripts/benchmark-first-semantic-visible.mjs +151 -0
- package/scripts/benchmark-gate.mjs +107 -0
- package/scripts/benchmark-generic-resumed-single-chunk-embed.mjs +104 -0
- package/scripts/benchmark-noop.mjs +24 -0
- package/scripts/benchmark-priority-ready-search.mjs +165 -0
- package/scripts/benchmark-repeat-search.mjs +148 -0
- package/scripts/benchmark-resumed-retry-burst.mjs +187 -0
- package/scripts/benchmark-resumed-single-chunk-success.mjs +154 -0
- package/scripts/benchmark-resumed-single-chunk.mjs +146 -0
- package/scripts/benchmark-single-priority-chunk-embed.mjs +145 -0
- package/scripts/benchmark-small-change.mjs +146 -0
- package/scripts/benchmark-stage-summary.mjs +88 -0
- package/scripts/lib/auto-continuation-state.mjs +34 -0
- package/scripts/lib/benchmark-query-packs.mjs +123 -0
- package/scripts/lib/benchmark-snapshot.mjs +109 -0
- package/scripts/lib/mcp-bench.mjs +455 -0
- package/src/architecture-query-fallback.js +50 -0
- package/src/background-definition-chunks.js +199 -0
- package/src/background-embedding-profile.js +64 -0
- package/src/background-fine-budget.js +18 -0
- package/src/background-fine-runtime.js +179 -0
- package/src/background-fine-selection.js +332 -0
- package/src/checkpoint-policy.js +16 -0
- package/src/conflict-policy.js +17 -0
- package/src/deferred-retry-delay.js +14 -0
- package/src/deferred-retry-status.js +10 -0
- package/src/embedding-attempt-ordinal.js +17 -0
- package/src/embedding-failure-penalty.js +60 -0
- package/src/embedding-failure-policy.js +52 -0
- package/src/embedding-flush-timeout.js +33 -0
- package/src/embedding-inflight-status.js +18 -0
- package/src/embedding-model-policy.js +44 -0
- package/src/embedding-next-switch.js +18 -0
- package/src/embedding-request-status-detail.js +25 -0
- package/src/embedding-request-status.js +22 -0
- package/src/embedding-selection-order.js +23 -0
- package/src/fine-run-queue.js +14 -0
- package/src/index.js +7970 -0
- package/src/job-supersession.js +25 -0
- package/src/priority-progress.js +20 -0
- package/src/priority-ready-anchor-coverage-normalize.js +18 -0
- package/src/priority-ready-anchor-coverage.js +23 -0
- package/src/priority-ready-hotspots.js +344 -0
- package/src/priority-ready-status.js +30 -0
- package/src/priority-ready-targets.js +45 -0
- package/src/priority-usable-attempt-plan.js +44 -0
- package/src/priority-usable-attempt-timeout.js +18 -0
- package/src/priority-usable-fast-path.js +11 -0
- package/src/priority-usable-probe-order.js +34 -0
- package/src/remote-strategy-failure-cache.js +55 -0
- package/src/resume-seed.js +9 -0
- package/src/semantic-first-checkpoint.js +8 -0
- package/src/semantic-slow-path.js +10 -0
- package/src/single-chunk-attempt-timeout.js +13 -0
- package/src/single-chunk-embedding-content.js +26 -0
- package/src/single-chunk-embedding-policy.js +18 -0
- package/src/single-chunk-provider-order.js +12 -0
- package/src/single-chunk-provider-policy.js +63 -0
- package/src/worker-lock-retry.js +24 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
import { cleanupBenchmarkRun, installBenchmarkSignalCleanup, normalizeBenchmarkResult, readIndexMeta, sleep, spawnMcpClient } from "./lib/mcp-bench.mjs";
|
|
8
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
9
|
+
|
|
10
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
11
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
12
|
+
const TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_TIMEOUT_MS || 30000);
|
|
13
|
+
const REPEAT_QUERY = "billing backend execution path RunExprWithRequest model pricing editor";
|
|
14
|
+
|
|
15
|
+
async function waitUntilPriorityReady(client, targetPath, timeoutMs, pollMs, indexHome) {
|
|
16
|
+
const startedAt = Date.now();
|
|
17
|
+
const deadline = startedAt + timeoutMs;
|
|
18
|
+
let finalText = "";
|
|
19
|
+
while (Date.now() < deadline) {
|
|
20
|
+
const text = await client.callTool("get_indexing_status", { path: targetPath });
|
|
21
|
+
finalText = text;
|
|
22
|
+
const meta = indexHome ? await readIndexMeta(indexHome, targetPath) : null;
|
|
23
|
+
const priorityReadyFromMeta = Boolean(meta?.priorityFineReady) && Number(meta?.fineSemanticChunkCount || 0) > 0;
|
|
24
|
+
const fullReadyFromMeta = Boolean(meta?.complete);
|
|
25
|
+
if (/^Priority semantic search is ready\b/.test(text) || /^Indexed\b/.test(text) || priorityReadyFromMeta || fullReadyFromMeta) {
|
|
26
|
+
return {
|
|
27
|
+
settled: true,
|
|
28
|
+
wallMs: Date.now() - startedAt,
|
|
29
|
+
finalText,
|
|
30
|
+
meta,
|
|
31
|
+
parsed: normalizeBenchmarkResult(targetPath, finalText, Date.now() - startedAt),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
await sleep(pollMs);
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
settled: false,
|
|
38
|
+
wallMs: Date.now() - startedAt,
|
|
39
|
+
finalText,
|
|
40
|
+
meta: indexHome ? await readIndexMeta(indexHome, targetPath) : null,
|
|
41
|
+
parsed: normalizeBenchmarkResult(targetPath, finalText, Date.now() - startedAt),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function anchorCoverageFromMeta(meta) {
|
|
46
|
+
const coverage = meta?.priorityReadyAnchorCoverage || {};
|
|
47
|
+
return {
|
|
48
|
+
total: Number(coverage.total || 0),
|
|
49
|
+
completed: Number(coverage.completed || 0),
|
|
50
|
+
completed_paths: Array.isArray(coverage.completed_paths) ? coverage.completed_paths : [],
|
|
51
|
+
pending_paths: Array.isArray(coverage.pending_paths) ? coverage.pending_paths : [],
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function main() {
|
|
56
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
57
|
+
const snapshotDir = snapshot.snapshotDir;
|
|
58
|
+
const isolatedIndexHome = await fs.mkdtemp(path.join(os.tmpdir(), "iderouter-bench-home-"));
|
|
59
|
+
const env = {
|
|
60
|
+
...process.env,
|
|
61
|
+
IDEROUTER_INDEX_HOME: isolatedIndexHome,
|
|
62
|
+
};
|
|
63
|
+
const { child, client } = spawnMcpClient(env);
|
|
64
|
+
const uninstallSignalCleanup = installBenchmarkSignalCleanup(() => ({
|
|
65
|
+
child,
|
|
66
|
+
indexHome: isolatedIndexHome,
|
|
67
|
+
targetPath: snapshotDir,
|
|
68
|
+
snapshotDir,
|
|
69
|
+
}));
|
|
70
|
+
|
|
71
|
+
try {
|
|
72
|
+
await client.initialize();
|
|
73
|
+
const startText = await client.callTool("index_codebase", { path: snapshotDir, wait: false, cloud: false });
|
|
74
|
+
const ready = await waitUntilPriorityReady(
|
|
75
|
+
client,
|
|
76
|
+
snapshotDir,
|
|
77
|
+
Math.max(TIMEOUT_MS, 180000),
|
|
78
|
+
POLL_MS,
|
|
79
|
+
isolatedIndexHome,
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
let first = null;
|
|
83
|
+
let second = null;
|
|
84
|
+
if (ready.settled) {
|
|
85
|
+
const startedFirst = Date.now();
|
|
86
|
+
const firstText = await client.callTool("search_code", {
|
|
87
|
+
path: snapshotDir,
|
|
88
|
+
query: REPEAT_QUERY,
|
|
89
|
+
limit: 5,
|
|
90
|
+
});
|
|
91
|
+
first = {
|
|
92
|
+
wall_ms: Date.now() - startedFirst,
|
|
93
|
+
status: firstText.split("\n")[0],
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
const startedSecond = Date.now();
|
|
97
|
+
const secondText = await client.callTool("search_code", {
|
|
98
|
+
path: snapshotDir,
|
|
99
|
+
query: REPEAT_QUERY,
|
|
100
|
+
limit: 5,
|
|
101
|
+
});
|
|
102
|
+
second = {
|
|
103
|
+
wall_ms: Date.now() - startedSecond,
|
|
104
|
+
status: secondText.split("\n")[0],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
process.stdout.write(
|
|
109
|
+
`${JSON.stringify(
|
|
110
|
+
{
|
|
111
|
+
path: TARGET_PATH,
|
|
112
|
+
snapshot_path: snapshotDir,
|
|
113
|
+
snapshot_mode: snapshot.mode,
|
|
114
|
+
isolated_index_home: isolatedIndexHome,
|
|
115
|
+
start_status: startText.split("\n")[0],
|
|
116
|
+
priority_ready: ready,
|
|
117
|
+
priority_ready_anchor_coverage: anchorCoverageFromMeta(ready.meta),
|
|
118
|
+
priority_ready_kind: /^Indexed\b/.test(ready.finalText || "")
|
|
119
|
+
? "full_ready"
|
|
120
|
+
: ((/^Priority semantic search is ready\b/.test(ready.finalText || "") && Number(ready?.meta?.fineSemanticChunkCount || 0) > 0) || ready?.settled)
|
|
121
|
+
? "priority_semantic"
|
|
122
|
+
: "not_ready",
|
|
123
|
+
repeat_query: REPEAT_QUERY,
|
|
124
|
+
first,
|
|
125
|
+
second,
|
|
126
|
+
speedup_ratio: first && second && second.wall_ms > 0
|
|
127
|
+
? Number((first.wall_ms / second.wall_ms).toFixed(2))
|
|
128
|
+
: null,
|
|
129
|
+
},
|
|
130
|
+
null,
|
|
131
|
+
2,
|
|
132
|
+
)}\n`,
|
|
133
|
+
);
|
|
134
|
+
} finally {
|
|
135
|
+
uninstallSignalCleanup();
|
|
136
|
+
await cleanupBenchmarkRun({
|
|
137
|
+
child,
|
|
138
|
+
indexHome: isolatedIndexHome,
|
|
139
|
+
targetPath: snapshotDir,
|
|
140
|
+
snapshotDir,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
main().catch((error) => {
|
|
146
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
147
|
+
process.exitCode = 1;
|
|
148
|
+
});
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
cleanupBenchmarkRun,
|
|
10
|
+
installBenchmarkSignalCleanup,
|
|
11
|
+
readIndexMeta,
|
|
12
|
+
readJobStatus,
|
|
13
|
+
sleep,
|
|
14
|
+
spawnMcpClient,
|
|
15
|
+
} from "./lib/mcp-bench.mjs";
|
|
16
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
17
|
+
|
|
18
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
19
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
20
|
+
const FIRST_PHASE_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_RETRY_BURST_FIRST_PHASE_TIMEOUT_MS || 30000);
|
|
21
|
+
const TOTAL_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_RETRY_BURST_TOTAL_TIMEOUT_MS || 90000);
|
|
22
|
+
const MAX_RETRIES = Math.max(1, Number(process.env.IDEROUTER_BENCH_RETRY_BURST_MAX_RETRIES || 4));
|
|
23
|
+
|
|
24
|
+
function snapshotState(meta, job, statusText) {
|
|
25
|
+
return {
|
|
26
|
+
status_text: String(statusText || "").split("\n")[0],
|
|
27
|
+
job_id: job?.id || "",
|
|
28
|
+
job_status: job?.status || "",
|
|
29
|
+
current_step: job?.currentStep || "",
|
|
30
|
+
embedding_model: job?.embeddingModel || job?.diagnostics?.embeddingModel || "",
|
|
31
|
+
embedding_model_source: job?.embeddingModelSource || job?.diagnostics?.embeddingModelSource || "",
|
|
32
|
+
progress: Number(job?.progress || 0),
|
|
33
|
+
embedded_count: Number(job?.embeddedCount || 0),
|
|
34
|
+
total_chunks: Number(job?.totalChunks || 0),
|
|
35
|
+
background_fine_deferred: Boolean(job?.backgroundFineDeferred),
|
|
36
|
+
semantic_retry_deferred: Boolean(job?.semanticRetryDeferred),
|
|
37
|
+
semantic_retry_count: Number(job?.semanticRetryCount || 0),
|
|
38
|
+
searchable_modes: job?.searchableModes || meta?.searchableModes || "",
|
|
39
|
+
fine_semantic_files: Number(meta?.fineSemanticFileCount || 0),
|
|
40
|
+
fine_semantic_chunks: Number(meta?.fineSemanticChunkCount || 0),
|
|
41
|
+
updated_at: job?.updatedAt || meta?.updatedAt || "",
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function waitForInitialDeferred(client, snapshotDir, indexHome) {
|
|
46
|
+
const startedAt = Date.now();
|
|
47
|
+
let lastText = "";
|
|
48
|
+
while (Date.now() - startedAt < FIRST_PHASE_TIMEOUT_MS) {
|
|
49
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
50
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
51
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
52
|
+
const state = snapshotState(meta, job, lastText);
|
|
53
|
+
if (state.semantic_retry_deferred) {
|
|
54
|
+
return {
|
|
55
|
+
settled: true,
|
|
56
|
+
wall_ms: Date.now() - startedAt,
|
|
57
|
+
state,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
await sleep(POLL_MS);
|
|
61
|
+
}
|
|
62
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
63
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
64
|
+
return {
|
|
65
|
+
settled: false,
|
|
66
|
+
wall_ms: Date.now() - startedAt,
|
|
67
|
+
state: snapshotState(meta, job, lastText),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function observeRetryBurst(client, snapshotDir, indexHome, baselineJobId) {
|
|
72
|
+
const startedAt = Date.now();
|
|
73
|
+
let lastText = "";
|
|
74
|
+
const attempts = [];
|
|
75
|
+
let lastRecordedJobId = baselineJobId;
|
|
76
|
+
|
|
77
|
+
while (Date.now() - startedAt < TOTAL_TIMEOUT_MS) {
|
|
78
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
79
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
80
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
81
|
+
const state = snapshotState(meta, job, lastText);
|
|
82
|
+
|
|
83
|
+
if (state.fine_semantic_chunks > 0 || state.fine_semantic_files > 0) {
|
|
84
|
+
return {
|
|
85
|
+
settled: true,
|
|
86
|
+
wall_ms: Date.now() - startedAt,
|
|
87
|
+
outcome: "semantic_visible",
|
|
88
|
+
attempts,
|
|
89
|
+
state,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const newDeferredCycle =
|
|
94
|
+
state.semantic_retry_deferred &&
|
|
95
|
+
state.job_id &&
|
|
96
|
+
state.job_id !== baselineJobId &&
|
|
97
|
+
state.job_id !== lastRecordedJobId;
|
|
98
|
+
|
|
99
|
+
if (newDeferredCycle) {
|
|
100
|
+
attempts.push({
|
|
101
|
+
job_id: state.job_id,
|
|
102
|
+
embedding_model: state.embedding_model,
|
|
103
|
+
embedding_model_source: state.embedding_model_source,
|
|
104
|
+
semantic_retry_count: state.semantic_retry_count,
|
|
105
|
+
wall_ms: Date.now() - startedAt,
|
|
106
|
+
status_text: state.status_text,
|
|
107
|
+
});
|
|
108
|
+
lastRecordedJobId = state.job_id;
|
|
109
|
+
if (attempts.length >= MAX_RETRIES) {
|
|
110
|
+
return {
|
|
111
|
+
settled: true,
|
|
112
|
+
wall_ms: Date.now() - startedAt,
|
|
113
|
+
outcome: "max_retries_observed",
|
|
114
|
+
attempts,
|
|
115
|
+
state,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
await sleep(POLL_MS);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
124
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
125
|
+
return {
|
|
126
|
+
settled: false,
|
|
127
|
+
wall_ms: Date.now() - startedAt,
|
|
128
|
+
outcome: "timeout",
|
|
129
|
+
attempts,
|
|
130
|
+
state: snapshotState(meta, job, lastText),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function main() {
|
|
135
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
136
|
+
const snapshotDir = snapshot.snapshotDir;
|
|
137
|
+
const isolatedIndexHome = await fs.mkdtemp(path.join(os.tmpdir(), "iderouter-bench-retry-burst-"));
|
|
138
|
+
const env = {
|
|
139
|
+
...process.env,
|
|
140
|
+
IDEROUTER_INDEX_HOME: isolatedIndexHome,
|
|
141
|
+
};
|
|
142
|
+
const { child, client } = spawnMcpClient(env);
|
|
143
|
+
const uninstallSignalCleanup = installBenchmarkSignalCleanup(() => ({
|
|
144
|
+
child,
|
|
145
|
+
indexHome: isolatedIndexHome,
|
|
146
|
+
targetPath: snapshotDir,
|
|
147
|
+
snapshotDir,
|
|
148
|
+
}));
|
|
149
|
+
|
|
150
|
+
try {
|
|
151
|
+
await client.initialize();
|
|
152
|
+
const startText = await client.callTool("index_codebase", { path: snapshotDir, wait: false, cloud: false });
|
|
153
|
+
const firstPhase = await waitForInitialDeferred(client, snapshotDir, isolatedIndexHome);
|
|
154
|
+
const burst = firstPhase.settled
|
|
155
|
+
? await observeRetryBurst(client, snapshotDir, isolatedIndexHome, firstPhase.state.job_id)
|
|
156
|
+
: { settled: false, wall_ms: 0, outcome: "no_first_phase", attempts: [], state: firstPhase.state };
|
|
157
|
+
|
|
158
|
+
process.stdout.write(
|
|
159
|
+
`${JSON.stringify(
|
|
160
|
+
{
|
|
161
|
+
path: TARGET_PATH,
|
|
162
|
+
snapshot_path: snapshotDir,
|
|
163
|
+
snapshot_mode: snapshot.mode,
|
|
164
|
+
isolated_index_home: isolatedIndexHome,
|
|
165
|
+
start_status: startText.split("\n")[0],
|
|
166
|
+
first_phase: firstPhase,
|
|
167
|
+
burst,
|
|
168
|
+
},
|
|
169
|
+
null,
|
|
170
|
+
2,
|
|
171
|
+
)}\n`,
|
|
172
|
+
);
|
|
173
|
+
} finally {
|
|
174
|
+
uninstallSignalCleanup();
|
|
175
|
+
await cleanupBenchmarkRun({
|
|
176
|
+
child,
|
|
177
|
+
indexHome: isolatedIndexHome,
|
|
178
|
+
targetPath: snapshotDir,
|
|
179
|
+
snapshotDir,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
main().catch((error) => {
|
|
185
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
186
|
+
process.exitCode = 1;
|
|
187
|
+
});
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
cleanupBenchmarkRun,
|
|
10
|
+
installBenchmarkSignalCleanup,
|
|
11
|
+
readIndexMeta,
|
|
12
|
+
readJobStatus,
|
|
13
|
+
sleep,
|
|
14
|
+
spawnMcpClient,
|
|
15
|
+
} from "./lib/mcp-bench.mjs";
|
|
16
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
17
|
+
|
|
18
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
19
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
20
|
+
const FIRST_PHASE_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_RESUMED_SINGLE_CHUNK_FIRST_PHASE_TIMEOUT_MS || 30000);
|
|
21
|
+
const RESUME_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_RESUMED_SINGLE_CHUNK_RESUME_TIMEOUT_MS || 30000);
|
|
22
|
+
|
|
23
|
+
function snapshotState(meta, job, statusText) {
|
|
24
|
+
return {
|
|
25
|
+
status_text: String(statusText || "").split("\n")[0],
|
|
26
|
+
job_id: job?.id || "",
|
|
27
|
+
job_status: job?.status || "",
|
|
28
|
+
current_step: job?.currentStep || "",
|
|
29
|
+
embedding_model: job?.embeddingModel || job?.diagnostics?.embeddingModel || "",
|
|
30
|
+
embedding_model_source: job?.embeddingModelSource || job?.diagnostics?.embeddingModelSource || "",
|
|
31
|
+
progress: Number(job?.progress || 0),
|
|
32
|
+
embedded_count: Number(job?.embeddedCount || 0),
|
|
33
|
+
total_chunks: Number(job?.totalChunks || 0),
|
|
34
|
+
background_fine_deferred: Boolean(job?.backgroundFineDeferred),
|
|
35
|
+
semantic_retry_deferred: Boolean(job?.semanticRetryDeferred),
|
|
36
|
+
searchable_modes: job?.searchableModes || meta?.searchableModes || "",
|
|
37
|
+
fine_semantic_files: Number(meta?.fineSemanticFileCount || 0),
|
|
38
|
+
fine_semantic_chunks: Number(meta?.fineSemanticChunkCount || 0),
|
|
39
|
+
updated_at: job?.updatedAt || meta?.updatedAt || "",
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function waitForFirstDeferredReady(client, snapshotDir, indexHome) {
|
|
44
|
+
const startedAt = Date.now();
|
|
45
|
+
let lastText = "";
|
|
46
|
+
while (Date.now() - startedAt < FIRST_PHASE_TIMEOUT_MS) {
|
|
47
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
48
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
49
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
50
|
+
const state = snapshotState(meta, job, lastText);
|
|
51
|
+
if (state.semantic_retry_deferred) {
|
|
52
|
+
return {
|
|
53
|
+
settled: true,
|
|
54
|
+
wall_ms: Date.now() - startedAt,
|
|
55
|
+
state,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
await sleep(POLL_MS);
|
|
59
|
+
}
|
|
60
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
61
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
62
|
+
return {
|
|
63
|
+
settled: false,
|
|
64
|
+
wall_ms: Date.now() - startedAt,
|
|
65
|
+
state: snapshotState(meta, job, lastText),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function waitForResumedOutcome(client, snapshotDir, indexHome, baselineJobId) {
|
|
70
|
+
const startedAt = Date.now();
|
|
71
|
+
let lastText = "";
|
|
72
|
+
while (Date.now() - startedAt < RESUME_TIMEOUT_MS) {
|
|
73
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
74
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
75
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
76
|
+
const state = snapshotState(meta, job, lastText);
|
|
77
|
+
const resumedJob = state.job_id && state.job_id !== baselineJobId;
|
|
78
|
+
const semanticVisible = state.fine_semantic_chunks > 0 || state.fine_semantic_files > 0;
|
|
79
|
+
const reDeferred = resumedJob && state.semantic_retry_deferred;
|
|
80
|
+
const failed = resumedJob && state.job_status === "failed";
|
|
81
|
+
if (semanticVisible || reDeferred || failed) {
|
|
82
|
+
return {
|
|
83
|
+
settled: true,
|
|
84
|
+
wall_ms: Date.now() - startedAt,
|
|
85
|
+
outcome: semanticVisible ? "semantic_visible" : reDeferred ? "re_deferred" : "failed",
|
|
86
|
+
state,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
await sleep(POLL_MS);
|
|
90
|
+
}
|
|
91
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
92
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
93
|
+
return {
|
|
94
|
+
settled: false,
|
|
95
|
+
wall_ms: Date.now() - startedAt,
|
|
96
|
+
outcome: "timeout",
|
|
97
|
+
state: snapshotState(meta, job, lastText),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function main() {
|
|
102
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
103
|
+
const snapshotDir = snapshot.snapshotDir;
|
|
104
|
+
const isolatedIndexHome = await fs.mkdtemp(path.join(os.tmpdir(), "iderouter-bench-resumed-success-"));
|
|
105
|
+
const env = {
|
|
106
|
+
...process.env,
|
|
107
|
+
IDEROUTER_INDEX_HOME: isolatedIndexHome,
|
|
108
|
+
};
|
|
109
|
+
const { child, client } = spawnMcpClient(env);
|
|
110
|
+
const uninstallSignalCleanup = installBenchmarkSignalCleanup(() => ({
|
|
111
|
+
child,
|
|
112
|
+
indexHome: isolatedIndexHome,
|
|
113
|
+
targetPath: snapshotDir,
|
|
114
|
+
snapshotDir,
|
|
115
|
+
}));
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
await client.initialize();
|
|
119
|
+
const startText = await client.callTool("index_codebase", { path: snapshotDir, wait: false, cloud: false });
|
|
120
|
+
const firstPhase = await waitForFirstDeferredReady(client, snapshotDir, isolatedIndexHome);
|
|
121
|
+
const resumedOutcome = firstPhase.settled
|
|
122
|
+
? await waitForResumedOutcome(client, snapshotDir, isolatedIndexHome, firstPhase.state.job_id)
|
|
123
|
+
: { settled: false, wall_ms: 0, outcome: "no_first_phase", state: firstPhase.state };
|
|
124
|
+
|
|
125
|
+
process.stdout.write(
|
|
126
|
+
`${JSON.stringify(
|
|
127
|
+
{
|
|
128
|
+
path: TARGET_PATH,
|
|
129
|
+
snapshot_path: snapshotDir,
|
|
130
|
+
snapshot_mode: snapshot.mode,
|
|
131
|
+
isolated_index_home: isolatedIndexHome,
|
|
132
|
+
start_status: startText.split("\n")[0],
|
|
133
|
+
first_phase: firstPhase,
|
|
134
|
+
resumed_outcome: resumedOutcome,
|
|
135
|
+
},
|
|
136
|
+
null,
|
|
137
|
+
2,
|
|
138
|
+
)}\n`,
|
|
139
|
+
);
|
|
140
|
+
} finally {
|
|
141
|
+
uninstallSignalCleanup();
|
|
142
|
+
await cleanupBenchmarkRun({
|
|
143
|
+
child,
|
|
144
|
+
indexHome: isolatedIndexHome,
|
|
145
|
+
targetPath: snapshotDir,
|
|
146
|
+
snapshotDir,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
main().catch((error) => {
|
|
152
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
153
|
+
process.exitCode = 1;
|
|
154
|
+
});
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
|
|
8
|
+
import { cleanupBenchmarkRun, installBenchmarkSignalCleanup, readIndexMeta, readJobStatus, sleep, spawnMcpClient } from "./lib/mcp-bench.mjs";
|
|
9
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
10
|
+
|
|
11
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
12
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
13
|
+
const FIRST_PHASE_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_RESUMED_SINGLE_CHUNK_FIRST_PHASE_TIMEOUT_MS || 30000);
|
|
14
|
+
const RESUME_TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_RESUMED_SINGLE_CHUNK_RESUME_TIMEOUT_MS || 30000);
|
|
15
|
+
|
|
16
|
+
function snapshotState(meta, job, statusText) {
|
|
17
|
+
return {
|
|
18
|
+
status_text: String(statusText || "").split("\n")[0],
|
|
19
|
+
job_id: job?.id || "",
|
|
20
|
+
job_status: job?.status || "",
|
|
21
|
+
current_step: job?.currentStep || "",
|
|
22
|
+
embedding_model: job?.embeddingModel || job?.diagnostics?.embeddingModel || "",
|
|
23
|
+
embedding_model_source: job?.embeddingModelSource || job?.diagnostics?.embeddingModelSource || "",
|
|
24
|
+
progress: Number(job?.progress || 0),
|
|
25
|
+
embedded_count: Number(job?.embeddedCount || 0),
|
|
26
|
+
total_chunks: Number(job?.totalChunks || 0),
|
|
27
|
+
background_fine_deferred: Boolean(job?.backgroundFineDeferred),
|
|
28
|
+
semantic_retry_deferred: Boolean(job?.semanticRetryDeferred),
|
|
29
|
+
searchable_modes: job?.searchableModes || meta?.searchableModes || "",
|
|
30
|
+
fine_semantic_files: Number(meta?.fineSemanticFileCount || 0),
|
|
31
|
+
fine_semantic_chunks: Number(meta?.fineSemanticChunkCount || 0),
|
|
32
|
+
updated_at: job?.updatedAt || meta?.updatedAt || "",
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function waitForDeferredReady(client, snapshotDir, indexHome) {
|
|
37
|
+
const startedAt = Date.now();
|
|
38
|
+
let lastText = "";
|
|
39
|
+
while (Date.now() - startedAt < FIRST_PHASE_TIMEOUT_MS) {
|
|
40
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
41
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
42
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
43
|
+
if (job?.semanticRetryDeferred) {
|
|
44
|
+
return {
|
|
45
|
+
settled: true,
|
|
46
|
+
wall_ms: Date.now() - startedAt,
|
|
47
|
+
state: snapshotState(meta, job, lastText),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
await sleep(POLL_MS);
|
|
51
|
+
}
|
|
52
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
53
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
54
|
+
return {
|
|
55
|
+
settled: false,
|
|
56
|
+
wall_ms: Date.now() - startedAt,
|
|
57
|
+
state: snapshotState(meta, job, lastText),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function waitForResumedSingleChunk(client, snapshotDir, indexHome, baselineJobId) {
|
|
62
|
+
const startedAt = Date.now();
|
|
63
|
+
let lastText = "";
|
|
64
|
+
while (Date.now() - startedAt < RESUME_TIMEOUT_MS) {
|
|
65
|
+
lastText = await client.callTool("get_indexing_status", { path: snapshotDir });
|
|
66
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
67
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
68
|
+
const state = snapshotState(meta, job, lastText);
|
|
69
|
+
if (
|
|
70
|
+
state.job_id &&
|
|
71
|
+
state.job_id !== baselineJobId &&
|
|
72
|
+
state.job_status === "indexing" &&
|
|
73
|
+
state.embedding_model_source.startsWith("previous_failure_fallback_deferred_resume") &&
|
|
74
|
+
state.total_chunks === 1
|
|
75
|
+
) {
|
|
76
|
+
return {
|
|
77
|
+
settled: true,
|
|
78
|
+
wall_ms: Date.now() - startedAt,
|
|
79
|
+
state,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
await sleep(POLL_MS);
|
|
83
|
+
}
|
|
84
|
+
const meta = await readIndexMeta(indexHome, snapshotDir);
|
|
85
|
+
const job = await readJobStatus(indexHome, snapshotDir);
|
|
86
|
+
return {
|
|
87
|
+
settled: false,
|
|
88
|
+
wall_ms: Date.now() - startedAt,
|
|
89
|
+
state: snapshotState(meta, job, lastText),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function main() {
|
|
94
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
95
|
+
const snapshotDir = snapshot.snapshotDir;
|
|
96
|
+
const isolatedIndexHome = await fs.mkdtemp(path.join(os.tmpdir(), "iderouter-bench-resumed-single-"));
|
|
97
|
+
const env = {
|
|
98
|
+
...process.env,
|
|
99
|
+
IDEROUTER_INDEX_HOME: isolatedIndexHome,
|
|
100
|
+
};
|
|
101
|
+
const { child, client } = spawnMcpClient(env);
|
|
102
|
+
const uninstallSignalCleanup = installBenchmarkSignalCleanup(() => ({
|
|
103
|
+
child,
|
|
104
|
+
indexHome: isolatedIndexHome,
|
|
105
|
+
targetPath: snapshotDir,
|
|
106
|
+
snapshotDir,
|
|
107
|
+
}));
|
|
108
|
+
|
|
109
|
+
try {
|
|
110
|
+
await client.initialize();
|
|
111
|
+
const startText = await client.callTool("index_codebase", { path: snapshotDir, wait: false, cloud: false });
|
|
112
|
+
const firstPhase = await waitForDeferredReady(client, snapshotDir, isolatedIndexHome);
|
|
113
|
+
const resumed = firstPhase.settled
|
|
114
|
+
? await waitForResumedSingleChunk(client, snapshotDir, isolatedIndexHome, firstPhase.state.job_id)
|
|
115
|
+
: { settled: false, wall_ms: 0, state: firstPhase.state };
|
|
116
|
+
|
|
117
|
+
process.stdout.write(
|
|
118
|
+
`${JSON.stringify(
|
|
119
|
+
{
|
|
120
|
+
path: TARGET_PATH,
|
|
121
|
+
snapshot_path: snapshotDir,
|
|
122
|
+
snapshot_mode: snapshot.mode,
|
|
123
|
+
isolated_index_home: isolatedIndexHome,
|
|
124
|
+
start_status: startText.split("\n")[0],
|
|
125
|
+
first_phase: firstPhase,
|
|
126
|
+
resumed_single_chunk: resumed,
|
|
127
|
+
},
|
|
128
|
+
null,
|
|
129
|
+
2,
|
|
130
|
+
)}\n`,
|
|
131
|
+
);
|
|
132
|
+
} finally {
|
|
133
|
+
uninstallSignalCleanup();
|
|
134
|
+
await cleanupBenchmarkRun({
|
|
135
|
+
child,
|
|
136
|
+
indexHome: isolatedIndexHome,
|
|
137
|
+
targetPath: snapshotDir,
|
|
138
|
+
snapshotDir,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
main().catch((error) => {
|
|
144
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
});
|