@iderouter/index-mcp 0.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/package.json +26 -0
- package/scripts/benchmark-all.mjs +177 -0
- package/scripts/benchmark-auto-continuation.mjs +188 -0
- package/scripts/benchmark-background-fine-resume.mjs +245 -0
- package/scripts/benchmark-background-fine-wait.mjs +76 -0
- package/scripts/benchmark-background-fine.mjs +132 -0
- package/scripts/benchmark-clean-snapshot.mjs +83 -0
- package/scripts/benchmark-coarse-ready-search.mjs +161 -0
- package/scripts/benchmark-deferred.mjs +62 -0
- package/scripts/benchmark-first-semantic-visible.mjs +151 -0
- package/scripts/benchmark-gate.mjs +107 -0
- package/scripts/benchmark-generic-resumed-single-chunk-embed.mjs +104 -0
- package/scripts/benchmark-noop.mjs +24 -0
- package/scripts/benchmark-priority-ready-search.mjs +165 -0
- package/scripts/benchmark-repeat-search.mjs +148 -0
- package/scripts/benchmark-resumed-retry-burst.mjs +187 -0
- package/scripts/benchmark-resumed-single-chunk-success.mjs +154 -0
- package/scripts/benchmark-resumed-single-chunk.mjs +146 -0
- package/scripts/benchmark-single-priority-chunk-embed.mjs +145 -0
- package/scripts/benchmark-small-change.mjs +146 -0
- package/scripts/benchmark-stage-summary.mjs +88 -0
- package/scripts/lib/auto-continuation-state.mjs +34 -0
- package/scripts/lib/benchmark-query-packs.mjs +123 -0
- package/scripts/lib/benchmark-snapshot.mjs +109 -0
- package/scripts/lib/mcp-bench.mjs +455 -0
- package/src/architecture-query-fallback.js +50 -0
- package/src/background-definition-chunks.js +199 -0
- package/src/background-embedding-profile.js +64 -0
- package/src/background-fine-budget.js +18 -0
- package/src/background-fine-runtime.js +179 -0
- package/src/background-fine-selection.js +332 -0
- package/src/checkpoint-policy.js +16 -0
- package/src/conflict-policy.js +17 -0
- package/src/deferred-retry-delay.js +14 -0
- package/src/deferred-retry-status.js +10 -0
- package/src/embedding-attempt-ordinal.js +17 -0
- package/src/embedding-failure-penalty.js +60 -0
- package/src/embedding-failure-policy.js +52 -0
- package/src/embedding-flush-timeout.js +33 -0
- package/src/embedding-inflight-status.js +18 -0
- package/src/embedding-model-policy.js +44 -0
- package/src/embedding-next-switch.js +18 -0
- package/src/embedding-request-status-detail.js +25 -0
- package/src/embedding-request-status.js +22 -0
- package/src/embedding-selection-order.js +23 -0
- package/src/fine-run-queue.js +14 -0
- package/src/index.js +7970 -0
- package/src/job-supersession.js +25 -0
- package/src/priority-progress.js +20 -0
- package/src/priority-ready-anchor-coverage-normalize.js +18 -0
- package/src/priority-ready-anchor-coverage.js +23 -0
- package/src/priority-ready-hotspots.js +344 -0
- package/src/priority-ready-status.js +30 -0
- package/src/priority-ready-targets.js +45 -0
- package/src/priority-usable-attempt-plan.js +44 -0
- package/src/priority-usable-attempt-timeout.js +18 -0
- package/src/priority-usable-fast-path.js +11 -0
- package/src/priority-usable-probe-order.js +34 -0
- package/src/remote-strategy-failure-cache.js +55 -0
- package/src/resume-seed.js +9 -0
- package/src/semantic-first-checkpoint.js +8 -0
- package/src/semantic-slow-path.js +10 -0
- package/src/single-chunk-attempt-timeout.js +13 -0
- package/src/single-chunk-embedding-content.js +26 -0
- package/src/single-chunk-embedding-policy.js +18 -0
- package/src/single-chunk-provider-order.js +12 -0
- package/src/single-chunk-provider-policy.js +63 -0
- package/src/worker-lock-retry.js +24 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import fs from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import process from "node:process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
|
|
9
|
+
import { createBenchmarkSnapshot } from "./lib/benchmark-snapshot.mjs";
|
|
10
|
+
|
|
11
|
+
const execFileAsync = promisify(execFile);
|
|
12
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
13
|
+
const MODEL = String(process.env.IDEROUTER_BENCH_SINGLE_CHUNK_MODEL || "openai/text-embedding-3-small").trim();
|
|
14
|
+
const TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_SINGLE_CHUNK_TIMEOUT_MS || 60000);
|
|
15
|
+
const RELATIVE_PATH = "relay/common/relay_info.go";
|
|
16
|
+
|
|
17
|
+
function compactAlphaNum(value) {
|
|
18
|
+
return String(value || "").toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function snippetWindow(content, start, size) {
|
|
22
|
+
const safeStart = Math.max(0, start);
|
|
23
|
+
const safeEnd = Math.min(content.length, safeStart + Math.max(1, size));
|
|
24
|
+
return {
|
|
25
|
+
start: safeStart,
|
|
26
|
+
end: safeEnd,
|
|
27
|
+
content: content.slice(safeStart, safeEnd).trim(),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function buildLineOffsets(content) {
|
|
32
|
+
const offsets = [0];
|
|
33
|
+
for (let i = 0; i < content.length; i += 1) {
|
|
34
|
+
if (content.charCodeAt(i) === 10) offsets.push(i + 1);
|
|
35
|
+
}
|
|
36
|
+
return offsets;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function lineForOffsetFromIndex(lineOffsets, offset) {
|
|
40
|
+
let low = 0;
|
|
41
|
+
let high = lineOffsets.length - 1;
|
|
42
|
+
while (low <= high) {
|
|
43
|
+
const mid = (low + high) >> 1;
|
|
44
|
+
if (lineOffsets[mid] <= offset) {
|
|
45
|
+
low = mid + 1;
|
|
46
|
+
} else {
|
|
47
|
+
high = mid - 1;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return Math.max(1, high + 1);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function extractPrimaryChunk(content, relativePath) {
|
|
54
|
+
const compactContent = compactAlphaNum(content);
|
|
55
|
+
const anchorTerm = "streamsupportedchannels";
|
|
56
|
+
if (!compactContent.includes(anchorTerm)) {
|
|
57
|
+
throw new Error(`anchor term ${anchorTerm} not found in ${relativePath}`);
|
|
58
|
+
}
|
|
59
|
+
const rawLower = String(content || "").toLowerCase();
|
|
60
|
+
const first = rawLower.indexOf(anchorTerm);
|
|
61
|
+
const chunkSize = 520;
|
|
62
|
+
const window = snippetWindow(content, Math.max(0, first), chunkSize);
|
|
63
|
+
const lineOffsets = buildLineOffsets(content);
|
|
64
|
+
return {
|
|
65
|
+
relativePath,
|
|
66
|
+
startLine: lineForOffsetFromIndex(lineOffsets, window.start),
|
|
67
|
+
endLine: lineForOffsetFromIndex(lineOffsets, window.end),
|
|
68
|
+
content: window.content,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function postEmbeddingWithCurl({ endpoint, apiKey, model, input, timeoutMs }) {
|
|
73
|
+
const timeoutSeconds = Math.max(1, Math.ceil(timeoutMs / 1000));
|
|
74
|
+
const marker = "\n__IDEROUTER_HTTP_STATUS__:";
|
|
75
|
+
const payload = JSON.stringify({
|
|
76
|
+
model,
|
|
77
|
+
input: [input],
|
|
78
|
+
});
|
|
79
|
+
const args = [
|
|
80
|
+
"-sS",
|
|
81
|
+
"-X", "POST",
|
|
82
|
+
"--connect-timeout", String(Math.min(15, timeoutSeconds)),
|
|
83
|
+
"--max-time", String(timeoutSeconds),
|
|
84
|
+
"-H", "Content-Type: application/json",
|
|
85
|
+
"-H", `Authorization: Bearer ${apiKey}`,
|
|
86
|
+
"--data-binary", "@-",
|
|
87
|
+
"-w", `${marker}%{http_code}`,
|
|
88
|
+
endpoint,
|
|
89
|
+
];
|
|
90
|
+
const startedAt = Date.now();
|
|
91
|
+
const { stdout } = await execFileAsync("curl", args, {
|
|
92
|
+
input: payload,
|
|
93
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
94
|
+
});
|
|
95
|
+
const wallMs = Date.now() - startedAt;
|
|
96
|
+
const output = String(stdout || "");
|
|
97
|
+
const markerIndex = output.lastIndexOf(marker);
|
|
98
|
+
if (markerIndex < 0) {
|
|
99
|
+
throw new Error("curl did not return HTTP status marker");
|
|
100
|
+
}
|
|
101
|
+
const responseBody = output.slice(0, markerIndex);
|
|
102
|
+
const status = Number(output.slice(markerIndex + marker.length).trim());
|
|
103
|
+
return {
|
|
104
|
+
status,
|
|
105
|
+
wall_ms: wallMs,
|
|
106
|
+
response_preview: responseBody.slice(0, 500),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function main() {
|
|
111
|
+
const apiKey = process.env.IDEROUTER_API_KEY;
|
|
112
|
+
if (!apiKey) {
|
|
113
|
+
throw new Error("IDEROUTER_API_KEY is required");
|
|
114
|
+
}
|
|
115
|
+
const baseURL = process.env.IDEROUTER_BASE_URL || "https://cursor.iderouter.com/v1";
|
|
116
|
+
const endpoint = `${baseURL.replace(/\/+$/, "")}/embeddings`;
|
|
117
|
+
const snapshot = await createBenchmarkSnapshot(TARGET_PATH, process.env);
|
|
118
|
+
const filePath = path.join(snapshot.snapshotDir, RELATIVE_PATH);
|
|
119
|
+
try {
|
|
120
|
+
const content = await fs.readFile(filePath, "utf8");
|
|
121
|
+
const chunk = extractPrimaryChunk(content, RELATIVE_PATH);
|
|
122
|
+
const result = await postEmbeddingWithCurl({
|
|
123
|
+
endpoint,
|
|
124
|
+
apiKey,
|
|
125
|
+
model: MODEL,
|
|
126
|
+
input: chunk.content,
|
|
127
|
+
timeoutMs: TIMEOUT_MS,
|
|
128
|
+
});
|
|
129
|
+
process.stdout.write(`${JSON.stringify({
|
|
130
|
+
path: TARGET_PATH,
|
|
131
|
+
snapshot_path: snapshot.snapshotDir,
|
|
132
|
+
model: MODEL,
|
|
133
|
+
timeout_ms: TIMEOUT_MS,
|
|
134
|
+
chunk,
|
|
135
|
+
result,
|
|
136
|
+
}, null, 2)}\n`);
|
|
137
|
+
} finally {
|
|
138
|
+
await fs.rm(snapshot.snapshotDir, { recursive: true, force: true });
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
main().catch((error) => {
|
|
143
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
144
|
+
process.exitCode = 1;
|
|
145
|
+
});
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import { execFile } from "node:child_process";
|
|
5
|
+
import crypto from "node:crypto";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import process from "node:process";
|
|
8
|
+
import { promisify } from "node:util";
|
|
9
|
+
import { runIndexBenchmark } from "./lib/mcp-bench.mjs";
|
|
10
|
+
|
|
11
|
+
const execFileAsync = promisify(execFile);
|
|
12
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
13
|
+
const TIMEOUT_MS = Number(process.env.IDEROUTER_BENCH_TIMEOUT_MS || 30000);
|
|
14
|
+
const POLL_MS = Number(process.env.IDEROUTER_BENCH_POLL_MS || 500);
|
|
15
|
+
const DEFAULT_EXTENSIONS = new Set([
|
|
16
|
+
".c", ".cc", ".cpp", ".cs", ".css", ".go", ".graphql", ".h", ".hpp", ".html",
|
|
17
|
+
".java", ".js", ".json", ".jsx", ".kt", ".kts", ".md", ".mjs", ".php", ".proto",
|
|
18
|
+
".py", ".rb", ".rs", ".scss", ".sh", ".sql", ".swift", ".toml", ".ts", ".tsx",
|
|
19
|
+
".vue", ".xml", ".yaml", ".yml",
|
|
20
|
+
]);
|
|
21
|
+
const PREFERRED_MUTATION_PATHS = [
|
|
22
|
+
"relay/common/relay_info.go",
|
|
23
|
+
"relay/compatible_handler.go",
|
|
24
|
+
"service/user_memory.go",
|
|
25
|
+
"pkg/billingexpr/run.go",
|
|
26
|
+
"service/tiered_settle.go",
|
|
27
|
+
"relay/helper/price.go",
|
|
28
|
+
];
|
|
29
|
+
const EXTENSION_PRIORITY = new Map([
|
|
30
|
+
[".go", 0],
|
|
31
|
+
[".ts", 1],
|
|
32
|
+
[".tsx", 2],
|
|
33
|
+
[".js", 3],
|
|
34
|
+
[".jsx", 4],
|
|
35
|
+
[".json", 5],
|
|
36
|
+
[".sql", 6],
|
|
37
|
+
[".yaml", 7],
|
|
38
|
+
[".yml", 8],
|
|
39
|
+
[".md", 20],
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
function repoId(codebasePath) {
|
|
43
|
+
return crypto.createHash("sha256").update(path.resolve(codebasePath)).digest("hex").slice(0, 16);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function loadIndexedFiles() {
|
|
47
|
+
const metaPath = path.join(process.env.HOME, ".iderouter/index-mcp/indexes", `${repoId(TARGET_PATH)}.meta.json`);
|
|
48
|
+
const content = await fs.readFile(metaPath, "utf8");
|
|
49
|
+
const meta = JSON.parse(content);
|
|
50
|
+
return Array.isArray(meta.files) ? meta.files.map((file) => file.relativePath).filter(Boolean) : [];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function gitDirtyPaths() {
|
|
54
|
+
const { stdout } = await execFileAsync(
|
|
55
|
+
"git",
|
|
56
|
+
["-C", TARGET_PATH, "status", "--porcelain=v1", "-z", "--untracked-files=all"],
|
|
57
|
+
{ maxBuffer: 16 * 1024 * 1024 },
|
|
58
|
+
);
|
|
59
|
+
return new Set(
|
|
60
|
+
String(stdout || "")
|
|
61
|
+
.split("\0")
|
|
62
|
+
.filter(Boolean)
|
|
63
|
+
.map((entry) => entry.slice(3).trim())
|
|
64
|
+
.filter(Boolean),
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function selectMutationFile() {
|
|
69
|
+
if (process.env.IDEROUTER_BENCH_MUTATION_FILE) {
|
|
70
|
+
return path.join(TARGET_PATH, process.env.IDEROUTER_BENCH_MUTATION_FILE);
|
|
71
|
+
}
|
|
72
|
+
const indexedFiles = await loadIndexedFiles();
|
|
73
|
+
const dirtyPaths = await gitDirtyPaths();
|
|
74
|
+
const preferred = PREFERRED_MUTATION_PATHS.find((relativePath) => indexedFiles.includes(relativePath) && !dirtyPaths.has(relativePath));
|
|
75
|
+
if (preferred) return path.join(TARGET_PATH, preferred);
|
|
76
|
+
const candidate = indexedFiles
|
|
77
|
+
.filter((relativePath) => {
|
|
78
|
+
const ext = path.extname(relativePath);
|
|
79
|
+
return DEFAULT_EXTENSIONS.has(ext) && !dirtyPaths.has(relativePath);
|
|
80
|
+
})
|
|
81
|
+
.sort((a, b) => candidateScore(a) - candidateScore(b))[0];
|
|
82
|
+
if (!candidate) {
|
|
83
|
+
throw new Error("No clean indexed file available for benchmark-small-change");
|
|
84
|
+
}
|
|
85
|
+
return path.join(TARGET_PATH, candidate);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function candidateScore(relativePath) {
|
|
89
|
+
const ext = path.extname(relativePath);
|
|
90
|
+
let score = EXTENSION_PRIORITY.get(ext) ?? 50;
|
|
91
|
+
if (relativePath.startsWith(".github/")) score += 100;
|
|
92
|
+
if (relativePath.includes("/docs/") || relativePath.endsWith(".md")) score += 30;
|
|
93
|
+
return score;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function mutateBenchmarkFile(benchFile) {
|
|
97
|
+
const previous = await fs.readFile(benchFile, "utf8");
|
|
98
|
+
const next = `${previous}\n<!-- iderouter bench ${new Date().toISOString()} -->\n`;
|
|
99
|
+
await fs.writeFile(benchFile, next, "utf8");
|
|
100
|
+
return async () => {
|
|
101
|
+
await fs.writeFile(benchFile, previous, "utf8");
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function gitStatusForMutationFile(benchFile) {
|
|
106
|
+
const relativePath = path.relative(TARGET_PATH, benchFile);
|
|
107
|
+
const { stdout } = await execFileAsync(
|
|
108
|
+
"git",
|
|
109
|
+
["-C", TARGET_PATH, "status", "--porcelain=v1", "--untracked-files=all", "--", relativePath],
|
|
110
|
+
);
|
|
111
|
+
return String(stdout || "").trim();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function main() {
|
|
115
|
+
const benchFile = await selectMutationFile();
|
|
116
|
+
const restore = await mutateBenchmarkFile(benchFile);
|
|
117
|
+
try {
|
|
118
|
+
const gitStatus = await gitStatusForMutationFile(benchFile);
|
|
119
|
+
const result = await runIndexBenchmark({
|
|
120
|
+
targetPath: TARGET_PATH,
|
|
121
|
+
timeoutMs: TIMEOUT_MS,
|
|
122
|
+
pollMs: POLL_MS,
|
|
123
|
+
startArgs: { path: TARGET_PATH, wait: false, cloud: false },
|
|
124
|
+
});
|
|
125
|
+
process.stdout.write(
|
|
126
|
+
`${JSON.stringify(
|
|
127
|
+
{
|
|
128
|
+
...result.parsed,
|
|
129
|
+
mutation_file: path.relative(TARGET_PATH, benchFile),
|
|
130
|
+
preflight_git_status: gitStatus,
|
|
131
|
+
expected_dirty: Boolean(gitStatus),
|
|
132
|
+
observed_index_change: !/files_changed=0\b/.test(result.finalText),
|
|
133
|
+
},
|
|
134
|
+
null,
|
|
135
|
+
2,
|
|
136
|
+
)}\n`,
|
|
137
|
+
);
|
|
138
|
+
} finally {
|
|
139
|
+
await restore();
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
main().catch((error) => {
|
|
144
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
145
|
+
process.exitCode = 1;
|
|
146
|
+
});
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
const TARGET_PATH = path.resolve(process.argv[2] || process.cwd());
|
|
10
|
+
const SCRIPT_DIR = path.resolve("scripts");
|
|
11
|
+
|
|
12
|
+
async function runScript(scriptName) {
|
|
13
|
+
const scriptPath = path.join(SCRIPT_DIR, scriptName);
|
|
14
|
+
const { stdout } = await execFileAsync(process.execPath, [scriptPath, TARGET_PATH], {
|
|
15
|
+
cwd: process.cwd(),
|
|
16
|
+
env: process.env,
|
|
17
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
18
|
+
});
|
|
19
|
+
return JSON.parse(String(stdout || "").trim());
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function main() {
|
|
23
|
+
const priority = await runScript("benchmark-priority-ready-search.mjs");
|
|
24
|
+
const resume = await runScript("benchmark-background-fine-resume.mjs");
|
|
25
|
+
const priorityTop1Hits = Array.isArray(priority?.searches)
|
|
26
|
+
? priority.searches.filter((item) => item.top1_hit).length
|
|
27
|
+
: 0;
|
|
28
|
+
const priorityTotalQueries = Array.isArray(priority?.searches) ? priority.searches.length : 0;
|
|
29
|
+
const priorityHitRatio = priorityTotalQueries > 0 ? priorityTop1Hits / priorityTotalQueries : 0;
|
|
30
|
+
const priorityReadyMs = Number(priority?.priority_ready?.parsed?.priority_ready_ms || 0);
|
|
31
|
+
const priorityReadyAnchorCoverage = priority?.priority_ready_anchor_coverage || {};
|
|
32
|
+
const resumeSemanticFiles = Number(resume?.summary?.cumulative_file_count || 0);
|
|
33
|
+
const resumeRoundsRun = Number(resume?.summary?.rounds_run || 0);
|
|
34
|
+
const resumeRounds = Array.isArray(resume?.rounds) ? resume.rounds : [];
|
|
35
|
+
const resumeDeferContinuity = resumeRoundsRun > 1
|
|
36
|
+
? resumeRounds.slice(1).filter((round) => Number(round?.fine_semantic_files || 0) > Number(resumeRounds[round.round - 2]?.fine_semantic_files || 0)).length / Math.max(1, resumeRoundsRun - 1)
|
|
37
|
+
: 0;
|
|
38
|
+
const priorityScore = Math.max(0, Math.min(1, priorityReadyMs > 0 ? 1 - ((priorityReadyMs - 6000) / 12000) : 0));
|
|
39
|
+
const coverageScore = Math.max(0, Math.min(1, resumeSemanticFiles / 80));
|
|
40
|
+
const compositeScore = Number(((priorityHitRatio * 0.4) + (priorityScore * 0.25) + (coverageScore * 0.2) + (resumeDeferContinuity * 0.15)).toFixed(3));
|
|
41
|
+
const compositeBand = compositeScore >= 0.9
|
|
42
|
+
? "strong"
|
|
43
|
+
: compositeScore >= 0.8
|
|
44
|
+
? "good"
|
|
45
|
+
: compositeScore >= 0.65
|
|
46
|
+
? "developing"
|
|
47
|
+
: "weak";
|
|
48
|
+
|
|
49
|
+
process.stdout.write(
|
|
50
|
+
`${JSON.stringify(
|
|
51
|
+
{
|
|
52
|
+
path: TARGET_PATH,
|
|
53
|
+
priority_ready_ms: priorityReadyMs || null,
|
|
54
|
+
priority_ready_wall_ms: priority?.priority_ready?.wallMs ?? null,
|
|
55
|
+
priority_top1_hits: priorityTop1Hits,
|
|
56
|
+
priority_total_queries: priorityTotalQueries,
|
|
57
|
+
priority_ready_anchor_completed: Number(priorityReadyAnchorCoverage.completed || 0),
|
|
58
|
+
priority_ready_anchor_total: Number(priorityReadyAnchorCoverage.total || 0),
|
|
59
|
+
priority_ready_anchor_completed_paths: Array.isArray(priorityReadyAnchorCoverage.completed_paths) ? priorityReadyAnchorCoverage.completed_paths : [],
|
|
60
|
+
priority_ready_anchor_pending_paths: Array.isArray(priorityReadyAnchorCoverage.pending_paths) ? priorityReadyAnchorCoverage.pending_paths : [],
|
|
61
|
+
resume_rounds_run: resumeRoundsRun,
|
|
62
|
+
resume_cursor_sequence: resume?.summary?.cursor_sequence ?? [],
|
|
63
|
+
resume_cumulative_semantic_files: resumeSemanticFiles,
|
|
64
|
+
composite_score: compositeScore,
|
|
65
|
+
composite_band: compositeBand,
|
|
66
|
+
resume_rounds: resumeRounds
|
|
67
|
+
? resumeRounds.map((round) => ({
|
|
68
|
+
round: round.round,
|
|
69
|
+
fine_semantic_files: round.fine_semantic_files,
|
|
70
|
+
fine_semantic_chunks: round.fine_semantic_chunks,
|
|
71
|
+
background_bucket_cursor: round.background_bucket_cursor,
|
|
72
|
+
priority_ready_anchor_coverage: round.priority_ready_anchor_coverage,
|
|
73
|
+
embedding_requests: round?.diagnostics?.embedding_requests ?? 0,
|
|
74
|
+
embedding_request_ms_total: round?.diagnostics?.embedding_request_ms_total ?? 0,
|
|
75
|
+
deferred: round.deferred,
|
|
76
|
+
}))
|
|
77
|
+
: [],
|
|
78
|
+
},
|
|
79
|
+
null,
|
|
80
|
+
2,
|
|
81
|
+
)}\n`,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
main().catch((error) => {
|
|
86
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
87
|
+
process.exitCode = 1;
|
|
88
|
+
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export function hasAdvanced(beforeState, afterState) {
|
|
2
|
+
return (
|
|
3
|
+
Number(afterState.fine_semantic_files || 0) > Number(beforeState.fine_semantic_files || 0) ||
|
|
4
|
+
Number(afterState.fine_semantic_chunks || 0) > Number(beforeState.fine_semantic_chunks || 0) ||
|
|
5
|
+
String(afterState.background_bucket_cursor || "") !== String(beforeState.background_bucket_cursor || "") ||
|
|
6
|
+
(
|
|
7
|
+
Number(afterState.total_chunks || 0) > 0 &&
|
|
8
|
+
String(afterState.job_status || "") === "indexing" &&
|
|
9
|
+
String(afterState.current_step || "").toLowerCase().includes("embedding batch")
|
|
10
|
+
)
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function isFullReadyState(state) {
|
|
15
|
+
const modes = String(state?.searchable_modes || "");
|
|
16
|
+
return modes.includes("full_semantic") || /^Indexed\b/.test(String(state?.status_text || ""));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function isCoarseDeferredReady(state) {
|
|
20
|
+
const text = String(state?.status_text || "");
|
|
21
|
+
return /^Coarse lexical index is ready\b/.test(text) &&
|
|
22
|
+
text.includes("semantic indexing is temporarily deferred");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function continuationAlreadyMaterialized(firstPhaseState) {
|
|
26
|
+
return (
|
|
27
|
+
isFullReadyState(firstPhaseState) ||
|
|
28
|
+
(
|
|
29
|
+
Number(firstPhaseState.continuation_pass_count || 0) > 0 &&
|
|
30
|
+
Number(firstPhaseState.fine_semantic_files || 0) > 3 &&
|
|
31
|
+
Number(firstPhaseState.fine_semantic_chunks || 0) > 3
|
|
32
|
+
)
|
|
33
|
+
);
|
|
34
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
const PACKS = {
|
|
2
|
+
core: [
|
|
3
|
+
{
|
|
4
|
+
name: "user_memory_flow",
|
|
5
|
+
query: "user memory flow",
|
|
6
|
+
expect_any: [
|
|
7
|
+
"service/user_memory.go",
|
|
8
|
+
"relay/compatible_handler.go",
|
|
9
|
+
"relay/claude_handler.go",
|
|
10
|
+
],
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
name: "remove_disabled_fields",
|
|
14
|
+
query: "relay_info compatible_handler remove_disabled_request_fields safety_identifier include_obfuscation",
|
|
15
|
+
expect_any: [
|
|
16
|
+
"relay/common/relay_info.go",
|
|
17
|
+
"relay/compatible_handler.go",
|
|
18
|
+
],
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
name: "billing_execution_path",
|
|
22
|
+
query: "billing backend execution path RunExprWithRequest model pricing editor",
|
|
23
|
+
expect_any: [
|
|
24
|
+
"pkg/billingexpr/run.go",
|
|
25
|
+
"relay/helper/price.go",
|
|
26
|
+
"service/tiered_settle.go",
|
|
27
|
+
],
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
name: "privacy_request_flow",
|
|
31
|
+
query: "relay request filtering privacy fields flow",
|
|
32
|
+
expect_any: [
|
|
33
|
+
"relay/common/relay_info.go",
|
|
34
|
+
"relay/compatible_handler.go",
|
|
35
|
+
],
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
name: "stream_options_support",
|
|
39
|
+
query: "stream options support handling",
|
|
40
|
+
expect_any: [
|
|
41
|
+
"relay/compatible_handler.go",
|
|
42
|
+
"relay/common/relay_info.go",
|
|
43
|
+
],
|
|
44
|
+
},
|
|
45
|
+
],
|
|
46
|
+
real_user: [
|
|
47
|
+
{
|
|
48
|
+
name: "relay_architecture",
|
|
49
|
+
query: "relay architecture overview request flow controller service model",
|
|
50
|
+
expect_any: [
|
|
51
|
+
"controller/relay.go",
|
|
52
|
+
"relay/compatible_handler.go",
|
|
53
|
+
"relay/common/relay_info.go",
|
|
54
|
+
],
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
name: "billing_settlement_chain",
|
|
58
|
+
query: "how tiered billing settlement works quota pricing chain",
|
|
59
|
+
expect_any: [
|
|
60
|
+
"pkg/billingexpr/run.go",
|
|
61
|
+
"service/tiered_settle.go",
|
|
62
|
+
"relay/helper/price.go",
|
|
63
|
+
],
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
name: "user_memory_entrypoints",
|
|
67
|
+
query: "where user memory gets injected into requests and prompts",
|
|
68
|
+
expect_any: [
|
|
69
|
+
"service/user_memory.go",
|
|
70
|
+
"relay/compatible_handler.go",
|
|
71
|
+
"controller/user_memory.go",
|
|
72
|
+
],
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
name: "stream_options_definitions",
|
|
76
|
+
query: "which files define and force stream options support",
|
|
77
|
+
expect_any: [
|
|
78
|
+
"relay/compatible_handler.go",
|
|
79
|
+
"relay/common/relay_info.go",
|
|
80
|
+
"relay/channel/ali/adaptor.go",
|
|
81
|
+
],
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
name: "privacy_field_filtering",
|
|
85
|
+
query: "how disabled request fields privacy identifiers are filtered before upstream relay",
|
|
86
|
+
expect_any: [
|
|
87
|
+
"relay/common/relay_info.go",
|
|
88
|
+
"relay/compatible_handler.go",
|
|
89
|
+
],
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
name: "agent_config_path",
|
|
93
|
+
query: "where agent defaults and model configuration are defined",
|
|
94
|
+
expect_any: [
|
|
95
|
+
"setting/iderouter_setting/agent.go",
|
|
96
|
+
"dto/iderouter_agent.go",
|
|
97
|
+
"controller/iderouter.go",
|
|
98
|
+
],
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
name: "index_mcp_config",
|
|
102
|
+
query: "where index mcp strategy and installation config are built",
|
|
103
|
+
expect_any: [
|
|
104
|
+
"service/iderouter_index.go",
|
|
105
|
+
"iderouter_frontend/lib/iderouter-api.ts",
|
|
106
|
+
"iderouter_index_mcp/src/index.js",
|
|
107
|
+
],
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
export function listBenchmarkQueryPackNames() {
|
|
113
|
+
return Object.keys(PACKS);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function getBenchmarkQueryPack(name = "core") {
|
|
117
|
+
const key = String(name || "core").trim().toLowerCase();
|
|
118
|
+
const pack = PACKS[key];
|
|
119
|
+
if (!pack) {
|
|
120
|
+
throw new Error(`Unknown benchmark query pack: ${name}`);
|
|
121
|
+
}
|
|
122
|
+
return pack.map((item) => ({ ...item, expect_any: [...item.expect_any] }));
|
|
123
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
|
|
7
|
+
const execFileAsync = promisify(execFile);
|
|
8
|
+
|
|
9
|
+
export function resolveBenchmarkSnapshotMode(env = process.env) {
|
|
10
|
+
const raw = String(env?.IDEROUTER_BENCH_SNAPSHOT_MODE || "").trim().toLowerCase();
|
|
11
|
+
if (raw === "head" || raw === "clean_head" || raw === "clean-head") return "head";
|
|
12
|
+
return "working_tree";
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function createSnapshotPlan(targetPath, env = process.env) {
|
|
16
|
+
const mode = resolveBenchmarkSnapshotMode(env);
|
|
17
|
+
return {
|
|
18
|
+
targetPath,
|
|
19
|
+
mode,
|
|
20
|
+
label: mode,
|
|
21
|
+
tempPrefix: mode === "head" ? "iderouter-bench-head-" : "iderouter-bench-worktree-",
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function splitNullTerminated(stdout) {
|
|
26
|
+
return String(stdout || "")
|
|
27
|
+
.split("\0")
|
|
28
|
+
.map((item) => item.trim())
|
|
29
|
+
.filter(Boolean);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function gitNullList(sourcePath, args) {
|
|
33
|
+
const { stdout } = await execFileAsync("git", ["-C", sourcePath, ...args], {
|
|
34
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
35
|
+
});
|
|
36
|
+
return splitNullTerminated(stdout);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function exportTrackedFilesFromIndex(sourcePath, destinationPath) {
|
|
40
|
+
await execFileAsync("git", [
|
|
41
|
+
"-C",
|
|
42
|
+
sourcePath,
|
|
43
|
+
"checkout-index",
|
|
44
|
+
"-a",
|
|
45
|
+
"-f",
|
|
46
|
+
`--prefix=${destinationPath}/`,
|
|
47
|
+
], {
|
|
48
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function overlayWorkingTreeFiles(sourcePath, destinationPath, relativePaths) {
|
|
53
|
+
for (const relativePath of relativePaths) {
|
|
54
|
+
const sourceFile = path.join(sourcePath, relativePath);
|
|
55
|
+
const destinationFile = path.join(destinationPath, relativePath);
|
|
56
|
+
await fs.mkdir(path.dirname(destinationFile), { recursive: true });
|
|
57
|
+
await fs.cp(sourceFile, destinationFile, {
|
|
58
|
+
force: true,
|
|
59
|
+
recursive: false,
|
|
60
|
+
verbatimSymlinks: true,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function removeDeletedWorkingTreeFiles(destinationPath, relativePaths) {
|
|
66
|
+
for (const relativePath of relativePaths) {
|
|
67
|
+
await fs.rm(path.join(destinationPath, relativePath), { force: true, recursive: true });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function copyWorkingTree(sourcePath, destinationPath) {
|
|
72
|
+
await exportTrackedFilesFromIndex(sourcePath, destinationPath);
|
|
73
|
+
const [modifiedTrackedFiles, untrackedFiles, deletedTrackedFiles] = await Promise.all([
|
|
74
|
+
gitNullList(sourcePath, ["ls-files", "-m", "-z"]),
|
|
75
|
+
gitNullList(sourcePath, ["ls-files", "-o", "--exclude-standard", "-z"]),
|
|
76
|
+
gitNullList(sourcePath, ["ls-files", "-d", "-z"]),
|
|
77
|
+
]);
|
|
78
|
+
await overlayWorkingTreeFiles(
|
|
79
|
+
sourcePath,
|
|
80
|
+
destinationPath,
|
|
81
|
+
Array.from(new Set([...modifiedTrackedFiles, ...untrackedFiles])),
|
|
82
|
+
);
|
|
83
|
+
await removeDeletedWorkingTreeFiles(destinationPath, deletedTrackedFiles);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function cloneHeadSnapshot(sourcePath, destinationPath) {
|
|
87
|
+
await execFileAsync("git", ["clone", "--shared", "--no-checkout", sourcePath, destinationPath], {
|
|
88
|
+
cwd: sourcePath,
|
|
89
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
90
|
+
});
|
|
91
|
+
await execFileAsync("git", ["-C", destinationPath, "checkout", "-f", "HEAD"], {
|
|
92
|
+
cwd: sourcePath,
|
|
93
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export async function createBenchmarkSnapshot(targetPath, env = process.env) {
|
|
98
|
+
const plan = createSnapshotPlan(targetPath, env);
|
|
99
|
+
const snapshotDir = await fs.mkdtemp(path.join(os.tmpdir(), plan.tempPrefix));
|
|
100
|
+
if (plan.mode === "head") {
|
|
101
|
+
await cloneHeadSnapshot(targetPath, snapshotDir);
|
|
102
|
+
} else {
|
|
103
|
+
await copyWorkingTree(targetPath, snapshotDir);
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
...plan,
|
|
107
|
+
snapshotDir,
|
|
108
|
+
};
|
|
109
|
+
}
|