opencode-codebase-index 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +5 -3
- package/dist/cli.cjs +2400 -1786
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2404 -1790
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +454 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +454 -10
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +459 -69
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +459 -69
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +4 -2
package/dist/cli.cjs
CHANGED
|
@@ -1082,6 +1082,11 @@ function metricDelta(current, baseline) {
|
|
|
1082
1082
|
};
|
|
1083
1083
|
}
|
|
1084
1084
|
function compareSummaries(current, baseline, againstPath) {
|
|
1085
|
+
if (current.datasetName !== baseline.datasetName || current.datasetVersion !== baseline.datasetVersion || current.queryCount !== baseline.queryCount) {
|
|
1086
|
+
throw new Error(
|
|
1087
|
+
`Cannot compare incompatible evaluation datasets: current=${current.datasetName}@${current.datasetVersion} (${current.queryCount} queries), baseline=${baseline.datasetName}@${baseline.datasetVersion} (${baseline.queryCount} queries) at ${againstPath}`
|
|
1088
|
+
);
|
|
1089
|
+
}
|
|
1085
1090
|
return {
|
|
1086
1091
|
againstPath,
|
|
1087
1092
|
deltas: {
|
|
@@ -1103,6 +1108,34 @@ function compareSummaries(current, baseline, againstPath) {
|
|
|
1103
1108
|
estimatedCostUsd: metricDelta(
|
|
1104
1109
|
current.metrics.embedding.estimatedCostUsd,
|
|
1105
1110
|
baseline.metrics.embedding.estimatedCostUsd
|
|
1111
|
+
),
|
|
1112
|
+
contextResponseTokensAverage: metricDelta(
|
|
1113
|
+
current.metrics.contextEfficiency.responseTokens.average,
|
|
1114
|
+
baseline.metrics.contextEfficiency.responseTokens.average
|
|
1115
|
+
),
|
|
1116
|
+
contextResponseTokensP95: metricDelta(
|
|
1117
|
+
current.metrics.contextEfficiency.responseTokens.p95,
|
|
1118
|
+
baseline.metrics.contextEfficiency.responseTokens.p95
|
|
1119
|
+
),
|
|
1120
|
+
contextResponseTokensMax: metricDelta(
|
|
1121
|
+
current.metrics.contextEfficiency.responseTokens.max,
|
|
1122
|
+
baseline.metrics.contextEfficiency.responseTokens.max
|
|
1123
|
+
),
|
|
1124
|
+
contextDuplicateCandidateRatio: metricDelta(
|
|
1125
|
+
current.metrics.contextEfficiency.duplicateCandidateRatio,
|
|
1126
|
+
baseline.metrics.contextEfficiency.duplicateCandidateRatio
|
|
1127
|
+
),
|
|
1128
|
+
contextSelectedFileRatio: metricDelta(
|
|
1129
|
+
current.metrics.contextEfficiency.selectedFileRatio,
|
|
1130
|
+
baseline.metrics.contextEfficiency.selectedFileRatio
|
|
1131
|
+
),
|
|
1132
|
+
contextHitAt5Per1kResponseTokens: metricDelta(
|
|
1133
|
+
current.metrics.contextEfficiency.hitAt5Per1kResponseTokens,
|
|
1134
|
+
baseline.metrics.contextEfficiency.hitAt5Per1kResponseTokens
|
|
1135
|
+
),
|
|
1136
|
+
contextMrrAt10Per1kResponseTokens: metricDelta(
|
|
1137
|
+
current.metrics.contextEfficiency.mrrAt10Per1kResponseTokens,
|
|
1138
|
+
baseline.metrics.contextEfficiency.mrrAt10Per1kResponseTokens
|
|
1106
1139
|
)
|
|
1107
1140
|
}
|
|
1108
1141
|
};
|
|
@@ -1140,6 +1173,25 @@ function validateSummary(summary, summaryPath, options) {
|
|
|
1140
1173
|
assertFiniteNumber(summary.metrics.latencyMs.p99, `${summaryPath}.metrics.latencyMs.p99`);
|
|
1141
1174
|
assertFiniteNumber(summary.metrics.embedding.callCount, `${summaryPath}.metrics.embedding.callCount`);
|
|
1142
1175
|
assertFiniteNumber(summary.metrics.embedding.estimatedCostUsd, `${summaryPath}.metrics.embedding.estimatedCostUsd`);
|
|
1176
|
+
if (!summary.metrics.contextEfficiency) {
|
|
1177
|
+
summary.metrics.contextEfficiency = {
|
|
1178
|
+
queryCount: 0,
|
|
1179
|
+
responseTokens: { total: 0, average: 0, p95: 0, max: 0 },
|
|
1180
|
+
duplicateCandidateRatio: 0,
|
|
1181
|
+
selectedFileRatio: 0,
|
|
1182
|
+
hitAt5Per1kResponseTokens: 0,
|
|
1183
|
+
mrrAt10Per1kResponseTokens: 0
|
|
1184
|
+
};
|
|
1185
|
+
}
|
|
1186
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.queryCount, `${summaryPath}.metrics.contextEfficiency.queryCount`);
|
|
1187
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.total, `${summaryPath}.metrics.contextEfficiency.responseTokens.total`);
|
|
1188
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.average, `${summaryPath}.metrics.contextEfficiency.responseTokens.average`);
|
|
1189
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.p95, `${summaryPath}.metrics.contextEfficiency.responseTokens.p95`);
|
|
1190
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.max, `${summaryPath}.metrics.contextEfficiency.responseTokens.max`);
|
|
1191
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.duplicateCandidateRatio, `${summaryPath}.metrics.contextEfficiency.duplicateCandidateRatio`);
|
|
1192
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.selectedFileRatio, `${summaryPath}.metrics.contextEfficiency.selectedFileRatio`);
|
|
1193
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.hitAt5Per1kResponseTokens, `${summaryPath}.metrics.contextEfficiency.hitAt5Per1kResponseTokens`);
|
|
1194
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.mrrAt10Per1kResponseTokens, `${summaryPath}.metrics.contextEfficiency.mrrAt10Per1kResponseTokens`);
|
|
1143
1195
|
return summary;
|
|
1144
1196
|
}
|
|
1145
1197
|
function formatPct(value) {
|
|
@@ -1214,6 +1266,15 @@ function createSummaryMarkdown(summary, comparison, gate, sweep) {
|
|
|
1214
1266
|
lines.push(`| Embedding calls | ${summary.metrics.embedding.callCount} |`);
|
|
1215
1267
|
lines.push(`| Embedding tokens | ${summary.metrics.tokenEstimate.embeddingTokensUsed} |`);
|
|
1216
1268
|
lines.push(`| Estimated embedding cost | ${formatUsd(summary.metrics.embedding.estimatedCostUsd)} |`);
|
|
1269
|
+
lines.push(`| Context queries | ${summary.metrics.contextEfficiency.queryCount} |`);
|
|
1270
|
+
lines.push(`| Context response tokens total | ${summary.metrics.contextEfficiency.responseTokens.total} |`);
|
|
1271
|
+
lines.push(`| Context response tokens avg | ${summary.metrics.contextEfficiency.responseTokens.average.toFixed(1)} |`);
|
|
1272
|
+
lines.push(`| Context response tokens p95 | ${summary.metrics.contextEfficiency.responseTokens.p95.toFixed(1)} |`);
|
|
1273
|
+
lines.push(`| Context response tokens max | ${summary.metrics.contextEfficiency.responseTokens.max.toFixed(1)} |`);
|
|
1274
|
+
lines.push(`| Context duplicate candidate ratio | ${formatPct(summary.metrics.contextEfficiency.duplicateCandidateRatio)} |`);
|
|
1275
|
+
lines.push(`| Context selected-file ratio | ${formatPct(summary.metrics.contextEfficiency.selectedFileRatio)} |`);
|
|
1276
|
+
lines.push(`| Context Hit@5 / 1k response tokens | ${summary.metrics.contextEfficiency.hitAt5Per1kResponseTokens.toFixed(4)} |`);
|
|
1277
|
+
lines.push(`| Context MRR@10 / 1k response tokens | ${summary.metrics.contextEfficiency.mrrAt10Per1kResponseTokens.toFixed(4)} |`);
|
|
1217
1278
|
lines.push("");
|
|
1218
1279
|
lines.push("## Failure Buckets");
|
|
1219
1280
|
lines.push("");
|
|
@@ -1257,6 +1318,27 @@ function createSummaryMarkdown(summary, comparison, gate, sweep) {
|
|
|
1257
1318
|
lines.push(
|
|
1258
1319
|
`| p95 latency (ms) | ${comparison.deltas.latencyP95Ms.baseline.toFixed(3)} | ${comparison.deltas.latencyP95Ms.current.toFixed(3)} | ${signed(comparison.deltas.latencyP95Ms.absolute, 3)} |`
|
|
1259
1320
|
);
|
|
1321
|
+
lines.push(
|
|
1322
|
+
`| Context response tokens avg | ${comparison.deltas.contextResponseTokensAverage.baseline.toFixed(1)} | ${comparison.deltas.contextResponseTokensAverage.current.toFixed(1)} | ${signed(comparison.deltas.contextResponseTokensAverage.absolute, 1)} |`
|
|
1323
|
+
);
|
|
1324
|
+
lines.push(
|
|
1325
|
+
`| Context response tokens p95 | ${comparison.deltas.contextResponseTokensP95.baseline.toFixed(1)} | ${comparison.deltas.contextResponseTokensP95.current.toFixed(1)} | ${signed(comparison.deltas.contextResponseTokensP95.absolute, 1)} |`
|
|
1326
|
+
);
|
|
1327
|
+
lines.push(
|
|
1328
|
+
`| Context response tokens max | ${comparison.deltas.contextResponseTokensMax.baseline.toFixed(1)} | ${comparison.deltas.contextResponseTokensMax.current.toFixed(1)} | ${signed(comparison.deltas.contextResponseTokensMax.absolute, 1)} |`
|
|
1329
|
+
);
|
|
1330
|
+
lines.push(
|
|
1331
|
+
`| Context duplicate candidate ratio | ${formatPct(comparison.deltas.contextDuplicateCandidateRatio.baseline)} | ${formatPct(comparison.deltas.contextDuplicateCandidateRatio.current)} | ${signed(comparison.deltas.contextDuplicateCandidateRatio.absolute)} |`
|
|
1332
|
+
);
|
|
1333
|
+
lines.push(
|
|
1334
|
+
`| Context selected-file ratio | ${formatPct(comparison.deltas.contextSelectedFileRatio.baseline)} | ${formatPct(comparison.deltas.contextSelectedFileRatio.current)} | ${signed(comparison.deltas.contextSelectedFileRatio.absolute)} |`
|
|
1335
|
+
);
|
|
1336
|
+
lines.push(
|
|
1337
|
+
`| Context Hit@5 / 1k response tokens | ${comparison.deltas.contextHitAt5Per1kResponseTokens.baseline.toFixed(4)} | ${comparison.deltas.contextHitAt5Per1kResponseTokens.current.toFixed(4)} | ${signed(comparison.deltas.contextHitAt5Per1kResponseTokens.absolute)} |`
|
|
1338
|
+
);
|
|
1339
|
+
lines.push(
|
|
1340
|
+
`| Context MRR@10 / 1k response tokens | ${comparison.deltas.contextMrrAt10Per1kResponseTokens.baseline.toFixed(4)} | ${comparison.deltas.contextMrrAt10Per1kResponseTokens.current.toFixed(4)} | ${signed(comparison.deltas.contextMrrAt10Per1kResponseTokens.absolute)} |`
|
|
1341
|
+
);
|
|
1260
1342
|
lines.push("");
|
|
1261
1343
|
}
|
|
1262
1344
|
if (gate) {
|
|
@@ -1303,8 +1385,8 @@ function buildPerQueryArtifact(perQuery) {
|
|
|
1303
1385
|
}
|
|
1304
1386
|
|
|
1305
1387
|
// src/eval/runner.ts
|
|
1306
|
-
var
|
|
1307
|
-
var
|
|
1388
|
+
var import_fs14 = require("fs");
|
|
1389
|
+
var path19 = __toESM(require("path"), 1);
|
|
1308
1390
|
var import_perf_hooks2 = require("perf_hooks");
|
|
1309
1391
|
|
|
1310
1392
|
// src/indexer/index.ts
|
|
@@ -5047,7 +5129,12 @@ function sameReclaimOwner(left, right) {
|
|
|
5047
5129
|
}
|
|
5048
5130
|
function publishJsonDirectory(finalPath, value) {
|
|
5049
5131
|
const candidatePath = `${finalPath}.candidate.${process.pid}.${(0, import_crypto.randomUUID)()}`;
|
|
5050
|
-
|
|
5132
|
+
try {
|
|
5133
|
+
(0, import_fs7.mkdirSync)(candidatePath, { mode: 448 });
|
|
5134
|
+
} catch (error) {
|
|
5135
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5136
|
+
throw error;
|
|
5137
|
+
}
|
|
5051
5138
|
try {
|
|
5052
5139
|
(0, import_fs7.writeFileSync)(path11.join(candidatePath, OWNER_FILE_NAME), JSON.stringify(value), {
|
|
5053
5140
|
encoding: "utf-8",
|
|
@@ -5060,8 +5147,12 @@ function publishJsonDirectory(finalPath, value) {
|
|
|
5060
5147
|
return true;
|
|
5061
5148
|
} catch (error) {
|
|
5062
5149
|
if ((0, import_fs7.existsSync)(finalPath)) return false;
|
|
5150
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5063
5151
|
throw error;
|
|
5064
5152
|
}
|
|
5153
|
+
} catch (error) {
|
|
5154
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5155
|
+
throw error;
|
|
5065
5156
|
} finally {
|
|
5066
5157
|
if ((0, import_fs7.existsSync)(candidatePath)) (0, import_fs7.rmSync)(candidatePath, { recursive: true, force: true });
|
|
5067
5158
|
}
|
|
@@ -6634,6 +6725,27 @@ function mergeTieredResults(symbolLane, hybridLane, limit) {
|
|
|
6634
6725
|
}
|
|
6635
6726
|
return out;
|
|
6636
6727
|
}
|
|
6728
|
+
function selectChunksWithFileCoverage(chunks, limit) {
|
|
6729
|
+
if (limit <= 0 || chunks.length === 0) {
|
|
6730
|
+
return [];
|
|
6731
|
+
}
|
|
6732
|
+
if (chunks.length <= limit) {
|
|
6733
|
+
return chunks;
|
|
6734
|
+
}
|
|
6735
|
+
if (limit === 1) {
|
|
6736
|
+
return [chunks[Math.floor((chunks.length - 1) / 2)]];
|
|
6737
|
+
}
|
|
6738
|
+
const selected = [];
|
|
6739
|
+
for (let index = 0; index < limit; index++) {
|
|
6740
|
+
const sourceIndex = Math.round(index * (chunks.length - 1) / (limit - 1));
|
|
6741
|
+
selected.push(chunks[sourceIndex]);
|
|
6742
|
+
}
|
|
6743
|
+
return selected;
|
|
6744
|
+
}
|
|
6745
|
+
function selectIndexableChunks(chunks, limit, semanticOnly) {
|
|
6746
|
+
const indexableChunks = semanticOnly ? chunks.filter((chunk) => chunk.chunkType !== "other") : chunks;
|
|
6747
|
+
return selectChunksWithFileCoverage(indexableChunks, limit);
|
|
6748
|
+
}
|
|
6637
6749
|
function matchesSearchFilters(candidate, options, minScore) {
|
|
6638
6750
|
if (candidate.score < minScore) return false;
|
|
6639
6751
|
if (options?.fileType) {
|
|
@@ -8372,7 +8484,6 @@ var Indexer = class {
|
|
|
8372
8484
|
const relativePath = path12.relative(this.projectRoot, parsed.path);
|
|
8373
8485
|
stats.parseFailures.push(relativePath);
|
|
8374
8486
|
}
|
|
8375
|
-
let fileChunkCount = 0;
|
|
8376
8487
|
let chunksToProcess = parsed.chunks;
|
|
8377
8488
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
8378
8489
|
const changedFile = changedFiles.find((f) => f.path === parsed.path);
|
|
@@ -8381,13 +8492,12 @@ var Indexer = class {
|
|
|
8381
8492
|
chunksToProcess = textChunks;
|
|
8382
8493
|
}
|
|
8383
8494
|
}
|
|
8495
|
+
chunksToProcess = selectIndexableChunks(
|
|
8496
|
+
chunksToProcess,
|
|
8497
|
+
this.config.indexing.maxChunksPerFile,
|
|
8498
|
+
this.config.indexing.semanticOnly
|
|
8499
|
+
);
|
|
8384
8500
|
for (const chunk of chunksToProcess) {
|
|
8385
|
-
if (fileChunkCount >= this.config.indexing.maxChunksPerFile) {
|
|
8386
|
-
break;
|
|
8387
|
-
}
|
|
8388
|
-
if (this.config.indexing.semanticOnly && chunk.chunkType === "other") {
|
|
8389
|
-
continue;
|
|
8390
|
-
}
|
|
8391
8501
|
const id = generateChunkId(parsed.path, chunk);
|
|
8392
8502
|
const contentHash = generateChunkHash(chunk);
|
|
8393
8503
|
const existingContentHash = existingChunks.get(id);
|
|
@@ -8411,7 +8521,6 @@ var Indexer = class {
|
|
|
8411
8521
|
blameSummary: blameMetadata.blameSummary
|
|
8412
8522
|
});
|
|
8413
8523
|
if (existingContentHash === contentHash) {
|
|
8414
|
-
fileChunkCount++;
|
|
8415
8524
|
continue;
|
|
8416
8525
|
}
|
|
8417
8526
|
const texts = createEmbeddingTexts(chunk, parsed.path, getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)).map((text) => ({
|
|
@@ -8436,7 +8545,6 @@ var Indexer = class {
|
|
|
8436
8545
|
contentHash,
|
|
8437
8546
|
metadata
|
|
8438
8547
|
});
|
|
8439
|
-
fileChunkCount++;
|
|
8440
8548
|
}
|
|
8441
8549
|
}
|
|
8442
8550
|
const retryableFailedChunks = this.collectRetryableFailedChunks(
|
|
@@ -9985,1955 +10093,2500 @@ var Indexer = class {
|
|
|
9985
10093
|
}
|
|
9986
10094
|
};
|
|
9987
10095
|
|
|
9988
|
-
// src/
|
|
9989
|
-
|
|
9990
|
-
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
|
|
10003
|
-
|
|
10004
|
-
|
|
10005
|
-
if (thresholds.minRawDistinctTop3Ratio !== void 0 && summary.metrics.rawDistinctTop3Ratio < thresholds.minRawDistinctTop3Ratio) {
|
|
10006
|
-
violations.push({
|
|
10007
|
-
metric: "minRawDistinctTop3Ratio",
|
|
10008
|
-
message: `Raw Distinct Top@3 ${summary.metrics.rawDistinctTop3Ratio.toFixed(4)} is below minimum ${thresholds.minRawDistinctTop3Ratio.toFixed(4)}`
|
|
10009
|
-
});
|
|
10096
|
+
// src/tools/operations.ts
|
|
10097
|
+
var import_fs11 = require("fs");
|
|
10098
|
+
var path17 = __toESM(require("path"), 1);
|
|
10099
|
+
|
|
10100
|
+
// src/config/merger.ts
|
|
10101
|
+
var import_fs9 = require("fs");
|
|
10102
|
+
var path14 = __toESM(require("path"), 1);
|
|
10103
|
+
|
|
10104
|
+
// src/config/rebase.ts
|
|
10105
|
+
var path13 = __toESM(require("path"), 1);
|
|
10106
|
+
function isWithinRoot(rootDir, targetPath) {
|
|
10107
|
+
const relativePath = path13.relative(rootDir, targetPath);
|
|
10108
|
+
return relativePath === "" || !relativePath.startsWith("..") && !path13.isAbsolute(relativePath);
|
|
10109
|
+
}
|
|
10110
|
+
function rebasePathEntries(values, fromDir, toDir) {
|
|
10111
|
+
if (!Array.isArray(values)) {
|
|
10112
|
+
return [];
|
|
10010
10113
|
}
|
|
10011
|
-
|
|
10012
|
-
|
|
10013
|
-
|
|
10014
|
-
|
|
10015
|
-
message: `Hit@5 drop ${comparison.deltas.hitAt5.absolute.toFixed(4)} exceeds allowed -${thresholds.hitAt5MaxDrop.toFixed(4)}`
|
|
10016
|
-
});
|
|
10017
|
-
}
|
|
10018
|
-
if (thresholds.mrrAt10MaxDrop !== void 0 && comparison.deltas.mrrAt10.absolute < -thresholds.mrrAt10MaxDrop) {
|
|
10019
|
-
violations.push({
|
|
10020
|
-
metric: "mrrAt10MaxDrop",
|
|
10021
|
-
message: `MRR@10 drop ${comparison.deltas.mrrAt10.absolute.toFixed(4)} exceeds allowed -${thresholds.mrrAt10MaxDrop.toFixed(4)}`
|
|
10022
|
-
});
|
|
10114
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
10115
|
+
const trimmed = value.trim();
|
|
10116
|
+
if (!trimmed || path13.isAbsolute(trimmed)) {
|
|
10117
|
+
return trimmed;
|
|
10023
10118
|
}
|
|
10024
|
-
|
|
10025
|
-
|
|
10026
|
-
|
|
10027
|
-
|
|
10028
|
-
|
|
10119
|
+
return normalizePathSeparators(path13.normalize(path13.relative(toDir, path13.resolve(fromDir, trimmed))));
|
|
10120
|
+
}).filter(Boolean);
|
|
10121
|
+
}
|
|
10122
|
+
function resolveInheritedKnowledgeBaseEntries(values, sourceRoot, targetRoot) {
|
|
10123
|
+
if (!Array.isArray(values)) {
|
|
10124
|
+
return [];
|
|
10125
|
+
}
|
|
10126
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
10127
|
+
const trimmed = value.trim();
|
|
10128
|
+
if (!trimmed) {
|
|
10129
|
+
return trimmed;
|
|
10029
10130
|
}
|
|
10030
|
-
if (
|
|
10031
|
-
|
|
10032
|
-
|
|
10033
|
-
const allowed = baselineP95 * thresholds.p95LatencyMaxMultiplier;
|
|
10034
|
-
if (summary.metrics.latencyMs.p95 > allowed) {
|
|
10035
|
-
violations.push({
|
|
10036
|
-
metric: "p95LatencyMaxMultiplier",
|
|
10037
|
-
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds allowed ${allowed.toFixed(3)}ms (${thresholds.p95LatencyMaxMultiplier.toFixed(2)}x baseline)`
|
|
10038
|
-
});
|
|
10039
|
-
}
|
|
10131
|
+
if (path13.isAbsolute(trimmed)) {
|
|
10132
|
+
if (isWithinRoot(sourceRoot, trimmed)) {
|
|
10133
|
+
return normalizePathSeparators(path13.normalize(path13.relative(sourceRoot, trimmed) || "."));
|
|
10040
10134
|
}
|
|
10135
|
+
return path13.normalize(trimmed);
|
|
10041
10136
|
}
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
|
|
10048
|
-
}
|
|
10049
|
-
return {
|
|
10050
|
-
passed: violations.length === 0,
|
|
10051
|
-
budgetName: budget.name,
|
|
10052
|
-
violations
|
|
10053
|
-
};
|
|
10137
|
+
const resolvedFromSource = path13.resolve(sourceRoot, trimmed);
|
|
10138
|
+
if (isWithinRoot(sourceRoot, resolvedFromSource)) {
|
|
10139
|
+
return normalizePathSeparators(path13.normalize(trimmed));
|
|
10140
|
+
}
|
|
10141
|
+
return normalizePathSeparators(path13.normalize(path13.relative(targetRoot, resolvedFromSource)));
|
|
10142
|
+
}).filter(Boolean);
|
|
10054
10143
|
}
|
|
10055
10144
|
|
|
10056
|
-
// src/
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10145
|
+
// src/config/merger.ts
|
|
10146
|
+
var PROJECT_OVERRIDE_KEYS = [
|
|
10147
|
+
"embeddingProvider",
|
|
10148
|
+
"customProvider",
|
|
10149
|
+
"embeddingModel",
|
|
10150
|
+
"reranker",
|
|
10151
|
+
"include",
|
|
10152
|
+
"exclude",
|
|
10153
|
+
"indexing",
|
|
10154
|
+
"search",
|
|
10155
|
+
"debug",
|
|
10156
|
+
"scope"
|
|
10157
|
+
];
|
|
10158
|
+
var MERGE_ARRAY_KEYS = ["knowledgeBases", "additionalInclude"];
|
|
10159
|
+
function isRecord(value) {
|
|
10160
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10069
10161
|
}
|
|
10070
|
-
function
|
|
10071
|
-
return
|
|
10162
|
+
function isStringArray2(value) {
|
|
10163
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
10072
10164
|
}
|
|
10073
|
-
function
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10080
|
-
unique.push(result);
|
|
10165
|
+
function applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key) {
|
|
10166
|
+
if (key in normalizedProjectConfig) {
|
|
10167
|
+
merged[key] = normalizedProjectConfig[key];
|
|
10168
|
+
return;
|
|
10169
|
+
}
|
|
10170
|
+
if (key in globalConfig) {
|
|
10171
|
+
merged[key] = globalConfig[key];
|
|
10081
10172
|
}
|
|
10082
|
-
return unique;
|
|
10083
|
-
}
|
|
10084
|
-
function distinctTopKRatio(results, k) {
|
|
10085
|
-
const top = results.slice(0, k);
|
|
10086
|
-
if (top.length === 0) return 0;
|
|
10087
|
-
const distinct = new Set(top.map((result) => normalizePath(result.filePath))).size;
|
|
10088
|
-
return distinct / top.length;
|
|
10089
10173
|
}
|
|
10090
|
-
function
|
|
10091
|
-
|
|
10092
|
-
const expected = normalizePath(expectedPath);
|
|
10093
|
-
if (actual === expected) return true;
|
|
10094
|
-
return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);
|
|
10174
|
+
function mergeUniqueStringArray(values) {
|
|
10175
|
+
return [...new Set(values.map((value) => String(value).trim()))];
|
|
10095
10176
|
}
|
|
10096
|
-
function
|
|
10097
|
-
|
|
10098
|
-
const
|
|
10099
|
-
|
|
10177
|
+
function normalizeKnowledgeBasePath(value) {
|
|
10178
|
+
let normalized = path14.normalize(String(value).trim());
|
|
10179
|
+
const root = path14.parse(normalized).root;
|
|
10180
|
+
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
10181
|
+
normalized = normalized.slice(0, -1);
|
|
10182
|
+
}
|
|
10183
|
+
return normalized;
|
|
10100
10184
|
}
|
|
10101
|
-
function
|
|
10102
|
-
return
|
|
10185
|
+
function mergeKnowledgeBasePaths(values) {
|
|
10186
|
+
return [...new Set(values.map((value) => normalizeKnowledgeBasePath(value)).filter((value) => value.length > 0))];
|
|
10103
10187
|
}
|
|
10104
|
-
function
|
|
10105
|
-
|
|
10106
|
-
|
|
10107
|
-
|
|
10108
|
-
|
|
10188
|
+
function validateConfigLayerShape(rawConfig, filePath) {
|
|
10189
|
+
if (!isRecord(rawConfig)) {
|
|
10190
|
+
throw new Error(`Config file ${filePath} must contain a JSON object at the root.`);
|
|
10191
|
+
}
|
|
10192
|
+
if (rawConfig.knowledgeBases !== void 0 && !isStringArray2(rawConfig.knowledgeBases)) {
|
|
10193
|
+
throw new Error(`Config file ${filePath} field 'knowledgeBases' must be an array of strings.`);
|
|
10194
|
+
}
|
|
10195
|
+
if (rawConfig.additionalInclude !== void 0 && !isStringArray2(rawConfig.additionalInclude)) {
|
|
10196
|
+
throw new Error(`Config file ${filePath} field 'additionalInclude' must be an array of strings.`);
|
|
10197
|
+
}
|
|
10198
|
+
if (rawConfig.include !== void 0 && !isStringArray2(rawConfig.include)) {
|
|
10199
|
+
throw new Error(`Config file ${filePath} field 'include' must be an array of strings.`);
|
|
10200
|
+
}
|
|
10201
|
+
if (rawConfig.exclude !== void 0 && !isStringArray2(rawConfig.exclude)) {
|
|
10202
|
+
throw new Error(`Config file ${filePath} field 'exclude' must be an array of strings.`);
|
|
10203
|
+
}
|
|
10204
|
+
for (const section of ["customProvider", "indexing", "search", "debug", "reranker"]) {
|
|
10205
|
+
const value = rawConfig[section];
|
|
10206
|
+
if (value !== void 0 && !isRecord(value)) {
|
|
10207
|
+
throw new Error(`Config file ${filePath} field '${section}' must be an object.`);
|
|
10109
10208
|
}
|
|
10110
10209
|
}
|
|
10111
|
-
return
|
|
10210
|
+
return rawConfig;
|
|
10112
10211
|
}
|
|
10113
|
-
function
|
|
10114
|
-
|
|
10115
|
-
|
|
10116
|
-
|
|
10117
|
-
|
|
10118
|
-
|
|
10119
|
-
|
|
10120
|
-
|
|
10121
|
-
(
|
|
10122
|
-
|
|
10123
|
-
|
|
10124
|
-
|
|
10212
|
+
function loadJsonFile(filePath) {
|
|
10213
|
+
if (!(0, import_fs9.existsSync)(filePath)) {
|
|
10214
|
+
return null;
|
|
10215
|
+
}
|
|
10216
|
+
try {
|
|
10217
|
+
const content = (0, import_fs9.readFileSync)(filePath, "utf-8");
|
|
10218
|
+
return validateConfigLayerShape(JSON.parse(content), filePath);
|
|
10219
|
+
} catch (error) {
|
|
10220
|
+
if (error instanceof Error && error.message.startsWith("Config file ")) {
|
|
10221
|
+
throw error;
|
|
10222
|
+
}
|
|
10223
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
10224
|
+
throw new Error(`Failed to load config file ${filePath}: ${message}`);
|
|
10225
|
+
}
|
|
10125
10226
|
}
|
|
10126
|
-
function
|
|
10127
|
-
|
|
10128
|
-
return lowered.includes("/docs/") || lowered.includes("/test/") || lowered.includes("/tests/") || lowered.includes("readme") || lowered.includes("/benchmarks/");
|
|
10227
|
+
function loadConfigFile(filePath) {
|
|
10228
|
+
return loadJsonFile(filePath);
|
|
10129
10229
|
}
|
|
10130
|
-
function
|
|
10131
|
-
const
|
|
10132
|
-
|
|
10133
|
-
|
|
10134
|
-
|
|
10135
|
-
|
|
10230
|
+
function materializeLocalProjectConfig(projectRoot, config, host = "opencode") {
|
|
10231
|
+
const localConfigPath = resolveWritableProjectConfigPath(projectRoot, host);
|
|
10232
|
+
(0, import_fs9.mkdirSync)(path14.dirname(localConfigPath), { recursive: true });
|
|
10233
|
+
(0, import_fs9.writeFileSync)(localConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
10234
|
+
return localConfigPath;
|
|
10235
|
+
}
|
|
10236
|
+
function loadProjectConfigLayer(projectRoot, host = "opencode") {
|
|
10237
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot, host);
|
|
10238
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
10239
|
+
if (!projectConfig) {
|
|
10240
|
+
return {};
|
|
10136
10241
|
}
|
|
10137
|
-
|
|
10138
|
-
|
|
10139
|
-
|
|
10242
|
+
const normalizedConfig = { ...projectConfig };
|
|
10243
|
+
const projectConfigBaseDir = path14.dirname(path14.dirname(projectConfigPath));
|
|
10244
|
+
if (Array.isArray(normalizedConfig.knowledgeBases)) {
|
|
10245
|
+
normalizedConfig.knowledgeBases = resolveInheritedKnowledgeBaseEntries(
|
|
10246
|
+
normalizedConfig.knowledgeBases,
|
|
10247
|
+
projectConfigBaseDir,
|
|
10248
|
+
projectRoot
|
|
10140
10249
|
);
|
|
10141
|
-
if (!hasSymbol) return "wrong-symbol";
|
|
10142
10250
|
}
|
|
10143
|
-
|
|
10144
|
-
|
|
10145
|
-
|
|
10251
|
+
return normalizedConfig;
|
|
10252
|
+
}
|
|
10253
|
+
function loadMergedConfig(projectRoot, host = "opencode") {
|
|
10254
|
+
const globalConfigPath = resolveGlobalConfigPath(host);
|
|
10255
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot, host);
|
|
10256
|
+
let globalConfig = null;
|
|
10257
|
+
let globalConfigError = null;
|
|
10258
|
+
try {
|
|
10259
|
+
globalConfig = loadJsonFile(globalConfigPath);
|
|
10260
|
+
} catch (error) {
|
|
10261
|
+
globalConfigError = error instanceof Error ? error : new Error(String(error));
|
|
10146
10262
|
}
|
|
10147
|
-
|
|
10148
|
-
|
|
10263
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
10264
|
+
const normalizedProjectConfig = loadProjectConfigLayer(projectRoot, host);
|
|
10265
|
+
if (globalConfigError) {
|
|
10266
|
+
if (!projectConfig) {
|
|
10267
|
+
throw globalConfigError;
|
|
10268
|
+
}
|
|
10269
|
+
globalConfig = null;
|
|
10149
10270
|
}
|
|
10150
|
-
|
|
10151
|
-
}
|
|
10152
|
-
|
|
10153
|
-
|
|
10154
|
-
|
|
10155
|
-
|
|
10156
|
-
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
|
|
10163
|
-
|
|
10164
|
-
|
|
10165
|
-
|
|
10166
|
-
|
|
10167
|
-
|
|
10168
|
-
|
|
10169
|
-
|
|
10170
|
-
|
|
10171
|
-
|
|
10172
|
-
}
|
|
10173
|
-
function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingTokensUsed, costPer1MTokensUsd) {
|
|
10174
|
-
const count = perQuery.length;
|
|
10175
|
-
const safeDiv = (value) => count === 0 ? 0 : value / count;
|
|
10176
|
-
const sum = {
|
|
10177
|
-
hitAt1: 0,
|
|
10178
|
-
hitAt3: 0,
|
|
10179
|
-
hitAt5: 0,
|
|
10180
|
-
hitAt10: 0,
|
|
10181
|
-
mrrAt10: 0,
|
|
10182
|
-
ndcgAt10: 0,
|
|
10183
|
-
distinctTop3Ratio: 0,
|
|
10184
|
-
rawDistinctTop3Ratio: 0
|
|
10185
|
-
};
|
|
10186
|
-
const failureBuckets = {
|
|
10187
|
-
"wrong-file": 0,
|
|
10188
|
-
"wrong-symbol": 0,
|
|
10189
|
-
"docs-tests-outranking-source": 0,
|
|
10190
|
-
"no-relevant-hit-top-k": 0
|
|
10191
|
-
};
|
|
10192
|
-
const latencies = perQuery.map((item) => item.latencyMs);
|
|
10193
|
-
for (const query of perQuery) {
|
|
10194
|
-
if (query.hitAt1) sum.hitAt1 += 1;
|
|
10195
|
-
if (query.hitAt3) sum.hitAt3 += 1;
|
|
10196
|
-
if (query.hitAt5) sum.hitAt5 += 1;
|
|
10197
|
-
if (query.hitAt10) sum.hitAt10 += 1;
|
|
10198
|
-
sum.mrrAt10 += query.reciprocalRankAt10;
|
|
10199
|
-
sum.ndcgAt10 += query.ndcgAt10;
|
|
10200
|
-
sum.distinctTop3Ratio += distinctTopKRatio(query.results, 3);
|
|
10201
|
-
sum.rawDistinctTop3Ratio += query.rawTop3DistinctRatio;
|
|
10202
|
-
if (query.failureBucket) {
|
|
10203
|
-
failureBuckets[query.failureBucket] += 1;
|
|
10271
|
+
if (!globalConfig && !projectConfig) {
|
|
10272
|
+
return {};
|
|
10273
|
+
}
|
|
10274
|
+
if (!projectConfig && globalConfig) {
|
|
10275
|
+
return globalConfig;
|
|
10276
|
+
}
|
|
10277
|
+
if (!globalConfig && projectConfig) {
|
|
10278
|
+
return normalizedProjectConfig;
|
|
10279
|
+
}
|
|
10280
|
+
if (!globalConfig || !projectConfig) {
|
|
10281
|
+
return globalConfig ?? normalizedProjectConfig;
|
|
10282
|
+
}
|
|
10283
|
+
const merged = { ...globalConfig };
|
|
10284
|
+
for (const key of PROJECT_OVERRIDE_KEYS) {
|
|
10285
|
+
applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key);
|
|
10286
|
+
}
|
|
10287
|
+
if (projectConfig) {
|
|
10288
|
+
for (const key of Object.keys(projectConfig)) {
|
|
10289
|
+
if (PROJECT_OVERRIDE_KEYS.includes(key) || MERGE_ARRAY_KEYS.includes(key)) {
|
|
10290
|
+
continue;
|
|
10291
|
+
}
|
|
10292
|
+
merged[key] = normalizedProjectConfig[key];
|
|
10204
10293
|
}
|
|
10205
10294
|
}
|
|
10206
|
-
const
|
|
10207
|
-
|
|
10208
|
-
|
|
10209
|
-
|
|
10210
|
-
|
|
10211
|
-
|
|
10212
|
-
|
|
10213
|
-
|
|
10214
|
-
|
|
10215
|
-
rawDistinctTop3Ratio: safeDiv(sum.rawDistinctTop3Ratio),
|
|
10216
|
-
latencyMs: {
|
|
10217
|
-
p50: percentile(latencies, 0.5),
|
|
10218
|
-
p95: percentile(latencies, 0.95),
|
|
10219
|
-
p99: percentile(latencies, 0.99)
|
|
10220
|
-
},
|
|
10221
|
-
tokenEstimate: {
|
|
10222
|
-
queryTokens,
|
|
10223
|
-
embeddingTokensUsed
|
|
10224
|
-
},
|
|
10225
|
-
embedding: {
|
|
10226
|
-
callCount: embeddingCallCount,
|
|
10227
|
-
estimatedCostUsd: embeddingTokensUsed / 1e6 * costPer1MTokensUsd,
|
|
10228
|
-
costPer1MTokensUsd
|
|
10229
|
-
},
|
|
10230
|
-
failureBuckets
|
|
10231
|
-
};
|
|
10295
|
+
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
10296
|
+
const projectKbs = projectConfig ? Array.isArray(normalizedProjectConfig.knowledgeBases) ? normalizedProjectConfig.knowledgeBases : [] : [];
|
|
10297
|
+
const allKbs = [...globalKbs, ...projectKbs];
|
|
10298
|
+
merged.knowledgeBases = mergeKnowledgeBasePaths(allKbs);
|
|
10299
|
+
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
10300
|
+
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
10301
|
+
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
10302
|
+
merged.additionalInclude = mergeUniqueStringArray(allAdditional);
|
|
10303
|
+
return merged;
|
|
10232
10304
|
}
|
|
10233
10305
|
|
|
10234
|
-
// src/
|
|
10235
|
-
var
|
|
10236
|
-
|
|
10237
|
-
|
|
10306
|
+
// src/tools/knowledge-base-paths.ts
|
|
10307
|
+
var path15 = __toESM(require("path"), 1);
|
|
10308
|
+
function resolveConfigPathValue(value, baseDir) {
|
|
10309
|
+
const trimmed = value.trim();
|
|
10310
|
+
if (!trimmed) {
|
|
10311
|
+
return trimmed;
|
|
10312
|
+
}
|
|
10313
|
+
const absolutePath = path15.isAbsolute(trimmed) ? trimmed : path15.resolve(baseDir, trimmed);
|
|
10314
|
+
return path15.normalize(absolutePath);
|
|
10315
|
+
}
|
|
10238
10316
|
|
|
10239
|
-
// src/
|
|
10240
|
-
var
|
|
10241
|
-
|
|
10242
|
-
|
|
10243
|
-
|
|
10317
|
+
// src/tools/utils.ts
|
|
10318
|
+
var import_tiktoken = require("tiktoken");
|
|
10319
|
+
var MIN_CONTEXT_PACK_TOKEN_BUDGET = 128;
|
|
10320
|
+
var MAX_CONTEXT_PACK_TOKEN_BUDGET = 4e3;
|
|
10321
|
+
var DEFAULT_CONTEXT_PACK_TOKEN_BUDGET = 1200;
|
|
10322
|
+
var CONTEXT_TOKENIZER = (0, import_tiktoken.get_encoding)("cl100k_base");
|
|
10323
|
+
function clampContextPackTokenBudget(tokenBudget) {
|
|
10324
|
+
if (tokenBudget === void 0 || !Number.isFinite(tokenBudget)) {
|
|
10325
|
+
return DEFAULT_CONTEXT_PACK_TOKEN_BUDGET;
|
|
10326
|
+
}
|
|
10327
|
+
return Math.min(
|
|
10328
|
+
MAX_CONTEXT_PACK_TOKEN_BUDGET,
|
|
10329
|
+
Math.max(MIN_CONTEXT_PACK_TOKEN_BUDGET, Math.floor(tokenBudget))
|
|
10330
|
+
);
|
|
10244
10331
|
}
|
|
10245
|
-
function
|
|
10246
|
-
|
|
10247
|
-
return [];
|
|
10248
|
-
}
|
|
10249
|
-
return values.filter((value) => typeof value === "string").map((value) => {
|
|
10250
|
-
const trimmed = value.trim();
|
|
10251
|
-
if (!trimmed || path13.isAbsolute(trimmed)) {
|
|
10252
|
-
return trimmed;
|
|
10253
|
-
}
|
|
10254
|
-
return normalizePathSeparators(path13.normalize(path13.relative(toDir, path13.resolve(fromDir, trimmed))));
|
|
10255
|
-
}).filter(Boolean);
|
|
10332
|
+
function countContextTokens(text) {
|
|
10333
|
+
return CONTEXT_TOKENIZER.encode(text).length;
|
|
10256
10334
|
}
|
|
10257
|
-
function
|
|
10258
|
-
|
|
10259
|
-
|
|
10335
|
+
function fitTextToContextBudget(text, tokenBudget) {
|
|
10336
|
+
const normalizedBudget = clampContextPackTokenBudget(tokenBudget);
|
|
10337
|
+
const tokenEstimate = countContextTokens(text);
|
|
10338
|
+
if (tokenEstimate <= normalizedBudget) {
|
|
10339
|
+
return {
|
|
10340
|
+
text,
|
|
10341
|
+
tokenBudget: normalizedBudget,
|
|
10342
|
+
tokenEstimate,
|
|
10343
|
+
truncated: false
|
|
10344
|
+
};
|
|
10260
10345
|
}
|
|
10261
|
-
|
|
10262
|
-
|
|
10263
|
-
|
|
10264
|
-
|
|
10265
|
-
|
|
10266
|
-
|
|
10267
|
-
|
|
10268
|
-
|
|
10269
|
-
|
|
10270
|
-
|
|
10271
|
-
|
|
10272
|
-
|
|
10273
|
-
|
|
10274
|
-
|
|
10275
|
-
|
|
10276
|
-
|
|
10277
|
-
}
|
|
10346
|
+
const suffix = "\n...[truncated to context token budget]";
|
|
10347
|
+
const codePoints = Array.from(text);
|
|
10348
|
+
let low = 0;
|
|
10349
|
+
let high = codePoints.length;
|
|
10350
|
+
while (low < high) {
|
|
10351
|
+
const middle = Math.ceil((low + high) / 2);
|
|
10352
|
+
const candidate = `${codePoints.slice(0, middle).join("").trimEnd()}${suffix}`;
|
|
10353
|
+
if (countContextTokens(candidate) <= normalizedBudget) low = middle;
|
|
10354
|
+
else high = middle - 1;
|
|
10355
|
+
}
|
|
10356
|
+
const fitted = `${codePoints.slice(0, low).join("").trimEnd()}${suffix}`;
|
|
10357
|
+
return {
|
|
10358
|
+
text: fitted,
|
|
10359
|
+
tokenBudget: normalizedBudget,
|
|
10360
|
+
tokenEstimate: countContextTokens(fitted),
|
|
10361
|
+
truncated: true
|
|
10362
|
+
};
|
|
10278
10363
|
}
|
|
10279
|
-
|
|
10280
|
-
|
|
10281
|
-
function isRecord(value) {
|
|
10282
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10364
|
+
function normalizedLineRange(result) {
|
|
10365
|
+
return result.startLine <= result.endLine ? { start: result.startLine, end: result.endLine } : { start: result.endLine, end: result.startLine };
|
|
10283
10366
|
}
|
|
10284
|
-
function
|
|
10285
|
-
return
|
|
10367
|
+
function rankContextCandidates(results) {
|
|
10368
|
+
return results.map((result, originalIndex) => ({ result, originalIndex })).sort((left, right) => right.result.score - left.result.score || left.originalIndex - right.originalIndex);
|
|
10286
10369
|
}
|
|
10287
|
-
function
|
|
10288
|
-
|
|
10289
|
-
|
|
10370
|
+
function deduplicateContextCandidates(candidates) {
|
|
10371
|
+
const acceptedByFile = /* @__PURE__ */ new Map();
|
|
10372
|
+
const deduplicated = [];
|
|
10373
|
+
for (const { result } of candidates) {
|
|
10374
|
+
const range = normalizedLineRange(result);
|
|
10375
|
+
const accepted = acceptedByFile.get(result.filePath) ?? [];
|
|
10376
|
+
if (accepted.some((item) => item.start <= range.end && range.start <= item.end)) {
|
|
10377
|
+
continue;
|
|
10378
|
+
}
|
|
10379
|
+
accepted.push(range);
|
|
10380
|
+
acceptedByFile.set(result.filePath, accepted);
|
|
10381
|
+
deduplicated.push(result);
|
|
10290
10382
|
}
|
|
10291
|
-
|
|
10292
|
-
|
|
10293
|
-
|
|
10383
|
+
return deduplicated;
|
|
10384
|
+
}
|
|
10385
|
+
function diversifyContextCandidates(results) {
|
|
10386
|
+
const byFile = /* @__PURE__ */ new Map();
|
|
10387
|
+
for (const result of results) {
|
|
10388
|
+
const bucket = byFile.get(result.filePath) ?? [];
|
|
10389
|
+
bucket.push(result);
|
|
10390
|
+
byFile.set(result.filePath, bucket);
|
|
10391
|
+
}
|
|
10392
|
+
const files = [...byFile.keys()];
|
|
10393
|
+
const diversified = [];
|
|
10394
|
+
for (let depth = 0; diversified.length < results.length; depth += 1) {
|
|
10395
|
+
for (const file of files) {
|
|
10396
|
+
const result = byFile.get(file)?.[depth];
|
|
10397
|
+
if (result) diversified.push(result);
|
|
10398
|
+
}
|
|
10399
|
+
}
|
|
10400
|
+
return diversified;
|
|
10401
|
+
}
|
|
10402
|
+
function compactEvidenceValue(value, maxChars) {
|
|
10403
|
+
const characters = [...value];
|
|
10404
|
+
if (characters.length <= maxChars) return value;
|
|
10405
|
+
return `\u2026${characters.slice(-(maxChars - 1)).join("")}`;
|
|
10406
|
+
}
|
|
10407
|
+
function formatContextEvidence(result, index) {
|
|
10408
|
+
const symbol = result.name ? ` ${JSON.stringify(compactEvidenceValue(result.name, 80))}` : "";
|
|
10409
|
+
const path27 = compactEvidenceValue(result.filePath, 120);
|
|
10410
|
+
return `[${index}] ${result.chunkType}${symbol} in ${path27}:${result.startLine}-${result.endLine} (score ${result.score.toFixed(2)})`;
|
|
10411
|
+
}
|
|
10412
|
+
function formatContextPack(heading, selected, candidateCount, duplicateCount, limitOmittedCount, budgetOmittedCount) {
|
|
10413
|
+
const lines = selected.map((result, index) => formatContextEvidence(result, index + 1));
|
|
10414
|
+
const notes = [];
|
|
10415
|
+
if (duplicateCount > 0) notes.push(`${duplicateCount} overlapping duplicate${duplicateCount === 1 ? "" : "s"} removed`);
|
|
10416
|
+
if (limitOmittedCount > 0) notes.push(`${limitOmittedCount} additional result${limitOmittedCount === 1 ? "" : "s"} excluded by result limit`);
|
|
10417
|
+
if (budgetOmittedCount > 0) notes.push(`${budgetOmittedCount} additional result${budgetOmittedCount === 1 ? "" : "s"} omitted by token budget`);
|
|
10418
|
+
const footer = notes.length > 0 ? `Selected ${selected.length} of ${candidateCount} candidates; ${notes.join("; ")}.` : `Selected ${selected.length} of ${candidateCount} candidates.`;
|
|
10419
|
+
return `${heading}
|
|
10420
|
+
|
|
10421
|
+
${lines.join("\n")}
|
|
10422
|
+
|
|
10423
|
+
${footer}`;
|
|
10424
|
+
}
|
|
10425
|
+
function buildContextPack(results, options = {}) {
|
|
10426
|
+
const requestedTokenBudget = options.tokenBudget ?? DEFAULT_CONTEXT_PACK_TOKEN_BUDGET;
|
|
10427
|
+
const tokenBudget = clampContextPackTokenBudget(options.tokenBudget);
|
|
10428
|
+
const heading = compactEvidenceValue(options.heading?.trim() || "Codebase evidence", 160);
|
|
10429
|
+
const maxResults = Math.max(0, Math.floor(options.maxResults ?? results.length));
|
|
10430
|
+
const candidateCount = results.length;
|
|
10431
|
+
const deduplicated = deduplicateContextCandidates(rankContextCandidates(results));
|
|
10432
|
+
const diversified = diversifyContextCandidates(deduplicated);
|
|
10433
|
+
const duplicateCount = candidateCount - deduplicated.length;
|
|
10434
|
+
const selectable = diversified.slice(0, maxResults);
|
|
10435
|
+
const limitOmittedCount = deduplicated.length - selectable.length;
|
|
10436
|
+
let selected = [];
|
|
10437
|
+
let text = formatContextPack(heading, selected, candidateCount, duplicateCount, limitOmittedCount, selectable.length);
|
|
10438
|
+
for (let count = 1; count <= selectable.length; count += 1) {
|
|
10439
|
+
const candidateSelection = selectable.slice(0, count);
|
|
10440
|
+
const budgetOmittedCount2 = selectable.length - candidateSelection.length;
|
|
10441
|
+
const candidateText = formatContextPack(
|
|
10442
|
+
heading,
|
|
10443
|
+
candidateSelection,
|
|
10444
|
+
candidateCount,
|
|
10445
|
+
duplicateCount,
|
|
10446
|
+
limitOmittedCount,
|
|
10447
|
+
budgetOmittedCount2
|
|
10448
|
+
);
|
|
10449
|
+
if (countContextTokens(candidateText) > tokenBudget) break;
|
|
10450
|
+
selected = candidateSelection;
|
|
10451
|
+
text = candidateText;
|
|
10294
10452
|
}
|
|
10295
|
-
|
|
10296
|
-
|
|
10297
|
-
|
|
10298
|
-
|
|
10299
|
-
|
|
10453
|
+
const fitted = fitTextToContextBudget(text, tokenBudget);
|
|
10454
|
+
const budgetOmittedCount = selectable.length - selected.length;
|
|
10455
|
+
const omittedCount = candidateCount - selected.length;
|
|
10456
|
+
return {
|
|
10457
|
+
requestedTokenBudget,
|
|
10458
|
+
tokenBudget,
|
|
10459
|
+
text: fitted.text,
|
|
10460
|
+
tokenEstimate: fitted.tokenEstimate,
|
|
10461
|
+
results: selected,
|
|
10462
|
+
candidateCount,
|
|
10463
|
+
deduplicatedCount: deduplicated.length,
|
|
10464
|
+
selectedCount: selected.length,
|
|
10465
|
+
omittedCount,
|
|
10466
|
+
duplicateCount,
|
|
10467
|
+
limitOmittedCount,
|
|
10468
|
+
budgetOmittedCount
|
|
10469
|
+
};
|
|
10470
|
+
}
|
|
10471
|
+
var MAX_CONTENT_LINES = 30;
|
|
10472
|
+
function truncateContent(content) {
|
|
10473
|
+
const lines = content.split("\n");
|
|
10474
|
+
if (lines.length <= MAX_CONTENT_LINES) return content;
|
|
10475
|
+
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
10476
|
+
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
10477
|
+
}
|
|
10478
|
+
function formatIndexStats(stats, verbose = false) {
|
|
10479
|
+
if (stats.resetCorruptedIndex) {
|
|
10480
|
+
return stats.warning ?? "Detected a corrupted local index and reset it during indexing. Run index_codebase again to rebuild search data.";
|
|
10300
10481
|
}
|
|
10301
|
-
|
|
10302
|
-
|
|
10482
|
+
const lines = [];
|
|
10483
|
+
if (stats.failedChunks > 0) {
|
|
10484
|
+
lines.push(`INDEXING WARNING: ${stats.failedChunks} chunks failed to embed.`);
|
|
10485
|
+
if (stats.failedBatchesPath) {
|
|
10486
|
+
lines.push(`Inspect failed batches at: ${stats.failedBatchesPath}`);
|
|
10487
|
+
}
|
|
10488
|
+
lines.push("");
|
|
10303
10489
|
}
|
|
10304
|
-
|
|
10305
|
-
|
|
10306
|
-
|
|
10307
|
-
|
|
10490
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
10491
|
+
lines.push(`${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
10492
|
+
} else if (stats.indexedChunks === 0) {
|
|
10493
|
+
lines.push(`${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
10494
|
+
} else {
|
|
10495
|
+
let main2 = `${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
10496
|
+
if (stats.existingChunks > 0) {
|
|
10497
|
+
main2 += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
10498
|
+
}
|
|
10499
|
+
lines.push(main2);
|
|
10500
|
+
if (stats.removedChunks > 0) {
|
|
10501
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
10502
|
+
}
|
|
10503
|
+
if (stats.failedChunks > 0) {
|
|
10504
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
10308
10505
|
}
|
|
10506
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
10309
10507
|
}
|
|
10310
|
-
|
|
10311
|
-
|
|
10312
|
-
|
|
10313
|
-
|
|
10314
|
-
|
|
10315
|
-
|
|
10316
|
-
|
|
10317
|
-
|
|
10508
|
+
if (verbose) {
|
|
10509
|
+
if (stats.skippedFiles.length > 0) {
|
|
10510
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
10511
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
10512
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
10513
|
+
lines.push("");
|
|
10514
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
10515
|
+
if (tooLarge.length > 0) {
|
|
10516
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
10517
|
+
}
|
|
10518
|
+
if (excluded.length > 0) {
|
|
10519
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
10520
|
+
}
|
|
10521
|
+
if (gitignored.length > 0) {
|
|
10522
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
10523
|
+
}
|
|
10524
|
+
}
|
|
10525
|
+
if (stats.parseFailures.length > 0) {
|
|
10526
|
+
lines.push("");
|
|
10527
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
10318
10528
|
}
|
|
10319
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
10320
|
-
throw new Error(`Failed to parse eval config JSON at ${filePath}: ${message}`);
|
|
10321
10529
|
}
|
|
10530
|
+
return lines.join("\n");
|
|
10322
10531
|
}
|
|
10323
|
-
function
|
|
10324
|
-
|
|
10325
|
-
|
|
10326
|
-
|
|
10327
|
-
|
|
10328
|
-
|
|
10329
|
-
|
|
10330
|
-
|
|
10331
|
-
|
|
10332
|
-
|
|
10333
|
-
|
|
10334
|
-
|
|
10335
|
-
|
|
10336
|
-
|
|
10337
|
-
|
|
10338
|
-
|
|
10339
|
-
);
|
|
10340
|
-
if (Array.isArray(config.knowledgeBases)) {
|
|
10341
|
-
config.knowledgeBases = rebaseEntries(config.knowledgeBases);
|
|
10532
|
+
function formatStatus(status) {
|
|
10533
|
+
if (!status.indexed) {
|
|
10534
|
+
if (status.warning) {
|
|
10535
|
+
return status.warning;
|
|
10536
|
+
}
|
|
10537
|
+
if (status.failedBatchesCount > 0) {
|
|
10538
|
+
const lines2 = [
|
|
10539
|
+
"Codebase is not indexed. The last indexing run left failed embedding batches.",
|
|
10540
|
+
"Fix the provider/model configuration, then rerun index_codebase normally to retry the saved failed batches. Use force=true only for a full rebuild or compatibility reset."
|
|
10541
|
+
];
|
|
10542
|
+
if (status.failedBatchesPath) {
|
|
10543
|
+
lines2.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
10544
|
+
}
|
|
10545
|
+
return lines2.join("\n");
|
|
10546
|
+
}
|
|
10547
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
10342
10548
|
}
|
|
10343
|
-
|
|
10344
|
-
|
|
10549
|
+
const lines = [
|
|
10550
|
+
`Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
10551
|
+
`Provider: ${status.provider}`,
|
|
10552
|
+
`Model: ${status.model}`,
|
|
10553
|
+
`Location: ${status.indexPath}`
|
|
10554
|
+
];
|
|
10555
|
+
if (status.currentBranch !== "default") {
|
|
10556
|
+
lines.push(`Current branch: ${status.currentBranch}`);
|
|
10557
|
+
lines.push(`Base branch: ${status.baseBranch}`);
|
|
10345
10558
|
}
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10351
|
-
|
|
10352
|
-
parseJsonConfigFile(fromPath),
|
|
10353
|
-
projectRoot,
|
|
10354
|
-
fromPath
|
|
10355
|
-
);
|
|
10559
|
+
if (status.failedBatchesCount > 0) {
|
|
10560
|
+
lines.push("");
|
|
10561
|
+
lines.push(`INDEXING WARNING: ${status.failedBatchesCount} failed embedding batch${status.failedBatchesCount === 1 ? " remains" : "es remain"}.`);
|
|
10562
|
+
if (status.failedBatchesPath) {
|
|
10563
|
+
lines.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
10564
|
+
}
|
|
10356
10565
|
}
|
|
10357
|
-
|
|
10358
|
-
|
|
10359
|
-
|
|
10360
|
-
parseJsonConfigFile(projectConfig),
|
|
10361
|
-
projectRoot,
|
|
10362
|
-
projectConfig
|
|
10363
|
-
);
|
|
10566
|
+
if (status.warning) {
|
|
10567
|
+
lines.push("");
|
|
10568
|
+
lines.push(`INDEX WARNING: ${status.warning}`);
|
|
10364
10569
|
}
|
|
10365
|
-
|
|
10366
|
-
|
|
10367
|
-
|
|
10570
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
10571
|
+
lines.push("");
|
|
10572
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
10573
|
+
if (status.compatibility.storedMetadata) {
|
|
10574
|
+
const stored = status.compatibility.storedMetadata;
|
|
10575
|
+
lines.push(`Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
10576
|
+
lines.push(`Current config: ${status.provider}/${status.model}`);
|
|
10577
|
+
}
|
|
10578
|
+
} else if (!status.compatibility) {
|
|
10579
|
+
lines.push(`Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
10580
|
+
} else {
|
|
10581
|
+
lines.push(`Compatibility: Index is compatible with the current provider and model.`);
|
|
10368
10582
|
}
|
|
10369
|
-
return
|
|
10370
|
-
}
|
|
10371
|
-
function getIndexRootPath(projectRoot, scope) {
|
|
10372
|
-
return scope === "global" ? getGlobalIndexPath() : resolveProjectIndexPath(projectRoot, scope);
|
|
10373
|
-
}
|
|
10374
|
-
function getLocalProjectIndexRoot(projectRoot) {
|
|
10375
|
-
return path14.join(projectRoot, ".opencode", "index");
|
|
10376
|
-
}
|
|
10377
|
-
function getLocalProjectConfigPath(projectRoot) {
|
|
10378
|
-
return path14.join(projectRoot, ".opencode", "codebase-index.json");
|
|
10583
|
+
return lines.join("\n");
|
|
10379
10584
|
}
|
|
10380
|
-
function
|
|
10381
|
-
|
|
10382
|
-
|
|
10383
|
-
|
|
10585
|
+
function formatProgressTitle(progress) {
|
|
10586
|
+
switch (progress.phase) {
|
|
10587
|
+
case "scanning":
|
|
10588
|
+
return "Scanning files...";
|
|
10589
|
+
case "parsing":
|
|
10590
|
+
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
10591
|
+
case "embedding":
|
|
10592
|
+
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
10593
|
+
case "storing":
|
|
10594
|
+
return "Storing index...";
|
|
10595
|
+
case "complete":
|
|
10596
|
+
return "Indexing complete";
|
|
10597
|
+
default:
|
|
10598
|
+
return "Indexing...";
|
|
10384
10599
|
}
|
|
10385
10600
|
}
|
|
10386
|
-
function
|
|
10387
|
-
|
|
10388
|
-
|
|
10389
|
-
if (
|
|
10390
|
-
return
|
|
10601
|
+
function calculatePercentage(progress) {
|
|
10602
|
+
if (progress.phase === "scanning") return 0;
|
|
10603
|
+
if (progress.phase === "complete") return 100;
|
|
10604
|
+
if (progress.phase === "parsing") {
|
|
10605
|
+
if (progress.totalFiles === 0) return 5;
|
|
10606
|
+
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
10391
10607
|
}
|
|
10392
|
-
if (
|
|
10393
|
-
return
|
|
10608
|
+
if (progress.phase === "embedding") {
|
|
10609
|
+
if (progress.totalChunks === 0) return 20;
|
|
10610
|
+
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
10394
10611
|
}
|
|
10395
|
-
|
|
10396
|
-
|
|
10397
|
-
projectRoot,
|
|
10398
|
-
resolvedConfigPath
|
|
10399
|
-
);
|
|
10400
|
-
(0, import_fs9.mkdirSync)(path14.dirname(localConfigPath), { recursive: true });
|
|
10401
|
-
(0, import_fs9.writeFileSync)(localConfigPath, JSON.stringify(sourceConfig, null, 2), "utf-8");
|
|
10402
|
-
return localConfigPath;
|
|
10612
|
+
if (progress.phase === "storing") return 95;
|
|
10613
|
+
return 0;
|
|
10403
10614
|
}
|
|
10404
|
-
function
|
|
10405
|
-
|
|
10406
|
-
|
|
10615
|
+
function formatCodebasePeek(results) {
|
|
10616
|
+
if (results.length === 0) {
|
|
10617
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
10618
|
+
}
|
|
10619
|
+
const formatted = results.map((r, idx) => {
|
|
10620
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
10621
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
10622
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})${formatBlame(r)}`;
|
|
10623
|
+
});
|
|
10624
|
+
return formatted.join("\n");
|
|
10407
10625
|
}
|
|
10408
|
-
function
|
|
10409
|
-
|
|
10410
|
-
|
|
10411
|
-
};
|
|
10412
|
-
if (overrides?.fusionStrategy !== void 0) {
|
|
10413
|
-
nextSearch.fusionStrategy = overrides.fusionStrategy;
|
|
10626
|
+
function formatHealthCheck(result) {
|
|
10627
|
+
if (result.resetCorruptedIndex) {
|
|
10628
|
+
return result.warning ?? "Detected a corrupted local index and reset it. Run index_codebase to rebuild search data.";
|
|
10414
10629
|
}
|
|
10415
|
-
if (
|
|
10416
|
-
|
|
10630
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0 && result.gcOrphanSymbols === 0 && result.gcOrphanCallEdges === 0) {
|
|
10631
|
+
return "Index is healthy. No stale entries found.";
|
|
10417
10632
|
}
|
|
10418
|
-
|
|
10419
|
-
|
|
10633
|
+
const lines = [];
|
|
10634
|
+
if (result.removed > 0) {
|
|
10635
|
+
lines.push(`Removed stale entries: ${result.removed}`);
|
|
10420
10636
|
}
|
|
10421
|
-
if (
|
|
10422
|
-
|
|
10637
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
10638
|
+
lines.push(`Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
10423
10639
|
}
|
|
10424
|
-
|
|
10425
|
-
|
|
10426
|
-
search: nextSearch
|
|
10427
|
-
};
|
|
10428
|
-
}
|
|
10429
|
-
function getEmbeddingCostPer1MTokens(embeddingProvider) {
|
|
10430
|
-
return embeddingProvider === "custom" || embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(embeddingProvider).costPer1MTokens;
|
|
10431
|
-
}
|
|
10432
|
-
|
|
10433
|
-
// src/eval/schema.ts
|
|
10434
|
-
var import_fs10 = require("fs");
|
|
10435
|
-
function parseJsonFile(filePath) {
|
|
10436
|
-
const content = (0, import_fs10.readFileSync)(filePath, "utf-8");
|
|
10437
|
-
try {
|
|
10438
|
-
return JSON.parse(content);
|
|
10439
|
-
} catch (error) {
|
|
10440
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
10441
|
-
throw new Error(`Failed to parse JSON from ${filePath}: ${message}`);
|
|
10640
|
+
if (result.gcOrphanChunks > 0) {
|
|
10641
|
+
lines.push(`Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
10442
10642
|
}
|
|
10443
|
-
|
|
10444
|
-
|
|
10445
|
-
return typeof value === "object" && value !== null;
|
|
10446
|
-
}
|
|
10447
|
-
function isStringArray3(value) {
|
|
10448
|
-
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
10449
|
-
}
|
|
10450
|
-
function asPositiveNumber(value, path27) {
|
|
10451
|
-
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
10452
|
-
throw new Error(`${path27} must be a non-negative number`);
|
|
10453
|
-
}
|
|
10454
|
-
return value;
|
|
10455
|
-
}
|
|
10456
|
-
function parseQueryType(value, path27) {
|
|
10457
|
-
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy") {
|
|
10458
|
-
return value;
|
|
10459
|
-
}
|
|
10460
|
-
throw new Error(
|
|
10461
|
-
`${path27} must be one of: definition, implementation-intent, similarity, keyword-heavy`
|
|
10462
|
-
);
|
|
10463
|
-
}
|
|
10464
|
-
function parseExpected(input, path27) {
|
|
10465
|
-
if (!isRecord2(input)) {
|
|
10466
|
-
throw new Error(`${path27} must be an object`);
|
|
10643
|
+
if (result.gcOrphanSymbols > 0) {
|
|
10644
|
+
lines.push(`Garbage collected orphan symbols: ${result.gcOrphanSymbols}`);
|
|
10467
10645
|
}
|
|
10468
|
-
|
|
10469
|
-
|
|
10470
|
-
const symbolRaw = input.symbol;
|
|
10471
|
-
const branchRaw = input.branch;
|
|
10472
|
-
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
10473
|
-
const acceptableFiles = isStringArray3(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
10474
|
-
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
10475
|
-
throw new Error(`${path27} must include either expected.filePath or expected.acceptableFiles`);
|
|
10646
|
+
if (result.gcOrphanCallEdges > 0) {
|
|
10647
|
+
lines.push(`Garbage collected orphan call edges: ${result.gcOrphanCallEdges}`);
|
|
10476
10648
|
}
|
|
10477
|
-
if (
|
|
10478
|
-
|
|
10649
|
+
if (result.filePaths.length > 0) {
|
|
10650
|
+
lines.push(`Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
10479
10651
|
}
|
|
10480
|
-
|
|
10481
|
-
|
|
10652
|
+
return lines.join("\n");
|
|
10653
|
+
}
|
|
10654
|
+
function formatCallGraphCallers(name, callers, relationshipType) {
|
|
10655
|
+
if (callers.length === 0) {
|
|
10656
|
+
return `No callers found for "${name}"${relationshipType ? ` with type ${relationshipType}` : ""}. It may not be called by any tracked function, or the index needs updating.`;
|
|
10482
10657
|
}
|
|
10483
|
-
|
|
10484
|
-
|
|
10658
|
+
const formatted = callers.map((edge, index) => {
|
|
10659
|
+
const confidence = edge.confidence !== "Direct" ? ` [${edge.confidence.toLowerCase()}]` : "";
|
|
10660
|
+
return `[${index + 1}] \u2190 from ${edge.fromSymbolName ?? "<unknown>"} in ${edge.fromSymbolFilePath ?? "<unknown file>"} [${edge.fromSymbolId}] (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? " [resolved]" : " [unresolved]"}`;
|
|
10661
|
+
});
|
|
10662
|
+
return `"${name}" is called by ${callers.length} function(s):
|
|
10663
|
+
|
|
10664
|
+
${formatted.join("\n")}`;
|
|
10665
|
+
}
|
|
10666
|
+
function formatCallGraphCallees(symbolId, callees, relationshipType) {
|
|
10667
|
+
if (callees.length === 0) {
|
|
10668
|
+
return `No callees found for symbol ${symbolId}${relationshipType ? ` with type ${relationshipType}` : ""}. The function may not call any other tracked functions.`;
|
|
10485
10669
|
}
|
|
10486
|
-
return {
|
|
10487
|
-
|
|
10488
|
-
|
|
10489
|
-
|
|
10490
|
-
branch: typeof branchRaw === "string" ? branchRaw : void 0
|
|
10491
|
-
};
|
|
10670
|
+
return callees.map((edge, index) => {
|
|
10671
|
+
const confidence = edge.confidence !== "Direct" ? ` [${edge.confidence.toLowerCase()}]` : "";
|
|
10672
|
+
return `[${index + 1}] \u2192 ${edge.targetName} (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? ` [resolved: ${edge.toSymbolId}]` : " [unresolved]"}`;
|
|
10673
|
+
}).join("\n");
|
|
10492
10674
|
}
|
|
10493
|
-
function
|
|
10494
|
-
|
|
10495
|
-
|
|
10496
|
-
throw new Error(`${path27} must be an object`);
|
|
10675
|
+
function formatCallGraphPath(from, to, path27) {
|
|
10676
|
+
if (path27.length === 0) {
|
|
10677
|
+
return `No path found between "${from}" and "${to}". They may be in disconnected components, or the call graph index needs updating.`;
|
|
10497
10678
|
}
|
|
10498
|
-
const
|
|
10499
|
-
|
|
10500
|
-
|
|
10501
|
-
|
|
10502
|
-
|
|
10503
|
-
|
|
10679
|
+
const formatted = path27.map((hop, index) => {
|
|
10680
|
+
const prefix = index === 0 ? "[start]" : `--${hop.callType}-->`;
|
|
10681
|
+
const location = hop.filePath ? ` (${hop.filePath}:${hop.line})` : "";
|
|
10682
|
+
return `${prefix} ${hop.symbolName}${location}`;
|
|
10683
|
+
});
|
|
10684
|
+
return `Path (${path27.length} hops):
|
|
10685
|
+
${formatted.join("\n")}`;
|
|
10686
|
+
}
|
|
10687
|
+
function formatResultHeader(result, index) {
|
|
10688
|
+
return result.name ? `[${index + 1}] ${result.chunkType} "${result.name}" in ${result.filePath}:${result.startLine}-${result.endLine}` : `[${index + 1}] ${result.chunkType} in ${result.filePath}:${result.startLine}-${result.endLine}`;
|
|
10689
|
+
}
|
|
10690
|
+
function formatBlame(result) {
|
|
10691
|
+
if (!result.blame) {
|
|
10692
|
+
return "";
|
|
10504
10693
|
}
|
|
10505
|
-
|
|
10506
|
-
|
|
10694
|
+
const date = new Date(result.blame.committedAt * 1e3).toISOString().slice(0, 10);
|
|
10695
|
+
return `
|
|
10696
|
+
${result.blame.sha.slice(0, 7)} | ${result.blame.author} | ${date} | ${result.blame.summary}`;
|
|
10697
|
+
}
|
|
10698
|
+
function formatDefinitionLookup(results, query) {
|
|
10699
|
+
if (results.length === 0) {
|
|
10700
|
+
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
10507
10701
|
}
|
|
10508
|
-
|
|
10509
|
-
|
|
10510
|
-
|
|
10511
|
-
|
|
10512
|
-
|
|
10513
|
-
|
|
10702
|
+
const formatted = results.map((r, idx) => {
|
|
10703
|
+
const header = formatResultHeader(r, idx);
|
|
10704
|
+
return `${header} (score: ${r.score.toFixed(2)})${formatBlame(r)}
|
|
10705
|
+
\`\`\`
|
|
10706
|
+
${truncateContent(r.content)}
|
|
10707
|
+
\`\`\``;
|
|
10708
|
+
});
|
|
10709
|
+
return formatted.join("\n\n");
|
|
10514
10710
|
}
|
|
10515
|
-
function
|
|
10516
|
-
|
|
10517
|
-
|
|
10711
|
+
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
10712
|
+
const formatted = results.map((r, idx) => {
|
|
10713
|
+
const header = formatResultHeader(r, idx);
|
|
10714
|
+
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
10715
|
+
return `${header} ${scoreLabel}${formatBlame(r)}
|
|
10716
|
+
\`\`\`
|
|
10717
|
+
${truncateContent(r.content)}
|
|
10718
|
+
\`\`\``;
|
|
10719
|
+
});
|
|
10720
|
+
return formatted.join("\n\n");
|
|
10721
|
+
}
|
|
10722
|
+
|
|
10723
|
+
// src/tools/config-state.ts
|
|
10724
|
+
var import_fs10 = require("fs");
|
|
10725
|
+
var path16 = __toESM(require("path"), 1);
|
|
10726
|
+
function normalizeKnowledgeBasePaths(config, projectRoot) {
|
|
10727
|
+
const normalized = { ...config };
|
|
10728
|
+
if (Array.isArray(normalized.knowledgeBases)) {
|
|
10729
|
+
normalized.knowledgeBases = normalized.knowledgeBases.map(
|
|
10730
|
+
(kb) => resolveConfigPathValue(kb, projectRoot)
|
|
10731
|
+
);
|
|
10518
10732
|
}
|
|
10519
|
-
|
|
10520
|
-
|
|
10521
|
-
|
|
10522
|
-
|
|
10523
|
-
|
|
10524
|
-
throw new Error(`${sourceLabel}.version must be a non-empty string`);
|
|
10733
|
+
return normalized;
|
|
10734
|
+
}
|
|
10735
|
+
function toConfigRecord(rawConfig) {
|
|
10736
|
+
if (!rawConfig || typeof rawConfig !== "object") {
|
|
10737
|
+
return {};
|
|
10525
10738
|
}
|
|
10526
|
-
|
|
10527
|
-
|
|
10739
|
+
return { ...rawConfig };
|
|
10740
|
+
}
|
|
10741
|
+
function loadRuntimeConfig(projectRoot, host = "opencode") {
|
|
10742
|
+
return normalizeKnowledgeBasePaths(toConfigRecord(loadMergedConfig(projectRoot, host)), projectRoot);
|
|
10743
|
+
}
|
|
10744
|
+
|
|
10745
|
+
// src/tools/operations.ts
|
|
10746
|
+
var indexerCache = /* @__PURE__ */ new Map();
|
|
10747
|
+
var configCache = /* @__PURE__ */ new Map();
|
|
10748
|
+
var defaultProjectRoots = /* @__PURE__ */ new Map();
|
|
10749
|
+
function getIndexBusyResult(error) {
|
|
10750
|
+
if (!isIndexLockContentionError(error)) return null;
|
|
10751
|
+
const owner = error.owner;
|
|
10752
|
+
const ownerText = owner ? `PID ${owner.pid}, operation ${owner.operation}, since ${owner.startedAt}` : "unreadable owner";
|
|
10753
|
+
if (error.reason === "legacy-lock") {
|
|
10754
|
+
return {
|
|
10755
|
+
kind: "busy",
|
|
10756
|
+
text: `INDEX_BUSY: legacy lock format detected (${ownerText}). Verify the PID and remove this lock manually only if it is stale.`
|
|
10757
|
+
};
|
|
10528
10758
|
}
|
|
10529
|
-
if (
|
|
10530
|
-
|
|
10759
|
+
if (error.reason === "unknown-owner") {
|
|
10760
|
+
return {
|
|
10761
|
+
kind: "busy",
|
|
10762
|
+
text: `INDEX_BUSY: unreadable or remote lock owner (${ownerText}). Automatic recovery was refused; manual verification is required.`
|
|
10763
|
+
};
|
|
10531
10764
|
}
|
|
10532
|
-
|
|
10533
|
-
|
|
10765
|
+
return { kind: "busy", text: `INDEX_BUSY: another index operation is already in progress (${ownerText}).` };
|
|
10766
|
+
}
|
|
10767
|
+
function getProjectRoot(projectRoot, host) {
|
|
10768
|
+
if (projectRoot) {
|
|
10769
|
+
return projectRoot;
|
|
10534
10770
|
}
|
|
10535
|
-
|
|
10536
|
-
|
|
10771
|
+
const root = defaultProjectRoots.get(host);
|
|
10772
|
+
if (!root) {
|
|
10773
|
+
throw new Error("Codebase index tools not initialized. Plugin may not be loaded correctly.");
|
|
10537
10774
|
}
|
|
10538
|
-
|
|
10539
|
-
|
|
10540
|
-
|
|
10541
|
-
|
|
10542
|
-
|
|
10543
|
-
|
|
10544
|
-
|
|
10775
|
+
return root;
|
|
10776
|
+
}
|
|
10777
|
+
function getIndexerCacheKey(projectRoot, host) {
|
|
10778
|
+
return `${host}::${projectRoot}`;
|
|
10779
|
+
}
|
|
10780
|
+
function getOrCreateIndexer(projectRoot, host) {
|
|
10781
|
+
const key = getIndexerCacheKey(projectRoot, host);
|
|
10782
|
+
const cached = indexerCache.get(key);
|
|
10783
|
+
if (cached) {
|
|
10784
|
+
return cached;
|
|
10545
10785
|
}
|
|
10546
|
-
|
|
10547
|
-
|
|
10548
|
-
|
|
10549
|
-
|
|
10550
|
-
|
|
10551
|
-
};
|
|
10786
|
+
const config = parseConfig(loadRuntimeConfig(projectRoot, host));
|
|
10787
|
+
const indexer = new Indexer(projectRoot, config, host);
|
|
10788
|
+
indexerCache.set(key, indexer);
|
|
10789
|
+
configCache.set(key, config);
|
|
10790
|
+
return indexer;
|
|
10552
10791
|
}
|
|
10553
|
-
function
|
|
10554
|
-
|
|
10555
|
-
|
|
10792
|
+
function initializeTools(projectRoot, config, host = "opencode") {
|
|
10793
|
+
defaultProjectRoots.set(host, projectRoot);
|
|
10794
|
+
const key = getIndexerCacheKey(projectRoot, host);
|
|
10795
|
+
const indexer = new Indexer(projectRoot, config, host);
|
|
10796
|
+
indexerCache.set(key, indexer);
|
|
10797
|
+
configCache.set(key, config);
|
|
10556
10798
|
}
|
|
10557
|
-
function
|
|
10558
|
-
|
|
10799
|
+
function getIndexerForProject(projectRoot, host = "opencode") {
|
|
10800
|
+
const root = getProjectRoot(projectRoot, host);
|
|
10801
|
+
return getOrCreateIndexer(root, host);
|
|
10559
10802
|
}
|
|
10560
|
-
function
|
|
10561
|
-
|
|
10562
|
-
|
|
10563
|
-
|
|
10564
|
-
|
|
10565
|
-
|
|
10566
|
-
|
|
10567
|
-
const
|
|
10568
|
-
|
|
10569
|
-
|
|
10570
|
-
|
|
10571
|
-
if (baselinePath !== void 0 && typeof baselinePath !== "string") {
|
|
10572
|
-
throw new Error(`${sourceLabel}.baselinePath must be a string when provided`);
|
|
10573
|
-
}
|
|
10574
|
-
if (!isRecord2(thresholds)) {
|
|
10575
|
-
throw new Error(`${sourceLabel}.thresholds must be an object`);
|
|
10803
|
+
function refreshIndexerForDirectory(projectRoot, host = "opencode", config = parseConfig(loadRuntimeConfig(projectRoot, host))) {
|
|
10804
|
+
const key = getIndexerCacheKey(projectRoot, host);
|
|
10805
|
+
const indexer = new Indexer(projectRoot, config, host);
|
|
10806
|
+
indexerCache.set(key, indexer);
|
|
10807
|
+
configCache.set(key, config);
|
|
10808
|
+
}
|
|
10809
|
+
function shouldForceLocalizeProjectIndex(projectRoot, host = "opencode") {
|
|
10810
|
+
const root = getProjectRoot(projectRoot, host);
|
|
10811
|
+
const localIndexPath = path17.join(root, getHostProjectIndexRelativePath(host));
|
|
10812
|
+
if ((0, import_fs11.existsSync)(localIndexPath)) {
|
|
10813
|
+
return false;
|
|
10576
10814
|
}
|
|
10577
|
-
|
|
10578
|
-
|
|
10579
|
-
baselinePath: typeof baselinePath === "string" ? baselinePath : void 0,
|
|
10580
|
-
failOnMissingBaseline: typeof failOnMissingBaseline === "boolean" ? failOnMissingBaseline : true,
|
|
10581
|
-
thresholds: {
|
|
10582
|
-
hitAt5MaxDrop: parseThresholdValue(
|
|
10583
|
-
thresholds.hitAt5MaxDrop,
|
|
10584
|
-
"hitAt5MaxDrop",
|
|
10585
|
-
sourceLabel
|
|
10586
|
-
),
|
|
10587
|
-
mrrAt10MaxDrop: parseThresholdValue(
|
|
10588
|
-
thresholds.mrrAt10MaxDrop,
|
|
10589
|
-
"mrrAt10MaxDrop",
|
|
10590
|
-
sourceLabel
|
|
10591
|
-
),
|
|
10592
|
-
rawDistinctTop3RatioMaxDrop: parseThresholdValue(
|
|
10593
|
-
thresholds.rawDistinctTop3RatioMaxDrop,
|
|
10594
|
-
"rawDistinctTop3RatioMaxDrop",
|
|
10595
|
-
sourceLabel
|
|
10596
|
-
),
|
|
10597
|
-
p95LatencyMaxMultiplier: parseThresholdValue(
|
|
10598
|
-
thresholds.p95LatencyMaxMultiplier,
|
|
10599
|
-
"p95LatencyMaxMultiplier",
|
|
10600
|
-
sourceLabel
|
|
10601
|
-
),
|
|
10602
|
-
p95LatencyMaxAbsoluteMs: parseThresholdValue(
|
|
10603
|
-
thresholds.p95LatencyMaxAbsoluteMs,
|
|
10604
|
-
"p95LatencyMaxAbsoluteMs",
|
|
10605
|
-
sourceLabel
|
|
10606
|
-
),
|
|
10607
|
-
minHitAt5: parseThresholdValue(
|
|
10608
|
-
thresholds.minHitAt5,
|
|
10609
|
-
"minHitAt5",
|
|
10610
|
-
sourceLabel
|
|
10611
|
-
),
|
|
10612
|
-
minMrrAt10: parseThresholdValue(
|
|
10613
|
-
thresholds.minMrrAt10,
|
|
10614
|
-
"minMrrAt10",
|
|
10615
|
-
sourceLabel
|
|
10616
|
-
),
|
|
10617
|
-
minRawDistinctTop3Ratio: parseThresholdValue(
|
|
10618
|
-
thresholds.minRawDistinctTop3Ratio,
|
|
10619
|
-
"minRawDistinctTop3Ratio",
|
|
10620
|
-
sourceLabel
|
|
10621
|
-
)
|
|
10622
|
-
}
|
|
10623
|
-
};
|
|
10815
|
+
const inheritedIndexPath = resolveWorktreeFallbackProjectIndexPath(root, host);
|
|
10816
|
+
return inheritedIndexPath !== null;
|
|
10624
10817
|
}
|
|
10625
|
-
function
|
|
10626
|
-
const
|
|
10627
|
-
return
|
|
10818
|
+
async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
10819
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10820
|
+
return indexer.search(query, options.limit, {
|
|
10821
|
+
fileType: options.fileType,
|
|
10822
|
+
directory: options.directory,
|
|
10823
|
+
chunkType: options.chunkType,
|
|
10824
|
+
contextLines: options.contextLines,
|
|
10825
|
+
metadataOnly: options.metadataOnly,
|
|
10826
|
+
definitionIntent: options.definitionIntent,
|
|
10827
|
+
blameAuthor: options.blameAuthor,
|
|
10828
|
+
blameSha: options.blameSha,
|
|
10829
|
+
blameSince: options.blameSince
|
|
10830
|
+
});
|
|
10628
10831
|
}
|
|
10629
|
-
|
|
10630
|
-
|
|
10631
|
-
|
|
10632
|
-
|
|
10633
|
-
|
|
10634
|
-
|
|
10635
|
-
|
|
10636
|
-
|
|
10637
|
-
|
|
10638
|
-
|
|
10639
|
-
|
|
10640
|
-
|
|
10641
|
-
|
|
10642
|
-
|
|
10643
|
-
|
|
10644
|
-
|
|
10645
|
-
|
|
10646
|
-
|
|
10647
|
-
|
|
10648
|
-
|
|
10649
|
-
|
|
10650
|
-
|
|
10651
|
-
}
|
|
10652
|
-
const start = import_perf_hooks2.performance.now();
|
|
10653
|
-
const result = await indexer.search(query.query, 10, {
|
|
10654
|
-
metadataOnly: true,
|
|
10655
|
-
filterByBranch: !!query.expected.branch
|
|
10656
|
-
});
|
|
10657
|
-
const elapsed = import_perf_hooks2.performance.now() - start;
|
|
10658
|
-
const materialized = result.map((item) => ({
|
|
10659
|
-
filePath: item.filePath,
|
|
10660
|
-
startLine: item.startLine,
|
|
10661
|
-
endLine: item.endLine,
|
|
10662
|
-
score: item.score,
|
|
10663
|
-
chunkType: item.chunkType,
|
|
10664
|
-
name: item.name
|
|
10665
|
-
}));
|
|
10666
|
-
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
10667
|
-
}
|
|
10668
|
-
const logger = indexer.getLogger();
|
|
10669
|
-
const metricSnapshot = logger.getMetrics();
|
|
10670
|
-
const costPer1MTokensUsd = getEmbeddingCostPer1MTokens(effectiveConfig.embeddingProvider);
|
|
10671
|
-
const summary = {
|
|
10672
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10673
|
-
projectRoot: options.projectRoot,
|
|
10674
|
-
datasetPath,
|
|
10675
|
-
datasetName: dataset.name,
|
|
10676
|
-
datasetVersion: dataset.version,
|
|
10677
|
-
queryCount: dataset.queries.length,
|
|
10678
|
-
topK: 10,
|
|
10679
|
-
searchConfig: {
|
|
10680
|
-
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
10681
|
-
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
10682
|
-
rrfK: effectiveConfig.search.rrfK,
|
|
10683
|
-
rerankTopN: effectiveConfig.search.rerankTopN
|
|
10684
|
-
},
|
|
10685
|
-
metrics: computeEvalMetrics(
|
|
10686
|
-
dataset.queries,
|
|
10687
|
-
perQuery,
|
|
10688
|
-
metricSnapshot.embeddingApiCalls,
|
|
10689
|
-
metricSnapshot.embeddingTokensUsed,
|
|
10690
|
-
costPer1MTokensUsd
|
|
10691
|
-
)
|
|
10692
|
-
};
|
|
10693
|
-
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
10694
|
-
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
10695
|
-
writeJson(path15.join(outputDir, "summary.json"), summary);
|
|
10696
|
-
writeJson(path15.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
10697
|
-
let comparison;
|
|
10698
|
-
if (againstPath) {
|
|
10699
|
-
const baseline = loadSummary(againstPath);
|
|
10700
|
-
comparison = compareSummaries(summary, baseline, againstPath);
|
|
10701
|
-
writeJson(path15.join(outputDir, "compare.json"), comparison);
|
|
10702
|
-
}
|
|
10703
|
-
let gate;
|
|
10704
|
-
if (options.ciMode) {
|
|
10705
|
-
if (!budgetPath) {
|
|
10706
|
-
throw new Error("CI mode requires --budget path");
|
|
10707
|
-
}
|
|
10708
|
-
const budget = loadBudget(budgetPath);
|
|
10709
|
-
if (!comparison && budget.baselinePath) {
|
|
10710
|
-
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
10711
|
-
if ((0, import_fs11.existsSync)(resolvedBaseline)) {
|
|
10712
|
-
const baselineSummary = loadSummary(resolvedBaseline);
|
|
10713
|
-
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
10714
|
-
writeJson(path15.join(outputDir, "compare.json"), comparison);
|
|
10715
|
-
} else if (budget.failOnMissingBaseline) {
|
|
10716
|
-
throw new Error(
|
|
10717
|
-
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
10718
|
-
);
|
|
10719
|
-
}
|
|
10720
|
-
}
|
|
10721
|
-
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
10832
|
+
async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
10833
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10834
|
+
return indexer.findSimilar(code, options.limit, {
|
|
10835
|
+
fileType: options.fileType,
|
|
10836
|
+
directory: options.directory,
|
|
10837
|
+
chunkType: options.chunkType,
|
|
10838
|
+
excludeFile: options.excludeFile
|
|
10839
|
+
});
|
|
10840
|
+
}
|
|
10841
|
+
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
10842
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10843
|
+
return indexer.search(query, options.limit, {
|
|
10844
|
+
fileType: options.fileType,
|
|
10845
|
+
directory: options.directory,
|
|
10846
|
+
definitionIntent: true
|
|
10847
|
+
});
|
|
10848
|
+
}
|
|
10849
|
+
async function getCallGraphData(projectRoot, host, params) {
|
|
10850
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10851
|
+
if (params.direction === "callees") {
|
|
10852
|
+
if (!params.symbolId) {
|
|
10853
|
+
return { direction: "callees", callees: [], callers: [] };
|
|
10722
10854
|
}
|
|
10723
|
-
const
|
|
10724
|
-
|
|
10725
|
-
return { outputDir, summary, perQuery, comparison, gate };
|
|
10726
|
-
} finally {
|
|
10727
|
-
await indexer.close();
|
|
10855
|
+
const callees = await indexer.getCallees(params.symbolId, params.relationshipType);
|
|
10856
|
+
return { direction: "callees", callees, callers: [] };
|
|
10728
10857
|
}
|
|
10858
|
+
const callers = await indexer.getCallers(params.name, params.relationshipType);
|
|
10859
|
+
return { direction: "callers", callers, callees: [] };
|
|
10729
10860
|
}
|
|
10730
|
-
async function
|
|
10731
|
-
const
|
|
10732
|
-
|
|
10733
|
-
|
|
10734
|
-
|
|
10735
|
-
const
|
|
10736
|
-
|
|
10737
|
-
|
|
10738
|
-
|
|
10739
|
-
|
|
10740
|
-
|
|
10741
|
-
|
|
10742
|
-
|
|
10743
|
-
|
|
10744
|
-
|
|
10745
|
-
|
|
10746
|
-
|
|
10747
|
-
|
|
10748
|
-
|
|
10749
|
-
|
|
10750
|
-
|
|
10751
|
-
|
|
10752
|
-
|
|
10753
|
-
|
|
10861
|
+
async function getCallGraphPath(projectRoot, host, from, to, maxDepth) {
|
|
10862
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10863
|
+
return indexer.findCallPath(from, to, maxDepth);
|
|
10864
|
+
}
|
|
10865
|
+
async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
10866
|
+
const root = getProjectRoot(projectRoot, host);
|
|
10867
|
+
const key = getIndexerCacheKey(root, host);
|
|
10868
|
+
let indexer = getIndexerForProject(root, host);
|
|
10869
|
+
const runtimeConfig = configCache.get(key);
|
|
10870
|
+
try {
|
|
10871
|
+
if (args.estimateOnly) {
|
|
10872
|
+
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
10873
|
+
}
|
|
10874
|
+
const runIndex = async (target) => {
|
|
10875
|
+
const operation = args.force ? target.forceIndex.bind(target) : target.index.bind(target);
|
|
10876
|
+
return operation((progress) => {
|
|
10877
|
+
if (onProgress) {
|
|
10878
|
+
return onProgress(formatProgressTitle(progress), {
|
|
10879
|
+
phase: progress.phase,
|
|
10880
|
+
filesProcessed: progress.filesProcessed,
|
|
10881
|
+
totalFiles: progress.totalFiles,
|
|
10882
|
+
chunksProcessed: progress.chunksProcessed,
|
|
10883
|
+
totalChunks: progress.totalChunks,
|
|
10884
|
+
percentage: calculatePercentage(progress)
|
|
10754
10885
|
});
|
|
10755
10886
|
}
|
|
10756
|
-
|
|
10887
|
+
return Promise.resolve();
|
|
10888
|
+
});
|
|
10889
|
+
};
|
|
10890
|
+
let stats;
|
|
10891
|
+
if (args.force && shouldForceLocalizeProjectIndex(root, host)) {
|
|
10892
|
+
const inheritedIndexPath = resolveProjectIndexPath(root, runtimeConfig.scope, host);
|
|
10893
|
+
stats = await withIndexLock(inheritedIndexPath, "force-index", async () => {
|
|
10894
|
+
materializeLocalProjectConfig(root, loadProjectConfigLayer(root, host), host);
|
|
10895
|
+
refreshIndexerForDirectory(root, host, runtimeConfig);
|
|
10896
|
+
indexer = getIndexerForProject(root, host);
|
|
10897
|
+
return runIndex(indexer);
|
|
10898
|
+
}, { completeRecoveries: false });
|
|
10899
|
+
} else {
|
|
10900
|
+
stats = await runIndex(indexer);
|
|
10757
10901
|
}
|
|
10902
|
+
return { kind: "stats", stats };
|
|
10903
|
+
} catch (error) {
|
|
10904
|
+
const busyResult = getIndexBusyResult(error);
|
|
10905
|
+
if (!busyResult) throw error;
|
|
10906
|
+
return busyResult;
|
|
10758
10907
|
}
|
|
10759
|
-
const bestByHitAt5 = [...runs].sort(
|
|
10760
|
-
(a, b) => b.summary.metrics.hitAt5 - a.summary.metrics.hitAt5
|
|
10761
|
-
)[0];
|
|
10762
|
-
const bestByMrrAt10 = [...runs].sort(
|
|
10763
|
-
(a, b) => b.summary.metrics.mrrAt10 - a.summary.metrics.mrrAt10
|
|
10764
|
-
)[0];
|
|
10765
|
-
const bestByP95Latency = [...runs].sort(
|
|
10766
|
-
(a, b) => a.summary.metrics.latencyMs.p95 - b.summary.metrics.latencyMs.p95
|
|
10767
|
-
)[0];
|
|
10768
|
-
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
10769
|
-
const failedGateRuns = runs.filter((run) => run.gate && !run.gate.passed).length;
|
|
10770
|
-
const gatePassed = failedGateRuns === 0;
|
|
10771
|
-
const aggregate = {
|
|
10772
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10773
|
-
againstPath: options.againstPath,
|
|
10774
|
-
runCount: runs.length,
|
|
10775
|
-
runs,
|
|
10776
|
-
gatePassed,
|
|
10777
|
-
failedGateRuns,
|
|
10778
|
-
bestByHitAt5,
|
|
10779
|
-
bestByMrrAt10,
|
|
10780
|
-
bestByP95Latency
|
|
10781
|
-
};
|
|
10782
|
-
writeJson(path15.join(outputDir, "compare.json"), aggregate);
|
|
10783
|
-
const md = createSummaryMarkdown(
|
|
10784
|
-
bestByHitAt5?.summary ?? runs[0].summary,
|
|
10785
|
-
bestByHitAt5?.comparison,
|
|
10786
|
-
void 0,
|
|
10787
|
-
aggregate
|
|
10788
|
-
);
|
|
10789
|
-
writeText(path15.join(outputDir, "summary.md"), md);
|
|
10790
|
-
writeJson(path15.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
10791
|
-
return { outputDir, aggregate };
|
|
10792
|
-
}
|
|
10793
|
-
|
|
10794
|
-
// src/eval/cli.ts
|
|
10795
|
-
var path17 = __toESM(require("path"), 1);
|
|
10796
|
-
|
|
10797
|
-
// src/eval/cli-parser.ts
|
|
10798
|
-
var path16 = __toESM(require("path"), 1);
|
|
10799
|
-
function printUsage() {
|
|
10800
|
-
console.log(`
|
|
10801
|
-
Usage:
|
|
10802
|
-
opencode-codebase-index-mcp eval run [options]
|
|
10803
|
-
opencode-codebase-index-mcp eval compare --against <summary.json> [options]
|
|
10804
|
-
opencode-codebase-index-mcp eval diff --current <summary.json> --against <summary.json> [options]
|
|
10805
|
-
|
|
10806
|
-
Options:
|
|
10807
|
-
--project <path> Project root (default: cwd)
|
|
10808
|
-
--config <path> Config JSON path
|
|
10809
|
-
--dataset <path> Golden dataset path (default: benchmarks/golden/small.json)
|
|
10810
|
-
--current <path> Current summary.json path (required for eval diff)
|
|
10811
|
-
--output <path> Output root dir (default: benchmarks/results)
|
|
10812
|
-
--against <path> Baseline summary.json to compare against
|
|
10813
|
-
--budget <path> Budget file for CI mode (default: benchmarks/budgets/default.json)
|
|
10814
|
-
--ci Enable CI gate mode
|
|
10815
|
-
--reindex Force reindex before eval
|
|
10816
|
-
|
|
10817
|
-
Search overrides:
|
|
10818
|
-
--fusionStrategy <rrf|weighted>
|
|
10819
|
-
--hybridWeight <0-1>
|
|
10820
|
-
--rrfK <number>
|
|
10821
|
-
--rerankTopN <number>
|
|
10822
|
-
|
|
10823
|
-
Sweep options (comma-separated values):
|
|
10824
|
-
--sweepFusionStrategy <rrf,weighted>
|
|
10825
|
-
--sweepHybridWeight <0.3,0.5,0.7>
|
|
10826
|
-
--sweepRrfK <30,60,90>
|
|
10827
|
-
--sweepRerankTopN <10,20,40>
|
|
10828
|
-
`);
|
|
10829
10908
|
}
|
|
10830
|
-
function
|
|
10831
|
-
const
|
|
10832
|
-
|
|
10833
|
-
throw new Error(`${flag} must be a number`);
|
|
10834
|
-
}
|
|
10835
|
-
return parsed;
|
|
10909
|
+
async function getIndexStatus(projectRoot, host) {
|
|
10910
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10911
|
+
return indexer.getStatus();
|
|
10836
10912
|
}
|
|
10837
|
-
function
|
|
10838
|
-
|
|
10913
|
+
async function getIndexHealthCheck(projectRoot, host) {
|
|
10914
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10915
|
+
return indexer.healthCheck();
|
|
10839
10916
|
}
|
|
10840
|
-
function
|
|
10841
|
-
|
|
10842
|
-
|
|
10843
|
-
|
|
10844
|
-
|
|
10845
|
-
|
|
10846
|
-
|
|
10847
|
-
parsed.push(candidate);
|
|
10917
|
+
async function runIndexHealthCheck(projectRoot, host) {
|
|
10918
|
+
try {
|
|
10919
|
+
return { kind: "health", health: await getIndexHealthCheck(projectRoot, host) };
|
|
10920
|
+
} catch (error) {
|
|
10921
|
+
const busyResult = getIndexBusyResult(error);
|
|
10922
|
+
if (!busyResult) throw error;
|
|
10923
|
+
return busyResult;
|
|
10848
10924
|
}
|
|
10849
|
-
return parsed;
|
|
10850
10925
|
}
|
|
10851
|
-
function
|
|
10852
|
-
|
|
10853
|
-
|
|
10854
|
-
|
|
10926
|
+
async function getPrImpact(projectRoot, host, params) {
|
|
10927
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10928
|
+
return indexer.getPrImpact({
|
|
10929
|
+
pr: params.pr,
|
|
10930
|
+
branch: params.branch,
|
|
10931
|
+
maxDepth: params.maxDepth,
|
|
10932
|
+
hubThreshold: params.hubThreshold,
|
|
10933
|
+
checkConflicts: params.checkConflicts,
|
|
10934
|
+
direction: params.direction
|
|
10935
|
+
});
|
|
10855
10936
|
}
|
|
10856
|
-
function
|
|
10857
|
-
const
|
|
10858
|
-
|
|
10859
|
-
|
|
10860
|
-
|
|
10861
|
-
|
|
10862
|
-
|
|
10863
|
-
|
|
10864
|
-
|
|
10865
|
-
};
|
|
10866
|
-
for (let i = 0; i < argv.length; i += 1) {
|
|
10867
|
-
const arg = argv[i];
|
|
10868
|
-
const next = argv[i + 1];
|
|
10869
|
-
if (arg === "--project" && next) {
|
|
10870
|
-
parsed.projectRoot = path16.resolve(cwd, next);
|
|
10871
|
-
i += 1;
|
|
10872
|
-
continue;
|
|
10873
|
-
}
|
|
10874
|
-
if (arg === "--config" && next) {
|
|
10875
|
-
parsed.configPath = path16.resolve(cwd, next);
|
|
10876
|
-
i += 1;
|
|
10877
|
-
continue;
|
|
10878
|
-
}
|
|
10879
|
-
if (arg === "--dataset" && next) {
|
|
10880
|
-
parsed.datasetPath = next;
|
|
10881
|
-
i += 1;
|
|
10882
|
-
continue;
|
|
10883
|
-
}
|
|
10884
|
-
if (arg === "--current" && next) {
|
|
10885
|
-
parsed.currentPath = next;
|
|
10886
|
-
i += 1;
|
|
10887
|
-
continue;
|
|
10888
|
-
}
|
|
10889
|
-
if (arg === "--output" && next) {
|
|
10890
|
-
parsed.outputRoot = next;
|
|
10891
|
-
i += 1;
|
|
10892
|
-
continue;
|
|
10893
|
-
}
|
|
10894
|
-
if (arg === "--against" && next) {
|
|
10895
|
-
parsed.againstPath = next;
|
|
10896
|
-
i += 1;
|
|
10897
|
-
continue;
|
|
10898
|
-
}
|
|
10899
|
-
if (arg === "--budget" && next) {
|
|
10900
|
-
parsed.budgetPath = next;
|
|
10901
|
-
i += 1;
|
|
10902
|
-
continue;
|
|
10903
|
-
}
|
|
10904
|
-
if (arg === "--ci") {
|
|
10905
|
-
parsed.ciMode = true;
|
|
10906
|
-
continue;
|
|
10907
|
-
}
|
|
10908
|
-
if (arg === "--reindex") {
|
|
10909
|
-
parsed.reindex = true;
|
|
10910
|
-
continue;
|
|
10911
|
-
}
|
|
10912
|
-
if (arg === "--fusionStrategy" && next) {
|
|
10913
|
-
if (next !== "rrf" && next !== "weighted") {
|
|
10914
|
-
throw new Error("--fusionStrategy must be rrf or weighted");
|
|
10915
|
-
}
|
|
10916
|
-
parsed.fusionStrategy = next;
|
|
10917
|
-
i += 1;
|
|
10918
|
-
continue;
|
|
10919
|
-
}
|
|
10920
|
-
if (arg === "--hybridWeight" && next) {
|
|
10921
|
-
parsed.hybridWeight = parseNumber(next, "--hybridWeight");
|
|
10922
|
-
i += 1;
|
|
10923
|
-
continue;
|
|
10924
|
-
}
|
|
10925
|
-
if (arg === "--rrfK" && next) {
|
|
10926
|
-
parsed.rrfK = parseNumber(next, "--rrfK");
|
|
10927
|
-
i += 1;
|
|
10928
|
-
continue;
|
|
10929
|
-
}
|
|
10930
|
-
if (arg === "--rerankTopN" && next) {
|
|
10931
|
-
parsed.rerankTopN = parseNumber(next, "--rerankTopN");
|
|
10932
|
-
i += 1;
|
|
10933
|
-
continue;
|
|
10934
|
-
}
|
|
10935
|
-
if (arg === "--sweepFusionStrategy" && next) {
|
|
10936
|
-
parsed.sweep.fusionStrategy = parseCsvFusion(next);
|
|
10937
|
-
i += 1;
|
|
10938
|
-
continue;
|
|
10939
|
-
}
|
|
10940
|
-
if (arg === "--sweepHybridWeight" && next) {
|
|
10941
|
-
parsed.sweep.hybridWeight = parseCsvNumbers(next, "--sweepHybridWeight");
|
|
10942
|
-
i += 1;
|
|
10943
|
-
continue;
|
|
10944
|
-
}
|
|
10945
|
-
if (arg === "--sweepRrfK" && next) {
|
|
10946
|
-
parsed.sweep.rrfK = parseCsvNumbers(next, "--sweepRrfK");
|
|
10947
|
-
i += 1;
|
|
10948
|
-
continue;
|
|
10949
|
-
}
|
|
10950
|
-
if (arg === "--sweepRerankTopN" && next) {
|
|
10951
|
-
parsed.sweep.rerankTopN = parseCsvNumbers(next, "--sweepRerankTopN");
|
|
10952
|
-
i += 1;
|
|
10953
|
-
continue;
|
|
10954
|
-
}
|
|
10937
|
+
async function getIndexMetrics(projectRoot, host) {
|
|
10938
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10939
|
+
const logger = indexer.getLogger();
|
|
10940
|
+
if (!logger.isEnabled()) {
|
|
10941
|
+
return {
|
|
10942
|
+
enabled: false,
|
|
10943
|
+
metricsEnabled: false,
|
|
10944
|
+
text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true,\n "metrics": true\n }\n}\n```'
|
|
10945
|
+
};
|
|
10955
10946
|
}
|
|
10956
|
-
|
|
10957
|
-
|
|
10958
|
-
|
|
10959
|
-
|
|
10960
|
-
|
|
10961
|
-
|
|
10962
|
-
const current = argv[i];
|
|
10963
|
-
const next = argv[i + 1];
|
|
10964
|
-
if (current === "--against" && next) {
|
|
10965
|
-
explicitAgainst = next;
|
|
10966
|
-
i += 1;
|
|
10967
|
-
continue;
|
|
10968
|
-
}
|
|
10969
|
-
filtered.push(current);
|
|
10947
|
+
if (!logger.isMetricsEnabled()) {
|
|
10948
|
+
return {
|
|
10949
|
+
enabled: true,
|
|
10950
|
+
metricsEnabled: false,
|
|
10951
|
+
text: 'Metrics collection is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true,\n "metrics": true\n }\n}\n```'
|
|
10952
|
+
};
|
|
10970
10953
|
}
|
|
10971
10954
|
return {
|
|
10972
|
-
|
|
10973
|
-
|
|
10955
|
+
enabled: true,
|
|
10956
|
+
metricsEnabled: true,
|
|
10957
|
+
text: logger.formatMetrics()
|
|
10974
10958
|
};
|
|
10975
10959
|
}
|
|
10976
|
-
function
|
|
10977
|
-
|
|
10978
|
-
|
|
10979
|
-
|
|
10980
|
-
|
|
10981
|
-
|
|
10982
|
-
|
|
10983
|
-
|
|
10984
|
-
|
|
10985
|
-
|
|
10986
|
-
|
|
10987
|
-
|
|
10988
|
-
|
|
10989
|
-
|
|
10990
|
-
|
|
10991
|
-
|
|
10992
|
-
}
|
|
10960
|
+
async function getIndexLogs(projectRoot, host, args) {
|
|
10961
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10962
|
+
const logger = indexer.getLogger();
|
|
10963
|
+
if (!logger.isEnabled()) {
|
|
10964
|
+
return {
|
|
10965
|
+
kind: "disabled",
|
|
10966
|
+
text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true\n }\n}\n```'
|
|
10967
|
+
};
|
|
10968
|
+
}
|
|
10969
|
+
let logs;
|
|
10970
|
+
if (args.category) {
|
|
10971
|
+
logs = logger.getLogsByCategory(args.category, args.limit);
|
|
10972
|
+
} else if (args.level) {
|
|
10973
|
+
logs = logger.getLogsByLevel(args.level, args.limit);
|
|
10974
|
+
} else {
|
|
10975
|
+
logs = logger.getLogs(args.limit);
|
|
10976
|
+
}
|
|
10977
|
+
if (logs.length === 0) {
|
|
10978
|
+
return {
|
|
10979
|
+
kind: "entries",
|
|
10980
|
+
text: "No logs recorded yet. Logs are captured during indexing and search operations."
|
|
10981
|
+
};
|
|
10982
|
+
}
|
|
10983
|
+
const text = logs.map((entry) => {
|
|
10984
|
+
const dataStr = entry.data ? ` ${JSON.stringify(entry.data)}` : "";
|
|
10985
|
+
return `[${entry.timestamp}] [${entry.level.toUpperCase()}] [${entry.category}] ${entry.message}${dataStr}`;
|
|
10986
|
+
}).join("\n");
|
|
10987
|
+
return { kind: "entries", text };
|
|
10993
10988
|
}
|
|
10994
10989
|
|
|
10995
|
-
// src/
|
|
10996
|
-
|
|
10997
|
-
|
|
10998
|
-
|
|
10999
|
-
|
|
11000
|
-
|
|
10990
|
+
// src/tools/symbol-inference.ts
|
|
10991
|
+
var IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g;
|
|
10992
|
+
var QUOTED_BACKTICK_RE = /`([^`]+)`/g;
|
|
10993
|
+
var QUOTED_SINGLE_RE = /'([^'\\]+)'/g;
|
|
10994
|
+
var QUOTED_DOUBLE_RE = /"([^"]+)"/g;
|
|
10995
|
+
var SYMBOL_LIKE_RE = /^(?:[A-Za-z_$][A-Za-z0-9_$]*)$/;
|
|
10996
|
+
var CAMEL_CASE_RE = /^[a-z_][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*$/;
|
|
10997
|
+
var PASCAL_CASE_RE = /^[A-Z][A-Za-z0-9_$]*$/;
|
|
10998
|
+
var SNAKE_CASE_RE = /^[a-z][a-z0-9_]*_[a-z0-9_]+$/;
|
|
10999
|
+
var DEFINITION_INTENT_RE = /\b(where|defined|definition|define|declaration|symbol|function|method|class|interface|type)\b/i;
|
|
11000
|
+
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
11001
|
+
"a",
|
|
11002
|
+
"an",
|
|
11003
|
+
"and",
|
|
11004
|
+
"are",
|
|
11005
|
+
"at",
|
|
11006
|
+
"for",
|
|
11007
|
+
"find",
|
|
11008
|
+
"how",
|
|
11009
|
+
"i",
|
|
11010
|
+
"in",
|
|
11011
|
+
"is",
|
|
11012
|
+
"it",
|
|
11013
|
+
"of",
|
|
11014
|
+
"on",
|
|
11015
|
+
"that",
|
|
11016
|
+
"the",
|
|
11017
|
+
"definition",
|
|
11018
|
+
"show",
|
|
11019
|
+
"to",
|
|
11020
|
+
"where",
|
|
11021
|
+
"which",
|
|
11022
|
+
"what",
|
|
11023
|
+
"you",
|
|
11024
|
+
"your",
|
|
11025
|
+
"with"
|
|
11026
|
+
]);
|
|
11027
|
+
function stripCallSuffix(token) {
|
|
11028
|
+
return token.replace(/\(\s*\)$/, "");
|
|
11029
|
+
}
|
|
11030
|
+
function isLikelySymbolName(token) {
|
|
11031
|
+
if (!SYMBOL_LIKE_RE.test(token)) {
|
|
11032
|
+
return false;
|
|
11001
11033
|
}
|
|
11002
|
-
if (
|
|
11003
|
-
|
|
11004
|
-
|
|
11005
|
-
|
|
11006
|
-
|
|
11007
|
-
|
|
11008
|
-
|
|
11009
|
-
|
|
11010
|
-
|
|
11011
|
-
|
|
11012
|
-
|
|
11013
|
-
console.error(
|
|
11014
|
-
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
11015
|
-
);
|
|
11016
|
-
return 1;
|
|
11017
|
-
}
|
|
11018
|
-
return 0;
|
|
11019
|
-
}
|
|
11020
|
-
const result = await runEvaluation(runOptions);
|
|
11021
|
-
console.log(`Eval run complete. Artifacts: ${result.outputDir}`);
|
|
11022
|
-
console.log(
|
|
11023
|
-
`Hit@5=${(result.summary.metrics.hitAt5 * 100).toFixed(2)}% MRR@10=${result.summary.metrics.mrrAt10.toFixed(4)} p95=${result.summary.metrics.latencyMs.p95.toFixed(3)}ms`
|
|
11024
|
-
);
|
|
11025
|
-
if (result.gate && !result.gate.passed) {
|
|
11026
|
-
for (const violation of result.gate.violations) {
|
|
11027
|
-
console.error(`[CI-GATE] ${violation.metric}: ${violation.message}`);
|
|
11028
|
-
}
|
|
11029
|
-
return 1;
|
|
11034
|
+
if (STOP_WORDS.has(token.toLowerCase())) {
|
|
11035
|
+
return false;
|
|
11036
|
+
}
|
|
11037
|
+
return CAMEL_CASE_RE.test(token) || PASCAL_CASE_RE.test(token) || SNAKE_CASE_RE.test(token);
|
|
11038
|
+
}
|
|
11039
|
+
function extractQuotedIdentifiers(query) {
|
|
11040
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
11041
|
+
for (const match of query.matchAll(QUOTED_BACKTICK_RE)) {
|
|
11042
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
11043
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
11044
|
+
identifiers.add(candidate);
|
|
11030
11045
|
}
|
|
11031
|
-
return 0;
|
|
11032
11046
|
}
|
|
11033
|
-
|
|
11034
|
-
const
|
|
11035
|
-
if (
|
|
11036
|
-
|
|
11047
|
+
for (const match of query.matchAll(QUOTED_SINGLE_RE)) {
|
|
11048
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
11049
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
11050
|
+
identifiers.add(candidate);
|
|
11037
11051
|
}
|
|
11038
|
-
|
|
11039
|
-
|
|
11040
|
-
|
|
11041
|
-
|
|
11042
|
-
|
|
11043
|
-
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
11044
|
-
console.error(
|
|
11045
|
-
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
11046
|
-
);
|
|
11047
|
-
return 1;
|
|
11048
|
-
}
|
|
11049
|
-
return 0;
|
|
11052
|
+
}
|
|
11053
|
+
for (const match of query.matchAll(QUOTED_DOUBLE_RE)) {
|
|
11054
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
11055
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
11056
|
+
identifiers.add(candidate);
|
|
11050
11057
|
}
|
|
11051
|
-
const result = await runEvaluation(runOptions);
|
|
11052
|
-
console.log(`Eval compare complete. Artifacts: ${result.outputDir}`);
|
|
11053
|
-
return 0;
|
|
11054
11058
|
}
|
|
11055
|
-
|
|
11056
|
-
|
|
11057
|
-
|
|
11058
|
-
|
|
11059
|
+
return [...identifiers];
|
|
11060
|
+
}
|
|
11061
|
+
function extractBareIdentifiers(query) {
|
|
11062
|
+
const unquoted = query.replace(QUOTED_BACKTICK_RE, " ").replace(QUOTED_SINGLE_RE, " ").replace(QUOTED_DOUBLE_RE, " ");
|
|
11063
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
11064
|
+
for (const match of unquoted.matchAll(IDENTIFIER_RE)) {
|
|
11065
|
+
const candidate = stripCallSuffix(match[0]);
|
|
11066
|
+
if (isLikelySymbolName(candidate)) {
|
|
11067
|
+
identifiers.add(candidate);
|
|
11059
11068
|
}
|
|
11060
|
-
|
|
11061
|
-
|
|
11069
|
+
}
|
|
11070
|
+
return [...identifiers];
|
|
11071
|
+
}
|
|
11072
|
+
function isSingleMeaningfulToken(query, symbol) {
|
|
11073
|
+
const tokens = query.replace(/[`'"()]/g, " ").split(/[^A-Za-z0-9_$]+/).map((token) => token.trim().toLowerCase()).filter((token) => token.length > 0).filter((token) => !STOP_WORDS.has(token));
|
|
11074
|
+
return tokens.length === 1 && tokens[0] === symbol.toLowerCase();
|
|
11075
|
+
}
|
|
11076
|
+
function inferExactSymbolFromQuery(query) {
|
|
11077
|
+
const quoted = extractQuotedIdentifiers(query);
|
|
11078
|
+
if (quoted.length === 1) {
|
|
11079
|
+
return quoted[0];
|
|
11080
|
+
}
|
|
11081
|
+
if (quoted.length > 1) {
|
|
11082
|
+
return void 0;
|
|
11083
|
+
}
|
|
11084
|
+
const candidates = extractBareIdentifiers(query);
|
|
11085
|
+
if (candidates.length !== 1) {
|
|
11086
|
+
return void 0;
|
|
11087
|
+
}
|
|
11088
|
+
const candidate = candidates[0];
|
|
11089
|
+
if (DEFINITION_INTENT_RE.test(query)) {
|
|
11090
|
+
return candidate;
|
|
11091
|
+
}
|
|
11092
|
+
if (isSingleMeaningfulToken(query, candidate)) {
|
|
11093
|
+
return candidate;
|
|
11094
|
+
}
|
|
11095
|
+
return void 0;
|
|
11096
|
+
}
|
|
11097
|
+
|
|
11098
|
+
// src/tools/context.ts
|
|
11099
|
+
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
11100
|
+
var MAX_CONTEXT_RESULT_LIMIT = 100;
|
|
11101
|
+
var MIN_CONTEXT_PATH_DEPTH = 1;
|
|
11102
|
+
var MAX_CONTEXT_PATH_DEPTH = 100;
|
|
11103
|
+
function locations(results) {
|
|
11104
|
+
return results.map((result) => ({
|
|
11105
|
+
filePath: result.filePath,
|
|
11106
|
+
startLine: result.startLine,
|
|
11107
|
+
endLine: result.endLine,
|
|
11108
|
+
score: result.score,
|
|
11109
|
+
chunkType: result.chunkType,
|
|
11110
|
+
name: result.name
|
|
11111
|
+
}));
|
|
11112
|
+
}
|
|
11113
|
+
function packedResult(route, routedQuery, pack) {
|
|
11114
|
+
return {
|
|
11115
|
+
text: pack.text,
|
|
11116
|
+
details: {
|
|
11117
|
+
route,
|
|
11118
|
+
routedQuery,
|
|
11119
|
+
tokenBudget: pack.tokenBudget,
|
|
11120
|
+
tokenEstimate: pack.tokenEstimate,
|
|
11121
|
+
candidateCount: pack.candidateCount,
|
|
11122
|
+
deduplicatedCount: pack.deduplicatedCount,
|
|
11123
|
+
selectedCount: pack.selectedCount,
|
|
11124
|
+
omittedCount: pack.omittedCount,
|
|
11125
|
+
duplicateCount: pack.duplicateCount,
|
|
11126
|
+
limitOmittedCount: pack.limitOmittedCount,
|
|
11127
|
+
budgetOmittedCount: pack.budgetOmittedCount,
|
|
11128
|
+
results: locations(pack.results)
|
|
11062
11129
|
}
|
|
11063
|
-
|
|
11064
|
-
|
|
11065
|
-
|
|
11066
|
-
|
|
11130
|
+
};
|
|
11131
|
+
}
|
|
11132
|
+
async function resolveSearchContext(input, operations) {
|
|
11133
|
+
const explicitSymbol = input.symbol ?? void 0;
|
|
11134
|
+
const limit = input.limit ?? 10;
|
|
11135
|
+
const tokenBudget = input.tokenBudget ?? void 0;
|
|
11136
|
+
const lookupSymbol = explicitSymbol ?? inferExactSymbolFromQuery(input.query);
|
|
11137
|
+
if (lookupSymbol) {
|
|
11138
|
+
const definitions = await operations.lookup(lookupSymbol, MAX_CONTEXT_RESULT_LIMIT);
|
|
11139
|
+
if (definitions.length > 0) {
|
|
11140
|
+
return packedResult("definition", lookupSymbol, buildContextPack(definitions, {
|
|
11141
|
+
tokenBudget,
|
|
11142
|
+
maxResults: limit,
|
|
11143
|
+
heading: `Definition evidence for ${JSON.stringify(lookupSymbol)}`
|
|
11144
|
+
}));
|
|
11067
11145
|
}
|
|
11068
|
-
if (
|
|
11069
|
-
|
|
11146
|
+
if (explicitSymbol) {
|
|
11147
|
+
const fitted = fitTextToContextBudget(
|
|
11148
|
+
formatDefinitionLookup(definitions, lookupSymbol),
|
|
11149
|
+
tokenBudget
|
|
11150
|
+
);
|
|
11151
|
+
return { text: fitted.text };
|
|
11070
11152
|
}
|
|
11071
|
-
|
|
11072
|
-
|
|
11073
|
-
|
|
11074
|
-
const
|
|
11075
|
-
|
|
11076
|
-
|
|
11077
|
-
const comparison = compareSummaries(
|
|
11078
|
-
currentSummary,
|
|
11079
|
-
baselineSummary,
|
|
11080
|
-
path17.resolve(parsed.projectRoot, parsed.againstPath)
|
|
11153
|
+
}
|
|
11154
|
+
const results = await operations.search(input.query, MAX_CONTEXT_RESULT_LIMIT);
|
|
11155
|
+
if (results.length === 0) {
|
|
11156
|
+
const fitted = fitTextToContextBudget(
|
|
11157
|
+
"No matching code found. Try a different query or run index_codebase first.",
|
|
11158
|
+
tokenBudget
|
|
11081
11159
|
);
|
|
11082
|
-
|
|
11083
|
-
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
11084
|
-
writeJson(path17.join(outputDir, "compare.json"), comparison);
|
|
11085
|
-
writeText(path17.join(outputDir, "summary.md"), summaryMd);
|
|
11086
|
-
writeJson(path17.join(outputDir, "summary.json"), currentSummary);
|
|
11087
|
-
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
11088
|
-
return 0;
|
|
11160
|
+
return { text: fitted.text };
|
|
11089
11161
|
}
|
|
11090
|
-
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
|
|
11095
|
-
|
|
11096
|
-
|
|
11097
|
-
|
|
11098
|
-
|
|
11099
|
-
|
|
11100
|
-
|
|
11101
|
-
|
|
11102
|
-
|
|
11103
|
-
|
|
11104
|
-
|
|
11105
|
-
|
|
11106
|
-
|
|
11107
|
-
|
|
11108
|
-
|
|
11109
|
-
|
|
11110
|
-
|
|
11111
|
-
|
|
11112
|
-
|
|
11113
|
-
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11118
|
-
|
|
11119
|
-
|
|
11120
|
-
(args) => ({
|
|
11121
|
-
messages: [{
|
|
11122
|
-
role: "user",
|
|
11123
|
-
content: {
|
|
11124
|
-
type: "text",
|
|
11125
|
-
text: `Find code related to: "${args.query}"
|
|
11126
|
-
|
|
11127
|
-
Use a hybrid approach:
|
|
11128
|
-
1. First use codebase_peek to find semantic matches by meaning
|
|
11129
|
-
2. Then use grep for exact identifier matches
|
|
11130
|
-
3. Combine results for comprehensive coverage`
|
|
11131
|
-
}
|
|
11132
|
-
}]
|
|
11133
|
-
})
|
|
11134
|
-
);
|
|
11135
|
-
server.prompt(
|
|
11136
|
-
"index",
|
|
11137
|
-
"Index the codebase for semantic search",
|
|
11138
|
-
{ options: import_zod.z.string().optional().describe("Options: 'force' to rebuild, 'estimate' to check costs") },
|
|
11139
|
-
(args) => {
|
|
11140
|
-
const opts = args.options?.toLowerCase() ?? "";
|
|
11141
|
-
let instruction = "Use the index_codebase tool to index the codebase for semantic search.";
|
|
11142
|
-
if (opts.includes("force")) {
|
|
11143
|
-
instruction = "Use the index_codebase tool with force=true to rebuild the entire index from scratch.";
|
|
11144
|
-
} else if (opts.includes("estimate")) {
|
|
11145
|
-
instruction = "Use the index_codebase tool with estimateOnly=true to check the cost estimate before indexing.";
|
|
11146
|
-
}
|
|
11162
|
+
return packedResult("conceptual", input.query, buildContextPack(results, {
|
|
11163
|
+
tokenBudget,
|
|
11164
|
+
maxResults: limit,
|
|
11165
|
+
heading: `Codebase evidence for ${JSON.stringify(input.query)}`
|
|
11166
|
+
}));
|
|
11167
|
+
}
|
|
11168
|
+
function fittedDetails(route, fitted) {
|
|
11169
|
+
return {
|
|
11170
|
+
route,
|
|
11171
|
+
tokenBudget: fitted.tokenBudget,
|
|
11172
|
+
tokenEstimate: fitted.tokenEstimate,
|
|
11173
|
+
truncated: fitted.truncated
|
|
11174
|
+
};
|
|
11175
|
+
}
|
|
11176
|
+
async function resolveCodebaseContext(projectRoot, host, input) {
|
|
11177
|
+
const from = input.from ?? void 0;
|
|
11178
|
+
const to = input.to ?? void 0;
|
|
11179
|
+
const symbol = input.symbol ?? void 0;
|
|
11180
|
+
const limit = input.limit ?? 10;
|
|
11181
|
+
const maxDepth = input.maxDepth ?? 10;
|
|
11182
|
+
const fileType = input.fileType ?? void 0;
|
|
11183
|
+
const directory = input.directory ?? void 0;
|
|
11184
|
+
const tokenBudget = input.tokenBudget ?? void 0;
|
|
11185
|
+
if (from && to) {
|
|
11186
|
+
const path27 = await getCallGraphPath(projectRoot, host, from, to, maxDepth);
|
|
11187
|
+
if (path27.length > 0) {
|
|
11188
|
+
const fitted2 = fitTextToContextBudget(
|
|
11189
|
+
formatCallGraphPath(from, to, path27),
|
|
11190
|
+
tokenBudget
|
|
11191
|
+
);
|
|
11147
11192
|
return {
|
|
11148
|
-
|
|
11149
|
-
|
|
11150
|
-
content: { type: "text", text: instruction }
|
|
11151
|
-
}]
|
|
11193
|
+
text: fitted2.text,
|
|
11194
|
+
details: fittedDetails("path", fitted2)
|
|
11152
11195
|
};
|
|
11153
11196
|
}
|
|
11154
|
-
|
|
11155
|
-
|
|
11156
|
-
|
|
11157
|
-
|
|
11158
|
-
|
|
11159
|
-
()
|
|
11160
|
-
|
|
11161
|
-
|
|
11162
|
-
|
|
11163
|
-
|
|
11164
|
-
|
|
11165
|
-
|
|
11166
|
-
|
|
11167
|
-
|
|
11168
|
-
|
|
11169
|
-
|
|
11170
|
-
|
|
11171
|
-
|
|
11172
|
-
|
|
11173
|
-
|
|
11174
|
-
|
|
11175
|
-
|
|
11176
|
-
|
|
11177
|
-
|
|
11178
|
-
|
|
11179
|
-
|
|
11180
|
-
|
|
11181
|
-
|
|
11182
|
-
|
|
11197
|
+
const { callers } = await getCallGraphData(projectRoot, host, {
|
|
11198
|
+
name: to,
|
|
11199
|
+
direction: "callers"
|
|
11200
|
+
});
|
|
11201
|
+
const directEdge = callers.find((edge) => edge.fromSymbolName === from);
|
|
11202
|
+
if (directEdge) {
|
|
11203
|
+
const location = directEdge.fromSymbolFilePath ? ` at ${directEdge.fromSymbolFilePath}:${directEdge.line}` : "";
|
|
11204
|
+
const fitted2 = fitTextToContextBudget(
|
|
11205
|
+
`Direct path: ${from} --${directEdge.callType}--> ${to}${location} (edge is ${directEdge.isResolved ? "resolved" : "unresolved"}).`,
|
|
11206
|
+
tokenBudget
|
|
11207
|
+
);
|
|
11208
|
+
return {
|
|
11209
|
+
text: fitted2.text,
|
|
11210
|
+
details: fittedDetails("direct-edge", fitted2)
|
|
11211
|
+
};
|
|
11212
|
+
}
|
|
11213
|
+
const fitted = fitTextToContextBudget(
|
|
11214
|
+
formatCallGraphPath(from, to, path27),
|
|
11215
|
+
tokenBudget
|
|
11216
|
+
);
|
|
11217
|
+
return {
|
|
11218
|
+
text: fitted.text,
|
|
11219
|
+
details: fittedDetails("path", fitted)
|
|
11220
|
+
};
|
|
11221
|
+
}
|
|
11222
|
+
return resolveSearchContext({ query: input.query, symbol, limit, tokenBudget }, {
|
|
11223
|
+
lookup: (lookupSymbol, retrievalLimit) => implementationLookup(projectRoot, host, lookupSymbol, {
|
|
11224
|
+
limit: retrievalLimit,
|
|
11225
|
+
fileType,
|
|
11226
|
+
directory
|
|
11227
|
+
}),
|
|
11228
|
+
search: (query, retrievalLimit) => searchCodebase(projectRoot, host, query, {
|
|
11229
|
+
limit: retrievalLimit,
|
|
11230
|
+
fileType,
|
|
11231
|
+
directory,
|
|
11232
|
+
metadataOnly: true
|
|
11183
11233
|
})
|
|
11184
|
-
);
|
|
11234
|
+
});
|
|
11185
11235
|
}
|
|
11186
11236
|
|
|
11187
|
-
// src/
|
|
11188
|
-
|
|
11189
|
-
|
|
11190
|
-
|
|
11191
|
-
|
|
11192
|
-
|
|
11193
|
-
|
|
11194
|
-
|
|
11195
|
-
|
|
11196
|
-
|
|
11197
|
-
}
|
|
11198
|
-
function formatIndexStats(stats, verbose = false) {
|
|
11199
|
-
if (stats.resetCorruptedIndex) {
|
|
11200
|
-
return stats.warning ?? "Detected a corrupted local index and reset it during indexing. Run index_codebase again to rebuild search data.";
|
|
11237
|
+
// src/eval/budget.ts
|
|
11238
|
+
function evaluateBudgetGate(budget, summary, comparison) {
|
|
11239
|
+
const BASELINE_P95_EPSILON_MS = 1e-3;
|
|
11240
|
+
const violations = [];
|
|
11241
|
+
const { thresholds } = budget;
|
|
11242
|
+
if (thresholds.minHitAt5 !== void 0 && summary.metrics.hitAt5 < thresholds.minHitAt5) {
|
|
11243
|
+
violations.push({
|
|
11244
|
+
metric: "minHitAt5",
|
|
11245
|
+
message: `Hit@5 ${summary.metrics.hitAt5.toFixed(4)} is below minimum ${thresholds.minHitAt5.toFixed(4)}`
|
|
11246
|
+
});
|
|
11201
11247
|
}
|
|
11202
|
-
|
|
11203
|
-
|
|
11204
|
-
|
|
11205
|
-
|
|
11206
|
-
|
|
11207
|
-
}
|
|
11208
|
-
lines.push("");
|
|
11248
|
+
if (thresholds.minMrrAt10 !== void 0 && summary.metrics.mrrAt10 < thresholds.minMrrAt10) {
|
|
11249
|
+
violations.push({
|
|
11250
|
+
metric: "minMrrAt10",
|
|
11251
|
+
message: `MRR@10 ${summary.metrics.mrrAt10.toFixed(4)} is below minimum ${thresholds.minMrrAt10.toFixed(4)}`
|
|
11252
|
+
});
|
|
11209
11253
|
}
|
|
11210
|
-
if (
|
|
11211
|
-
|
|
11212
|
-
|
|
11213
|
-
|
|
11214
|
-
|
|
11215
|
-
|
|
11216
|
-
|
|
11217
|
-
|
|
11254
|
+
if (thresholds.minRawDistinctTop3Ratio !== void 0 && summary.metrics.rawDistinctTop3Ratio < thresholds.minRawDistinctTop3Ratio) {
|
|
11255
|
+
violations.push({
|
|
11256
|
+
metric: "minRawDistinctTop3Ratio",
|
|
11257
|
+
message: `Raw Distinct Top@3 ${summary.metrics.rawDistinctTop3Ratio.toFixed(4)} is below minimum ${thresholds.minRawDistinctTop3Ratio.toFixed(4)}`
|
|
11258
|
+
});
|
|
11259
|
+
}
|
|
11260
|
+
if (comparison) {
|
|
11261
|
+
if (thresholds.hitAt5MaxDrop !== void 0 && comparison.deltas.hitAt5.absolute < -thresholds.hitAt5MaxDrop) {
|
|
11262
|
+
violations.push({
|
|
11263
|
+
metric: "hitAt5MaxDrop",
|
|
11264
|
+
message: `Hit@5 drop ${comparison.deltas.hitAt5.absolute.toFixed(4)} exceeds allowed -${thresholds.hitAt5MaxDrop.toFixed(4)}`
|
|
11265
|
+
});
|
|
11218
11266
|
}
|
|
11219
|
-
|
|
11220
|
-
|
|
11221
|
-
|
|
11267
|
+
if (thresholds.mrrAt10MaxDrop !== void 0 && comparison.deltas.mrrAt10.absolute < -thresholds.mrrAt10MaxDrop) {
|
|
11268
|
+
violations.push({
|
|
11269
|
+
metric: "mrrAt10MaxDrop",
|
|
11270
|
+
message: `MRR@10 drop ${comparison.deltas.mrrAt10.absolute.toFixed(4)} exceeds allowed -${thresholds.mrrAt10MaxDrop.toFixed(4)}`
|
|
11271
|
+
});
|
|
11222
11272
|
}
|
|
11223
|
-
if (
|
|
11224
|
-
|
|
11273
|
+
if (thresholds.rawDistinctTop3RatioMaxDrop !== void 0 && comparison.deltas.rawDistinctTop3Ratio.absolute < -thresholds.rawDistinctTop3RatioMaxDrop) {
|
|
11274
|
+
violations.push({
|
|
11275
|
+
metric: "rawDistinctTop3RatioMaxDrop",
|
|
11276
|
+
message: `Raw Distinct Top@3 drop ${comparison.deltas.rawDistinctTop3Ratio.absolute.toFixed(4)} exceeds allowed -${thresholds.rawDistinctTop3RatioMaxDrop.toFixed(4)}`
|
|
11277
|
+
});
|
|
11225
11278
|
}
|
|
11226
|
-
|
|
11227
|
-
|
|
11228
|
-
|
|
11229
|
-
|
|
11230
|
-
|
|
11231
|
-
|
|
11232
|
-
|
|
11233
|
-
|
|
11234
|
-
|
|
11235
|
-
|
|
11236
|
-
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
11237
|
-
}
|
|
11238
|
-
if (excluded.length > 0) {
|
|
11239
|
-
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
11240
|
-
}
|
|
11241
|
-
if (gitignored.length > 0) {
|
|
11242
|
-
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
11279
|
+
if (thresholds.p95LatencyMaxMultiplier !== void 0) {
|
|
11280
|
+
const baselineP95 = comparison.deltas.latencyP95Ms.baseline;
|
|
11281
|
+
if (baselineP95 > BASELINE_P95_EPSILON_MS) {
|
|
11282
|
+
const allowed = baselineP95 * thresholds.p95LatencyMaxMultiplier;
|
|
11283
|
+
if (summary.metrics.latencyMs.p95 > allowed) {
|
|
11284
|
+
violations.push({
|
|
11285
|
+
metric: "p95LatencyMaxMultiplier",
|
|
11286
|
+
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds allowed ${allowed.toFixed(3)}ms (${thresholds.p95LatencyMaxMultiplier.toFixed(2)}x baseline)`
|
|
11287
|
+
});
|
|
11288
|
+
}
|
|
11243
11289
|
}
|
|
11244
11290
|
}
|
|
11245
|
-
|
|
11246
|
-
|
|
11247
|
-
|
|
11291
|
+
}
|
|
11292
|
+
if (thresholds.p95LatencyMaxAbsoluteMs !== void 0 && summary.metrics.latencyMs.p95 > thresholds.p95LatencyMaxAbsoluteMs) {
|
|
11293
|
+
violations.push({
|
|
11294
|
+
metric: "p95LatencyMaxAbsoluteMs",
|
|
11295
|
+
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds absolute maximum ${thresholds.p95LatencyMaxAbsoluteMs.toFixed(3)}ms`
|
|
11296
|
+
});
|
|
11297
|
+
}
|
|
11298
|
+
const context = summary.metrics.contextEfficiency;
|
|
11299
|
+
if (context.queryCount > 0) {
|
|
11300
|
+
if (thresholds.maxContextResponseTokensAverage !== void 0 && context.responseTokens.average > thresholds.maxContextResponseTokensAverage) {
|
|
11301
|
+
violations.push({
|
|
11302
|
+
metric: "maxContextResponseTokensAverage",
|
|
11303
|
+
message: `Context response average ${context.responseTokens.average.toFixed(1)} tokens exceeds maximum ${thresholds.maxContextResponseTokensAverage.toFixed(1)}`
|
|
11304
|
+
});
|
|
11305
|
+
}
|
|
11306
|
+
if (thresholds.maxContextResponseTokensP95 !== void 0 && context.responseTokens.p95 > thresholds.maxContextResponseTokensP95) {
|
|
11307
|
+
violations.push({
|
|
11308
|
+
metric: "maxContextResponseTokensP95",
|
|
11309
|
+
message: `Context response p95 ${context.responseTokens.p95.toFixed(1)} tokens exceeds maximum ${thresholds.maxContextResponseTokensP95.toFixed(1)}`
|
|
11310
|
+
});
|
|
11311
|
+
}
|
|
11312
|
+
if (thresholds.maxContextResponseTokensMax !== void 0 && context.responseTokens.max > thresholds.maxContextResponseTokensMax) {
|
|
11313
|
+
violations.push({
|
|
11314
|
+
metric: "maxContextResponseTokensMax",
|
|
11315
|
+
message: `Context response maximum ${context.responseTokens.max.toFixed(1)} tokens exceeds maximum ${thresholds.maxContextResponseTokensMax.toFixed(1)}`
|
|
11316
|
+
});
|
|
11317
|
+
}
|
|
11318
|
+
if (thresholds.maxContextDuplicateCandidateRatio !== void 0 && context.duplicateCandidateRatio > thresholds.maxContextDuplicateCandidateRatio) {
|
|
11319
|
+
violations.push({
|
|
11320
|
+
metric: "maxContextDuplicateCandidateRatio",
|
|
11321
|
+
message: `Context duplicate candidate ratio ${context.duplicateCandidateRatio.toFixed(4)} exceeds maximum ${thresholds.maxContextDuplicateCandidateRatio.toFixed(4)}`
|
|
11322
|
+
});
|
|
11323
|
+
}
|
|
11324
|
+
if (thresholds.minContextSelectedFileRatio !== void 0 && context.selectedFileRatio < thresholds.minContextSelectedFileRatio) {
|
|
11325
|
+
violations.push({
|
|
11326
|
+
metric: "minContextSelectedFileRatio",
|
|
11327
|
+
message: `Context selected-file ratio ${context.selectedFileRatio.toFixed(4)} is below minimum ${thresholds.minContextSelectedFileRatio.toFixed(4)}`
|
|
11328
|
+
});
|
|
11329
|
+
}
|
|
11330
|
+
if (thresholds.minContextHitAt5Per1kResponseTokens !== void 0 && context.hitAt5Per1kResponseTokens < thresholds.minContextHitAt5Per1kResponseTokens) {
|
|
11331
|
+
violations.push({
|
|
11332
|
+
metric: "minContextHitAt5Per1kResponseTokens",
|
|
11333
|
+
message: `Context Hit@5 per 1k response tokens ${context.hitAt5Per1kResponseTokens.toFixed(4)} is below minimum ${thresholds.minContextHitAt5Per1kResponseTokens.toFixed(4)}`
|
|
11334
|
+
});
|
|
11335
|
+
}
|
|
11336
|
+
if (thresholds.minContextMrrAt10Per1kResponseTokens !== void 0 && context.mrrAt10Per1kResponseTokens < thresholds.minContextMrrAt10Per1kResponseTokens) {
|
|
11337
|
+
violations.push({
|
|
11338
|
+
metric: "minContextMrrAt10Per1kResponseTokens",
|
|
11339
|
+
message: `Context MRR@10 per 1k response tokens ${context.mrrAt10Per1kResponseTokens.toFixed(4)} is below minimum ${thresholds.minContextMrrAt10Per1kResponseTokens.toFixed(4)}`
|
|
11340
|
+
});
|
|
11248
11341
|
}
|
|
11249
11342
|
}
|
|
11250
|
-
return
|
|
11343
|
+
return {
|
|
11344
|
+
passed: violations.length === 0,
|
|
11345
|
+
budgetName: budget.name,
|
|
11346
|
+
violations
|
|
11347
|
+
};
|
|
11348
|
+
}
|
|
11349
|
+
|
|
11350
|
+
// src/eval/metrics.ts
|
|
11351
|
+
function percentile(values, p) {
|
|
11352
|
+
if (values.length === 0) return 0;
|
|
11353
|
+
if (values.length === 1) return values[0];
|
|
11354
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
11355
|
+
const x = p * (sorted.length - 1);
|
|
11356
|
+
const lowerIndex = Math.floor(x);
|
|
11357
|
+
const upperIndex = Math.ceil(x);
|
|
11358
|
+
if (lowerIndex === upperIndex) {
|
|
11359
|
+
return sorted[lowerIndex];
|
|
11360
|
+
}
|
|
11361
|
+
const fraction = x - lowerIndex;
|
|
11362
|
+
return sorted[lowerIndex] + fraction * (sorted[upperIndex] - sorted[lowerIndex]);
|
|
11363
|
+
}
|
|
11364
|
+
function normalizePath(input) {
|
|
11365
|
+
return normalizePathSeparators(input);
|
|
11366
|
+
}
|
|
11367
|
+
function uniqueResultsByPath(results) {
|
|
11368
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11369
|
+
const unique = [];
|
|
11370
|
+
for (const result of results) {
|
|
11371
|
+
const normalized = normalizePath(result.filePath);
|
|
11372
|
+
if (seen.has(normalized)) continue;
|
|
11373
|
+
seen.add(normalized);
|
|
11374
|
+
unique.push(result);
|
|
11375
|
+
}
|
|
11376
|
+
return unique;
|
|
11377
|
+
}
|
|
11378
|
+
function distinctTopKRatio(results, k) {
|
|
11379
|
+
const top = results.slice(0, k);
|
|
11380
|
+
if (top.length === 0) return 0;
|
|
11381
|
+
const distinct = new Set(top.map((result) => normalizePath(result.filePath))).size;
|
|
11382
|
+
return distinct / top.length;
|
|
11383
|
+
}
|
|
11384
|
+
function pathMatchesExpected(actualPath, expectedPath) {
|
|
11385
|
+
const actual = normalizePath(actualPath);
|
|
11386
|
+
const expected = normalizePath(expectedPath);
|
|
11387
|
+
if (actual === expected) return true;
|
|
11388
|
+
return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);
|
|
11389
|
+
}
|
|
11390
|
+
function getRelevantPaths(query) {
|
|
11391
|
+
const fromExact = query.expected.filePath ? [query.expected.filePath] : [];
|
|
11392
|
+
const fromAcceptable = query.expected.acceptableFiles ?? [];
|
|
11393
|
+
return Array.from(/* @__PURE__ */ new Set([...fromExact, ...fromAcceptable]));
|
|
11394
|
+
}
|
|
11395
|
+
function isRelevantResult(filePath, relevantPaths) {
|
|
11396
|
+
return relevantPaths.some((expected) => pathMatchesExpected(filePath, expected));
|
|
11397
|
+
}
|
|
11398
|
+
function reciprocalRankAtK(results, relevantPaths, k) {
|
|
11399
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
11400
|
+
for (let i = 0; i < top.length; i += 1) {
|
|
11401
|
+
if (isRelevantResult(top[i].filePath, relevantPaths)) {
|
|
11402
|
+
return 1 / (i + 1);
|
|
11403
|
+
}
|
|
11404
|
+
}
|
|
11405
|
+
return 0;
|
|
11406
|
+
}
|
|
11407
|
+
function ndcgAtK(results, relevantPaths, k) {
|
|
11408
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
11409
|
+
const dcg = top.reduce((sum, result, i) => {
|
|
11410
|
+
const rel = isRelevantResult(result.filePath, relevantPaths) ? 1 : 0;
|
|
11411
|
+
return sum + rel / Math.log2(i + 2);
|
|
11412
|
+
}, 0);
|
|
11413
|
+
const idealLen = Math.min(k, relevantPaths.length);
|
|
11414
|
+
const idcg = Array.from({ length: idealLen }, (_, i) => 1 / Math.log2(i + 2)).reduce(
|
|
11415
|
+
(sum, value) => sum + value,
|
|
11416
|
+
0
|
|
11417
|
+
);
|
|
11418
|
+
return idcg === 0 ? 0 : dcg / idcg;
|
|
11251
11419
|
}
|
|
11252
|
-
function
|
|
11253
|
-
|
|
11254
|
-
|
|
11255
|
-
|
|
11256
|
-
|
|
11257
|
-
|
|
11258
|
-
|
|
11259
|
-
|
|
11260
|
-
|
|
11261
|
-
|
|
11262
|
-
if (status.failedBatchesPath) {
|
|
11263
|
-
lines2.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
11264
|
-
}
|
|
11265
|
-
return lines2.join("\n");
|
|
11266
|
-
}
|
|
11267
|
-
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
11420
|
+
function isDocsOrTestsPath(filePath) {
|
|
11421
|
+
const lowered = normalizePath(filePath).toLowerCase();
|
|
11422
|
+
return lowered.includes("/docs/") || lowered.includes("/test/") || lowered.includes("/tests/") || lowered.includes("readme") || lowered.includes("/benchmarks/");
|
|
11423
|
+
}
|
|
11424
|
+
function classifyFailureBucket(query, results, k) {
|
|
11425
|
+
const relevantPaths = getRelevantPaths(query);
|
|
11426
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
11427
|
+
const hasRelevantTopK = top.some((result) => isRelevantResult(result.filePath, relevantPaths));
|
|
11428
|
+
if (!hasRelevantTopK) {
|
|
11429
|
+
return "no-relevant-hit-top-k";
|
|
11268
11430
|
}
|
|
11269
|
-
|
|
11270
|
-
|
|
11271
|
-
|
|
11272
|
-
|
|
11273
|
-
|
|
11274
|
-
];
|
|
11275
|
-
if (status.currentBranch !== "default") {
|
|
11276
|
-
lines.push(`Current branch: ${status.currentBranch}`);
|
|
11277
|
-
lines.push(`Base branch: ${status.baseBranch}`);
|
|
11431
|
+
if (query.expected.symbol) {
|
|
11432
|
+
const hasSymbol = top.some(
|
|
11433
|
+
(result) => isRelevantResult(result.filePath, relevantPaths) && result.name === query.expected.symbol
|
|
11434
|
+
);
|
|
11435
|
+
if (!hasSymbol) return "wrong-symbol";
|
|
11278
11436
|
}
|
|
11279
|
-
|
|
11280
|
-
|
|
11281
|
-
|
|
11282
|
-
if (status.failedBatchesPath) {
|
|
11283
|
-
lines.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
11284
|
-
}
|
|
11437
|
+
const top1 = top[0];
|
|
11438
|
+
if (top1 && !isRelevantResult(top1.filePath, relevantPaths) && isDocsOrTestsPath(top1.filePath)) {
|
|
11439
|
+
return "docs-tests-outranking-source";
|
|
11285
11440
|
}
|
|
11286
|
-
if (
|
|
11287
|
-
|
|
11288
|
-
lines.push(`INDEX WARNING: ${status.warning}`);
|
|
11441
|
+
if (top1 && !isRelevantResult(top1.filePath, relevantPaths)) {
|
|
11442
|
+
return "wrong-file";
|
|
11289
11443
|
}
|
|
11290
|
-
|
|
11291
|
-
|
|
11292
|
-
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11296
|
-
|
|
11444
|
+
return void 0;
|
|
11445
|
+
}
|
|
11446
|
+
function buildPerQueryResult(query, results, latencyMs, k, route = {
|
|
11447
|
+
resolvedRoute: "search",
|
|
11448
|
+
routedQuery: query.query
|
|
11449
|
+
}, context) {
|
|
11450
|
+
const relevantPaths = getRelevantPaths(query);
|
|
11451
|
+
const deduped = uniqueResultsByPath(results);
|
|
11452
|
+
const hitAt = (cutoff) => deduped.slice(0, cutoff).some((result) => isRelevantResult(result.filePath, relevantPaths));
|
|
11453
|
+
const perQuery = {
|
|
11454
|
+
id: query.id,
|
|
11455
|
+
query: query.query,
|
|
11456
|
+
queryType: query.queryType,
|
|
11457
|
+
retrievalMode: query.retrievalMode ?? "search",
|
|
11458
|
+
resolvedRoute: route.resolvedRoute,
|
|
11459
|
+
routedQuery: route.routedQuery,
|
|
11460
|
+
latencyMs,
|
|
11461
|
+
hitAt1: hitAt(1),
|
|
11462
|
+
hitAt3: hitAt(3),
|
|
11463
|
+
hitAt5: hitAt(5),
|
|
11464
|
+
hitAt10: hitAt(10),
|
|
11465
|
+
reciprocalRankAt10: reciprocalRankAtK(deduped, relevantPaths, 10),
|
|
11466
|
+
ndcgAt10: ndcgAtK(deduped, relevantPaths, 10),
|
|
11467
|
+
failureBucket: classifyFailureBucket(query, results, k),
|
|
11468
|
+
rawTop3DistinctRatio: distinctTopKRatio(results, 3),
|
|
11469
|
+
tokenBudget: context?.tokenBudget,
|
|
11470
|
+
responseTokens: context?.responseTokens ?? 0,
|
|
11471
|
+
candidateCount: context?.candidateCount ?? results.length,
|
|
11472
|
+
deduplicatedCount: context?.deduplicatedCount ?? results.length,
|
|
11473
|
+
selectedCount: results.length,
|
|
11474
|
+
omittedCount: context?.omittedCount ?? 0,
|
|
11475
|
+
duplicateCandidateRatio: context && context.candidateCount > 0 ? (context.candidateCount - context.deduplicatedCount) / context.candidateCount : 0,
|
|
11476
|
+
selectedFileRatio: results.length === 0 ? 0 : new Set(results.map((result) => normalizePath(result.filePath))).size / results.length,
|
|
11477
|
+
results: deduped
|
|
11478
|
+
};
|
|
11479
|
+
return perQuery;
|
|
11480
|
+
}
|
|
11481
|
+
function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingTokensUsed, costPer1MTokensUsd) {
|
|
11482
|
+
const count = perQuery.length;
|
|
11483
|
+
const safeDiv = (value) => count === 0 ? 0 : value / count;
|
|
11484
|
+
const sum = {
|
|
11485
|
+
hitAt1: 0,
|
|
11486
|
+
hitAt3: 0,
|
|
11487
|
+
hitAt5: 0,
|
|
11488
|
+
hitAt10: 0,
|
|
11489
|
+
mrrAt10: 0,
|
|
11490
|
+
ndcgAt10: 0,
|
|
11491
|
+
distinctTop3Ratio: 0,
|
|
11492
|
+
rawDistinctTop3Ratio: 0
|
|
11493
|
+
};
|
|
11494
|
+
const failureBuckets = {
|
|
11495
|
+
"wrong-file": 0,
|
|
11496
|
+
"wrong-symbol": 0,
|
|
11497
|
+
"docs-tests-outranking-source": 0,
|
|
11498
|
+
"no-relevant-hit-top-k": 0
|
|
11499
|
+
};
|
|
11500
|
+
const latencies = perQuery.map((item) => item.latencyMs);
|
|
11501
|
+
const contextQueries = perQuery.filter((item) => item.retrievalMode === "context");
|
|
11502
|
+
const contextResponseTokens = contextQueries.map((item) => item.responseTokens);
|
|
11503
|
+
const totalContextResponseTokens = contextResponseTokens.reduce((sum2, value) => sum2 + value, 0);
|
|
11504
|
+
const contextTokenUnits = totalContextResponseTokens / 1e3;
|
|
11505
|
+
for (const query of perQuery) {
|
|
11506
|
+
if (query.hitAt1) sum.hitAt1 += 1;
|
|
11507
|
+
if (query.hitAt3) sum.hitAt3 += 1;
|
|
11508
|
+
if (query.hitAt5) sum.hitAt5 += 1;
|
|
11509
|
+
if (query.hitAt10) sum.hitAt10 += 1;
|
|
11510
|
+
sum.mrrAt10 += query.reciprocalRankAt10;
|
|
11511
|
+
sum.ndcgAt10 += query.ndcgAt10;
|
|
11512
|
+
sum.distinctTop3Ratio += distinctTopKRatio(query.results, 3);
|
|
11513
|
+
sum.rawDistinctTop3Ratio += query.rawTop3DistinctRatio;
|
|
11514
|
+
if (query.failureBucket) {
|
|
11515
|
+
failureBuckets[query.failureBucket] += 1;
|
|
11297
11516
|
}
|
|
11298
|
-
} else if (!status.compatibility) {
|
|
11299
|
-
lines.push(`Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
11300
|
-
} else {
|
|
11301
|
-
lines.push(`Compatibility: Index is compatible with the current provider and model.`);
|
|
11302
11517
|
}
|
|
11303
|
-
|
|
11518
|
+
const queryTokens = queries.reduce((acc, q) => acc + estimateTokens(q.query), 0);
|
|
11519
|
+
return {
|
|
11520
|
+
hitAt1: safeDiv(sum.hitAt1),
|
|
11521
|
+
hitAt3: safeDiv(sum.hitAt3),
|
|
11522
|
+
hitAt5: safeDiv(sum.hitAt5),
|
|
11523
|
+
hitAt10: safeDiv(sum.hitAt10),
|
|
11524
|
+
mrrAt10: safeDiv(sum.mrrAt10),
|
|
11525
|
+
ndcgAt10: safeDiv(sum.ndcgAt10),
|
|
11526
|
+
distinctTop3Ratio: safeDiv(sum.distinctTop3Ratio),
|
|
11527
|
+
rawDistinctTop3Ratio: safeDiv(sum.rawDistinctTop3Ratio),
|
|
11528
|
+
latencyMs: {
|
|
11529
|
+
p50: percentile(latencies, 0.5),
|
|
11530
|
+
p95: percentile(latencies, 0.95),
|
|
11531
|
+
p99: percentile(latencies, 0.99)
|
|
11532
|
+
},
|
|
11533
|
+
tokenEstimate: {
|
|
11534
|
+
queryTokens,
|
|
11535
|
+
embeddingTokensUsed
|
|
11536
|
+
},
|
|
11537
|
+
embedding: {
|
|
11538
|
+
callCount: embeddingCallCount,
|
|
11539
|
+
estimatedCostUsd: embeddingTokensUsed / 1e6 * costPer1MTokensUsd,
|
|
11540
|
+
costPer1MTokensUsd
|
|
11541
|
+
},
|
|
11542
|
+
contextEfficiency: {
|
|
11543
|
+
queryCount: contextQueries.length,
|
|
11544
|
+
responseTokens: {
|
|
11545
|
+
total: totalContextResponseTokens,
|
|
11546
|
+
average: contextQueries.length === 0 ? 0 : totalContextResponseTokens / contextQueries.length,
|
|
11547
|
+
p95: percentile(contextResponseTokens, 0.95),
|
|
11548
|
+
max: contextResponseTokens.length === 0 ? 0 : Math.max(...contextResponseTokens)
|
|
11549
|
+
},
|
|
11550
|
+
duplicateCandidateRatio: contextQueries.length === 0 ? 0 : contextQueries.reduce((sum2, item) => sum2 + item.duplicateCandidateRatio, 0) / contextQueries.length,
|
|
11551
|
+
selectedFileRatio: contextQueries.length === 0 ? 0 : contextQueries.reduce((sum2, item) => sum2 + item.selectedFileRatio, 0) / contextQueries.length,
|
|
11552
|
+
hitAt5Per1kResponseTokens: contextTokenUnits === 0 ? 0 : contextQueries.filter((item) => item.hitAt5).length / contextTokenUnits,
|
|
11553
|
+
mrrAt10Per1kResponseTokens: contextTokenUnits === 0 ? 0 : contextQueries.reduce((sum2, item) => sum2 + item.reciprocalRankAt10, 0) / contextTokenUnits
|
|
11554
|
+
},
|
|
11555
|
+
failureBuckets
|
|
11556
|
+
};
|
|
11304
11557
|
}
|
|
11305
|
-
|
|
11306
|
-
|
|
11307
|
-
|
|
11308
|
-
|
|
11309
|
-
|
|
11310
|
-
|
|
11311
|
-
|
|
11312
|
-
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
11313
|
-
case "storing":
|
|
11314
|
-
return "Storing index...";
|
|
11315
|
-
case "complete":
|
|
11316
|
-
return "Indexing complete";
|
|
11317
|
-
default:
|
|
11318
|
-
return "Indexing...";
|
|
11319
|
-
}
|
|
11558
|
+
|
|
11559
|
+
// src/eval/runner-config.ts
|
|
11560
|
+
var import_fs12 = require("fs");
|
|
11561
|
+
var os5 = __toESM(require("os"), 1);
|
|
11562
|
+
var path18 = __toESM(require("path"), 1);
|
|
11563
|
+
function isRecord2(value) {
|
|
11564
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
11320
11565
|
}
|
|
11321
|
-
function
|
|
11322
|
-
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11326
|
-
|
|
11566
|
+
function isStringArray3(value) {
|
|
11567
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
11568
|
+
}
|
|
11569
|
+
function validateEvalConfigShape(rawConfig, filePath) {
|
|
11570
|
+
if (!isRecord2(rawConfig)) {
|
|
11571
|
+
throw new Error(`Eval config at ${filePath} must contain a JSON object at the root.`);
|
|
11327
11572
|
}
|
|
11328
|
-
|
|
11329
|
-
|
|
11330
|
-
|
|
11573
|
+
const config = rawConfig;
|
|
11574
|
+
if (config.knowledgeBases !== void 0 && !isStringArray3(config.knowledgeBases)) {
|
|
11575
|
+
throw new Error(`Eval config at ${filePath} field 'knowledgeBases' must be an array of strings.`);
|
|
11331
11576
|
}
|
|
11332
|
-
if (
|
|
11333
|
-
|
|
11334
|
-
}
|
|
11335
|
-
function formatCodebasePeek(results) {
|
|
11336
|
-
if (results.length === 0) {
|
|
11337
|
-
return "No matching code found. Try a different query or run index_codebase first.";
|
|
11577
|
+
if (config.additionalInclude !== void 0 && !isStringArray3(config.additionalInclude)) {
|
|
11578
|
+
throw new Error(`Eval config at ${filePath} field 'additionalInclude' must be an array of strings.`);
|
|
11338
11579
|
}
|
|
11339
|
-
|
|
11340
|
-
|
|
11341
|
-
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
11342
|
-
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})${formatBlame(r)}`;
|
|
11343
|
-
});
|
|
11344
|
-
return formatted.join("\n");
|
|
11345
|
-
}
|
|
11346
|
-
function formatHealthCheck(result) {
|
|
11347
|
-
if (result.resetCorruptedIndex) {
|
|
11348
|
-
return result.warning ?? "Detected a corrupted local index and reset it. Run index_codebase to rebuild search data.";
|
|
11580
|
+
if (config.include !== void 0 && !isStringArray3(config.include)) {
|
|
11581
|
+
throw new Error(`Eval config at ${filePath} field 'include' must be an array of strings.`);
|
|
11349
11582
|
}
|
|
11350
|
-
if (
|
|
11351
|
-
|
|
11583
|
+
if (config.exclude !== void 0 && !isStringArray3(config.exclude)) {
|
|
11584
|
+
throw new Error(`Eval config at ${filePath} field 'exclude' must be an array of strings.`);
|
|
11352
11585
|
}
|
|
11353
|
-
const
|
|
11354
|
-
|
|
11355
|
-
|
|
11586
|
+
for (const section of ["customProvider", "indexing", "search", "debug", "reranker"]) {
|
|
11587
|
+
const value = config[section];
|
|
11588
|
+
if (value !== void 0 && !isRecord2(value)) {
|
|
11589
|
+
throw new Error(`Eval config at ${filePath} field '${section}' must be an object.`);
|
|
11590
|
+
}
|
|
11356
11591
|
}
|
|
11357
|
-
|
|
11358
|
-
|
|
11592
|
+
return config;
|
|
11593
|
+
}
|
|
11594
|
+
function parseJsonConfigFile(filePath) {
|
|
11595
|
+
try {
|
|
11596
|
+
return validateEvalConfigShape(JSON.parse((0, import_fs12.readFileSync)(filePath, "utf-8")), filePath);
|
|
11597
|
+
} catch (error) {
|
|
11598
|
+
if (error instanceof Error && error.message.startsWith("Eval config at ")) {
|
|
11599
|
+
throw error;
|
|
11600
|
+
}
|
|
11601
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
11602
|
+
throw new Error(`Failed to parse eval config JSON at ${filePath}: ${message}`);
|
|
11359
11603
|
}
|
|
11360
|
-
|
|
11361
|
-
|
|
11604
|
+
}
|
|
11605
|
+
function toAbsolute(projectRoot, maybeRelative) {
|
|
11606
|
+
return path18.isAbsolute(maybeRelative) ? maybeRelative : path18.join(projectRoot, maybeRelative);
|
|
11607
|
+
}
|
|
11608
|
+
function isProjectScopedConfigPath(configPath) {
|
|
11609
|
+
return path18.basename(configPath) === "codebase-index.json" && path18.basename(path18.dirname(configPath)) === ".opencode";
|
|
11610
|
+
}
|
|
11611
|
+
function normalizeEvalConfigKnowledgeBases(rawConfig, projectRoot, resolvedConfigPath) {
|
|
11612
|
+
const config = rawConfig && typeof rawConfig === "object" ? { ...rawConfig } : {};
|
|
11613
|
+
const rebaseEntries = (values) => isProjectScopedConfigPath(resolvedConfigPath) ? resolveInheritedKnowledgeBaseEntries(
|
|
11614
|
+
values,
|
|
11615
|
+
path18.dirname(path18.dirname(resolvedConfigPath)),
|
|
11616
|
+
projectRoot
|
|
11617
|
+
) : rebasePathEntries(
|
|
11618
|
+
values,
|
|
11619
|
+
path18.dirname(resolvedConfigPath),
|
|
11620
|
+
projectRoot
|
|
11621
|
+
);
|
|
11622
|
+
if (Array.isArray(config.knowledgeBases)) {
|
|
11623
|
+
config.knowledgeBases = rebaseEntries(config.knowledgeBases);
|
|
11362
11624
|
}
|
|
11363
|
-
if (
|
|
11364
|
-
|
|
11625
|
+
if (Array.isArray(config.additionalInclude)) {
|
|
11626
|
+
config.additionalInclude = rebaseEntries(config.additionalInclude);
|
|
11365
11627
|
}
|
|
11366
|
-
|
|
11367
|
-
|
|
11628
|
+
return config;
|
|
11629
|
+
}
|
|
11630
|
+
function loadRawConfig(projectRoot, configPath) {
|
|
11631
|
+
const fromPath = configPath ? toAbsolute(projectRoot, configPath) : null;
|
|
11632
|
+
if (fromPath && (0, import_fs12.existsSync)(fromPath)) {
|
|
11633
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
11634
|
+
parseJsonConfigFile(fromPath),
|
|
11635
|
+
projectRoot,
|
|
11636
|
+
fromPath
|
|
11637
|
+
);
|
|
11368
11638
|
}
|
|
11369
|
-
|
|
11370
|
-
|
|
11639
|
+
const projectConfig = resolveProjectConfigPath(projectRoot);
|
|
11640
|
+
if ((0, import_fs12.existsSync)(projectConfig)) {
|
|
11641
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
11642
|
+
parseJsonConfigFile(projectConfig),
|
|
11643
|
+
projectRoot,
|
|
11644
|
+
projectConfig
|
|
11645
|
+
);
|
|
11371
11646
|
}
|
|
11372
|
-
|
|
11373
|
-
|
|
11374
|
-
|
|
11375
|
-
if (callers.length === 0) {
|
|
11376
|
-
return `No callers found for "${name}"${relationshipType ? ` with type ${relationshipType}` : ""}. It may not be called by any tracked function, or the index needs updating.`;
|
|
11647
|
+
const globalConfig = path18.join(os5.homedir(), ".config", "opencode", "codebase-index.json");
|
|
11648
|
+
if ((0, import_fs12.existsSync)(globalConfig)) {
|
|
11649
|
+
return parseJsonConfigFile(globalConfig);
|
|
11377
11650
|
}
|
|
11378
|
-
|
|
11379
|
-
const confidence = edge.confidence !== "Direct" ? ` [${edge.confidence.toLowerCase()}]` : "";
|
|
11380
|
-
return `[${index + 1}] \u2190 from ${edge.fromSymbolName ?? "<unknown>"} in ${edge.fromSymbolFilePath ?? "<unknown file>"} [${edge.fromSymbolId}] (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? " [resolved]" : " [unresolved]"}`;
|
|
11381
|
-
});
|
|
11382
|
-
return `"${name}" is called by ${callers.length} function(s):
|
|
11383
|
-
|
|
11384
|
-
${formatted.join("\n")}`;
|
|
11651
|
+
return {};
|
|
11385
11652
|
}
|
|
11386
|
-
function
|
|
11387
|
-
|
|
11388
|
-
return `No callees found for symbol ${symbolId}${relationshipType ? ` with type ${relationshipType}` : ""}. The function may not call any other tracked functions.`;
|
|
11389
|
-
}
|
|
11390
|
-
return callees.map((edge, index) => {
|
|
11391
|
-
const confidence = edge.confidence !== "Direct" ? ` [${edge.confidence.toLowerCase()}]` : "";
|
|
11392
|
-
return `[${index + 1}] \u2192 ${edge.targetName} (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? ` [resolved: ${edge.toSymbolId}]` : " [unresolved]"}`;
|
|
11393
|
-
}).join("\n");
|
|
11653
|
+
function getIndexRootPath(projectRoot, scope) {
|
|
11654
|
+
return scope === "global" ? getGlobalIndexPath() : resolveProjectIndexPath(projectRoot, scope);
|
|
11394
11655
|
}
|
|
11395
|
-
function
|
|
11396
|
-
|
|
11397
|
-
return `No path found between "${from}" and "${to}". They may be in disconnected components, or the call graph index needs updating.`;
|
|
11398
|
-
}
|
|
11399
|
-
const formatted = path27.map((hop, index) => {
|
|
11400
|
-
const prefix = index === 0 ? "[start]" : `--${hop.callType}-->`;
|
|
11401
|
-
const location = hop.filePath ? ` (${hop.filePath}:${hop.line})` : "";
|
|
11402
|
-
return `${prefix} ${hop.symbolName}${location}`;
|
|
11403
|
-
});
|
|
11404
|
-
return `Path (${path27.length} hops):
|
|
11405
|
-
${formatted.join("\n")}`;
|
|
11656
|
+
function getLocalProjectIndexRoot(projectRoot) {
|
|
11657
|
+
return path18.join(projectRoot, ".opencode", "index");
|
|
11406
11658
|
}
|
|
11407
|
-
function
|
|
11408
|
-
return
|
|
11659
|
+
function getLocalProjectConfigPath(projectRoot) {
|
|
11660
|
+
return path18.join(projectRoot, ".opencode", "codebase-index.json");
|
|
11409
11661
|
}
|
|
11410
|
-
function
|
|
11411
|
-
|
|
11412
|
-
|
|
11662
|
+
function clearIndexRoot(projectRoot, scope) {
|
|
11663
|
+
const indexRoot = scope === "global" ? getIndexRootPath(projectRoot, scope) : getLocalProjectIndexRoot(projectRoot);
|
|
11664
|
+
if ((0, import_fs12.existsSync)(indexRoot)) {
|
|
11665
|
+
(0, import_fs12.rmSync)(indexRoot, { recursive: true, force: true });
|
|
11413
11666
|
}
|
|
11414
|
-
const date = new Date(result.blame.committedAt * 1e3).toISOString().slice(0, 10);
|
|
11415
|
-
return `
|
|
11416
|
-
${result.blame.sha.slice(0, 7)} | ${result.blame.author} | ${date} | ${result.blame.summary}`;
|
|
11417
11667
|
}
|
|
11418
|
-
function
|
|
11419
|
-
|
|
11420
|
-
|
|
11668
|
+
function ensureLocalEvalProjectConfig(projectRoot, configPath) {
|
|
11669
|
+
const localConfigPath = getLocalProjectConfigPath(projectRoot);
|
|
11670
|
+
const resolvedConfigPath = configPath ? toAbsolute(projectRoot, configPath) : resolveProjectConfigPath(projectRoot);
|
|
11671
|
+
if (!configPath && (0, import_fs12.existsSync)(localConfigPath)) {
|
|
11672
|
+
return localConfigPath;
|
|
11421
11673
|
}
|
|
11422
|
-
|
|
11423
|
-
|
|
11424
|
-
return `${header} (score: ${r.score.toFixed(2)})${formatBlame(r)}
|
|
11425
|
-
\`\`\`
|
|
11426
|
-
${truncateContent(r.content)}
|
|
11427
|
-
\`\`\``;
|
|
11428
|
-
});
|
|
11429
|
-
return formatted.join("\n\n");
|
|
11430
|
-
}
|
|
11431
|
-
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
11432
|
-
const formatted = results.map((r, idx) => {
|
|
11433
|
-
const header = formatResultHeader(r, idx);
|
|
11434
|
-
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
11435
|
-
return `${header} ${scoreLabel}${formatBlame(r)}
|
|
11436
|
-
\`\`\`
|
|
11437
|
-
${truncateContent(r.content)}
|
|
11438
|
-
\`\`\``;
|
|
11439
|
-
});
|
|
11440
|
-
return formatted.join("\n\n");
|
|
11441
|
-
}
|
|
11442
|
-
|
|
11443
|
-
// src/tools/format-pr-impact.ts
|
|
11444
|
-
function formatPrImpact(result) {
|
|
11445
|
-
const lines = [];
|
|
11446
|
-
lines.push(`\u2192 Files changed: ${result.changedFiles.length}`);
|
|
11447
|
-
for (const file of result.changedFiles) {
|
|
11448
|
-
lines.push(` - ${file}`);
|
|
11674
|
+
if (!(0, import_fs12.existsSync)(resolvedConfigPath) || resolvedConfigPath === localConfigPath) {
|
|
11675
|
+
return resolvedConfigPath;
|
|
11449
11676
|
}
|
|
11450
|
-
const
|
|
11451
|
-
|
|
11452
|
-
|
|
11453
|
-
|
|
11454
|
-
`\u2192 Symbols affected: ${result.totalAffected} (${directCount} direct, ${transitiveCount} transitive ${directionLabel})`
|
|
11677
|
+
const sourceConfig = normalizeEvalConfigKnowledgeBases(
|
|
11678
|
+
parseJsonConfigFile(resolvedConfigPath),
|
|
11679
|
+
projectRoot,
|
|
11680
|
+
resolvedConfigPath
|
|
11455
11681
|
);
|
|
11456
|
-
|
|
11457
|
-
|
|
11458
|
-
|
|
11459
|
-
|
|
11460
|
-
|
|
11461
|
-
|
|
11462
|
-
|
|
11463
|
-
|
|
11464
|
-
|
|
11682
|
+
(0, import_fs12.mkdirSync)(path18.dirname(localConfigPath), { recursive: true });
|
|
11683
|
+
(0, import_fs12.writeFileSync)(localConfigPath, JSON.stringify(sourceConfig, null, 2), "utf-8");
|
|
11684
|
+
return localConfigPath;
|
|
11685
|
+
}
|
|
11686
|
+
function loadParsedConfig(projectRoot, configPath) {
|
|
11687
|
+
const raw = loadRawConfig(projectRoot, configPath);
|
|
11688
|
+
return parseConfig(raw);
|
|
11689
|
+
}
|
|
11690
|
+
function resolveSearchConfig(parsedConfig, overrides) {
|
|
11691
|
+
const nextSearch = {
|
|
11692
|
+
...parsedConfig.search
|
|
11693
|
+
};
|
|
11694
|
+
if (overrides?.fusionStrategy !== void 0) {
|
|
11695
|
+
nextSearch.fusionStrategy = overrides.fusionStrategy;
|
|
11465
11696
|
}
|
|
11466
|
-
if (
|
|
11467
|
-
|
|
11468
|
-
`\u2192 Communities touched: ${result.communities.map((c) => c.label).join(", ")}`
|
|
11469
|
-
);
|
|
11470
|
-
for (const community of result.communities) {
|
|
11471
|
-
lines.push(
|
|
11472
|
-
` - ${community.label}: ${community.symbolCount} symbols`
|
|
11473
|
-
);
|
|
11474
|
-
}
|
|
11475
|
-
} else {
|
|
11476
|
-
lines.push("\u2192 Communities touched: none");
|
|
11697
|
+
if (overrides?.hybridWeight !== void 0) {
|
|
11698
|
+
nextSearch.hybridWeight = overrides.hybridWeight;
|
|
11477
11699
|
}
|
|
11478
|
-
|
|
11479
|
-
|
|
11480
|
-
lines.push(" Hub nodes in change scope:");
|
|
11481
|
-
for (const hub of result.hubNodes) {
|
|
11482
|
-
lines.push(` - ${hub.name} (${hub.callerCount} callers) at ${hub.filePath}`);
|
|
11483
|
-
}
|
|
11700
|
+
if (overrides?.rrfK !== void 0) {
|
|
11701
|
+
nextSearch.rrfK = overrides.rrfK;
|
|
11484
11702
|
}
|
|
11485
|
-
if (
|
|
11486
|
-
|
|
11487
|
-
(c) => `PR #${c.pr} (also touches ${c.overlappingCommunities.join(", ")} community)`
|
|
11488
|
-
);
|
|
11489
|
-
lines.push(`\u2192 Potential conflicts with: ${conflictList.join(", ")}`);
|
|
11703
|
+
if (overrides?.rerankTopN !== void 0) {
|
|
11704
|
+
nextSearch.rerankTopN = overrides.rerankTopN;
|
|
11490
11705
|
}
|
|
11491
|
-
return
|
|
11706
|
+
return {
|
|
11707
|
+
...parsedConfig,
|
|
11708
|
+
search: nextSearch
|
|
11709
|
+
};
|
|
11710
|
+
}
|
|
11711
|
+
function getEmbeddingCostPer1MTokens(embeddingProvider) {
|
|
11712
|
+
return embeddingProvider === "custom" || embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(embeddingProvider).costPer1MTokens;
|
|
11492
11713
|
}
|
|
11493
11714
|
|
|
11494
|
-
// src/
|
|
11495
|
-
var
|
|
11496
|
-
|
|
11497
|
-
|
|
11498
|
-
|
|
11499
|
-
|
|
11500
|
-
|
|
11501
|
-
|
|
11502
|
-
|
|
11503
|
-
|
|
11504
|
-
|
|
11505
|
-
"reranker",
|
|
11506
|
-
"include",
|
|
11507
|
-
"exclude",
|
|
11508
|
-
"indexing",
|
|
11509
|
-
"search",
|
|
11510
|
-
"debug",
|
|
11511
|
-
"scope"
|
|
11512
|
-
];
|
|
11513
|
-
var MERGE_ARRAY_KEYS = ["knowledgeBases", "additionalInclude"];
|
|
11715
|
+
// src/eval/schema.ts
|
|
11716
|
+
var import_fs13 = require("fs");
|
|
11717
|
+
function parseJsonFile(filePath) {
|
|
11718
|
+
const content = (0, import_fs13.readFileSync)(filePath, "utf-8");
|
|
11719
|
+
try {
|
|
11720
|
+
return JSON.parse(content);
|
|
11721
|
+
} catch (error) {
|
|
11722
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
11723
|
+
throw new Error(`Failed to parse JSON from ${filePath}: ${message}`);
|
|
11724
|
+
}
|
|
11725
|
+
}
|
|
11514
11726
|
function isRecord3(value) {
|
|
11515
|
-
return typeof value === "object" && value !== null
|
|
11727
|
+
return typeof value === "object" && value !== null;
|
|
11516
11728
|
}
|
|
11517
11729
|
function isStringArray4(value) {
|
|
11518
11730
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
11519
11731
|
}
|
|
11520
|
-
function
|
|
11521
|
-
if (
|
|
11522
|
-
|
|
11523
|
-
return;
|
|
11732
|
+
function asPositiveNumber(value, path27) {
|
|
11733
|
+
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
11734
|
+
throw new Error(`${path27} must be a non-negative number`);
|
|
11524
11735
|
}
|
|
11525
|
-
|
|
11526
|
-
|
|
11736
|
+
return value;
|
|
11737
|
+
}
|
|
11738
|
+
function parseQueryType(value, path27) {
|
|
11739
|
+
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy" || value === "conceptual") {
|
|
11740
|
+
return value;
|
|
11527
11741
|
}
|
|
11742
|
+
throw new Error(
|
|
11743
|
+
`${path27} must be one of: definition, implementation-intent, similarity, keyword-heavy, conceptual`
|
|
11744
|
+
);
|
|
11528
11745
|
}
|
|
11529
|
-
function
|
|
11530
|
-
|
|
11746
|
+
function parseRetrievalMode(value, path27) {
|
|
11747
|
+
if (value === void 0 || value === "search") return "search";
|
|
11748
|
+
if (value === "context") return value;
|
|
11749
|
+
throw new Error(`${path27} must be one of: search, context`);
|
|
11531
11750
|
}
|
|
11532
|
-
function
|
|
11533
|
-
|
|
11534
|
-
|
|
11535
|
-
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
11536
|
-
normalized = normalized.slice(0, -1);
|
|
11751
|
+
function parseExpected(input, path27) {
|
|
11752
|
+
if (!isRecord3(input)) {
|
|
11753
|
+
throw new Error(`${path27} must be an object`);
|
|
11537
11754
|
}
|
|
11538
|
-
|
|
11755
|
+
const filePathRaw = input.filePath;
|
|
11756
|
+
const acceptableFilesRaw = input.acceptableFiles;
|
|
11757
|
+
const symbolRaw = input.symbol;
|
|
11758
|
+
const branchRaw = input.branch;
|
|
11759
|
+
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
11760
|
+
const acceptableFiles = isStringArray4(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
11761
|
+
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
11762
|
+
throw new Error(`${path27} must include either expected.filePath or expected.acceptableFiles`);
|
|
11763
|
+
}
|
|
11764
|
+
if (acceptableFilesRaw !== void 0 && !isStringArray4(acceptableFilesRaw)) {
|
|
11765
|
+
throw new Error(`${path27}.acceptableFiles must be an array of strings`);
|
|
11766
|
+
}
|
|
11767
|
+
if (symbolRaw !== void 0 && typeof symbolRaw !== "string") {
|
|
11768
|
+
throw new Error(`${path27}.symbol must be a string when provided`);
|
|
11769
|
+
}
|
|
11770
|
+
if (branchRaw !== void 0 && typeof branchRaw !== "string") {
|
|
11771
|
+
throw new Error(`${path27}.branch must be a string when provided`);
|
|
11772
|
+
}
|
|
11773
|
+
return {
|
|
11774
|
+
filePath,
|
|
11775
|
+
acceptableFiles,
|
|
11776
|
+
symbol: typeof symbolRaw === "string" ? symbolRaw : void 0,
|
|
11777
|
+
branch: typeof branchRaw === "string" ? branchRaw : void 0
|
|
11778
|
+
};
|
|
11539
11779
|
}
|
|
11540
|
-
function
|
|
11541
|
-
|
|
11780
|
+
function parseQuery(input, index) {
|
|
11781
|
+
const path27 = `queries[${index}]`;
|
|
11782
|
+
if (!isRecord3(input)) {
|
|
11783
|
+
throw new Error(`${path27} must be an object`);
|
|
11784
|
+
}
|
|
11785
|
+
const id = input.id;
|
|
11786
|
+
const query = input.query;
|
|
11787
|
+
const queryType = input.queryType;
|
|
11788
|
+
const retrievalMode = input.retrievalMode;
|
|
11789
|
+
const expected = input.expected;
|
|
11790
|
+
if (typeof id !== "string" || id.trim().length === 0) {
|
|
11791
|
+
throw new Error(`${path27}.id must be a non-empty string`);
|
|
11792
|
+
}
|
|
11793
|
+
if (typeof query !== "string" || query.trim().length === 0) {
|
|
11794
|
+
throw new Error(`${path27}.query must be a non-empty string`);
|
|
11795
|
+
}
|
|
11796
|
+
return {
|
|
11797
|
+
id,
|
|
11798
|
+
query,
|
|
11799
|
+
queryType: parseQueryType(queryType, `${path27}.queryType`),
|
|
11800
|
+
retrievalMode: parseRetrievalMode(retrievalMode, `${path27}.retrievalMode`),
|
|
11801
|
+
expected: parseExpected(expected, `${path27}.expected`)
|
|
11802
|
+
};
|
|
11542
11803
|
}
|
|
11543
|
-
function
|
|
11544
|
-
if (!isRecord3(
|
|
11545
|
-
throw new Error(
|
|
11546
|
-
}
|
|
11547
|
-
if (rawConfig.knowledgeBases !== void 0 && !isStringArray4(rawConfig.knowledgeBases)) {
|
|
11548
|
-
throw new Error(`Config file ${filePath} field 'knowledgeBases' must be an array of strings.`);
|
|
11804
|
+
function parseGoldenDataset(raw, sourceLabel) {
|
|
11805
|
+
if (!isRecord3(raw)) {
|
|
11806
|
+
throw new Error(`${sourceLabel} must be a JSON object`);
|
|
11549
11807
|
}
|
|
11550
|
-
|
|
11551
|
-
|
|
11808
|
+
const version = raw.version;
|
|
11809
|
+
const name = raw.name;
|
|
11810
|
+
const description = raw.description;
|
|
11811
|
+
const queriesRaw = raw.queries;
|
|
11812
|
+
if (typeof version !== "string" || version.trim().length === 0) {
|
|
11813
|
+
throw new Error(`${sourceLabel}.version must be a non-empty string`);
|
|
11552
11814
|
}
|
|
11553
|
-
if (
|
|
11554
|
-
throw new Error(
|
|
11815
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
11816
|
+
throw new Error(`${sourceLabel}.name must be a non-empty string`);
|
|
11555
11817
|
}
|
|
11556
|
-
if (
|
|
11557
|
-
throw new Error(
|
|
11818
|
+
if (description !== void 0 && typeof description !== "string") {
|
|
11819
|
+
throw new Error(`${sourceLabel}.description must be a string when provided`);
|
|
11558
11820
|
}
|
|
11559
|
-
|
|
11560
|
-
|
|
11561
|
-
if (value !== void 0 && !isRecord3(value)) {
|
|
11562
|
-
throw new Error(`Config file ${filePath} field '${section}' must be an object.`);
|
|
11563
|
-
}
|
|
11821
|
+
if (!Array.isArray(queriesRaw)) {
|
|
11822
|
+
throw new Error(`${sourceLabel}.queries must be an array`);
|
|
11564
11823
|
}
|
|
11565
|
-
|
|
11566
|
-
}
|
|
11567
|
-
function loadJsonFile(filePath) {
|
|
11568
|
-
if (!(0, import_fs12.existsSync)(filePath)) {
|
|
11569
|
-
return null;
|
|
11824
|
+
if (queriesRaw.length === 0) {
|
|
11825
|
+
throw new Error(`${sourceLabel}.queries must contain at least one query`);
|
|
11570
11826
|
}
|
|
11571
|
-
|
|
11572
|
-
|
|
11573
|
-
|
|
11574
|
-
|
|
11575
|
-
|
|
11576
|
-
throw error;
|
|
11827
|
+
const queries = queriesRaw.map((query, idx) => parseQuery(query, idx));
|
|
11828
|
+
const idSet = /* @__PURE__ */ new Set();
|
|
11829
|
+
for (const query of queries) {
|
|
11830
|
+
if (idSet.has(query.id)) {
|
|
11831
|
+
throw new Error(`${sourceLabel}.queries has duplicate id: ${query.id}`);
|
|
11577
11832
|
}
|
|
11578
|
-
|
|
11579
|
-
throw new Error(`Failed to load config file ${filePath}: ${message}`);
|
|
11833
|
+
idSet.add(query.id);
|
|
11580
11834
|
}
|
|
11835
|
+
return {
|
|
11836
|
+
version,
|
|
11837
|
+
name,
|
|
11838
|
+
description: typeof description === "string" ? description : void 0,
|
|
11839
|
+
queries
|
|
11840
|
+
};
|
|
11581
11841
|
}
|
|
11582
|
-
function
|
|
11583
|
-
|
|
11584
|
-
|
|
11585
|
-
function materializeLocalProjectConfig(projectRoot, config, host = "opencode") {
|
|
11586
|
-
const localConfigPath = resolveWritableProjectConfigPath(projectRoot, host);
|
|
11587
|
-
(0, import_fs12.mkdirSync)(path18.dirname(localConfigPath), { recursive: true });
|
|
11588
|
-
(0, import_fs12.writeFileSync)(localConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
11589
|
-
return localConfigPath;
|
|
11842
|
+
function loadGoldenDataset(datasetPath) {
|
|
11843
|
+
const parsed = parseJsonFile(datasetPath);
|
|
11844
|
+
return parseGoldenDataset(parsed, datasetPath);
|
|
11590
11845
|
}
|
|
11591
|
-
function
|
|
11592
|
-
|
|
11593
|
-
const projectConfig = loadJsonFile(projectConfigPath);
|
|
11594
|
-
if (!projectConfig) {
|
|
11595
|
-
return {};
|
|
11596
|
-
}
|
|
11597
|
-
const normalizedConfig = { ...projectConfig };
|
|
11598
|
-
const projectConfigBaseDir = path18.dirname(path18.dirname(projectConfigPath));
|
|
11599
|
-
if (Array.isArray(normalizedConfig.knowledgeBases)) {
|
|
11600
|
-
normalizedConfig.knowledgeBases = resolveInheritedKnowledgeBaseEntries(
|
|
11601
|
-
normalizedConfig.knowledgeBases,
|
|
11602
|
-
projectConfigBaseDir,
|
|
11603
|
-
projectRoot
|
|
11604
|
-
);
|
|
11605
|
-
}
|
|
11606
|
-
return normalizedConfig;
|
|
11846
|
+
function parseThresholdValue(value, fieldName, sourceLabel) {
|
|
11847
|
+
return value === void 0 ? void 0 : asPositiveNumber(value, `${sourceLabel}.thresholds.${fieldName}`);
|
|
11607
11848
|
}
|
|
11608
|
-
function
|
|
11609
|
-
|
|
11610
|
-
|
|
11611
|
-
let globalConfig = null;
|
|
11612
|
-
let globalConfigError = null;
|
|
11613
|
-
try {
|
|
11614
|
-
globalConfig = loadJsonFile(globalConfigPath);
|
|
11615
|
-
} catch (error) {
|
|
11616
|
-
globalConfigError = error instanceof Error ? error : new Error(String(error));
|
|
11617
|
-
}
|
|
11618
|
-
const projectConfig = loadJsonFile(projectConfigPath);
|
|
11619
|
-
const normalizedProjectConfig = loadProjectConfigLayer(projectRoot, host);
|
|
11620
|
-
if (globalConfigError) {
|
|
11621
|
-
if (!projectConfig) {
|
|
11622
|
-
throw globalConfigError;
|
|
11623
|
-
}
|
|
11624
|
-
globalConfig = null;
|
|
11625
|
-
}
|
|
11626
|
-
if (!globalConfig && !projectConfig) {
|
|
11627
|
-
return {};
|
|
11849
|
+
function parseBudget(raw, sourceLabel) {
|
|
11850
|
+
if (!isRecord3(raw)) {
|
|
11851
|
+
throw new Error(`${sourceLabel} must be a JSON object`);
|
|
11628
11852
|
}
|
|
11629
|
-
|
|
11630
|
-
|
|
11853
|
+
const name = raw.name;
|
|
11854
|
+
const baselinePath = raw.baselinePath;
|
|
11855
|
+
const failOnMissingBaseline = raw.failOnMissingBaseline;
|
|
11856
|
+
const thresholds = raw.thresholds;
|
|
11857
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
11858
|
+
throw new Error(`${sourceLabel}.name must be a non-empty string`);
|
|
11631
11859
|
}
|
|
11632
|
-
if (
|
|
11633
|
-
|
|
11860
|
+
if (baselinePath !== void 0 && typeof baselinePath !== "string") {
|
|
11861
|
+
throw new Error(`${sourceLabel}.baselinePath must be a string when provided`);
|
|
11634
11862
|
}
|
|
11635
|
-
if (!
|
|
11636
|
-
|
|
11863
|
+
if (!isRecord3(thresholds)) {
|
|
11864
|
+
throw new Error(`${sourceLabel}.thresholds must be an object`);
|
|
11637
11865
|
}
|
|
11638
|
-
|
|
11639
|
-
|
|
11640
|
-
|
|
11866
|
+
return {
|
|
11867
|
+
name,
|
|
11868
|
+
baselinePath: typeof baselinePath === "string" ? baselinePath : void 0,
|
|
11869
|
+
failOnMissingBaseline: typeof failOnMissingBaseline === "boolean" ? failOnMissingBaseline : true,
|
|
11870
|
+
thresholds: {
|
|
11871
|
+
hitAt5MaxDrop: parseThresholdValue(
|
|
11872
|
+
thresholds.hitAt5MaxDrop,
|
|
11873
|
+
"hitAt5MaxDrop",
|
|
11874
|
+
sourceLabel
|
|
11875
|
+
),
|
|
11876
|
+
mrrAt10MaxDrop: parseThresholdValue(
|
|
11877
|
+
thresholds.mrrAt10MaxDrop,
|
|
11878
|
+
"mrrAt10MaxDrop",
|
|
11879
|
+
sourceLabel
|
|
11880
|
+
),
|
|
11881
|
+
rawDistinctTop3RatioMaxDrop: parseThresholdValue(
|
|
11882
|
+
thresholds.rawDistinctTop3RatioMaxDrop,
|
|
11883
|
+
"rawDistinctTop3RatioMaxDrop",
|
|
11884
|
+
sourceLabel
|
|
11885
|
+
),
|
|
11886
|
+
p95LatencyMaxMultiplier: parseThresholdValue(
|
|
11887
|
+
thresholds.p95LatencyMaxMultiplier,
|
|
11888
|
+
"p95LatencyMaxMultiplier",
|
|
11889
|
+
sourceLabel
|
|
11890
|
+
),
|
|
11891
|
+
p95LatencyMaxAbsoluteMs: parseThresholdValue(
|
|
11892
|
+
thresholds.p95LatencyMaxAbsoluteMs,
|
|
11893
|
+
"p95LatencyMaxAbsoluteMs",
|
|
11894
|
+
sourceLabel
|
|
11895
|
+
),
|
|
11896
|
+
minHitAt5: parseThresholdValue(
|
|
11897
|
+
thresholds.minHitAt5,
|
|
11898
|
+
"minHitAt5",
|
|
11899
|
+
sourceLabel
|
|
11900
|
+
),
|
|
11901
|
+
minMrrAt10: parseThresholdValue(
|
|
11902
|
+
thresholds.minMrrAt10,
|
|
11903
|
+
"minMrrAt10",
|
|
11904
|
+
sourceLabel
|
|
11905
|
+
),
|
|
11906
|
+
minRawDistinctTop3Ratio: parseThresholdValue(
|
|
11907
|
+
thresholds.minRawDistinctTop3Ratio,
|
|
11908
|
+
"minRawDistinctTop3Ratio",
|
|
11909
|
+
sourceLabel
|
|
11910
|
+
),
|
|
11911
|
+
maxContextResponseTokensAverage: parseThresholdValue(
|
|
11912
|
+
thresholds.maxContextResponseTokensAverage,
|
|
11913
|
+
"maxContextResponseTokensAverage",
|
|
11914
|
+
sourceLabel
|
|
11915
|
+
),
|
|
11916
|
+
maxContextResponseTokensP95: parseThresholdValue(
|
|
11917
|
+
thresholds.maxContextResponseTokensP95,
|
|
11918
|
+
"maxContextResponseTokensP95",
|
|
11919
|
+
sourceLabel
|
|
11920
|
+
),
|
|
11921
|
+
maxContextResponseTokensMax: parseThresholdValue(
|
|
11922
|
+
thresholds.maxContextResponseTokensMax,
|
|
11923
|
+
"maxContextResponseTokensMax",
|
|
11924
|
+
sourceLabel
|
|
11925
|
+
),
|
|
11926
|
+
maxContextDuplicateCandidateRatio: parseThresholdValue(
|
|
11927
|
+
thresholds.maxContextDuplicateCandidateRatio,
|
|
11928
|
+
"maxContextDuplicateCandidateRatio",
|
|
11929
|
+
sourceLabel
|
|
11930
|
+
),
|
|
11931
|
+
minContextSelectedFileRatio: parseThresholdValue(
|
|
11932
|
+
thresholds.minContextSelectedFileRatio,
|
|
11933
|
+
"minContextSelectedFileRatio",
|
|
11934
|
+
sourceLabel
|
|
11935
|
+
),
|
|
11936
|
+
minContextHitAt5Per1kResponseTokens: parseThresholdValue(
|
|
11937
|
+
thresholds.minContextHitAt5Per1kResponseTokens,
|
|
11938
|
+
"minContextHitAt5Per1kResponseTokens",
|
|
11939
|
+
sourceLabel
|
|
11940
|
+
),
|
|
11941
|
+
minContextMrrAt10Per1kResponseTokens: parseThresholdValue(
|
|
11942
|
+
thresholds.minContextMrrAt10Per1kResponseTokens,
|
|
11943
|
+
"minContextMrrAt10Per1kResponseTokens",
|
|
11944
|
+
sourceLabel
|
|
11945
|
+
)
|
|
11946
|
+
}
|
|
11947
|
+
};
|
|
11948
|
+
}
|
|
11949
|
+
function loadBudget(budgetPath) {
|
|
11950
|
+
const parsed = parseJsonFile(budgetPath);
|
|
11951
|
+
return parseBudget(parsed, budgetPath);
|
|
11952
|
+
}
|
|
11953
|
+
|
|
11954
|
+
// src/eval/runner.ts
|
|
11955
|
+
async function runEvaluation(options) {
|
|
11956
|
+
const datasetPath = toAbsolute(options.projectRoot, options.datasetPath);
|
|
11957
|
+
const againstPath = options.againstPath ? toAbsolute(options.projectRoot, options.againstPath) : void 0;
|
|
11958
|
+
const budgetPath = options.budgetPath ? toAbsolute(options.projectRoot, options.budgetPath) : void 0;
|
|
11959
|
+
const dataset = loadGoldenDataset(datasetPath);
|
|
11960
|
+
const resolvedEvalConfigPath = options.reindex ? ensureLocalEvalProjectConfig(options.projectRoot, options.configPath) : options.configPath;
|
|
11961
|
+
const parsedConfig = loadParsedConfig(options.projectRoot, resolvedEvalConfigPath);
|
|
11962
|
+
const effectiveConfig = resolveSearchConfig(parsedConfig, options.searchOverrides);
|
|
11963
|
+
if (options.reindex) {
|
|
11964
|
+
clearIndexRoot(options.projectRoot, effectiveConfig.scope);
|
|
11641
11965
|
}
|
|
11642
|
-
|
|
11643
|
-
|
|
11644
|
-
|
|
11645
|
-
|
|
11966
|
+
const indexer = new Indexer(options.projectRoot, effectiveConfig);
|
|
11967
|
+
try {
|
|
11968
|
+
await indexer.index();
|
|
11969
|
+
const perQuery = [];
|
|
11970
|
+
for (const query of dataset.queries) {
|
|
11971
|
+
if (query.expected.branch && query.expected.branch !== indexer.getCurrentBranch()) {
|
|
11972
|
+
throw new Error(
|
|
11973
|
+
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
11974
|
+
);
|
|
11975
|
+
}
|
|
11976
|
+
const start = import_perf_hooks2.performance.now();
|
|
11977
|
+
const contextResult = query.retrievalMode === "context" ? await resolveSearchContext({
|
|
11978
|
+
query: query.query,
|
|
11979
|
+
limit: 10,
|
|
11980
|
+
tokenBudget: DEFAULT_CONTEXT_PACK_TOKEN_BUDGET
|
|
11981
|
+
}, {
|
|
11982
|
+
lookup: (symbol, limit) => indexer.search(symbol, limit, {
|
|
11983
|
+
metadataOnly: true,
|
|
11984
|
+
filterByBranch: !!query.expected.branch,
|
|
11985
|
+
definitionIntent: true
|
|
11986
|
+
}),
|
|
11987
|
+
search: (searchQuery, limit) => indexer.search(searchQuery, limit, {
|
|
11988
|
+
metadataOnly: true,
|
|
11989
|
+
filterByBranch: !!query.expected.branch,
|
|
11990
|
+
definitionIntent: false
|
|
11991
|
+
})
|
|
11992
|
+
}) : void 0;
|
|
11993
|
+
const result = contextResult?.details?.results ?? await indexer.search(query.query, 10, {
|
|
11994
|
+
metadataOnly: true,
|
|
11995
|
+
filterByBranch: !!query.expected.branch
|
|
11996
|
+
});
|
|
11997
|
+
const elapsed = import_perf_hooks2.performance.now() - start;
|
|
11998
|
+
const resolvedRoute = contextResult?.details?.route === "definition" ? "definition" : "search";
|
|
11999
|
+
const routedQuery = contextResult?.details?.routedQuery ?? query.query;
|
|
12000
|
+
const materialized = result.map((item) => ({
|
|
12001
|
+
filePath: item.filePath,
|
|
12002
|
+
startLine: item.startLine,
|
|
12003
|
+
endLine: item.endLine,
|
|
12004
|
+
score: item.score,
|
|
12005
|
+
chunkType: item.chunkType,
|
|
12006
|
+
name: item.name
|
|
12007
|
+
}));
|
|
12008
|
+
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10, {
|
|
12009
|
+
resolvedRoute,
|
|
12010
|
+
routedQuery
|
|
12011
|
+
}, contextResult?.details ? {
|
|
12012
|
+
tokenBudget: contextResult.details.tokenBudget,
|
|
12013
|
+
responseTokens: contextResult.details.tokenEstimate,
|
|
12014
|
+
candidateCount: contextResult.details.candidateCount ?? 0,
|
|
12015
|
+
deduplicatedCount: contextResult.details.deduplicatedCount ?? 0,
|
|
12016
|
+
omittedCount: contextResult.details.omittedCount ?? 0
|
|
12017
|
+
} : void 0));
|
|
12018
|
+
}
|
|
12019
|
+
const logger = indexer.getLogger();
|
|
12020
|
+
const metricSnapshot = logger.getMetrics();
|
|
12021
|
+
const costPer1MTokensUsd = getEmbeddingCostPer1MTokens(effectiveConfig.embeddingProvider);
|
|
12022
|
+
const summary = {
|
|
12023
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12024
|
+
projectRoot: options.projectRoot,
|
|
12025
|
+
datasetPath,
|
|
12026
|
+
datasetName: dataset.name,
|
|
12027
|
+
datasetVersion: dataset.version,
|
|
12028
|
+
queryCount: dataset.queries.length,
|
|
12029
|
+
topK: 10,
|
|
12030
|
+
searchConfig: {
|
|
12031
|
+
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
12032
|
+
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
12033
|
+
rrfK: effectiveConfig.search.rrfK,
|
|
12034
|
+
rerankTopN: effectiveConfig.search.rerankTopN
|
|
12035
|
+
},
|
|
12036
|
+
metrics: computeEvalMetrics(
|
|
12037
|
+
dataset.queries,
|
|
12038
|
+
perQuery,
|
|
12039
|
+
metricSnapshot.embeddingApiCalls,
|
|
12040
|
+
metricSnapshot.embeddingTokensUsed,
|
|
12041
|
+
costPer1MTokensUsd
|
|
12042
|
+
)
|
|
12043
|
+
};
|
|
12044
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
12045
|
+
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
12046
|
+
writeJson(path19.join(outputDir, "summary.json"), summary);
|
|
12047
|
+
writeJson(path19.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
12048
|
+
let comparison;
|
|
12049
|
+
if (againstPath) {
|
|
12050
|
+
const baseline = loadSummary(againstPath);
|
|
12051
|
+
comparison = compareSummaries(summary, baseline, againstPath);
|
|
12052
|
+
writeJson(path19.join(outputDir, "compare.json"), comparison);
|
|
12053
|
+
}
|
|
12054
|
+
let gate;
|
|
12055
|
+
if (options.ciMode) {
|
|
12056
|
+
if (!budgetPath) {
|
|
12057
|
+
throw new Error("CI mode requires --budget path");
|
|
11646
12058
|
}
|
|
11647
|
-
|
|
12059
|
+
const budget = loadBudget(budgetPath);
|
|
12060
|
+
if (!comparison && budget.baselinePath) {
|
|
12061
|
+
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
12062
|
+
if ((0, import_fs14.existsSync)(resolvedBaseline)) {
|
|
12063
|
+
const baselineSummary = loadSummary(resolvedBaseline);
|
|
12064
|
+
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
12065
|
+
writeJson(path19.join(outputDir, "compare.json"), comparison);
|
|
12066
|
+
} else if (budget.failOnMissingBaseline) {
|
|
12067
|
+
throw new Error(
|
|
12068
|
+
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
12069
|
+
);
|
|
12070
|
+
}
|
|
12071
|
+
}
|
|
12072
|
+
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
11648
12073
|
}
|
|
12074
|
+
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
12075
|
+
writeText(path19.join(outputDir, "summary.md"), markdown);
|
|
12076
|
+
return { outputDir, summary, perQuery, comparison, gate };
|
|
12077
|
+
} finally {
|
|
12078
|
+
await indexer.close();
|
|
11649
12079
|
}
|
|
11650
|
-
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
11651
|
-
const projectKbs = projectConfig ? Array.isArray(normalizedProjectConfig.knowledgeBases) ? normalizedProjectConfig.knowledgeBases : [] : [];
|
|
11652
|
-
const allKbs = [...globalKbs, ...projectKbs];
|
|
11653
|
-
merged.knowledgeBases = mergeKnowledgeBasePaths(allKbs);
|
|
11654
|
-
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
11655
|
-
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
11656
|
-
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
11657
|
-
merged.additionalInclude = mergeUniqueStringArray(allAdditional);
|
|
11658
|
-
return merged;
|
|
11659
12080
|
}
|
|
11660
|
-
|
|
11661
|
-
|
|
11662
|
-
|
|
11663
|
-
|
|
11664
|
-
const
|
|
11665
|
-
|
|
11666
|
-
|
|
12081
|
+
async function runSweep(options, sweep) {
|
|
12082
|
+
const fusionValues = sweep.fusionStrategy && sweep.fusionStrategy.length > 0 ? [...sweep.fusionStrategy] : [void 0];
|
|
12083
|
+
const weightValues = sweep.hybridWeight && sweep.hybridWeight.length > 0 ? [...sweep.hybridWeight] : [void 0];
|
|
12084
|
+
const rrfValues = sweep.rrfK && sweep.rrfK.length > 0 ? [...sweep.rrfK] : [void 0];
|
|
12085
|
+
const rerankValues = sweep.rerankTopN && sweep.rerankTopN.length > 0 ? [...sweep.rerankTopN] : [void 0];
|
|
12086
|
+
const runs = [];
|
|
12087
|
+
for (const fusion of fusionValues) {
|
|
12088
|
+
for (const hybridWeight of weightValues) {
|
|
12089
|
+
for (const rrfK of rrfValues) {
|
|
12090
|
+
for (const rerankTopN of rerankValues) {
|
|
12091
|
+
const run = await runEvaluation({
|
|
12092
|
+
...options,
|
|
12093
|
+
searchOverrides: {
|
|
12094
|
+
...fusion !== void 0 ? { fusionStrategy: fusion } : {},
|
|
12095
|
+
...hybridWeight !== void 0 ? { hybridWeight } : {},
|
|
12096
|
+
...rrfK !== void 0 ? { rrfK } : {},
|
|
12097
|
+
...rerankTopN !== void 0 ? { rerankTopN } : {}
|
|
12098
|
+
}
|
|
12099
|
+
});
|
|
12100
|
+
runs.push({
|
|
12101
|
+
searchConfig: run.summary.searchConfig,
|
|
12102
|
+
summary: run.summary,
|
|
12103
|
+
comparison: run.comparison,
|
|
12104
|
+
gate: run.gate
|
|
12105
|
+
});
|
|
12106
|
+
}
|
|
12107
|
+
}
|
|
12108
|
+
}
|
|
11667
12109
|
}
|
|
11668
|
-
const
|
|
11669
|
-
|
|
12110
|
+
const bestByHitAt5 = [...runs].sort(
|
|
12111
|
+
(a, b) => b.summary.metrics.hitAt5 - a.summary.metrics.hitAt5
|
|
12112
|
+
)[0];
|
|
12113
|
+
const bestByMrrAt10 = [...runs].sort(
|
|
12114
|
+
(a, b) => b.summary.metrics.mrrAt10 - a.summary.metrics.mrrAt10
|
|
12115
|
+
)[0];
|
|
12116
|
+
const bestByP95Latency = [...runs].sort(
|
|
12117
|
+
(a, b) => a.summary.metrics.latencyMs.p95 - b.summary.metrics.latencyMs.p95
|
|
12118
|
+
)[0];
|
|
12119
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
12120
|
+
const failedGateRuns = runs.filter((run) => run.gate && !run.gate.passed).length;
|
|
12121
|
+
const gatePassed = failedGateRuns === 0;
|
|
12122
|
+
const aggregate = {
|
|
12123
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12124
|
+
againstPath: options.againstPath,
|
|
12125
|
+
runCount: runs.length,
|
|
12126
|
+
runs,
|
|
12127
|
+
gatePassed,
|
|
12128
|
+
failedGateRuns,
|
|
12129
|
+
bestByHitAt5,
|
|
12130
|
+
bestByMrrAt10,
|
|
12131
|
+
bestByP95Latency
|
|
12132
|
+
};
|
|
12133
|
+
writeJson(path19.join(outputDir, "compare.json"), aggregate);
|
|
12134
|
+
const md = createSummaryMarkdown(
|
|
12135
|
+
bestByHitAt5?.summary ?? runs[0].summary,
|
|
12136
|
+
bestByHitAt5?.comparison,
|
|
12137
|
+
void 0,
|
|
12138
|
+
aggregate
|
|
12139
|
+
);
|
|
12140
|
+
writeText(path19.join(outputDir, "summary.md"), md);
|
|
12141
|
+
writeJson(path19.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
12142
|
+
return { outputDir, aggregate };
|
|
11670
12143
|
}
|
|
11671
12144
|
|
|
11672
|
-
// src/
|
|
11673
|
-
var
|
|
12145
|
+
// src/eval/cli.ts
|
|
12146
|
+
var path21 = __toESM(require("path"), 1);
|
|
12147
|
+
|
|
12148
|
+
// src/eval/cli-parser.ts
|
|
11674
12149
|
var path20 = __toESM(require("path"), 1);
|
|
11675
|
-
function
|
|
11676
|
-
|
|
11677
|
-
|
|
11678
|
-
|
|
11679
|
-
|
|
11680
|
-
|
|
11681
|
-
|
|
11682
|
-
|
|
12150
|
+
function printUsage() {
|
|
12151
|
+
console.log(`
|
|
12152
|
+
Usage:
|
|
12153
|
+
opencode-codebase-index-mcp eval run [options]
|
|
12154
|
+
opencode-codebase-index-mcp eval compare --against <summary.json> [options]
|
|
12155
|
+
opencode-codebase-index-mcp eval diff --current <summary.json> --against <summary.json> [options]
|
|
12156
|
+
|
|
12157
|
+
Options:
|
|
12158
|
+
--project <path> Project root (default: cwd)
|
|
12159
|
+
--config <path> Config JSON path
|
|
12160
|
+
--dataset <path> Golden dataset path (default: benchmarks/golden/small.json)
|
|
12161
|
+
--current <path> Current summary.json path (required for eval diff)
|
|
12162
|
+
--output <path> Output root dir (default: benchmarks/results)
|
|
12163
|
+
--against <path> Baseline summary.json to compare against
|
|
12164
|
+
--budget <path> Budget file for CI mode (default: benchmarks/budgets/default.json)
|
|
12165
|
+
--ci Enable CI gate mode
|
|
12166
|
+
--reindex Force reindex before eval
|
|
12167
|
+
|
|
12168
|
+
Search overrides:
|
|
12169
|
+
--fusionStrategy <rrf|weighted>
|
|
12170
|
+
--hybridWeight <0-1>
|
|
12171
|
+
--rrfK <number>
|
|
12172
|
+
--rerankTopN <number>
|
|
12173
|
+
|
|
12174
|
+
Sweep options (comma-separated values):
|
|
12175
|
+
--sweepFusionStrategy <rrf,weighted>
|
|
12176
|
+
--sweepHybridWeight <0.3,0.5,0.7>
|
|
12177
|
+
--sweepRrfK <30,60,90>
|
|
12178
|
+
--sweepRerankTopN <10,20,40>
|
|
12179
|
+
`);
|
|
11683
12180
|
}
|
|
11684
|
-
function
|
|
11685
|
-
|
|
11686
|
-
|
|
12181
|
+
function parseNumber(value, flag) {
|
|
12182
|
+
const parsed = Number(value);
|
|
12183
|
+
if (Number.isNaN(parsed)) {
|
|
12184
|
+
throw new Error(`${flag} must be a number`);
|
|
11687
12185
|
}
|
|
11688
|
-
return
|
|
12186
|
+
return parsed;
|
|
11689
12187
|
}
|
|
11690
|
-
function
|
|
11691
|
-
return
|
|
12188
|
+
function parseCsvNumbers(value, flag) {
|
|
12189
|
+
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0).map((item) => parseNumber(item, flag));
|
|
11692
12190
|
}
|
|
11693
|
-
|
|
11694
|
-
|
|
11695
|
-
|
|
11696
|
-
|
|
11697
|
-
|
|
11698
|
-
|
|
11699
|
-
|
|
11700
|
-
|
|
11701
|
-
const ownerText = owner ? `PID ${owner.pid}, operation ${owner.operation}, since ${owner.startedAt}` : "unreadable owner";
|
|
11702
|
-
if (error.reason === "legacy-lock") {
|
|
11703
|
-
return {
|
|
11704
|
-
kind: "busy",
|
|
11705
|
-
text: `INDEX_BUSY: legacy lock format detected (${ownerText}). Verify the PID and remove this lock manually only if it is stale.`
|
|
11706
|
-
};
|
|
12191
|
+
function parseCsvFusion(value) {
|
|
12192
|
+
const values = value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
12193
|
+
const parsed = [];
|
|
12194
|
+
for (const candidate of values) {
|
|
12195
|
+
if (candidate !== "rrf" && candidate !== "weighted") {
|
|
12196
|
+
throw new Error("--sweepFusionStrategy accepts only rrf,weighted");
|
|
12197
|
+
}
|
|
12198
|
+
parsed.push(candidate);
|
|
11707
12199
|
}
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
|
|
11712
|
-
|
|
12200
|
+
return parsed;
|
|
12201
|
+
}
|
|
12202
|
+
function hasSweepOptions(sweep) {
|
|
12203
|
+
return Boolean(
|
|
12204
|
+
sweep.fusionStrategy && sweep.fusionStrategy.length > 0 || sweep.hybridWeight && sweep.hybridWeight.length > 0 || sweep.rrfK && sweep.rrfK.length > 0 || sweep.rerankTopN && sweep.rerankTopN.length > 0
|
|
12205
|
+
);
|
|
12206
|
+
}
|
|
12207
|
+
function parseEvalArgs(argv, cwd) {
|
|
12208
|
+
const parsed = {
|
|
12209
|
+
projectRoot: cwd,
|
|
12210
|
+
datasetPath: "benchmarks/golden/small.json",
|
|
12211
|
+
outputRoot: "benchmarks/results",
|
|
12212
|
+
budgetPath: "benchmarks/budgets/default.json",
|
|
12213
|
+
ciMode: false,
|
|
12214
|
+
reindex: false,
|
|
12215
|
+
sweep: {}
|
|
12216
|
+
};
|
|
12217
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
12218
|
+
const arg = argv[i];
|
|
12219
|
+
const next = argv[i + 1];
|
|
12220
|
+
if (arg === "--project" && next) {
|
|
12221
|
+
parsed.projectRoot = path20.resolve(cwd, next);
|
|
12222
|
+
i += 1;
|
|
12223
|
+
continue;
|
|
12224
|
+
}
|
|
12225
|
+
if (arg === "--config" && next) {
|
|
12226
|
+
parsed.configPath = path20.resolve(cwd, next);
|
|
12227
|
+
i += 1;
|
|
12228
|
+
continue;
|
|
12229
|
+
}
|
|
12230
|
+
if (arg === "--dataset" && next) {
|
|
12231
|
+
parsed.datasetPath = next;
|
|
12232
|
+
i += 1;
|
|
12233
|
+
continue;
|
|
12234
|
+
}
|
|
12235
|
+
if (arg === "--current" && next) {
|
|
12236
|
+
parsed.currentPath = next;
|
|
12237
|
+
i += 1;
|
|
12238
|
+
continue;
|
|
12239
|
+
}
|
|
12240
|
+
if (arg === "--output" && next) {
|
|
12241
|
+
parsed.outputRoot = next;
|
|
12242
|
+
i += 1;
|
|
12243
|
+
continue;
|
|
12244
|
+
}
|
|
12245
|
+
if (arg === "--against" && next) {
|
|
12246
|
+
parsed.againstPath = next;
|
|
12247
|
+
i += 1;
|
|
12248
|
+
continue;
|
|
12249
|
+
}
|
|
12250
|
+
if (arg === "--budget" && next) {
|
|
12251
|
+
parsed.budgetPath = next;
|
|
12252
|
+
i += 1;
|
|
12253
|
+
continue;
|
|
12254
|
+
}
|
|
12255
|
+
if (arg === "--ci") {
|
|
12256
|
+
parsed.ciMode = true;
|
|
12257
|
+
continue;
|
|
12258
|
+
}
|
|
12259
|
+
if (arg === "--reindex") {
|
|
12260
|
+
parsed.reindex = true;
|
|
12261
|
+
continue;
|
|
12262
|
+
}
|
|
12263
|
+
if (arg === "--fusionStrategy" && next) {
|
|
12264
|
+
if (next !== "rrf" && next !== "weighted") {
|
|
12265
|
+
throw new Error("--fusionStrategy must be rrf or weighted");
|
|
12266
|
+
}
|
|
12267
|
+
parsed.fusionStrategy = next;
|
|
12268
|
+
i += 1;
|
|
12269
|
+
continue;
|
|
12270
|
+
}
|
|
12271
|
+
if (arg === "--hybridWeight" && next) {
|
|
12272
|
+
parsed.hybridWeight = parseNumber(next, "--hybridWeight");
|
|
12273
|
+
i += 1;
|
|
12274
|
+
continue;
|
|
12275
|
+
}
|
|
12276
|
+
if (arg === "--rrfK" && next) {
|
|
12277
|
+
parsed.rrfK = parseNumber(next, "--rrfK");
|
|
12278
|
+
i += 1;
|
|
12279
|
+
continue;
|
|
12280
|
+
}
|
|
12281
|
+
if (arg === "--rerankTopN" && next) {
|
|
12282
|
+
parsed.rerankTopN = parseNumber(next, "--rerankTopN");
|
|
12283
|
+
i += 1;
|
|
12284
|
+
continue;
|
|
12285
|
+
}
|
|
12286
|
+
if (arg === "--sweepFusionStrategy" && next) {
|
|
12287
|
+
parsed.sweep.fusionStrategy = parseCsvFusion(next);
|
|
12288
|
+
i += 1;
|
|
12289
|
+
continue;
|
|
12290
|
+
}
|
|
12291
|
+
if (arg === "--sweepHybridWeight" && next) {
|
|
12292
|
+
parsed.sweep.hybridWeight = parseCsvNumbers(next, "--sweepHybridWeight");
|
|
12293
|
+
i += 1;
|
|
12294
|
+
continue;
|
|
12295
|
+
}
|
|
12296
|
+
if (arg === "--sweepRrfK" && next) {
|
|
12297
|
+
parsed.sweep.rrfK = parseCsvNumbers(next, "--sweepRrfK");
|
|
12298
|
+
i += 1;
|
|
12299
|
+
continue;
|
|
12300
|
+
}
|
|
12301
|
+
if (arg === "--sweepRerankTopN" && next) {
|
|
12302
|
+
parsed.sweep.rerankTopN = parseCsvNumbers(next, "--sweepRerankTopN");
|
|
12303
|
+
i += 1;
|
|
12304
|
+
continue;
|
|
12305
|
+
}
|
|
11713
12306
|
}
|
|
11714
|
-
return
|
|
12307
|
+
return parsed;
|
|
11715
12308
|
}
|
|
11716
|
-
function
|
|
11717
|
-
|
|
11718
|
-
|
|
11719
|
-
|
|
11720
|
-
|
|
11721
|
-
|
|
11722
|
-
|
|
12309
|
+
function parseEvalSubcommandOptions(argv, cwd) {
|
|
12310
|
+
let explicitAgainst;
|
|
12311
|
+
const filtered = [];
|
|
12312
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
12313
|
+
const current = argv[i];
|
|
12314
|
+
const next = argv[i + 1];
|
|
12315
|
+
if (current === "--against" && next) {
|
|
12316
|
+
explicitAgainst = next;
|
|
12317
|
+
i += 1;
|
|
12318
|
+
continue;
|
|
12319
|
+
}
|
|
12320
|
+
filtered.push(current);
|
|
11723
12321
|
}
|
|
11724
|
-
return
|
|
12322
|
+
return {
|
|
12323
|
+
parsed: parseEvalArgs(filtered, cwd),
|
|
12324
|
+
explicitAgainst
|
|
12325
|
+
};
|
|
11725
12326
|
}
|
|
11726
|
-
function
|
|
11727
|
-
return
|
|
12327
|
+
function toRunOptions(parsed) {
|
|
12328
|
+
return {
|
|
12329
|
+
projectRoot: parsed.projectRoot,
|
|
12330
|
+
configPath: parsed.configPath,
|
|
12331
|
+
datasetPath: parsed.datasetPath,
|
|
12332
|
+
outputRoot: parsed.outputRoot,
|
|
12333
|
+
againstPath: parsed.againstPath,
|
|
12334
|
+
budgetPath: parsed.budgetPath,
|
|
12335
|
+
ciMode: parsed.ciMode,
|
|
12336
|
+
reindex: parsed.reindex,
|
|
12337
|
+
searchOverrides: {
|
|
12338
|
+
...parsed.fusionStrategy !== void 0 ? { fusionStrategy: parsed.fusionStrategy } : {},
|
|
12339
|
+
...parsed.hybridWeight !== void 0 ? { hybridWeight: parsed.hybridWeight } : {},
|
|
12340
|
+
...parsed.rrfK !== void 0 ? { rrfK: parsed.rrfK } : {},
|
|
12341
|
+
...parsed.rerankTopN !== void 0 ? { rerankTopN: parsed.rerankTopN } : {}
|
|
12342
|
+
}
|
|
12343
|
+
};
|
|
11728
12344
|
}
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11732
|
-
|
|
11733
|
-
|
|
12345
|
+
|
|
12346
|
+
// src/eval/cli.ts
|
|
12347
|
+
async function handleEvalCommand(args, cwd) {
|
|
12348
|
+
const subcommand = args[0];
|
|
12349
|
+
if (!subcommand || subcommand === "--help" || subcommand === "-h") {
|
|
12350
|
+
printUsage();
|
|
12351
|
+
return 0;
|
|
11734
12352
|
}
|
|
11735
|
-
|
|
11736
|
-
|
|
11737
|
-
|
|
11738
|
-
|
|
11739
|
-
|
|
11740
|
-
|
|
11741
|
-
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
|
|
11745
|
-
|
|
11746
|
-
|
|
11747
|
-
}
|
|
11748
|
-
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
11752
|
-
|
|
11753
|
-
|
|
11754
|
-
|
|
11755
|
-
|
|
11756
|
-
|
|
11757
|
-
|
|
11758
|
-
|
|
11759
|
-
|
|
11760
|
-
|
|
11761
|
-
|
|
11762
|
-
|
|
12353
|
+
if (subcommand === "run") {
|
|
12354
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
12355
|
+
if (explicitAgainst) {
|
|
12356
|
+
parsed.againstPath = explicitAgainst;
|
|
12357
|
+
}
|
|
12358
|
+
const runOptions = toRunOptions(parsed);
|
|
12359
|
+
if (hasSweepOptions(parsed.sweep)) {
|
|
12360
|
+
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
12361
|
+
console.log(`Eval sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
12362
|
+
console.log(`Sweep runs: ${sweep.aggregate.runCount}`);
|
|
12363
|
+
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
12364
|
+
console.error(
|
|
12365
|
+
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
12366
|
+
);
|
|
12367
|
+
return 1;
|
|
12368
|
+
}
|
|
12369
|
+
return 0;
|
|
12370
|
+
}
|
|
12371
|
+
const result = await runEvaluation(runOptions);
|
|
12372
|
+
console.log(`Eval run complete. Artifacts: ${result.outputDir}`);
|
|
12373
|
+
console.log(
|
|
12374
|
+
`Hit@5=${(result.summary.metrics.hitAt5 * 100).toFixed(2)}% MRR@10=${result.summary.metrics.mrrAt10.toFixed(4)} p95=${result.summary.metrics.latencyMs.p95.toFixed(3)}ms`
|
|
12375
|
+
);
|
|
12376
|
+
if (result.gate && !result.gate.passed) {
|
|
12377
|
+
for (const violation of result.gate.violations) {
|
|
12378
|
+
console.error(`[CI-GATE] ${violation.metric}: ${violation.message}`);
|
|
12379
|
+
}
|
|
12380
|
+
return 1;
|
|
12381
|
+
}
|
|
12382
|
+
return 0;
|
|
11763
12383
|
}
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11769
|
-
return indexer.search(query, options.limit, {
|
|
11770
|
-
fileType: options.fileType,
|
|
11771
|
-
directory: options.directory,
|
|
11772
|
-
chunkType: options.chunkType,
|
|
11773
|
-
contextLines: options.contextLines,
|
|
11774
|
-
metadataOnly: options.metadataOnly,
|
|
11775
|
-
definitionIntent: options.definitionIntent,
|
|
11776
|
-
blameAuthor: options.blameAuthor,
|
|
11777
|
-
blameSha: options.blameSha,
|
|
11778
|
-
blameSince: options.blameSince
|
|
11779
|
-
});
|
|
11780
|
-
}
|
|
11781
|
-
async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
11782
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11783
|
-
return indexer.findSimilar(code, options.limit, {
|
|
11784
|
-
fileType: options.fileType,
|
|
11785
|
-
directory: options.directory,
|
|
11786
|
-
chunkType: options.chunkType,
|
|
11787
|
-
excludeFile: options.excludeFile
|
|
11788
|
-
});
|
|
11789
|
-
}
|
|
11790
|
-
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
11791
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11792
|
-
return indexer.search(query, options.limit, {
|
|
11793
|
-
fileType: options.fileType,
|
|
11794
|
-
directory: options.directory,
|
|
11795
|
-
definitionIntent: true
|
|
11796
|
-
});
|
|
11797
|
-
}
|
|
11798
|
-
async function getCallGraphData(projectRoot, host, params) {
|
|
11799
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11800
|
-
if (params.direction === "callees") {
|
|
11801
|
-
if (!params.symbolId) {
|
|
11802
|
-
return { direction: "callees", callees: [], callers: [] };
|
|
12384
|
+
if (subcommand === "compare") {
|
|
12385
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
12386
|
+
if (!explicitAgainst) {
|
|
12387
|
+
throw new Error("eval compare requires --against <baseline summary.json>");
|
|
11803
12388
|
}
|
|
11804
|
-
|
|
11805
|
-
|
|
12389
|
+
parsed.againstPath = explicitAgainst;
|
|
12390
|
+
const runOptions = toRunOptions(parsed);
|
|
12391
|
+
if (hasSweepOptions(parsed.sweep)) {
|
|
12392
|
+
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
12393
|
+
console.log(`Eval compare sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
12394
|
+
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
12395
|
+
console.error(
|
|
12396
|
+
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
12397
|
+
);
|
|
12398
|
+
return 1;
|
|
12399
|
+
}
|
|
12400
|
+
return 0;
|
|
12401
|
+
}
|
|
12402
|
+
const result = await runEvaluation(runOptions);
|
|
12403
|
+
console.log(`Eval compare complete. Artifacts: ${result.outputDir}`);
|
|
12404
|
+
return 0;
|
|
11806
12405
|
}
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
11810
|
-
|
|
11811
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11812
|
-
return indexer.findCallPath(from, to, maxDepth);
|
|
11813
|
-
}
|
|
11814
|
-
async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
11815
|
-
const root = getProjectRoot(projectRoot, host);
|
|
11816
|
-
const key = getIndexerCacheKey(root, host);
|
|
11817
|
-
let indexer = getIndexerForProject(root, host);
|
|
11818
|
-
const runtimeConfig = configCache.get(key);
|
|
11819
|
-
try {
|
|
11820
|
-
if (args.estimateOnly) {
|
|
11821
|
-
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
12406
|
+
if (subcommand === "diff") {
|
|
12407
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
12408
|
+
if (!explicitAgainst) {
|
|
12409
|
+
throw new Error("eval diff requires --against <baseline summary.json>");
|
|
11822
12410
|
}
|
|
11823
|
-
|
|
11824
|
-
|
|
11825
|
-
|
|
11826
|
-
|
|
11827
|
-
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
totalFiles: progress.totalFiles,
|
|
11831
|
-
chunksProcessed: progress.chunksProcessed,
|
|
11832
|
-
totalChunks: progress.totalChunks,
|
|
11833
|
-
percentage: calculatePercentage(progress)
|
|
11834
|
-
});
|
|
11835
|
-
}
|
|
11836
|
-
return Promise.resolve();
|
|
11837
|
-
});
|
|
11838
|
-
};
|
|
11839
|
-
let stats;
|
|
11840
|
-
if (args.force && shouldForceLocalizeProjectIndex(root, host)) {
|
|
11841
|
-
const inheritedIndexPath = resolveProjectIndexPath(root, runtimeConfig.scope, host);
|
|
11842
|
-
stats = await withIndexLock(inheritedIndexPath, "force-index", async () => {
|
|
11843
|
-
materializeLocalProjectConfig(root, loadProjectConfigLayer(root, host), host);
|
|
11844
|
-
refreshIndexerForDirectory(root, host, runtimeConfig);
|
|
11845
|
-
indexer = getIndexerForProject(root, host);
|
|
11846
|
-
return runIndex(indexer);
|
|
11847
|
-
}, { completeRecoveries: false });
|
|
11848
|
-
} else {
|
|
11849
|
-
stats = await runIndex(indexer);
|
|
12411
|
+
if (!parsed.currentPath) {
|
|
12412
|
+
throw new Error("eval diff requires --current <current summary.json>");
|
|
12413
|
+
}
|
|
12414
|
+
parsed.againstPath = explicitAgainst;
|
|
12415
|
+
const currentPath = parsed.currentPath;
|
|
12416
|
+
if (!currentPath.endsWith(".json")) {
|
|
12417
|
+
throw new Error("eval diff --current must point to a summary JSON file");
|
|
11850
12418
|
}
|
|
11851
|
-
|
|
11852
|
-
|
|
11853
|
-
|
|
11854
|
-
|
|
11855
|
-
|
|
11856
|
-
|
|
11857
|
-
|
|
11858
|
-
|
|
11859
|
-
|
|
11860
|
-
|
|
11861
|
-
|
|
11862
|
-
|
|
11863
|
-
|
|
11864
|
-
|
|
11865
|
-
|
|
11866
|
-
|
|
11867
|
-
|
|
11868
|
-
|
|
11869
|
-
|
|
11870
|
-
|
|
11871
|
-
|
|
11872
|
-
return busyResult;
|
|
12419
|
+
if (!parsed.againstPath.endsWith(".json")) {
|
|
12420
|
+
throw new Error("eval diff --against must point to a summary JSON file");
|
|
12421
|
+
}
|
|
12422
|
+
const currentSummary = loadSummary(path21.resolve(parsed.projectRoot, currentPath), {
|
|
12423
|
+
allowLegacyDiversityMetrics: true
|
|
12424
|
+
});
|
|
12425
|
+
const baselineSummary = loadSummary(path21.resolve(parsed.projectRoot, parsed.againstPath), {
|
|
12426
|
+
allowLegacyDiversityMetrics: true
|
|
12427
|
+
});
|
|
12428
|
+
const comparison = compareSummaries(
|
|
12429
|
+
currentSummary,
|
|
12430
|
+
baselineSummary,
|
|
12431
|
+
path21.resolve(parsed.projectRoot, parsed.againstPath)
|
|
12432
|
+
);
|
|
12433
|
+
const outputDir = createRunDirectory(path21.resolve(parsed.projectRoot, parsed.outputRoot));
|
|
12434
|
+
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
12435
|
+
writeJson(path21.join(outputDir, "compare.json"), comparison);
|
|
12436
|
+
writeText(path21.join(outputDir, "summary.md"), summaryMd);
|
|
12437
|
+
writeJson(path21.join(outputDir, "summary.json"), currentSummary);
|
|
12438
|
+
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
12439
|
+
return 0;
|
|
11873
12440
|
}
|
|
12441
|
+
throw new Error(`Unknown eval subcommand: ${subcommand}`);
|
|
11874
12442
|
}
|
|
11875
|
-
|
|
11876
|
-
|
|
11877
|
-
|
|
11878
|
-
|
|
11879
|
-
|
|
11880
|
-
|
|
11881
|
-
|
|
11882
|
-
|
|
11883
|
-
|
|
11884
|
-
|
|
12443
|
+
|
|
12444
|
+
// src/mcp-server.ts
|
|
12445
|
+
var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
12446
|
+
var import_fs15 = require("fs");
|
|
12447
|
+
|
|
12448
|
+
// src/mcp-server/register-prompts.ts
|
|
12449
|
+
var import_zod = require("zod");
|
|
12450
|
+
function registerMcpPrompts(server) {
|
|
12451
|
+
server.prompt(
|
|
12452
|
+
"search",
|
|
12453
|
+
"Search codebase by meaning using semantic search",
|
|
12454
|
+
{ query: import_zod.z.string().describe("What to search for in the codebase") },
|
|
12455
|
+
(args) => ({
|
|
12456
|
+
messages: [{
|
|
12457
|
+
role: "user",
|
|
12458
|
+
content: {
|
|
12459
|
+
type: "text",
|
|
12460
|
+
text: `Search the codebase for: "${args.query}"
|
|
12461
|
+
|
|
12462
|
+
Use the codebase_search tool with this query. If you need just locations first, use codebase_peek instead to save tokens.`
|
|
12463
|
+
}
|
|
12464
|
+
}]
|
|
12465
|
+
})
|
|
12466
|
+
);
|
|
12467
|
+
server.prompt(
|
|
12468
|
+
"find",
|
|
12469
|
+
"Find code using hybrid approach (semantic + grep)",
|
|
12470
|
+
{ query: import_zod.z.string().describe("What to find in the codebase") },
|
|
12471
|
+
(args) => ({
|
|
12472
|
+
messages: [{
|
|
12473
|
+
role: "user",
|
|
12474
|
+
content: {
|
|
12475
|
+
type: "text",
|
|
12476
|
+
text: `Find code related to: "${args.query}"
|
|
12477
|
+
|
|
12478
|
+
Use a hybrid approach:
|
|
12479
|
+
1. First use codebase_peek to find semantic matches by meaning
|
|
12480
|
+
2. Then use grep for exact identifier matches
|
|
12481
|
+
3. Combine results for comprehensive coverage`
|
|
12482
|
+
}
|
|
12483
|
+
}]
|
|
12484
|
+
})
|
|
12485
|
+
);
|
|
12486
|
+
server.prompt(
|
|
12487
|
+
"index",
|
|
12488
|
+
"Index the codebase for semantic search",
|
|
12489
|
+
{ options: import_zod.z.string().optional().describe("Options: 'force' to rebuild, 'estimate' to check costs") },
|
|
12490
|
+
(args) => {
|
|
12491
|
+
const opts = args.options?.toLowerCase() ?? "";
|
|
12492
|
+
let instruction = "Use the index_codebase tool to index the codebase for semantic search.";
|
|
12493
|
+
if (opts.includes("force")) {
|
|
12494
|
+
instruction = "Use the index_codebase tool with force=true to rebuild the entire index from scratch.";
|
|
12495
|
+
} else if (opts.includes("estimate")) {
|
|
12496
|
+
instruction = "Use the index_codebase tool with estimateOnly=true to check the cost estimate before indexing.";
|
|
12497
|
+
}
|
|
12498
|
+
return {
|
|
12499
|
+
messages: [{
|
|
12500
|
+
role: "user",
|
|
12501
|
+
content: { type: "text", text: instruction }
|
|
12502
|
+
}]
|
|
12503
|
+
};
|
|
12504
|
+
}
|
|
12505
|
+
);
|
|
12506
|
+
server.prompt(
|
|
12507
|
+
"status",
|
|
12508
|
+
"Check if the codebase is indexed and ready",
|
|
12509
|
+
{},
|
|
12510
|
+
() => ({
|
|
12511
|
+
messages: [{
|
|
12512
|
+
role: "user",
|
|
12513
|
+
content: {
|
|
12514
|
+
type: "text",
|
|
12515
|
+
text: "Use the index_status tool to check if the codebase index is ready and show its current state."
|
|
12516
|
+
}
|
|
12517
|
+
}]
|
|
12518
|
+
})
|
|
12519
|
+
);
|
|
12520
|
+
server.prompt(
|
|
12521
|
+
"definition",
|
|
12522
|
+
"Find where a symbol is defined in the codebase",
|
|
12523
|
+
{ query: import_zod.z.string().describe("Symbol name or description to find the definition of") },
|
|
12524
|
+
(args) => ({
|
|
12525
|
+
messages: [{
|
|
12526
|
+
role: "user",
|
|
12527
|
+
content: {
|
|
12528
|
+
type: "text",
|
|
12529
|
+
text: `Find the definition of: "${args.query}"
|
|
12530
|
+
|
|
12531
|
+
Use the implementation_lookup tool to find where this symbol is defined. This prioritizes real implementation files over tests, docs, and examples. If no definition is found, fall back to codebase_search for broader discovery.`
|
|
12532
|
+
}
|
|
12533
|
+
}]
|
|
12534
|
+
})
|
|
12535
|
+
);
|
|
11885
12536
|
}
|
|
11886
|
-
|
|
11887
|
-
|
|
11888
|
-
|
|
11889
|
-
|
|
11890
|
-
|
|
11891
|
-
|
|
11892
|
-
|
|
11893
|
-
|
|
11894
|
-
|
|
12537
|
+
|
|
12538
|
+
// src/mcp-server/register-tools.ts
|
|
12539
|
+
var import_zod2 = require("zod");
|
|
12540
|
+
|
|
12541
|
+
// src/tools/format-pr-impact.ts
|
|
12542
|
+
function formatPrImpact(result) {
|
|
12543
|
+
const lines = [];
|
|
12544
|
+
lines.push(`\u2192 Files changed: ${result.changedFiles.length}`);
|
|
12545
|
+
for (const file of result.changedFiles) {
|
|
12546
|
+
lines.push(` - ${file}`);
|
|
11895
12547
|
}
|
|
11896
|
-
|
|
11897
|
-
|
|
11898
|
-
|
|
11899
|
-
|
|
11900
|
-
|
|
11901
|
-
|
|
12548
|
+
const directCount = result.directSymbols.length;
|
|
12549
|
+
const transitiveCount = result.transitiveCallers.length;
|
|
12550
|
+
const directionLabel = result.direction === "callees" ? "callees" : result.direction === "both" ? "reachable (callers + callees)" : "callers";
|
|
12551
|
+
lines.push(
|
|
12552
|
+
`\u2192 Symbols affected: ${result.totalAffected} (${directCount} direct, ${transitiveCount} transitive ${directionLabel})`
|
|
12553
|
+
);
|
|
12554
|
+
if (directCount > 0) {
|
|
12555
|
+
lines.push(
|
|
12556
|
+
` Direct: ${result.directSymbols.map((s) => `${s.name} (${s.kind})`).join(", ")}`
|
|
12557
|
+
);
|
|
11902
12558
|
}
|
|
11903
|
-
|
|
11904
|
-
|
|
11905
|
-
|
|
11906
|
-
|
|
11907
|
-
};
|
|
11908
|
-
}
|
|
11909
|
-
async function getIndexLogs(projectRoot, host, args) {
|
|
11910
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11911
|
-
const logger = indexer.getLogger();
|
|
11912
|
-
if (!logger.isEnabled()) {
|
|
11913
|
-
return {
|
|
11914
|
-
kind: "disabled",
|
|
11915
|
-
text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true\n }\n}\n```'
|
|
11916
|
-
};
|
|
12559
|
+
if (transitiveCount > 0) {
|
|
12560
|
+
lines.push(
|
|
12561
|
+
` Transitive ${directionLabel}: ${result.transitiveCallers.map((s) => s.name).join(", ")}`
|
|
12562
|
+
);
|
|
11917
12563
|
}
|
|
11918
|
-
|
|
11919
|
-
|
|
11920
|
-
|
|
11921
|
-
|
|
11922
|
-
|
|
12564
|
+
if (result.communities.length > 0) {
|
|
12565
|
+
lines.push(
|
|
12566
|
+
`\u2192 Communities touched: ${result.communities.map((c) => c.label).join(", ")}`
|
|
12567
|
+
);
|
|
12568
|
+
for (const community of result.communities) {
|
|
12569
|
+
lines.push(
|
|
12570
|
+
` - ${community.label}: ${community.symbolCount} symbols`
|
|
12571
|
+
);
|
|
12572
|
+
}
|
|
11923
12573
|
} else {
|
|
11924
|
-
|
|
12574
|
+
lines.push("\u2192 Communities touched: none");
|
|
11925
12575
|
}
|
|
11926
|
-
|
|
11927
|
-
|
|
11928
|
-
|
|
11929
|
-
|
|
11930
|
-
};
|
|
12576
|
+
lines.push(`\u2192 Risk: ${result.riskLevel} \u2014 ${result.riskReason}`);
|
|
12577
|
+
if (result.hubNodes.length > 0) {
|
|
12578
|
+
lines.push(" Hub nodes in change scope:");
|
|
12579
|
+
for (const hub of result.hubNodes) {
|
|
12580
|
+
lines.push(` - ${hub.name} (${hub.callerCount} callers) at ${hub.filePath}`);
|
|
12581
|
+
}
|
|
11931
12582
|
}
|
|
11932
|
-
|
|
11933
|
-
const
|
|
11934
|
-
|
|
11935
|
-
|
|
11936
|
-
|
|
12583
|
+
if (result.conflictingPRs && result.conflictingPRs.length > 0) {
|
|
12584
|
+
const conflictList = result.conflictingPRs.map(
|
|
12585
|
+
(c) => `PR #${c.pr} (also touches ${c.overlappingCommunities.join(", ")} community)`
|
|
12586
|
+
);
|
|
12587
|
+
lines.push(`\u2192 Potential conflicts with: ${conflictList.join(", ")}`);
|
|
12588
|
+
}
|
|
12589
|
+
return lines.join("\n");
|
|
11937
12590
|
}
|
|
11938
12591
|
|
|
11939
12592
|
// src/mcp-server/shared.ts
|
|
@@ -11958,66 +12611,27 @@ function allowNullAsUndefined(schema) {
|
|
|
11958
12611
|
function registerMcpTools(server, runtime) {
|
|
11959
12612
|
server.tool(
|
|
11960
12613
|
"codebase_context",
|
|
11961
|
-
"PREFERRED FIRST TOOL for any question about this repository. Use before built-in code search, grep, shell search, or broad file reads. Provide from+to for a dependency path, symbol for a definition, or only query for low-token conceptual discovery. Use call_graph directly for callers or callees.",
|
|
12614
|
+
"PREFERRED FIRST TOOL for any question about this repository. Returns a deduplicated, file-diverse evidence pack within tokenBudget. Use before built-in code search, grep, shell search, or broad file reads. Provide from+to for a dependency path, symbol for a definition, or only query for low-token conceptual discovery. Use call_graph directly for callers or callees.",
|
|
11962
12615
|
{
|
|
11963
12616
|
query: import_zod2.z.string().describe("The codebase question or behavior to locate. Always provide the user's repository question here."),
|
|
11964
12617
|
from: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Source symbol. For dependency-path questions, extract the first endpoint and provide it here."),
|
|
11965
12618
|
to: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Target symbol. For dependency-path questions, extract the second endpoint and provide it here."),
|
|
11966
12619
|
symbol: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Exact symbol for an authoritative definition lookup. Omit when from and to are supplied."),
|
|
11967
|
-
limit: allowNullAsUndefined(
|
|
11968
|
-
|
|
12620
|
+
limit: allowNullAsUndefined(
|
|
12621
|
+
import_zod2.z.number().int().min(MIN_CONTEXT_RESULT_LIMIT).max(MAX_CONTEXT_RESULT_LIMIT).optional().default(10)
|
|
12622
|
+
).describe(`Maximum number of search or definition results (${MIN_CONTEXT_RESULT_LIMIT}-${MAX_CONTEXT_RESULT_LIMIT})`),
|
|
12623
|
+
maxDepth: allowNullAsUndefined(
|
|
12624
|
+
import_zod2.z.number().int().min(MIN_CONTEXT_PATH_DEPTH).max(MAX_CONTEXT_PATH_DEPTH).optional().default(10)
|
|
12625
|
+
).describe(`Maximum call-graph traversal depth for from/to path lookup (${MIN_CONTEXT_PATH_DEPTH}-${MAX_CONTEXT_PATH_DEPTH})`),
|
|
11969
12626
|
fileType: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
11970
|
-
directory: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')")
|
|
12627
|
+
directory: allowNullAsUndefined(import_zod2.z.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
12628
|
+
tokenBudget: allowNullAsUndefined(
|
|
12629
|
+
import_zod2.z.number().int().min(MIN_CONTEXT_PACK_TOKEN_BUDGET).max(MAX_CONTEXT_PACK_TOKEN_BUDGET).optional().default(DEFAULT_CONTEXT_PACK_TOKEN_BUDGET)
|
|
12630
|
+
).describe(`Maximum response tokens for this context pack (${MIN_CONTEXT_PACK_TOKEN_BUDGET}-${MAX_CONTEXT_PACK_TOKEN_BUDGET})`)
|
|
11971
12631
|
},
|
|
11972
12632
|
async (args) => {
|
|
11973
|
-
|
|
11974
|
-
|
|
11975
|
-
if (path27.length > 0) {
|
|
11976
|
-
return { content: [{ type: "text", text: formatCallGraphPath(args.from, args.to, path27) }] };
|
|
11977
|
-
}
|
|
11978
|
-
const { callers } = await getCallGraphData(runtime.projectRoot, runtime.host, {
|
|
11979
|
-
name: args.to,
|
|
11980
|
-
direction: "callers"
|
|
11981
|
-
});
|
|
11982
|
-
const directEdge = callers.find((edge) => edge.fromSymbolName === args.from);
|
|
11983
|
-
if (directEdge) {
|
|
11984
|
-
const location = directEdge.fromSymbolFilePath ? ` at ${directEdge.fromSymbolFilePath}:${directEdge.line}` : "";
|
|
11985
|
-
return {
|
|
11986
|
-
content: [{
|
|
11987
|
-
type: "text",
|
|
11988
|
-
text: `Direct path: ${args.from} --${directEdge.callType}--> ${args.to}${location} (edge is ${directEdge.isResolved ? "resolved" : "unresolved"}).`
|
|
11989
|
-
}]
|
|
11990
|
-
};
|
|
11991
|
-
}
|
|
11992
|
-
return { content: [{ type: "text", text: formatCallGraphPath(args.from, args.to, path27) }] };
|
|
11993
|
-
}
|
|
11994
|
-
if (args.symbol) {
|
|
11995
|
-
const results2 = await implementationLookup(runtime.projectRoot, runtime.host, args.symbol, {
|
|
11996
|
-
limit: args.limit ?? 10,
|
|
11997
|
-
fileType: args.fileType,
|
|
11998
|
-
directory: args.directory
|
|
11999
|
-
});
|
|
12000
|
-
return { content: [{ type: "text", text: formatDefinitionLookup(results2, args.symbol) }] };
|
|
12001
|
-
}
|
|
12002
|
-
const results = await searchCodebase(runtime.projectRoot, runtime.host, args.query, {
|
|
12003
|
-
limit: args.limit ?? 10,
|
|
12004
|
-
fileType: args.fileType,
|
|
12005
|
-
directory: args.directory,
|
|
12006
|
-
metadataOnly: true
|
|
12007
|
-
});
|
|
12008
|
-
if (results.length === 0) {
|
|
12009
|
-
return { content: [{ type: "text", text: "No matching code found. Try a different query or run index_codebase first." }] };
|
|
12010
|
-
}
|
|
12011
|
-
return {
|
|
12012
|
-
content: [{
|
|
12013
|
-
type: "text",
|
|
12014
|
-
text: `Found ${results.length} locations for "${args.query}":
|
|
12015
|
-
|
|
12016
|
-
${formatCodebasePeek(results)}
|
|
12017
|
-
|
|
12018
|
-
Use implementation_lookup for an authoritative definition, or call call_graph/call_graph_path once you have a symbol.`
|
|
12019
|
-
}]
|
|
12020
|
-
};
|
|
12633
|
+
const result = await resolveCodebaseContext(runtime.projectRoot, runtime.host, args);
|
|
12634
|
+
return { content: [{ type: "text", text: result.text }] };
|
|
12021
12635
|
}
|
|
12022
12636
|
);
|
|
12023
12637
|
server.tool(
|
|
@@ -12272,7 +12886,7 @@ ${formatSearchResults(results)}` }] };
|
|
|
12272
12886
|
var import_meta2 = {};
|
|
12273
12887
|
function getServerInstructions(host) {
|
|
12274
12888
|
const hostText = `host ${host}`;
|
|
12275
|
-
return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns
|
|
12889
|
+
return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns a token-budgeted location pack and routes to definitions or call-graph helpers when symbol intent is present. Keep the default tokenBudget for normal discovery, then use implementation_lookup, codebase_search, or a targeted file read only for selected locations that need source content. Use codebase_peek for direct conceptual location lookup. For exact identifiers or exhaustive matches, use grep. After identifying symbols, use call_graph or call_graph_path to trace dependencies. If the index is unavailable, run index_codebase, then retry the retrieval tool.`;
|
|
12276
12890
|
}
|
|
12277
12891
|
function getPackageVersion() {
|
|
12278
12892
|
const raw = JSON.parse((0, import_fs15.readFileSync)(new URL("../package.json", import_meta2.url), "utf-8"));
|