opencode-codebase-index 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +5 -3
- package/dist/cli.cjs +2400 -1786
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2404 -1790
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +454 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +454 -10
- package/dist/index.js.map +1 -1
- package/dist/pi-extension.cjs +459 -69
- package/dist/pi-extension.cjs.map +1 -1
- package/dist/pi-extension.js +459 -69
- package/dist/pi-extension.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +4 -2
package/dist/cli.js
CHANGED
|
@@ -1070,6 +1070,11 @@ function metricDelta(current, baseline) {
|
|
|
1070
1070
|
};
|
|
1071
1071
|
}
|
|
1072
1072
|
function compareSummaries(current, baseline, againstPath) {
|
|
1073
|
+
if (current.datasetName !== baseline.datasetName || current.datasetVersion !== baseline.datasetVersion || current.queryCount !== baseline.queryCount) {
|
|
1074
|
+
throw new Error(
|
|
1075
|
+
`Cannot compare incompatible evaluation datasets: current=${current.datasetName}@${current.datasetVersion} (${current.queryCount} queries), baseline=${baseline.datasetName}@${baseline.datasetVersion} (${baseline.queryCount} queries) at ${againstPath}`
|
|
1076
|
+
);
|
|
1077
|
+
}
|
|
1073
1078
|
return {
|
|
1074
1079
|
againstPath,
|
|
1075
1080
|
deltas: {
|
|
@@ -1091,6 +1096,34 @@ function compareSummaries(current, baseline, againstPath) {
|
|
|
1091
1096
|
estimatedCostUsd: metricDelta(
|
|
1092
1097
|
current.metrics.embedding.estimatedCostUsd,
|
|
1093
1098
|
baseline.metrics.embedding.estimatedCostUsd
|
|
1099
|
+
),
|
|
1100
|
+
contextResponseTokensAverage: metricDelta(
|
|
1101
|
+
current.metrics.contextEfficiency.responseTokens.average,
|
|
1102
|
+
baseline.metrics.contextEfficiency.responseTokens.average
|
|
1103
|
+
),
|
|
1104
|
+
contextResponseTokensP95: metricDelta(
|
|
1105
|
+
current.metrics.contextEfficiency.responseTokens.p95,
|
|
1106
|
+
baseline.metrics.contextEfficiency.responseTokens.p95
|
|
1107
|
+
),
|
|
1108
|
+
contextResponseTokensMax: metricDelta(
|
|
1109
|
+
current.metrics.contextEfficiency.responseTokens.max,
|
|
1110
|
+
baseline.metrics.contextEfficiency.responseTokens.max
|
|
1111
|
+
),
|
|
1112
|
+
contextDuplicateCandidateRatio: metricDelta(
|
|
1113
|
+
current.metrics.contextEfficiency.duplicateCandidateRatio,
|
|
1114
|
+
baseline.metrics.contextEfficiency.duplicateCandidateRatio
|
|
1115
|
+
),
|
|
1116
|
+
contextSelectedFileRatio: metricDelta(
|
|
1117
|
+
current.metrics.contextEfficiency.selectedFileRatio,
|
|
1118
|
+
baseline.metrics.contextEfficiency.selectedFileRatio
|
|
1119
|
+
),
|
|
1120
|
+
contextHitAt5Per1kResponseTokens: metricDelta(
|
|
1121
|
+
current.metrics.contextEfficiency.hitAt5Per1kResponseTokens,
|
|
1122
|
+
baseline.metrics.contextEfficiency.hitAt5Per1kResponseTokens
|
|
1123
|
+
),
|
|
1124
|
+
contextMrrAt10Per1kResponseTokens: metricDelta(
|
|
1125
|
+
current.metrics.contextEfficiency.mrrAt10Per1kResponseTokens,
|
|
1126
|
+
baseline.metrics.contextEfficiency.mrrAt10Per1kResponseTokens
|
|
1094
1127
|
)
|
|
1095
1128
|
}
|
|
1096
1129
|
};
|
|
@@ -1128,6 +1161,25 @@ function validateSummary(summary, summaryPath, options) {
|
|
|
1128
1161
|
assertFiniteNumber(summary.metrics.latencyMs.p99, `${summaryPath}.metrics.latencyMs.p99`);
|
|
1129
1162
|
assertFiniteNumber(summary.metrics.embedding.callCount, `${summaryPath}.metrics.embedding.callCount`);
|
|
1130
1163
|
assertFiniteNumber(summary.metrics.embedding.estimatedCostUsd, `${summaryPath}.metrics.embedding.estimatedCostUsd`);
|
|
1164
|
+
if (!summary.metrics.contextEfficiency) {
|
|
1165
|
+
summary.metrics.contextEfficiency = {
|
|
1166
|
+
queryCount: 0,
|
|
1167
|
+
responseTokens: { total: 0, average: 0, p95: 0, max: 0 },
|
|
1168
|
+
duplicateCandidateRatio: 0,
|
|
1169
|
+
selectedFileRatio: 0,
|
|
1170
|
+
hitAt5Per1kResponseTokens: 0,
|
|
1171
|
+
mrrAt10Per1kResponseTokens: 0
|
|
1172
|
+
};
|
|
1173
|
+
}
|
|
1174
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.queryCount, `${summaryPath}.metrics.contextEfficiency.queryCount`);
|
|
1175
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.total, `${summaryPath}.metrics.contextEfficiency.responseTokens.total`);
|
|
1176
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.average, `${summaryPath}.metrics.contextEfficiency.responseTokens.average`);
|
|
1177
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.p95, `${summaryPath}.metrics.contextEfficiency.responseTokens.p95`);
|
|
1178
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.responseTokens.max, `${summaryPath}.metrics.contextEfficiency.responseTokens.max`);
|
|
1179
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.duplicateCandidateRatio, `${summaryPath}.metrics.contextEfficiency.duplicateCandidateRatio`);
|
|
1180
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.selectedFileRatio, `${summaryPath}.metrics.contextEfficiency.selectedFileRatio`);
|
|
1181
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.hitAt5Per1kResponseTokens, `${summaryPath}.metrics.contextEfficiency.hitAt5Per1kResponseTokens`);
|
|
1182
|
+
assertFiniteNumber(summary.metrics.contextEfficiency.mrrAt10Per1kResponseTokens, `${summaryPath}.metrics.contextEfficiency.mrrAt10Per1kResponseTokens`);
|
|
1131
1183
|
return summary;
|
|
1132
1184
|
}
|
|
1133
1185
|
function formatPct(value) {
|
|
@@ -1202,6 +1254,15 @@ function createSummaryMarkdown(summary, comparison, gate, sweep) {
|
|
|
1202
1254
|
lines.push(`| Embedding calls | ${summary.metrics.embedding.callCount} |`);
|
|
1203
1255
|
lines.push(`| Embedding tokens | ${summary.metrics.tokenEstimate.embeddingTokensUsed} |`);
|
|
1204
1256
|
lines.push(`| Estimated embedding cost | ${formatUsd(summary.metrics.embedding.estimatedCostUsd)} |`);
|
|
1257
|
+
lines.push(`| Context queries | ${summary.metrics.contextEfficiency.queryCount} |`);
|
|
1258
|
+
lines.push(`| Context response tokens total | ${summary.metrics.contextEfficiency.responseTokens.total} |`);
|
|
1259
|
+
lines.push(`| Context response tokens avg | ${summary.metrics.contextEfficiency.responseTokens.average.toFixed(1)} |`);
|
|
1260
|
+
lines.push(`| Context response tokens p95 | ${summary.metrics.contextEfficiency.responseTokens.p95.toFixed(1)} |`);
|
|
1261
|
+
lines.push(`| Context response tokens max | ${summary.metrics.contextEfficiency.responseTokens.max.toFixed(1)} |`);
|
|
1262
|
+
lines.push(`| Context duplicate candidate ratio | ${formatPct(summary.metrics.contextEfficiency.duplicateCandidateRatio)} |`);
|
|
1263
|
+
lines.push(`| Context selected-file ratio | ${formatPct(summary.metrics.contextEfficiency.selectedFileRatio)} |`);
|
|
1264
|
+
lines.push(`| Context Hit@5 / 1k response tokens | ${summary.metrics.contextEfficiency.hitAt5Per1kResponseTokens.toFixed(4)} |`);
|
|
1265
|
+
lines.push(`| Context MRR@10 / 1k response tokens | ${summary.metrics.contextEfficiency.mrrAt10Per1kResponseTokens.toFixed(4)} |`);
|
|
1205
1266
|
lines.push("");
|
|
1206
1267
|
lines.push("## Failure Buckets");
|
|
1207
1268
|
lines.push("");
|
|
@@ -1245,6 +1306,27 @@ function createSummaryMarkdown(summary, comparison, gate, sweep) {
|
|
|
1245
1306
|
lines.push(
|
|
1246
1307
|
`| p95 latency (ms) | ${comparison.deltas.latencyP95Ms.baseline.toFixed(3)} | ${comparison.deltas.latencyP95Ms.current.toFixed(3)} | ${signed(comparison.deltas.latencyP95Ms.absolute, 3)} |`
|
|
1247
1308
|
);
|
|
1309
|
+
lines.push(
|
|
1310
|
+
`| Context response tokens avg | ${comparison.deltas.contextResponseTokensAverage.baseline.toFixed(1)} | ${comparison.deltas.contextResponseTokensAverage.current.toFixed(1)} | ${signed(comparison.deltas.contextResponseTokensAverage.absolute, 1)} |`
|
|
1311
|
+
);
|
|
1312
|
+
lines.push(
|
|
1313
|
+
`| Context response tokens p95 | ${comparison.deltas.contextResponseTokensP95.baseline.toFixed(1)} | ${comparison.deltas.contextResponseTokensP95.current.toFixed(1)} | ${signed(comparison.deltas.contextResponseTokensP95.absolute, 1)} |`
|
|
1314
|
+
);
|
|
1315
|
+
lines.push(
|
|
1316
|
+
`| Context response tokens max | ${comparison.deltas.contextResponseTokensMax.baseline.toFixed(1)} | ${comparison.deltas.contextResponseTokensMax.current.toFixed(1)} | ${signed(comparison.deltas.contextResponseTokensMax.absolute, 1)} |`
|
|
1317
|
+
);
|
|
1318
|
+
lines.push(
|
|
1319
|
+
`| Context duplicate candidate ratio | ${formatPct(comparison.deltas.contextDuplicateCandidateRatio.baseline)} | ${formatPct(comparison.deltas.contextDuplicateCandidateRatio.current)} | ${signed(comparison.deltas.contextDuplicateCandidateRatio.absolute)} |`
|
|
1320
|
+
);
|
|
1321
|
+
lines.push(
|
|
1322
|
+
`| Context selected-file ratio | ${formatPct(comparison.deltas.contextSelectedFileRatio.baseline)} | ${formatPct(comparison.deltas.contextSelectedFileRatio.current)} | ${signed(comparison.deltas.contextSelectedFileRatio.absolute)} |`
|
|
1323
|
+
);
|
|
1324
|
+
lines.push(
|
|
1325
|
+
`| Context Hit@5 / 1k response tokens | ${comparison.deltas.contextHitAt5Per1kResponseTokens.baseline.toFixed(4)} | ${comparison.deltas.contextHitAt5Per1kResponseTokens.current.toFixed(4)} | ${signed(comparison.deltas.contextHitAt5Per1kResponseTokens.absolute)} |`
|
|
1326
|
+
);
|
|
1327
|
+
lines.push(
|
|
1328
|
+
`| Context MRR@10 / 1k response tokens | ${comparison.deltas.contextMrrAt10Per1kResponseTokens.baseline.toFixed(4)} | ${comparison.deltas.contextMrrAt10Per1kResponseTokens.current.toFixed(4)} | ${signed(comparison.deltas.contextMrrAt10Per1kResponseTokens.absolute)} |`
|
|
1329
|
+
);
|
|
1248
1330
|
lines.push("");
|
|
1249
1331
|
}
|
|
1250
1332
|
if (gate) {
|
|
@@ -1291,8 +1373,8 @@ function buildPerQueryArtifact(perQuery) {
|
|
|
1291
1373
|
}
|
|
1292
1374
|
|
|
1293
1375
|
// src/eval/runner.ts
|
|
1294
|
-
import { existsSync as
|
|
1295
|
-
import * as
|
|
1376
|
+
import { existsSync as existsSync12 } from "fs";
|
|
1377
|
+
import * as path19 from "path";
|
|
1296
1378
|
import { performance as performance3 } from "perf_hooks";
|
|
1297
1379
|
|
|
1298
1380
|
// src/indexer/index.ts
|
|
@@ -5044,7 +5126,12 @@ function sameReclaimOwner(left, right) {
|
|
|
5044
5126
|
}
|
|
5045
5127
|
function publishJsonDirectory(finalPath, value) {
|
|
5046
5128
|
const candidatePath = `${finalPath}.candidate.${process.pid}.${randomUUID()}`;
|
|
5047
|
-
|
|
5129
|
+
try {
|
|
5130
|
+
mkdirSync2(candidatePath, { mode: 448 });
|
|
5131
|
+
} catch (error) {
|
|
5132
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5133
|
+
throw error;
|
|
5134
|
+
}
|
|
5048
5135
|
try {
|
|
5049
5136
|
writeFileSync2(path11.join(candidatePath, OWNER_FILE_NAME), JSON.stringify(value), {
|
|
5050
5137
|
encoding: "utf-8",
|
|
@@ -5057,8 +5144,12 @@ function publishJsonDirectory(finalPath, value) {
|
|
|
5057
5144
|
return true;
|
|
5058
5145
|
} catch (error) {
|
|
5059
5146
|
if (existsSync6(finalPath)) return false;
|
|
5147
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5060
5148
|
throw error;
|
|
5061
5149
|
}
|
|
5150
|
+
} catch (error) {
|
|
5151
|
+
if (getErrorCode(error) === "ENOENT") return false;
|
|
5152
|
+
throw error;
|
|
5062
5153
|
} finally {
|
|
5063
5154
|
if (existsSync6(candidatePath)) rmSync(candidatePath, { recursive: true, force: true });
|
|
5064
5155
|
}
|
|
@@ -6631,6 +6722,27 @@ function mergeTieredResults(symbolLane, hybridLane, limit) {
|
|
|
6631
6722
|
}
|
|
6632
6723
|
return out;
|
|
6633
6724
|
}
|
|
6725
|
+
function selectChunksWithFileCoverage(chunks, limit) {
|
|
6726
|
+
if (limit <= 0 || chunks.length === 0) {
|
|
6727
|
+
return [];
|
|
6728
|
+
}
|
|
6729
|
+
if (chunks.length <= limit) {
|
|
6730
|
+
return chunks;
|
|
6731
|
+
}
|
|
6732
|
+
if (limit === 1) {
|
|
6733
|
+
return [chunks[Math.floor((chunks.length - 1) / 2)]];
|
|
6734
|
+
}
|
|
6735
|
+
const selected = [];
|
|
6736
|
+
for (let index = 0; index < limit; index++) {
|
|
6737
|
+
const sourceIndex = Math.round(index * (chunks.length - 1) / (limit - 1));
|
|
6738
|
+
selected.push(chunks[sourceIndex]);
|
|
6739
|
+
}
|
|
6740
|
+
return selected;
|
|
6741
|
+
}
|
|
6742
|
+
function selectIndexableChunks(chunks, limit, semanticOnly) {
|
|
6743
|
+
const indexableChunks = semanticOnly ? chunks.filter((chunk) => chunk.chunkType !== "other") : chunks;
|
|
6744
|
+
return selectChunksWithFileCoverage(indexableChunks, limit);
|
|
6745
|
+
}
|
|
6634
6746
|
function matchesSearchFilters(candidate, options, minScore) {
|
|
6635
6747
|
if (candidate.score < minScore) return false;
|
|
6636
6748
|
if (options?.fileType) {
|
|
@@ -8369,7 +8481,6 @@ var Indexer = class {
|
|
|
8369
8481
|
const relativePath = path12.relative(this.projectRoot, parsed.path);
|
|
8370
8482
|
stats.parseFailures.push(relativePath);
|
|
8371
8483
|
}
|
|
8372
|
-
let fileChunkCount = 0;
|
|
8373
8484
|
let chunksToProcess = parsed.chunks;
|
|
8374
8485
|
if (this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile) {
|
|
8375
8486
|
const changedFile = changedFiles.find((f) => f.path === parsed.path);
|
|
@@ -8378,13 +8489,12 @@ var Indexer = class {
|
|
|
8378
8489
|
chunksToProcess = textChunks;
|
|
8379
8490
|
}
|
|
8380
8491
|
}
|
|
8492
|
+
chunksToProcess = selectIndexableChunks(
|
|
8493
|
+
chunksToProcess,
|
|
8494
|
+
this.config.indexing.maxChunksPerFile,
|
|
8495
|
+
this.config.indexing.semanticOnly
|
|
8496
|
+
);
|
|
8381
8497
|
for (const chunk of chunksToProcess) {
|
|
8382
|
-
if (fileChunkCount >= this.config.indexing.maxChunksPerFile) {
|
|
8383
|
-
break;
|
|
8384
|
-
}
|
|
8385
|
-
if (this.config.indexing.semanticOnly && chunk.chunkType === "other") {
|
|
8386
|
-
continue;
|
|
8387
|
-
}
|
|
8388
8498
|
const id = generateChunkId(parsed.path, chunk);
|
|
8389
8499
|
const contentHash = generateChunkHash(chunk);
|
|
8390
8500
|
const existingContentHash = existingChunks.get(id);
|
|
@@ -8408,7 +8518,6 @@ var Indexer = class {
|
|
|
8408
8518
|
blameSummary: blameMetadata.blameSummary
|
|
8409
8519
|
});
|
|
8410
8520
|
if (existingContentHash === contentHash) {
|
|
8411
|
-
fileChunkCount++;
|
|
8412
8521
|
continue;
|
|
8413
8522
|
}
|
|
8414
8523
|
const texts = createEmbeddingTexts(chunk, parsed.path, getSafeEmbeddingChunkTokenLimit(configuredProviderInfo)).map((text) => ({
|
|
@@ -8433,7 +8542,6 @@ var Indexer = class {
|
|
|
8433
8542
|
contentHash,
|
|
8434
8543
|
metadata
|
|
8435
8544
|
});
|
|
8436
|
-
fileChunkCount++;
|
|
8437
8545
|
}
|
|
8438
8546
|
}
|
|
8439
8547
|
const retryableFailedChunks = this.collectRetryableFailedChunks(
|
|
@@ -9982,1955 +10090,2500 @@ var Indexer = class {
|
|
|
9982
10090
|
}
|
|
9983
10091
|
};
|
|
9984
10092
|
|
|
9985
|
-
// src/
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
9990
|
-
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
if (thresholds.minRawDistinctTop3Ratio !== void 0 && summary.metrics.rawDistinctTop3Ratio < thresholds.minRawDistinctTop3Ratio) {
|
|
10003
|
-
violations.push({
|
|
10004
|
-
metric: "minRawDistinctTop3Ratio",
|
|
10005
|
-
message: `Raw Distinct Top@3 ${summary.metrics.rawDistinctTop3Ratio.toFixed(4)} is below minimum ${thresholds.minRawDistinctTop3Ratio.toFixed(4)}`
|
|
10006
|
-
});
|
|
10093
|
+
// src/tools/operations.ts
|
|
10094
|
+
import { existsSync as existsSync10, realpathSync as realpathSync2, statSync as statSync4 } from "fs";
|
|
10095
|
+
import * as path17 from "path";
|
|
10096
|
+
|
|
10097
|
+
// src/config/merger.ts
|
|
10098
|
+
import { existsSync as existsSync8, mkdirSync as mkdirSync4, readFileSync as readFileSync8, writeFileSync as writeFileSync4 } from "fs";
|
|
10099
|
+
import * as path14 from "path";
|
|
10100
|
+
|
|
10101
|
+
// src/config/rebase.ts
|
|
10102
|
+
import * as path13 from "path";
|
|
10103
|
+
function isWithinRoot(rootDir, targetPath) {
|
|
10104
|
+
const relativePath = path13.relative(rootDir, targetPath);
|
|
10105
|
+
return relativePath === "" || !relativePath.startsWith("..") && !path13.isAbsolute(relativePath);
|
|
10106
|
+
}
|
|
10107
|
+
function rebasePathEntries(values, fromDir, toDir) {
|
|
10108
|
+
if (!Array.isArray(values)) {
|
|
10109
|
+
return [];
|
|
10007
10110
|
}
|
|
10008
|
-
|
|
10009
|
-
|
|
10010
|
-
|
|
10011
|
-
|
|
10012
|
-
message: `Hit@5 drop ${comparison.deltas.hitAt5.absolute.toFixed(4)} exceeds allowed -${thresholds.hitAt5MaxDrop.toFixed(4)}`
|
|
10013
|
-
});
|
|
10014
|
-
}
|
|
10015
|
-
if (thresholds.mrrAt10MaxDrop !== void 0 && comparison.deltas.mrrAt10.absolute < -thresholds.mrrAt10MaxDrop) {
|
|
10016
|
-
violations.push({
|
|
10017
|
-
metric: "mrrAt10MaxDrop",
|
|
10018
|
-
message: `MRR@10 drop ${comparison.deltas.mrrAt10.absolute.toFixed(4)} exceeds allowed -${thresholds.mrrAt10MaxDrop.toFixed(4)}`
|
|
10019
|
-
});
|
|
10111
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
10112
|
+
const trimmed = value.trim();
|
|
10113
|
+
if (!trimmed || path13.isAbsolute(trimmed)) {
|
|
10114
|
+
return trimmed;
|
|
10020
10115
|
}
|
|
10021
|
-
|
|
10022
|
-
|
|
10023
|
-
|
|
10024
|
-
|
|
10025
|
-
|
|
10116
|
+
return normalizePathSeparators(path13.normalize(path13.relative(toDir, path13.resolve(fromDir, trimmed))));
|
|
10117
|
+
}).filter(Boolean);
|
|
10118
|
+
}
|
|
10119
|
+
function resolveInheritedKnowledgeBaseEntries(values, sourceRoot, targetRoot) {
|
|
10120
|
+
if (!Array.isArray(values)) {
|
|
10121
|
+
return [];
|
|
10122
|
+
}
|
|
10123
|
+
return values.filter((value) => typeof value === "string").map((value) => {
|
|
10124
|
+
const trimmed = value.trim();
|
|
10125
|
+
if (!trimmed) {
|
|
10126
|
+
return trimmed;
|
|
10026
10127
|
}
|
|
10027
|
-
if (
|
|
10028
|
-
|
|
10029
|
-
|
|
10030
|
-
const allowed = baselineP95 * thresholds.p95LatencyMaxMultiplier;
|
|
10031
|
-
if (summary.metrics.latencyMs.p95 > allowed) {
|
|
10032
|
-
violations.push({
|
|
10033
|
-
metric: "p95LatencyMaxMultiplier",
|
|
10034
|
-
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds allowed ${allowed.toFixed(3)}ms (${thresholds.p95LatencyMaxMultiplier.toFixed(2)}x baseline)`
|
|
10035
|
-
});
|
|
10036
|
-
}
|
|
10128
|
+
if (path13.isAbsolute(trimmed)) {
|
|
10129
|
+
if (isWithinRoot(sourceRoot, trimmed)) {
|
|
10130
|
+
return normalizePathSeparators(path13.normalize(path13.relative(sourceRoot, trimmed) || "."));
|
|
10037
10131
|
}
|
|
10132
|
+
return path13.normalize(trimmed);
|
|
10038
10133
|
}
|
|
10039
|
-
|
|
10040
|
-
|
|
10041
|
-
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
}
|
|
10046
|
-
return {
|
|
10047
|
-
passed: violations.length === 0,
|
|
10048
|
-
budgetName: budget.name,
|
|
10049
|
-
violations
|
|
10050
|
-
};
|
|
10134
|
+
const resolvedFromSource = path13.resolve(sourceRoot, trimmed);
|
|
10135
|
+
if (isWithinRoot(sourceRoot, resolvedFromSource)) {
|
|
10136
|
+
return normalizePathSeparators(path13.normalize(trimmed));
|
|
10137
|
+
}
|
|
10138
|
+
return normalizePathSeparators(path13.normalize(path13.relative(targetRoot, resolvedFromSource)));
|
|
10139
|
+
}).filter(Boolean);
|
|
10051
10140
|
}
|
|
10052
10141
|
|
|
10053
|
-
// src/
|
|
10054
|
-
|
|
10055
|
-
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10142
|
+
// src/config/merger.ts
|
|
10143
|
+
var PROJECT_OVERRIDE_KEYS = [
|
|
10144
|
+
"embeddingProvider",
|
|
10145
|
+
"customProvider",
|
|
10146
|
+
"embeddingModel",
|
|
10147
|
+
"reranker",
|
|
10148
|
+
"include",
|
|
10149
|
+
"exclude",
|
|
10150
|
+
"indexing",
|
|
10151
|
+
"search",
|
|
10152
|
+
"debug",
|
|
10153
|
+
"scope"
|
|
10154
|
+
];
|
|
10155
|
+
var MERGE_ARRAY_KEYS = ["knowledgeBases", "additionalInclude"];
|
|
10156
|
+
function isRecord(value) {
|
|
10157
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10066
10158
|
}
|
|
10067
|
-
function
|
|
10068
|
-
return
|
|
10159
|
+
function isStringArray2(value) {
|
|
10160
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
10069
10161
|
}
|
|
10070
|
-
function
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
unique.push(result);
|
|
10162
|
+
function applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key) {
|
|
10163
|
+
if (key in normalizedProjectConfig) {
|
|
10164
|
+
merged[key] = normalizedProjectConfig[key];
|
|
10165
|
+
return;
|
|
10166
|
+
}
|
|
10167
|
+
if (key in globalConfig) {
|
|
10168
|
+
merged[key] = globalConfig[key];
|
|
10078
10169
|
}
|
|
10079
|
-
return unique;
|
|
10080
|
-
}
|
|
10081
|
-
function distinctTopKRatio(results, k) {
|
|
10082
|
-
const top = results.slice(0, k);
|
|
10083
|
-
if (top.length === 0) return 0;
|
|
10084
|
-
const distinct = new Set(top.map((result) => normalizePath(result.filePath))).size;
|
|
10085
|
-
return distinct / top.length;
|
|
10086
10170
|
}
|
|
10087
|
-
function
|
|
10088
|
-
|
|
10089
|
-
const expected = normalizePath(expectedPath);
|
|
10090
|
-
if (actual === expected) return true;
|
|
10091
|
-
return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);
|
|
10171
|
+
function mergeUniqueStringArray(values) {
|
|
10172
|
+
return [...new Set(values.map((value) => String(value).trim()))];
|
|
10092
10173
|
}
|
|
10093
|
-
function
|
|
10094
|
-
|
|
10095
|
-
const
|
|
10096
|
-
|
|
10174
|
+
function normalizeKnowledgeBasePath(value) {
|
|
10175
|
+
let normalized = path14.normalize(String(value).trim());
|
|
10176
|
+
const root = path14.parse(normalized).root;
|
|
10177
|
+
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
10178
|
+
normalized = normalized.slice(0, -1);
|
|
10179
|
+
}
|
|
10180
|
+
return normalized;
|
|
10097
10181
|
}
|
|
10098
|
-
function
|
|
10099
|
-
return
|
|
10182
|
+
function mergeKnowledgeBasePaths(values) {
|
|
10183
|
+
return [...new Set(values.map((value) => normalizeKnowledgeBasePath(value)).filter((value) => value.length > 0))];
|
|
10100
10184
|
}
|
|
10101
|
-
function
|
|
10102
|
-
|
|
10103
|
-
|
|
10104
|
-
|
|
10105
|
-
|
|
10185
|
+
function validateConfigLayerShape(rawConfig, filePath) {
|
|
10186
|
+
if (!isRecord(rawConfig)) {
|
|
10187
|
+
throw new Error(`Config file ${filePath} must contain a JSON object at the root.`);
|
|
10188
|
+
}
|
|
10189
|
+
if (rawConfig.knowledgeBases !== void 0 && !isStringArray2(rawConfig.knowledgeBases)) {
|
|
10190
|
+
throw new Error(`Config file ${filePath} field 'knowledgeBases' must be an array of strings.`);
|
|
10191
|
+
}
|
|
10192
|
+
if (rawConfig.additionalInclude !== void 0 && !isStringArray2(rawConfig.additionalInclude)) {
|
|
10193
|
+
throw new Error(`Config file ${filePath} field 'additionalInclude' must be an array of strings.`);
|
|
10194
|
+
}
|
|
10195
|
+
if (rawConfig.include !== void 0 && !isStringArray2(rawConfig.include)) {
|
|
10196
|
+
throw new Error(`Config file ${filePath} field 'include' must be an array of strings.`);
|
|
10197
|
+
}
|
|
10198
|
+
if (rawConfig.exclude !== void 0 && !isStringArray2(rawConfig.exclude)) {
|
|
10199
|
+
throw new Error(`Config file ${filePath} field 'exclude' must be an array of strings.`);
|
|
10200
|
+
}
|
|
10201
|
+
for (const section of ["customProvider", "indexing", "search", "debug", "reranker"]) {
|
|
10202
|
+
const value = rawConfig[section];
|
|
10203
|
+
if (value !== void 0 && !isRecord(value)) {
|
|
10204
|
+
throw new Error(`Config file ${filePath} field '${section}' must be an object.`);
|
|
10106
10205
|
}
|
|
10107
10206
|
}
|
|
10108
|
-
return
|
|
10207
|
+
return rawConfig;
|
|
10109
10208
|
}
|
|
10110
|
-
function
|
|
10111
|
-
|
|
10112
|
-
|
|
10113
|
-
|
|
10114
|
-
|
|
10115
|
-
|
|
10116
|
-
|
|
10117
|
-
|
|
10118
|
-
(
|
|
10119
|
-
|
|
10120
|
-
|
|
10121
|
-
|
|
10209
|
+
function loadJsonFile(filePath) {
|
|
10210
|
+
if (!existsSync8(filePath)) {
|
|
10211
|
+
return null;
|
|
10212
|
+
}
|
|
10213
|
+
try {
|
|
10214
|
+
const content = readFileSync8(filePath, "utf-8");
|
|
10215
|
+
return validateConfigLayerShape(JSON.parse(content), filePath);
|
|
10216
|
+
} catch (error) {
|
|
10217
|
+
if (error instanceof Error && error.message.startsWith("Config file ")) {
|
|
10218
|
+
throw error;
|
|
10219
|
+
}
|
|
10220
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
10221
|
+
throw new Error(`Failed to load config file ${filePath}: ${message}`);
|
|
10222
|
+
}
|
|
10122
10223
|
}
|
|
10123
|
-
function
|
|
10124
|
-
|
|
10125
|
-
return lowered.includes("/docs/") || lowered.includes("/test/") || lowered.includes("/tests/") || lowered.includes("readme") || lowered.includes("/benchmarks/");
|
|
10224
|
+
function loadConfigFile(filePath) {
|
|
10225
|
+
return loadJsonFile(filePath);
|
|
10126
10226
|
}
|
|
10127
|
-
function
|
|
10128
|
-
const
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10132
|
-
|
|
10227
|
+
function materializeLocalProjectConfig(projectRoot, config, host = "opencode") {
|
|
10228
|
+
const localConfigPath = resolveWritableProjectConfigPath(projectRoot, host);
|
|
10229
|
+
mkdirSync4(path14.dirname(localConfigPath), { recursive: true });
|
|
10230
|
+
writeFileSync4(localConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
10231
|
+
return localConfigPath;
|
|
10232
|
+
}
|
|
10233
|
+
function loadProjectConfigLayer(projectRoot, host = "opencode") {
|
|
10234
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot, host);
|
|
10235
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
10236
|
+
if (!projectConfig) {
|
|
10237
|
+
return {};
|
|
10133
10238
|
}
|
|
10134
|
-
|
|
10135
|
-
|
|
10136
|
-
|
|
10239
|
+
const normalizedConfig = { ...projectConfig };
|
|
10240
|
+
const projectConfigBaseDir = path14.dirname(path14.dirname(projectConfigPath));
|
|
10241
|
+
if (Array.isArray(normalizedConfig.knowledgeBases)) {
|
|
10242
|
+
normalizedConfig.knowledgeBases = resolveInheritedKnowledgeBaseEntries(
|
|
10243
|
+
normalizedConfig.knowledgeBases,
|
|
10244
|
+
projectConfigBaseDir,
|
|
10245
|
+
projectRoot
|
|
10137
10246
|
);
|
|
10138
|
-
if (!hasSymbol) return "wrong-symbol";
|
|
10139
10247
|
}
|
|
10140
|
-
|
|
10141
|
-
|
|
10142
|
-
|
|
10248
|
+
return normalizedConfig;
|
|
10249
|
+
}
|
|
10250
|
+
function loadMergedConfig(projectRoot, host = "opencode") {
|
|
10251
|
+
const globalConfigPath = resolveGlobalConfigPath(host);
|
|
10252
|
+
const projectConfigPath = resolveProjectConfigPath(projectRoot, host);
|
|
10253
|
+
let globalConfig = null;
|
|
10254
|
+
let globalConfigError = null;
|
|
10255
|
+
try {
|
|
10256
|
+
globalConfig = loadJsonFile(globalConfigPath);
|
|
10257
|
+
} catch (error) {
|
|
10258
|
+
globalConfigError = error instanceof Error ? error : new Error(String(error));
|
|
10143
10259
|
}
|
|
10144
|
-
|
|
10145
|
-
|
|
10260
|
+
const projectConfig = loadJsonFile(projectConfigPath);
|
|
10261
|
+
const normalizedProjectConfig = loadProjectConfigLayer(projectRoot, host);
|
|
10262
|
+
if (globalConfigError) {
|
|
10263
|
+
if (!projectConfig) {
|
|
10264
|
+
throw globalConfigError;
|
|
10265
|
+
}
|
|
10266
|
+
globalConfig = null;
|
|
10146
10267
|
}
|
|
10147
|
-
|
|
10148
|
-
}
|
|
10149
|
-
|
|
10150
|
-
|
|
10151
|
-
|
|
10152
|
-
|
|
10153
|
-
|
|
10154
|
-
|
|
10155
|
-
|
|
10156
|
-
|
|
10157
|
-
|
|
10158
|
-
|
|
10159
|
-
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
|
|
10163
|
-
|
|
10164
|
-
|
|
10165
|
-
|
|
10166
|
-
|
|
10167
|
-
|
|
10168
|
-
|
|
10169
|
-
}
|
|
10170
|
-
function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingTokensUsed, costPer1MTokensUsd) {
|
|
10171
|
-
const count = perQuery.length;
|
|
10172
|
-
const safeDiv = (value) => count === 0 ? 0 : value / count;
|
|
10173
|
-
const sum = {
|
|
10174
|
-
hitAt1: 0,
|
|
10175
|
-
hitAt3: 0,
|
|
10176
|
-
hitAt5: 0,
|
|
10177
|
-
hitAt10: 0,
|
|
10178
|
-
mrrAt10: 0,
|
|
10179
|
-
ndcgAt10: 0,
|
|
10180
|
-
distinctTop3Ratio: 0,
|
|
10181
|
-
rawDistinctTop3Ratio: 0
|
|
10182
|
-
};
|
|
10183
|
-
const failureBuckets = {
|
|
10184
|
-
"wrong-file": 0,
|
|
10185
|
-
"wrong-symbol": 0,
|
|
10186
|
-
"docs-tests-outranking-source": 0,
|
|
10187
|
-
"no-relevant-hit-top-k": 0
|
|
10188
|
-
};
|
|
10189
|
-
const latencies = perQuery.map((item) => item.latencyMs);
|
|
10190
|
-
for (const query of perQuery) {
|
|
10191
|
-
if (query.hitAt1) sum.hitAt1 += 1;
|
|
10192
|
-
if (query.hitAt3) sum.hitAt3 += 1;
|
|
10193
|
-
if (query.hitAt5) sum.hitAt5 += 1;
|
|
10194
|
-
if (query.hitAt10) sum.hitAt10 += 1;
|
|
10195
|
-
sum.mrrAt10 += query.reciprocalRankAt10;
|
|
10196
|
-
sum.ndcgAt10 += query.ndcgAt10;
|
|
10197
|
-
sum.distinctTop3Ratio += distinctTopKRatio(query.results, 3);
|
|
10198
|
-
sum.rawDistinctTop3Ratio += query.rawTop3DistinctRatio;
|
|
10199
|
-
if (query.failureBucket) {
|
|
10200
|
-
failureBuckets[query.failureBucket] += 1;
|
|
10268
|
+
if (!globalConfig && !projectConfig) {
|
|
10269
|
+
return {};
|
|
10270
|
+
}
|
|
10271
|
+
if (!projectConfig && globalConfig) {
|
|
10272
|
+
return globalConfig;
|
|
10273
|
+
}
|
|
10274
|
+
if (!globalConfig && projectConfig) {
|
|
10275
|
+
return normalizedProjectConfig;
|
|
10276
|
+
}
|
|
10277
|
+
if (!globalConfig || !projectConfig) {
|
|
10278
|
+
return globalConfig ?? normalizedProjectConfig;
|
|
10279
|
+
}
|
|
10280
|
+
const merged = { ...globalConfig };
|
|
10281
|
+
for (const key of PROJECT_OVERRIDE_KEYS) {
|
|
10282
|
+
applyProjectOverride(merged, normalizedProjectConfig, globalConfig, key);
|
|
10283
|
+
}
|
|
10284
|
+
if (projectConfig) {
|
|
10285
|
+
for (const key of Object.keys(projectConfig)) {
|
|
10286
|
+
if (PROJECT_OVERRIDE_KEYS.includes(key) || MERGE_ARRAY_KEYS.includes(key)) {
|
|
10287
|
+
continue;
|
|
10288
|
+
}
|
|
10289
|
+
merged[key] = normalizedProjectConfig[key];
|
|
10201
10290
|
}
|
|
10202
10291
|
}
|
|
10203
|
-
const
|
|
10204
|
-
|
|
10205
|
-
|
|
10206
|
-
|
|
10207
|
-
|
|
10208
|
-
|
|
10209
|
-
|
|
10210
|
-
|
|
10211
|
-
|
|
10212
|
-
rawDistinctTop3Ratio: safeDiv(sum.rawDistinctTop3Ratio),
|
|
10213
|
-
latencyMs: {
|
|
10214
|
-
p50: percentile(latencies, 0.5),
|
|
10215
|
-
p95: percentile(latencies, 0.95),
|
|
10216
|
-
p99: percentile(latencies, 0.99)
|
|
10217
|
-
},
|
|
10218
|
-
tokenEstimate: {
|
|
10219
|
-
queryTokens,
|
|
10220
|
-
embeddingTokensUsed
|
|
10221
|
-
},
|
|
10222
|
-
embedding: {
|
|
10223
|
-
callCount: embeddingCallCount,
|
|
10224
|
-
estimatedCostUsd: embeddingTokensUsed / 1e6 * costPer1MTokensUsd,
|
|
10225
|
-
costPer1MTokensUsd
|
|
10226
|
-
},
|
|
10227
|
-
failureBuckets
|
|
10228
|
-
};
|
|
10292
|
+
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
10293
|
+
const projectKbs = projectConfig ? Array.isArray(normalizedProjectConfig.knowledgeBases) ? normalizedProjectConfig.knowledgeBases : [] : [];
|
|
10294
|
+
const allKbs = [...globalKbs, ...projectKbs];
|
|
10295
|
+
merged.knowledgeBases = mergeKnowledgeBasePaths(allKbs);
|
|
10296
|
+
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
10297
|
+
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
10298
|
+
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
10299
|
+
merged.additionalInclude = mergeUniqueStringArray(allAdditional);
|
|
10300
|
+
return merged;
|
|
10229
10301
|
}
|
|
10230
10302
|
|
|
10231
|
-
// src/
|
|
10232
|
-
import
|
|
10233
|
-
|
|
10234
|
-
|
|
10303
|
+
// src/tools/knowledge-base-paths.ts
|
|
10304
|
+
import * as path15 from "path";
|
|
10305
|
+
function resolveConfigPathValue(value, baseDir) {
|
|
10306
|
+
const trimmed = value.trim();
|
|
10307
|
+
if (!trimmed) {
|
|
10308
|
+
return trimmed;
|
|
10309
|
+
}
|
|
10310
|
+
const absolutePath = path15.isAbsolute(trimmed) ? trimmed : path15.resolve(baseDir, trimmed);
|
|
10311
|
+
return path15.normalize(absolutePath);
|
|
10312
|
+
}
|
|
10235
10313
|
|
|
10236
|
-
// src/
|
|
10237
|
-
import
|
|
10238
|
-
|
|
10239
|
-
|
|
10240
|
-
|
|
10314
|
+
// src/tools/utils.ts
|
|
10315
|
+
import { get_encoding } from "tiktoken";
|
|
10316
|
+
var MIN_CONTEXT_PACK_TOKEN_BUDGET = 128;
|
|
10317
|
+
var MAX_CONTEXT_PACK_TOKEN_BUDGET = 4e3;
|
|
10318
|
+
var DEFAULT_CONTEXT_PACK_TOKEN_BUDGET = 1200;
|
|
10319
|
+
var CONTEXT_TOKENIZER = get_encoding("cl100k_base");
|
|
10320
|
+
function clampContextPackTokenBudget(tokenBudget) {
|
|
10321
|
+
if (tokenBudget === void 0 || !Number.isFinite(tokenBudget)) {
|
|
10322
|
+
return DEFAULT_CONTEXT_PACK_TOKEN_BUDGET;
|
|
10323
|
+
}
|
|
10324
|
+
return Math.min(
|
|
10325
|
+
MAX_CONTEXT_PACK_TOKEN_BUDGET,
|
|
10326
|
+
Math.max(MIN_CONTEXT_PACK_TOKEN_BUDGET, Math.floor(tokenBudget))
|
|
10327
|
+
);
|
|
10241
10328
|
}
|
|
10242
|
-
function
|
|
10243
|
-
|
|
10244
|
-
return [];
|
|
10245
|
-
}
|
|
10246
|
-
return values.filter((value) => typeof value === "string").map((value) => {
|
|
10247
|
-
const trimmed = value.trim();
|
|
10248
|
-
if (!trimmed || path13.isAbsolute(trimmed)) {
|
|
10249
|
-
return trimmed;
|
|
10250
|
-
}
|
|
10251
|
-
return normalizePathSeparators(path13.normalize(path13.relative(toDir, path13.resolve(fromDir, trimmed))));
|
|
10252
|
-
}).filter(Boolean);
|
|
10329
|
+
function countContextTokens(text) {
|
|
10330
|
+
return CONTEXT_TOKENIZER.encode(text).length;
|
|
10253
10331
|
}
|
|
10254
|
-
function
|
|
10255
|
-
|
|
10256
|
-
|
|
10332
|
+
function fitTextToContextBudget(text, tokenBudget) {
|
|
10333
|
+
const normalizedBudget = clampContextPackTokenBudget(tokenBudget);
|
|
10334
|
+
const tokenEstimate = countContextTokens(text);
|
|
10335
|
+
if (tokenEstimate <= normalizedBudget) {
|
|
10336
|
+
return {
|
|
10337
|
+
text,
|
|
10338
|
+
tokenBudget: normalizedBudget,
|
|
10339
|
+
tokenEstimate,
|
|
10340
|
+
truncated: false
|
|
10341
|
+
};
|
|
10257
10342
|
}
|
|
10258
|
-
|
|
10259
|
-
|
|
10260
|
-
|
|
10261
|
-
|
|
10262
|
-
|
|
10263
|
-
|
|
10264
|
-
|
|
10265
|
-
|
|
10266
|
-
|
|
10267
|
-
|
|
10268
|
-
|
|
10269
|
-
|
|
10270
|
-
|
|
10271
|
-
|
|
10272
|
-
|
|
10273
|
-
|
|
10274
|
-
}
|
|
10343
|
+
const suffix = "\n...[truncated to context token budget]";
|
|
10344
|
+
const codePoints = Array.from(text);
|
|
10345
|
+
let low = 0;
|
|
10346
|
+
let high = codePoints.length;
|
|
10347
|
+
while (low < high) {
|
|
10348
|
+
const middle = Math.ceil((low + high) / 2);
|
|
10349
|
+
const candidate = `${codePoints.slice(0, middle).join("").trimEnd()}${suffix}`;
|
|
10350
|
+
if (countContextTokens(candidate) <= normalizedBudget) low = middle;
|
|
10351
|
+
else high = middle - 1;
|
|
10352
|
+
}
|
|
10353
|
+
const fitted = `${codePoints.slice(0, low).join("").trimEnd()}${suffix}`;
|
|
10354
|
+
return {
|
|
10355
|
+
text: fitted,
|
|
10356
|
+
tokenBudget: normalizedBudget,
|
|
10357
|
+
tokenEstimate: countContextTokens(fitted),
|
|
10358
|
+
truncated: true
|
|
10359
|
+
};
|
|
10275
10360
|
}
|
|
10276
|
-
|
|
10277
|
-
|
|
10278
|
-
function isRecord(value) {
|
|
10279
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10361
|
+
function normalizedLineRange(result) {
|
|
10362
|
+
return result.startLine <= result.endLine ? { start: result.startLine, end: result.endLine } : { start: result.endLine, end: result.startLine };
|
|
10280
10363
|
}
|
|
10281
|
-
function
|
|
10282
|
-
return
|
|
10364
|
+
function rankContextCandidates(results) {
|
|
10365
|
+
return results.map((result, originalIndex) => ({ result, originalIndex })).sort((left, right) => right.result.score - left.result.score || left.originalIndex - right.originalIndex);
|
|
10283
10366
|
}
|
|
10284
|
-
function
|
|
10285
|
-
|
|
10286
|
-
|
|
10287
|
-
}
|
|
10288
|
-
|
|
10289
|
-
|
|
10290
|
-
|
|
10291
|
-
|
|
10292
|
-
if (config.additionalInclude !== void 0 && !isStringArray2(config.additionalInclude)) {
|
|
10293
|
-
throw new Error(`Eval config at ${filePath} field 'additionalInclude' must be an array of strings.`);
|
|
10294
|
-
}
|
|
10295
|
-
if (config.include !== void 0 && !isStringArray2(config.include)) {
|
|
10296
|
-
throw new Error(`Eval config at ${filePath} field 'include' must be an array of strings.`);
|
|
10297
|
-
}
|
|
10298
|
-
if (config.exclude !== void 0 && !isStringArray2(config.exclude)) {
|
|
10299
|
-
throw new Error(`Eval config at ${filePath} field 'exclude' must be an array of strings.`);
|
|
10300
|
-
}
|
|
10301
|
-
for (const section of ["customProvider", "indexing", "search", "debug", "reranker"]) {
|
|
10302
|
-
const value = config[section];
|
|
10303
|
-
if (value !== void 0 && !isRecord(value)) {
|
|
10304
|
-
throw new Error(`Eval config at ${filePath} field '${section}' must be an object.`);
|
|
10367
|
+
function deduplicateContextCandidates(candidates) {
|
|
10368
|
+
const acceptedByFile = /* @__PURE__ */ new Map();
|
|
10369
|
+
const deduplicated = [];
|
|
10370
|
+
for (const { result } of candidates) {
|
|
10371
|
+
const range = normalizedLineRange(result);
|
|
10372
|
+
const accepted = acceptedByFile.get(result.filePath) ?? [];
|
|
10373
|
+
if (accepted.some((item) => item.start <= range.end && range.start <= item.end)) {
|
|
10374
|
+
continue;
|
|
10305
10375
|
}
|
|
10376
|
+
accepted.push(range);
|
|
10377
|
+
acceptedByFile.set(result.filePath, accepted);
|
|
10378
|
+
deduplicated.push(result);
|
|
10306
10379
|
}
|
|
10307
|
-
return
|
|
10380
|
+
return deduplicated;
|
|
10308
10381
|
}
|
|
10309
|
-
function
|
|
10310
|
-
|
|
10311
|
-
|
|
10312
|
-
|
|
10313
|
-
|
|
10314
|
-
|
|
10315
|
-
|
|
10316
|
-
|
|
10317
|
-
|
|
10382
|
+
function diversifyContextCandidates(results) {
|
|
10383
|
+
const byFile = /* @__PURE__ */ new Map();
|
|
10384
|
+
for (const result of results) {
|
|
10385
|
+
const bucket = byFile.get(result.filePath) ?? [];
|
|
10386
|
+
bucket.push(result);
|
|
10387
|
+
byFile.set(result.filePath, bucket);
|
|
10388
|
+
}
|
|
10389
|
+
const files = [...byFile.keys()];
|
|
10390
|
+
const diversified = [];
|
|
10391
|
+
for (let depth = 0; diversified.length < results.length; depth += 1) {
|
|
10392
|
+
for (const file of files) {
|
|
10393
|
+
const result = byFile.get(file)?.[depth];
|
|
10394
|
+
if (result) diversified.push(result);
|
|
10395
|
+
}
|
|
10396
|
+
}
|
|
10397
|
+
return diversified;
|
|
10398
|
+
}
|
|
10399
|
+
function compactEvidenceValue(value, maxChars) {
|
|
10400
|
+
const characters = [...value];
|
|
10401
|
+
if (characters.length <= maxChars) return value;
|
|
10402
|
+
return `\u2026${characters.slice(-(maxChars - 1)).join("")}`;
|
|
10403
|
+
}
|
|
10404
|
+
function formatContextEvidence(result, index) {
|
|
10405
|
+
const symbol = result.name ? ` ${JSON.stringify(compactEvidenceValue(result.name, 80))}` : "";
|
|
10406
|
+
const path27 = compactEvidenceValue(result.filePath, 120);
|
|
10407
|
+
return `[${index}] ${result.chunkType}${symbol} in ${path27}:${result.startLine}-${result.endLine} (score ${result.score.toFixed(2)})`;
|
|
10408
|
+
}
|
|
10409
|
+
function formatContextPack(heading, selected, candidateCount, duplicateCount, limitOmittedCount, budgetOmittedCount) {
|
|
10410
|
+
const lines = selected.map((result, index) => formatContextEvidence(result, index + 1));
|
|
10411
|
+
const notes = [];
|
|
10412
|
+
if (duplicateCount > 0) notes.push(`${duplicateCount} overlapping duplicate${duplicateCount === 1 ? "" : "s"} removed`);
|
|
10413
|
+
if (limitOmittedCount > 0) notes.push(`${limitOmittedCount} additional result${limitOmittedCount === 1 ? "" : "s"} excluded by result limit`);
|
|
10414
|
+
if (budgetOmittedCount > 0) notes.push(`${budgetOmittedCount} additional result${budgetOmittedCount === 1 ? "" : "s"} omitted by token budget`);
|
|
10415
|
+
const footer = notes.length > 0 ? `Selected ${selected.length} of ${candidateCount} candidates; ${notes.join("; ")}.` : `Selected ${selected.length} of ${candidateCount} candidates.`;
|
|
10416
|
+
return `${heading}
|
|
10417
|
+
|
|
10418
|
+
${lines.join("\n")}
|
|
10419
|
+
|
|
10420
|
+
${footer}`;
|
|
10421
|
+
}
|
|
10422
|
+
function buildContextPack(results, options = {}) {
|
|
10423
|
+
const requestedTokenBudget = options.tokenBudget ?? DEFAULT_CONTEXT_PACK_TOKEN_BUDGET;
|
|
10424
|
+
const tokenBudget = clampContextPackTokenBudget(options.tokenBudget);
|
|
10425
|
+
const heading = compactEvidenceValue(options.heading?.trim() || "Codebase evidence", 160);
|
|
10426
|
+
const maxResults = Math.max(0, Math.floor(options.maxResults ?? results.length));
|
|
10427
|
+
const candidateCount = results.length;
|
|
10428
|
+
const deduplicated = deduplicateContextCandidates(rankContextCandidates(results));
|
|
10429
|
+
const diversified = diversifyContextCandidates(deduplicated);
|
|
10430
|
+
const duplicateCount = candidateCount - deduplicated.length;
|
|
10431
|
+
const selectable = diversified.slice(0, maxResults);
|
|
10432
|
+
const limitOmittedCount = deduplicated.length - selectable.length;
|
|
10433
|
+
let selected = [];
|
|
10434
|
+
let text = formatContextPack(heading, selected, candidateCount, duplicateCount, limitOmittedCount, selectable.length);
|
|
10435
|
+
for (let count = 1; count <= selectable.length; count += 1) {
|
|
10436
|
+
const candidateSelection = selectable.slice(0, count);
|
|
10437
|
+
const budgetOmittedCount2 = selectable.length - candidateSelection.length;
|
|
10438
|
+
const candidateText = formatContextPack(
|
|
10439
|
+
heading,
|
|
10440
|
+
candidateSelection,
|
|
10441
|
+
candidateCount,
|
|
10442
|
+
duplicateCount,
|
|
10443
|
+
limitOmittedCount,
|
|
10444
|
+
budgetOmittedCount2
|
|
10445
|
+
);
|
|
10446
|
+
if (countContextTokens(candidateText) > tokenBudget) break;
|
|
10447
|
+
selected = candidateSelection;
|
|
10448
|
+
text = candidateText;
|
|
10318
10449
|
}
|
|
10450
|
+
const fitted = fitTextToContextBudget(text, tokenBudget);
|
|
10451
|
+
const budgetOmittedCount = selectable.length - selected.length;
|
|
10452
|
+
const omittedCount = candidateCount - selected.length;
|
|
10453
|
+
return {
|
|
10454
|
+
requestedTokenBudget,
|
|
10455
|
+
tokenBudget,
|
|
10456
|
+
text: fitted.text,
|
|
10457
|
+
tokenEstimate: fitted.tokenEstimate,
|
|
10458
|
+
results: selected,
|
|
10459
|
+
candidateCount,
|
|
10460
|
+
deduplicatedCount: deduplicated.length,
|
|
10461
|
+
selectedCount: selected.length,
|
|
10462
|
+
omittedCount,
|
|
10463
|
+
duplicateCount,
|
|
10464
|
+
limitOmittedCount,
|
|
10465
|
+
budgetOmittedCount
|
|
10466
|
+
};
|
|
10319
10467
|
}
|
|
10320
|
-
|
|
10321
|
-
|
|
10322
|
-
|
|
10323
|
-
|
|
10324
|
-
return
|
|
10468
|
+
var MAX_CONTENT_LINES = 30;
|
|
10469
|
+
function truncateContent(content) {
|
|
10470
|
+
const lines = content.split("\n");
|
|
10471
|
+
if (lines.length <= MAX_CONTENT_LINES) return content;
|
|
10472
|
+
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
10473
|
+
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
10325
10474
|
}
|
|
10326
|
-
function
|
|
10327
|
-
|
|
10328
|
-
|
|
10329
|
-
values,
|
|
10330
|
-
path14.dirname(path14.dirname(resolvedConfigPath)),
|
|
10331
|
-
projectRoot
|
|
10332
|
-
) : rebasePathEntries(
|
|
10333
|
-
values,
|
|
10334
|
-
path14.dirname(resolvedConfigPath),
|
|
10335
|
-
projectRoot
|
|
10336
|
-
);
|
|
10337
|
-
if (Array.isArray(config.knowledgeBases)) {
|
|
10338
|
-
config.knowledgeBases = rebaseEntries(config.knowledgeBases);
|
|
10475
|
+
function formatIndexStats(stats, verbose = false) {
|
|
10476
|
+
if (stats.resetCorruptedIndex) {
|
|
10477
|
+
return stats.warning ?? "Detected a corrupted local index and reset it during indexing. Run index_codebase again to rebuild search data.";
|
|
10339
10478
|
}
|
|
10340
|
-
|
|
10341
|
-
|
|
10479
|
+
const lines = [];
|
|
10480
|
+
if (stats.failedChunks > 0) {
|
|
10481
|
+
lines.push(`INDEXING WARNING: ${stats.failedChunks} chunks failed to embed.`);
|
|
10482
|
+
if (stats.failedBatchesPath) {
|
|
10483
|
+
lines.push(`Inspect failed batches at: ${stats.failedBatchesPath}`);
|
|
10484
|
+
}
|
|
10485
|
+
lines.push("");
|
|
10342
10486
|
}
|
|
10343
|
-
|
|
10487
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
10488
|
+
lines.push(`${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
10489
|
+
} else if (stats.indexedChunks === 0) {
|
|
10490
|
+
lines.push(`${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
10491
|
+
} else {
|
|
10492
|
+
let main2 = `${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
10493
|
+
if (stats.existingChunks > 0) {
|
|
10494
|
+
main2 += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
10495
|
+
}
|
|
10496
|
+
lines.push(main2);
|
|
10497
|
+
if (stats.removedChunks > 0) {
|
|
10498
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
10499
|
+
}
|
|
10500
|
+
if (stats.failedChunks > 0) {
|
|
10501
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
10502
|
+
}
|
|
10503
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
10504
|
+
}
|
|
10505
|
+
if (verbose) {
|
|
10506
|
+
if (stats.skippedFiles.length > 0) {
|
|
10507
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
10508
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
10509
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
10510
|
+
lines.push("");
|
|
10511
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
10512
|
+
if (tooLarge.length > 0) {
|
|
10513
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
10514
|
+
}
|
|
10515
|
+
if (excluded.length > 0) {
|
|
10516
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
10517
|
+
}
|
|
10518
|
+
if (gitignored.length > 0) {
|
|
10519
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
10520
|
+
}
|
|
10521
|
+
}
|
|
10522
|
+
if (stats.parseFailures.length > 0) {
|
|
10523
|
+
lines.push("");
|
|
10524
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
10525
|
+
}
|
|
10526
|
+
}
|
|
10527
|
+
return lines.join("\n");
|
|
10344
10528
|
}
|
|
10345
|
-
function
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10351
|
-
|
|
10352
|
-
|
|
10529
|
+
function formatStatus(status) {
|
|
10530
|
+
if (!status.indexed) {
|
|
10531
|
+
if (status.warning) {
|
|
10532
|
+
return status.warning;
|
|
10533
|
+
}
|
|
10534
|
+
if (status.failedBatchesCount > 0) {
|
|
10535
|
+
const lines2 = [
|
|
10536
|
+
"Codebase is not indexed. The last indexing run left failed embedding batches.",
|
|
10537
|
+
"Fix the provider/model configuration, then rerun index_codebase normally to retry the saved failed batches. Use force=true only for a full rebuild or compatibility reset."
|
|
10538
|
+
];
|
|
10539
|
+
if (status.failedBatchesPath) {
|
|
10540
|
+
lines2.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
10541
|
+
}
|
|
10542
|
+
return lines2.join("\n");
|
|
10543
|
+
}
|
|
10544
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
10353
10545
|
}
|
|
10354
|
-
const
|
|
10355
|
-
|
|
10356
|
-
|
|
10357
|
-
|
|
10358
|
-
|
|
10359
|
-
|
|
10360
|
-
|
|
10546
|
+
const lines = [
|
|
10547
|
+
`Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
10548
|
+
`Provider: ${status.provider}`,
|
|
10549
|
+
`Model: ${status.model}`,
|
|
10550
|
+
`Location: ${status.indexPath}`
|
|
10551
|
+
];
|
|
10552
|
+
if (status.currentBranch !== "default") {
|
|
10553
|
+
lines.push(`Current branch: ${status.currentBranch}`);
|
|
10554
|
+
lines.push(`Base branch: ${status.baseBranch}`);
|
|
10361
10555
|
}
|
|
10362
|
-
|
|
10363
|
-
|
|
10364
|
-
|
|
10556
|
+
if (status.failedBatchesCount > 0) {
|
|
10557
|
+
lines.push("");
|
|
10558
|
+
lines.push(`INDEXING WARNING: ${status.failedBatchesCount} failed embedding batch${status.failedBatchesCount === 1 ? " remains" : "es remain"}.`);
|
|
10559
|
+
if (status.failedBatchesPath) {
|
|
10560
|
+
lines.push(`Failed batches: ${status.failedBatchesPath}`);
|
|
10561
|
+
}
|
|
10365
10562
|
}
|
|
10366
|
-
|
|
10367
|
-
|
|
10368
|
-
|
|
10369
|
-
|
|
10370
|
-
|
|
10371
|
-
|
|
10372
|
-
|
|
10373
|
-
|
|
10374
|
-
|
|
10375
|
-
|
|
10563
|
+
if (status.warning) {
|
|
10564
|
+
lines.push("");
|
|
10565
|
+
lines.push(`INDEX WARNING: ${status.warning}`);
|
|
10566
|
+
}
|
|
10567
|
+
if (status.compatibility && !status.compatibility.compatible) {
|
|
10568
|
+
lines.push("");
|
|
10569
|
+
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
10570
|
+
if (status.compatibility.storedMetadata) {
|
|
10571
|
+
const stored = status.compatibility.storedMetadata;
|
|
10572
|
+
lines.push(`Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
10573
|
+
lines.push(`Current config: ${status.provider}/${status.model}`);
|
|
10574
|
+
}
|
|
10575
|
+
} else if (!status.compatibility) {
|
|
10576
|
+
lines.push(`Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
10577
|
+
} else {
|
|
10578
|
+
lines.push(`Compatibility: Index is compatible with the current provider and model.`);
|
|
10579
|
+
}
|
|
10580
|
+
return lines.join("\n");
|
|
10376
10581
|
}
|
|
10377
|
-
function
|
|
10378
|
-
|
|
10379
|
-
|
|
10380
|
-
|
|
10582
|
+
function formatProgressTitle(progress) {
|
|
10583
|
+
switch (progress.phase) {
|
|
10584
|
+
case "scanning":
|
|
10585
|
+
return "Scanning files...";
|
|
10586
|
+
case "parsing":
|
|
10587
|
+
return `Parsing: ${progress.filesProcessed}/${progress.totalFiles} files`;
|
|
10588
|
+
case "embedding":
|
|
10589
|
+
return `Embedding: ${progress.chunksProcessed}/${progress.totalChunks} chunks`;
|
|
10590
|
+
case "storing":
|
|
10591
|
+
return "Storing index...";
|
|
10592
|
+
case "complete":
|
|
10593
|
+
return "Indexing complete";
|
|
10594
|
+
default:
|
|
10595
|
+
return "Indexing...";
|
|
10381
10596
|
}
|
|
10382
10597
|
}
|
|
10383
|
-
function
|
|
10384
|
-
|
|
10385
|
-
|
|
10386
|
-
if (
|
|
10387
|
-
return
|
|
10598
|
+
function calculatePercentage(progress) {
|
|
10599
|
+
if (progress.phase === "scanning") return 0;
|
|
10600
|
+
if (progress.phase === "complete") return 100;
|
|
10601
|
+
if (progress.phase === "parsing") {
|
|
10602
|
+
if (progress.totalFiles === 0) return 5;
|
|
10603
|
+
return Math.round(5 + progress.filesProcessed / progress.totalFiles * 15);
|
|
10388
10604
|
}
|
|
10389
|
-
if (
|
|
10390
|
-
return
|
|
10605
|
+
if (progress.phase === "embedding") {
|
|
10606
|
+
if (progress.totalChunks === 0) return 20;
|
|
10607
|
+
return Math.round(20 + progress.chunksProcessed / progress.totalChunks * 70);
|
|
10391
10608
|
}
|
|
10392
|
-
|
|
10393
|
-
|
|
10394
|
-
projectRoot,
|
|
10395
|
-
resolvedConfigPath
|
|
10396
|
-
);
|
|
10397
|
-
mkdirSync4(path14.dirname(localConfigPath), { recursive: true });
|
|
10398
|
-
writeFileSync4(localConfigPath, JSON.stringify(sourceConfig, null, 2), "utf-8");
|
|
10399
|
-
return localConfigPath;
|
|
10609
|
+
if (progress.phase === "storing") return 95;
|
|
10610
|
+
return 0;
|
|
10400
10611
|
}
|
|
10401
|
-
function
|
|
10402
|
-
|
|
10403
|
-
|
|
10612
|
+
function formatCodebasePeek(results) {
|
|
10613
|
+
if (results.length === 0) {
|
|
10614
|
+
return "No matching code found. Try a different query or run index_codebase first.";
|
|
10615
|
+
}
|
|
10616
|
+
const formatted = results.map((r, idx) => {
|
|
10617
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
10618
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
10619
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})${formatBlame(r)}`;
|
|
10620
|
+
});
|
|
10621
|
+
return formatted.join("\n");
|
|
10404
10622
|
}
|
|
10405
|
-
function
|
|
10406
|
-
|
|
10407
|
-
|
|
10408
|
-
};
|
|
10409
|
-
if (overrides?.fusionStrategy !== void 0) {
|
|
10410
|
-
nextSearch.fusionStrategy = overrides.fusionStrategy;
|
|
10623
|
+
function formatHealthCheck(result) {
|
|
10624
|
+
if (result.resetCorruptedIndex) {
|
|
10625
|
+
return result.warning ?? "Detected a corrupted local index and reset it. Run index_codebase to rebuild search data.";
|
|
10411
10626
|
}
|
|
10412
|
-
if (
|
|
10413
|
-
|
|
10627
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0 && result.gcOrphanSymbols === 0 && result.gcOrphanCallEdges === 0) {
|
|
10628
|
+
return "Index is healthy. No stale entries found.";
|
|
10414
10629
|
}
|
|
10415
|
-
|
|
10416
|
-
|
|
10630
|
+
const lines = [];
|
|
10631
|
+
if (result.removed > 0) {
|
|
10632
|
+
lines.push(`Removed stale entries: ${result.removed}`);
|
|
10417
10633
|
}
|
|
10418
|
-
if (
|
|
10419
|
-
|
|
10634
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
10635
|
+
lines.push(`Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
10420
10636
|
}
|
|
10421
|
-
|
|
10422
|
-
|
|
10423
|
-
search: nextSearch
|
|
10424
|
-
};
|
|
10425
|
-
}
|
|
10426
|
-
function getEmbeddingCostPer1MTokens(embeddingProvider) {
|
|
10427
|
-
return embeddingProvider === "custom" || embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(embeddingProvider).costPer1MTokens;
|
|
10428
|
-
}
|
|
10429
|
-
|
|
10430
|
-
// src/eval/schema.ts
|
|
10431
|
-
import { readFileSync as readFileSync9 } from "fs";
|
|
10432
|
-
function parseJsonFile(filePath) {
|
|
10433
|
-
const content = readFileSync9(filePath, "utf-8");
|
|
10434
|
-
try {
|
|
10435
|
-
return JSON.parse(content);
|
|
10436
|
-
} catch (error) {
|
|
10437
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
10438
|
-
throw new Error(`Failed to parse JSON from ${filePath}: ${message}`);
|
|
10637
|
+
if (result.gcOrphanChunks > 0) {
|
|
10638
|
+
lines.push(`Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
10439
10639
|
}
|
|
10440
|
-
|
|
10441
|
-
|
|
10442
|
-
return typeof value === "object" && value !== null;
|
|
10443
|
-
}
|
|
10444
|
-
function isStringArray3(value) {
|
|
10445
|
-
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
10446
|
-
}
|
|
10447
|
-
function asPositiveNumber(value, path27) {
|
|
10448
|
-
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
10449
|
-
throw new Error(`${path27} must be a non-negative number`);
|
|
10450
|
-
}
|
|
10451
|
-
return value;
|
|
10452
|
-
}
|
|
10453
|
-
function parseQueryType(value, path27) {
|
|
10454
|
-
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy") {
|
|
10455
|
-
return value;
|
|
10456
|
-
}
|
|
10457
|
-
throw new Error(
|
|
10458
|
-
`${path27} must be one of: definition, implementation-intent, similarity, keyword-heavy`
|
|
10459
|
-
);
|
|
10460
|
-
}
|
|
10461
|
-
function parseExpected(input, path27) {
|
|
10462
|
-
if (!isRecord2(input)) {
|
|
10463
|
-
throw new Error(`${path27} must be an object`);
|
|
10640
|
+
if (result.gcOrphanSymbols > 0) {
|
|
10641
|
+
lines.push(`Garbage collected orphan symbols: ${result.gcOrphanSymbols}`);
|
|
10464
10642
|
}
|
|
10465
|
-
|
|
10466
|
-
|
|
10467
|
-
const symbolRaw = input.symbol;
|
|
10468
|
-
const branchRaw = input.branch;
|
|
10469
|
-
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
10470
|
-
const acceptableFiles = isStringArray3(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
10471
|
-
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
10472
|
-
throw new Error(`${path27} must include either expected.filePath or expected.acceptableFiles`);
|
|
10643
|
+
if (result.gcOrphanCallEdges > 0) {
|
|
10644
|
+
lines.push(`Garbage collected orphan call edges: ${result.gcOrphanCallEdges}`);
|
|
10473
10645
|
}
|
|
10474
|
-
if (
|
|
10475
|
-
|
|
10646
|
+
if (result.filePaths.length > 0) {
|
|
10647
|
+
lines.push(`Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
10476
10648
|
}
|
|
10477
|
-
|
|
10478
|
-
|
|
10649
|
+
return lines.join("\n");
|
|
10650
|
+
}
|
|
10651
|
+
function formatCallGraphCallers(name, callers, relationshipType) {
|
|
10652
|
+
if (callers.length === 0) {
|
|
10653
|
+
return `No callers found for "${name}"${relationshipType ? ` with type ${relationshipType}` : ""}. It may not be called by any tracked function, or the index needs updating.`;
|
|
10479
10654
|
}
|
|
10480
|
-
|
|
10481
|
-
|
|
10655
|
+
const formatted = callers.map((edge, index) => {
|
|
10656
|
+
const confidence = edge.confidence !== "Direct" ? ` [${edge.confidence.toLowerCase()}]` : "";
|
|
10657
|
+
return `[${index + 1}] \u2190 from ${edge.fromSymbolName ?? "<unknown>"} in ${edge.fromSymbolFilePath ?? "<unknown file>"} [${edge.fromSymbolId}] (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? " [resolved]" : " [unresolved]"}`;
|
|
10658
|
+
});
|
|
10659
|
+
return `"${name}" is called by ${callers.length} function(s):
|
|
10660
|
+
|
|
10661
|
+
${formatted.join("\n")}`;
|
|
10662
|
+
}
|
|
10663
|
+
function formatCallGraphCallees(symbolId, callees, relationshipType) {
|
|
10664
|
+
if (callees.length === 0) {
|
|
10665
|
+
return `No callees found for symbol ${symbolId}${relationshipType ? ` with type ${relationshipType}` : ""}. The function may not call any other tracked functions.`;
|
|
10482
10666
|
}
|
|
10483
|
-
return {
|
|
10484
|
-
|
|
10485
|
-
|
|
10486
|
-
|
|
10487
|
-
branch: typeof branchRaw === "string" ? branchRaw : void 0
|
|
10488
|
-
};
|
|
10667
|
+
return callees.map((edge, index) => {
|
|
10668
|
+
const confidence = edge.confidence !== "Direct" ? ` [${edge.confidence.toLowerCase()}]` : "";
|
|
10669
|
+
return `[${index + 1}] \u2192 ${edge.targetName} (${edge.callType})${confidence} at line ${edge.line}${edge.isResolved ? ` [resolved: ${edge.toSymbolId}]` : " [unresolved]"}`;
|
|
10670
|
+
}).join("\n");
|
|
10489
10671
|
}
|
|
10490
|
-
function
|
|
10491
|
-
|
|
10492
|
-
|
|
10493
|
-
throw new Error(`${path27} must be an object`);
|
|
10672
|
+
function formatCallGraphPath(from, to, path27) {
|
|
10673
|
+
if (path27.length === 0) {
|
|
10674
|
+
return `No path found between "${from}" and "${to}". They may be in disconnected components, or the call graph index needs updating.`;
|
|
10494
10675
|
}
|
|
10495
|
-
const
|
|
10496
|
-
|
|
10497
|
-
|
|
10498
|
-
|
|
10499
|
-
|
|
10500
|
-
|
|
10676
|
+
const formatted = path27.map((hop, index) => {
|
|
10677
|
+
const prefix = index === 0 ? "[start]" : `--${hop.callType}-->`;
|
|
10678
|
+
const location = hop.filePath ? ` (${hop.filePath}:${hop.line})` : "";
|
|
10679
|
+
return `${prefix} ${hop.symbolName}${location}`;
|
|
10680
|
+
});
|
|
10681
|
+
return `Path (${path27.length} hops):
|
|
10682
|
+
${formatted.join("\n")}`;
|
|
10683
|
+
}
|
|
10684
|
+
function formatResultHeader(result, index) {
|
|
10685
|
+
return result.name ? `[${index + 1}] ${result.chunkType} "${result.name}" in ${result.filePath}:${result.startLine}-${result.endLine}` : `[${index + 1}] ${result.chunkType} in ${result.filePath}:${result.startLine}-${result.endLine}`;
|
|
10686
|
+
}
|
|
10687
|
+
function formatBlame(result) {
|
|
10688
|
+
if (!result.blame) {
|
|
10689
|
+
return "";
|
|
10501
10690
|
}
|
|
10502
|
-
|
|
10503
|
-
|
|
10691
|
+
const date = new Date(result.blame.committedAt * 1e3).toISOString().slice(0, 10);
|
|
10692
|
+
return `
|
|
10693
|
+
${result.blame.sha.slice(0, 7)} | ${result.blame.author} | ${date} | ${result.blame.summary}`;
|
|
10694
|
+
}
|
|
10695
|
+
function formatDefinitionLookup(results, query) {
|
|
10696
|
+
if (results.length === 0) {
|
|
10697
|
+
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
10504
10698
|
}
|
|
10505
|
-
|
|
10506
|
-
|
|
10507
|
-
|
|
10508
|
-
|
|
10509
|
-
|
|
10510
|
-
|
|
10699
|
+
const formatted = results.map((r, idx) => {
|
|
10700
|
+
const header = formatResultHeader(r, idx);
|
|
10701
|
+
return `${header} (score: ${r.score.toFixed(2)})${formatBlame(r)}
|
|
10702
|
+
\`\`\`
|
|
10703
|
+
${truncateContent(r.content)}
|
|
10704
|
+
\`\`\``;
|
|
10705
|
+
});
|
|
10706
|
+
return formatted.join("\n\n");
|
|
10511
10707
|
}
|
|
10512
|
-
function
|
|
10513
|
-
|
|
10514
|
-
|
|
10708
|
+
function formatSearchResults(results, scoreFormat = "similarity") {
|
|
10709
|
+
const formatted = results.map((r, idx) => {
|
|
10710
|
+
const header = formatResultHeader(r, idx);
|
|
10711
|
+
const scoreLabel = scoreFormat === "similarity" ? `(similarity: ${(r.score * 100).toFixed(1)}%)` : `(score: ${r.score.toFixed(2)})`;
|
|
10712
|
+
return `${header} ${scoreLabel}${formatBlame(r)}
|
|
10713
|
+
\`\`\`
|
|
10714
|
+
${truncateContent(r.content)}
|
|
10715
|
+
\`\`\``;
|
|
10716
|
+
});
|
|
10717
|
+
return formatted.join("\n\n");
|
|
10718
|
+
}
|
|
10719
|
+
|
|
10720
|
+
// src/tools/config-state.ts
|
|
10721
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync5, writeFileSync as writeFileSync5 } from "fs";
|
|
10722
|
+
import * as path16 from "path";
|
|
10723
|
+
function normalizeKnowledgeBasePaths(config, projectRoot) {
|
|
10724
|
+
const normalized = { ...config };
|
|
10725
|
+
if (Array.isArray(normalized.knowledgeBases)) {
|
|
10726
|
+
normalized.knowledgeBases = normalized.knowledgeBases.map(
|
|
10727
|
+
(kb) => resolveConfigPathValue(kb, projectRoot)
|
|
10728
|
+
);
|
|
10515
10729
|
}
|
|
10516
|
-
|
|
10517
|
-
|
|
10518
|
-
|
|
10519
|
-
|
|
10520
|
-
|
|
10521
|
-
throw new Error(`${sourceLabel}.version must be a non-empty string`);
|
|
10730
|
+
return normalized;
|
|
10731
|
+
}
|
|
10732
|
+
function toConfigRecord(rawConfig) {
|
|
10733
|
+
if (!rawConfig || typeof rawConfig !== "object") {
|
|
10734
|
+
return {};
|
|
10522
10735
|
}
|
|
10523
|
-
|
|
10524
|
-
|
|
10736
|
+
return { ...rawConfig };
|
|
10737
|
+
}
|
|
10738
|
+
function loadRuntimeConfig(projectRoot, host = "opencode") {
|
|
10739
|
+
return normalizeKnowledgeBasePaths(toConfigRecord(loadMergedConfig(projectRoot, host)), projectRoot);
|
|
10740
|
+
}
|
|
10741
|
+
|
|
10742
|
+
// src/tools/operations.ts
|
|
10743
|
+
var indexerCache = /* @__PURE__ */ new Map();
|
|
10744
|
+
var configCache = /* @__PURE__ */ new Map();
|
|
10745
|
+
var defaultProjectRoots = /* @__PURE__ */ new Map();
|
|
10746
|
+
function getIndexBusyResult(error) {
|
|
10747
|
+
if (!isIndexLockContentionError(error)) return null;
|
|
10748
|
+
const owner = error.owner;
|
|
10749
|
+
const ownerText = owner ? `PID ${owner.pid}, operation ${owner.operation}, since ${owner.startedAt}` : "unreadable owner";
|
|
10750
|
+
if (error.reason === "legacy-lock") {
|
|
10751
|
+
return {
|
|
10752
|
+
kind: "busy",
|
|
10753
|
+
text: `INDEX_BUSY: legacy lock format detected (${ownerText}). Verify the PID and remove this lock manually only if it is stale.`
|
|
10754
|
+
};
|
|
10525
10755
|
}
|
|
10526
|
-
if (
|
|
10527
|
-
|
|
10756
|
+
if (error.reason === "unknown-owner") {
|
|
10757
|
+
return {
|
|
10758
|
+
kind: "busy",
|
|
10759
|
+
text: `INDEX_BUSY: unreadable or remote lock owner (${ownerText}). Automatic recovery was refused; manual verification is required.`
|
|
10760
|
+
};
|
|
10528
10761
|
}
|
|
10529
|
-
|
|
10530
|
-
|
|
10762
|
+
return { kind: "busy", text: `INDEX_BUSY: another index operation is already in progress (${ownerText}).` };
|
|
10763
|
+
}
|
|
10764
|
+
function getProjectRoot(projectRoot, host) {
|
|
10765
|
+
if (projectRoot) {
|
|
10766
|
+
return projectRoot;
|
|
10531
10767
|
}
|
|
10532
|
-
|
|
10533
|
-
|
|
10768
|
+
const root = defaultProjectRoots.get(host);
|
|
10769
|
+
if (!root) {
|
|
10770
|
+
throw new Error("Codebase index tools not initialized. Plugin may not be loaded correctly.");
|
|
10534
10771
|
}
|
|
10535
|
-
|
|
10536
|
-
|
|
10537
|
-
|
|
10538
|
-
|
|
10539
|
-
|
|
10540
|
-
|
|
10541
|
-
|
|
10772
|
+
return root;
|
|
10773
|
+
}
|
|
10774
|
+
function getIndexerCacheKey(projectRoot, host) {
|
|
10775
|
+
return `${host}::${projectRoot}`;
|
|
10776
|
+
}
|
|
10777
|
+
function getOrCreateIndexer(projectRoot, host) {
|
|
10778
|
+
const key = getIndexerCacheKey(projectRoot, host);
|
|
10779
|
+
const cached = indexerCache.get(key);
|
|
10780
|
+
if (cached) {
|
|
10781
|
+
return cached;
|
|
10542
10782
|
}
|
|
10543
|
-
|
|
10544
|
-
|
|
10545
|
-
|
|
10546
|
-
|
|
10547
|
-
|
|
10548
|
-
};
|
|
10783
|
+
const config = parseConfig(loadRuntimeConfig(projectRoot, host));
|
|
10784
|
+
const indexer = new Indexer(projectRoot, config, host);
|
|
10785
|
+
indexerCache.set(key, indexer);
|
|
10786
|
+
configCache.set(key, config);
|
|
10787
|
+
return indexer;
|
|
10549
10788
|
}
|
|
10550
|
-
function
|
|
10551
|
-
|
|
10552
|
-
|
|
10789
|
+
function initializeTools(projectRoot, config, host = "opencode") {
|
|
10790
|
+
defaultProjectRoots.set(host, projectRoot);
|
|
10791
|
+
const key = getIndexerCacheKey(projectRoot, host);
|
|
10792
|
+
const indexer = new Indexer(projectRoot, config, host);
|
|
10793
|
+
indexerCache.set(key, indexer);
|
|
10794
|
+
configCache.set(key, config);
|
|
10553
10795
|
}
|
|
10554
|
-
function
|
|
10555
|
-
|
|
10796
|
+
function getIndexerForProject(projectRoot, host = "opencode") {
|
|
10797
|
+
const root = getProjectRoot(projectRoot, host);
|
|
10798
|
+
return getOrCreateIndexer(root, host);
|
|
10556
10799
|
}
|
|
10557
|
-
function
|
|
10558
|
-
|
|
10559
|
-
|
|
10560
|
-
|
|
10561
|
-
|
|
10562
|
-
|
|
10563
|
-
|
|
10564
|
-
const
|
|
10565
|
-
|
|
10566
|
-
|
|
10567
|
-
|
|
10568
|
-
if (baselinePath !== void 0 && typeof baselinePath !== "string") {
|
|
10569
|
-
throw new Error(`${sourceLabel}.baselinePath must be a string when provided`);
|
|
10570
|
-
}
|
|
10571
|
-
if (!isRecord2(thresholds)) {
|
|
10572
|
-
throw new Error(`${sourceLabel}.thresholds must be an object`);
|
|
10800
|
+
function refreshIndexerForDirectory(projectRoot, host = "opencode", config = parseConfig(loadRuntimeConfig(projectRoot, host))) {
|
|
10801
|
+
const key = getIndexerCacheKey(projectRoot, host);
|
|
10802
|
+
const indexer = new Indexer(projectRoot, config, host);
|
|
10803
|
+
indexerCache.set(key, indexer);
|
|
10804
|
+
configCache.set(key, config);
|
|
10805
|
+
}
|
|
10806
|
+
function shouldForceLocalizeProjectIndex(projectRoot, host = "opencode") {
|
|
10807
|
+
const root = getProjectRoot(projectRoot, host);
|
|
10808
|
+
const localIndexPath = path17.join(root, getHostProjectIndexRelativePath(host));
|
|
10809
|
+
if (existsSync10(localIndexPath)) {
|
|
10810
|
+
return false;
|
|
10573
10811
|
}
|
|
10574
|
-
|
|
10575
|
-
|
|
10576
|
-
baselinePath: typeof baselinePath === "string" ? baselinePath : void 0,
|
|
10577
|
-
failOnMissingBaseline: typeof failOnMissingBaseline === "boolean" ? failOnMissingBaseline : true,
|
|
10578
|
-
thresholds: {
|
|
10579
|
-
hitAt5MaxDrop: parseThresholdValue(
|
|
10580
|
-
thresholds.hitAt5MaxDrop,
|
|
10581
|
-
"hitAt5MaxDrop",
|
|
10582
|
-
sourceLabel
|
|
10583
|
-
),
|
|
10584
|
-
mrrAt10MaxDrop: parseThresholdValue(
|
|
10585
|
-
thresholds.mrrAt10MaxDrop,
|
|
10586
|
-
"mrrAt10MaxDrop",
|
|
10587
|
-
sourceLabel
|
|
10588
|
-
),
|
|
10589
|
-
rawDistinctTop3RatioMaxDrop: parseThresholdValue(
|
|
10590
|
-
thresholds.rawDistinctTop3RatioMaxDrop,
|
|
10591
|
-
"rawDistinctTop3RatioMaxDrop",
|
|
10592
|
-
sourceLabel
|
|
10593
|
-
),
|
|
10594
|
-
p95LatencyMaxMultiplier: parseThresholdValue(
|
|
10595
|
-
thresholds.p95LatencyMaxMultiplier,
|
|
10596
|
-
"p95LatencyMaxMultiplier",
|
|
10597
|
-
sourceLabel
|
|
10598
|
-
),
|
|
10599
|
-
p95LatencyMaxAbsoluteMs: parseThresholdValue(
|
|
10600
|
-
thresholds.p95LatencyMaxAbsoluteMs,
|
|
10601
|
-
"p95LatencyMaxAbsoluteMs",
|
|
10602
|
-
sourceLabel
|
|
10603
|
-
),
|
|
10604
|
-
minHitAt5: parseThresholdValue(
|
|
10605
|
-
thresholds.minHitAt5,
|
|
10606
|
-
"minHitAt5",
|
|
10607
|
-
sourceLabel
|
|
10608
|
-
),
|
|
10609
|
-
minMrrAt10: parseThresholdValue(
|
|
10610
|
-
thresholds.minMrrAt10,
|
|
10611
|
-
"minMrrAt10",
|
|
10612
|
-
sourceLabel
|
|
10613
|
-
),
|
|
10614
|
-
minRawDistinctTop3Ratio: parseThresholdValue(
|
|
10615
|
-
thresholds.minRawDistinctTop3Ratio,
|
|
10616
|
-
"minRawDistinctTop3Ratio",
|
|
10617
|
-
sourceLabel
|
|
10618
|
-
)
|
|
10619
|
-
}
|
|
10620
|
-
};
|
|
10812
|
+
const inheritedIndexPath = resolveWorktreeFallbackProjectIndexPath(root, host);
|
|
10813
|
+
return inheritedIndexPath !== null;
|
|
10621
10814
|
}
|
|
10622
|
-
function
|
|
10623
|
-
const
|
|
10624
|
-
return
|
|
10815
|
+
async function searchCodebase(projectRoot, host, query, options = {}) {
|
|
10816
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10817
|
+
return indexer.search(query, options.limit, {
|
|
10818
|
+
fileType: options.fileType,
|
|
10819
|
+
directory: options.directory,
|
|
10820
|
+
chunkType: options.chunkType,
|
|
10821
|
+
contextLines: options.contextLines,
|
|
10822
|
+
metadataOnly: options.metadataOnly,
|
|
10823
|
+
definitionIntent: options.definitionIntent,
|
|
10824
|
+
blameAuthor: options.blameAuthor,
|
|
10825
|
+
blameSha: options.blameSha,
|
|
10826
|
+
blameSince: options.blameSince
|
|
10827
|
+
});
|
|
10625
10828
|
}
|
|
10626
|
-
|
|
10627
|
-
|
|
10628
|
-
|
|
10629
|
-
|
|
10630
|
-
|
|
10631
|
-
|
|
10632
|
-
|
|
10633
|
-
|
|
10634
|
-
|
|
10635
|
-
|
|
10636
|
-
|
|
10637
|
-
|
|
10638
|
-
|
|
10639
|
-
|
|
10640
|
-
|
|
10641
|
-
|
|
10642
|
-
|
|
10643
|
-
|
|
10644
|
-
|
|
10645
|
-
|
|
10646
|
-
|
|
10647
|
-
|
|
10648
|
-
}
|
|
10649
|
-
const start = performance3.now();
|
|
10650
|
-
const result = await indexer.search(query.query, 10, {
|
|
10651
|
-
metadataOnly: true,
|
|
10652
|
-
filterByBranch: !!query.expected.branch
|
|
10653
|
-
});
|
|
10654
|
-
const elapsed = performance3.now() - start;
|
|
10655
|
-
const materialized = result.map((item) => ({
|
|
10656
|
-
filePath: item.filePath,
|
|
10657
|
-
startLine: item.startLine,
|
|
10658
|
-
endLine: item.endLine,
|
|
10659
|
-
score: item.score,
|
|
10660
|
-
chunkType: item.chunkType,
|
|
10661
|
-
name: item.name
|
|
10662
|
-
}));
|
|
10663
|
-
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10));
|
|
10664
|
-
}
|
|
10665
|
-
const logger = indexer.getLogger();
|
|
10666
|
-
const metricSnapshot = logger.getMetrics();
|
|
10667
|
-
const costPer1MTokensUsd = getEmbeddingCostPer1MTokens(effectiveConfig.embeddingProvider);
|
|
10668
|
-
const summary = {
|
|
10669
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10670
|
-
projectRoot: options.projectRoot,
|
|
10671
|
-
datasetPath,
|
|
10672
|
-
datasetName: dataset.name,
|
|
10673
|
-
datasetVersion: dataset.version,
|
|
10674
|
-
queryCount: dataset.queries.length,
|
|
10675
|
-
topK: 10,
|
|
10676
|
-
searchConfig: {
|
|
10677
|
-
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
10678
|
-
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
10679
|
-
rrfK: effectiveConfig.search.rrfK,
|
|
10680
|
-
rerankTopN: effectiveConfig.search.rerankTopN
|
|
10681
|
-
},
|
|
10682
|
-
metrics: computeEvalMetrics(
|
|
10683
|
-
dataset.queries,
|
|
10684
|
-
perQuery,
|
|
10685
|
-
metricSnapshot.embeddingApiCalls,
|
|
10686
|
-
metricSnapshot.embeddingTokensUsed,
|
|
10687
|
-
costPer1MTokensUsd
|
|
10688
|
-
)
|
|
10689
|
-
};
|
|
10690
|
-
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
10691
|
-
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
10692
|
-
writeJson(path15.join(outputDir, "summary.json"), summary);
|
|
10693
|
-
writeJson(path15.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
10694
|
-
let comparison;
|
|
10695
|
-
if (againstPath) {
|
|
10696
|
-
const baseline = loadSummary(againstPath);
|
|
10697
|
-
comparison = compareSummaries(summary, baseline, againstPath);
|
|
10698
|
-
writeJson(path15.join(outputDir, "compare.json"), comparison);
|
|
10699
|
-
}
|
|
10700
|
-
let gate;
|
|
10701
|
-
if (options.ciMode) {
|
|
10702
|
-
if (!budgetPath) {
|
|
10703
|
-
throw new Error("CI mode requires --budget path");
|
|
10704
|
-
}
|
|
10705
|
-
const budget = loadBudget(budgetPath);
|
|
10706
|
-
if (!comparison && budget.baselinePath) {
|
|
10707
|
-
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
10708
|
-
if (existsSync9(resolvedBaseline)) {
|
|
10709
|
-
const baselineSummary = loadSummary(resolvedBaseline);
|
|
10710
|
-
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
10711
|
-
writeJson(path15.join(outputDir, "compare.json"), comparison);
|
|
10712
|
-
} else if (budget.failOnMissingBaseline) {
|
|
10713
|
-
throw new Error(
|
|
10714
|
-
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
10715
|
-
);
|
|
10716
|
-
}
|
|
10717
|
-
}
|
|
10718
|
-
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
10829
|
+
async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
10830
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10831
|
+
return indexer.findSimilar(code, options.limit, {
|
|
10832
|
+
fileType: options.fileType,
|
|
10833
|
+
directory: options.directory,
|
|
10834
|
+
chunkType: options.chunkType,
|
|
10835
|
+
excludeFile: options.excludeFile
|
|
10836
|
+
});
|
|
10837
|
+
}
|
|
10838
|
+
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
10839
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10840
|
+
return indexer.search(query, options.limit, {
|
|
10841
|
+
fileType: options.fileType,
|
|
10842
|
+
directory: options.directory,
|
|
10843
|
+
definitionIntent: true
|
|
10844
|
+
});
|
|
10845
|
+
}
|
|
10846
|
+
async function getCallGraphData(projectRoot, host, params) {
|
|
10847
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10848
|
+
if (params.direction === "callees") {
|
|
10849
|
+
if (!params.symbolId) {
|
|
10850
|
+
return { direction: "callees", callees: [], callers: [] };
|
|
10719
10851
|
}
|
|
10720
|
-
const
|
|
10721
|
-
|
|
10722
|
-
return { outputDir, summary, perQuery, comparison, gate };
|
|
10723
|
-
} finally {
|
|
10724
|
-
await indexer.close();
|
|
10852
|
+
const callees = await indexer.getCallees(params.symbolId, params.relationshipType);
|
|
10853
|
+
return { direction: "callees", callees, callers: [] };
|
|
10725
10854
|
}
|
|
10855
|
+
const callers = await indexer.getCallers(params.name, params.relationshipType);
|
|
10856
|
+
return { direction: "callers", callers, callees: [] };
|
|
10726
10857
|
}
|
|
10727
|
-
async function
|
|
10728
|
-
const
|
|
10729
|
-
|
|
10730
|
-
|
|
10731
|
-
|
|
10732
|
-
const
|
|
10733
|
-
|
|
10734
|
-
|
|
10735
|
-
|
|
10736
|
-
|
|
10737
|
-
|
|
10738
|
-
|
|
10739
|
-
|
|
10740
|
-
|
|
10741
|
-
|
|
10742
|
-
|
|
10743
|
-
|
|
10744
|
-
|
|
10745
|
-
|
|
10746
|
-
|
|
10747
|
-
|
|
10748
|
-
|
|
10749
|
-
|
|
10750
|
-
|
|
10858
|
+
async function getCallGraphPath(projectRoot, host, from, to, maxDepth) {
|
|
10859
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10860
|
+
return indexer.findCallPath(from, to, maxDepth);
|
|
10861
|
+
}
|
|
10862
|
+
async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
10863
|
+
const root = getProjectRoot(projectRoot, host);
|
|
10864
|
+
const key = getIndexerCacheKey(root, host);
|
|
10865
|
+
let indexer = getIndexerForProject(root, host);
|
|
10866
|
+
const runtimeConfig = configCache.get(key);
|
|
10867
|
+
try {
|
|
10868
|
+
if (args.estimateOnly) {
|
|
10869
|
+
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
10870
|
+
}
|
|
10871
|
+
const runIndex = async (target) => {
|
|
10872
|
+
const operation = args.force ? target.forceIndex.bind(target) : target.index.bind(target);
|
|
10873
|
+
return operation((progress) => {
|
|
10874
|
+
if (onProgress) {
|
|
10875
|
+
return onProgress(formatProgressTitle(progress), {
|
|
10876
|
+
phase: progress.phase,
|
|
10877
|
+
filesProcessed: progress.filesProcessed,
|
|
10878
|
+
totalFiles: progress.totalFiles,
|
|
10879
|
+
chunksProcessed: progress.chunksProcessed,
|
|
10880
|
+
totalChunks: progress.totalChunks,
|
|
10881
|
+
percentage: calculatePercentage(progress)
|
|
10751
10882
|
});
|
|
10752
10883
|
}
|
|
10753
|
-
|
|
10884
|
+
return Promise.resolve();
|
|
10885
|
+
});
|
|
10886
|
+
};
|
|
10887
|
+
let stats;
|
|
10888
|
+
if (args.force && shouldForceLocalizeProjectIndex(root, host)) {
|
|
10889
|
+
const inheritedIndexPath = resolveProjectIndexPath(root, runtimeConfig.scope, host);
|
|
10890
|
+
stats = await withIndexLock(inheritedIndexPath, "force-index", async () => {
|
|
10891
|
+
materializeLocalProjectConfig(root, loadProjectConfigLayer(root, host), host);
|
|
10892
|
+
refreshIndexerForDirectory(root, host, runtimeConfig);
|
|
10893
|
+
indexer = getIndexerForProject(root, host);
|
|
10894
|
+
return runIndex(indexer);
|
|
10895
|
+
}, { completeRecoveries: false });
|
|
10896
|
+
} else {
|
|
10897
|
+
stats = await runIndex(indexer);
|
|
10754
10898
|
}
|
|
10899
|
+
return { kind: "stats", stats };
|
|
10900
|
+
} catch (error) {
|
|
10901
|
+
const busyResult = getIndexBusyResult(error);
|
|
10902
|
+
if (!busyResult) throw error;
|
|
10903
|
+
return busyResult;
|
|
10755
10904
|
}
|
|
10756
|
-
const bestByHitAt5 = [...runs].sort(
|
|
10757
|
-
(a, b) => b.summary.metrics.hitAt5 - a.summary.metrics.hitAt5
|
|
10758
|
-
)[0];
|
|
10759
|
-
const bestByMrrAt10 = [...runs].sort(
|
|
10760
|
-
(a, b) => b.summary.metrics.mrrAt10 - a.summary.metrics.mrrAt10
|
|
10761
|
-
)[0];
|
|
10762
|
-
const bestByP95Latency = [...runs].sort(
|
|
10763
|
-
(a, b) => a.summary.metrics.latencyMs.p95 - b.summary.metrics.latencyMs.p95
|
|
10764
|
-
)[0];
|
|
10765
|
-
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
10766
|
-
const failedGateRuns = runs.filter((run) => run.gate && !run.gate.passed).length;
|
|
10767
|
-
const gatePassed = failedGateRuns === 0;
|
|
10768
|
-
const aggregate = {
|
|
10769
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
10770
|
-
againstPath: options.againstPath,
|
|
10771
|
-
runCount: runs.length,
|
|
10772
|
-
runs,
|
|
10773
|
-
gatePassed,
|
|
10774
|
-
failedGateRuns,
|
|
10775
|
-
bestByHitAt5,
|
|
10776
|
-
bestByMrrAt10,
|
|
10777
|
-
bestByP95Latency
|
|
10778
|
-
};
|
|
10779
|
-
writeJson(path15.join(outputDir, "compare.json"), aggregate);
|
|
10780
|
-
const md = createSummaryMarkdown(
|
|
10781
|
-
bestByHitAt5?.summary ?? runs[0].summary,
|
|
10782
|
-
bestByHitAt5?.comparison,
|
|
10783
|
-
void 0,
|
|
10784
|
-
aggregate
|
|
10785
|
-
);
|
|
10786
|
-
writeText(path15.join(outputDir, "summary.md"), md);
|
|
10787
|
-
writeJson(path15.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
10788
|
-
return { outputDir, aggregate };
|
|
10789
10905
|
}
|
|
10790
|
-
|
|
10791
|
-
|
|
10792
|
-
|
|
10793
|
-
|
|
10794
|
-
// src/eval/cli-parser.ts
|
|
10795
|
-
import * as path16 from "path";
|
|
10796
|
-
function printUsage() {
|
|
10797
|
-
console.log(`
|
|
10798
|
-
Usage:
|
|
10799
|
-
opencode-codebase-index-mcp eval run [options]
|
|
10800
|
-
opencode-codebase-index-mcp eval compare --against <summary.json> [options]
|
|
10801
|
-
opencode-codebase-index-mcp eval diff --current <summary.json> --against <summary.json> [options]
|
|
10802
|
-
|
|
10803
|
-
Options:
|
|
10804
|
-
--project <path> Project root (default: cwd)
|
|
10805
|
-
--config <path> Config JSON path
|
|
10806
|
-
--dataset <path> Golden dataset path (default: benchmarks/golden/small.json)
|
|
10807
|
-
--current <path> Current summary.json path (required for eval diff)
|
|
10808
|
-
--output <path> Output root dir (default: benchmarks/results)
|
|
10809
|
-
--against <path> Baseline summary.json to compare against
|
|
10810
|
-
--budget <path> Budget file for CI mode (default: benchmarks/budgets/default.json)
|
|
10811
|
-
--ci Enable CI gate mode
|
|
10812
|
-
--reindex Force reindex before eval
|
|
10813
|
-
|
|
10814
|
-
Search overrides:
|
|
10815
|
-
--fusionStrategy <rrf|weighted>
|
|
10816
|
-
--hybridWeight <0-1>
|
|
10817
|
-
--rrfK <number>
|
|
10818
|
-
--rerankTopN <number>
|
|
10819
|
-
|
|
10820
|
-
Sweep options (comma-separated values):
|
|
10821
|
-
--sweepFusionStrategy <rrf,weighted>
|
|
10822
|
-
--sweepHybridWeight <0.3,0.5,0.7>
|
|
10823
|
-
--sweepRrfK <30,60,90>
|
|
10824
|
-
--sweepRerankTopN <10,20,40>
|
|
10825
|
-
`);
|
|
10826
|
-
}
|
|
10827
|
-
function parseNumber(value, flag) {
|
|
10828
|
-
const parsed = Number(value);
|
|
10829
|
-
if (Number.isNaN(parsed)) {
|
|
10830
|
-
throw new Error(`${flag} must be a number`);
|
|
10831
|
-
}
|
|
10832
|
-
return parsed;
|
|
10906
|
+
async function getIndexStatus(projectRoot, host) {
|
|
10907
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10908
|
+
return indexer.getStatus();
|
|
10833
10909
|
}
|
|
10834
|
-
function
|
|
10835
|
-
|
|
10910
|
+
async function getIndexHealthCheck(projectRoot, host) {
|
|
10911
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10912
|
+
return indexer.healthCheck();
|
|
10836
10913
|
}
|
|
10837
|
-
function
|
|
10838
|
-
|
|
10839
|
-
|
|
10840
|
-
|
|
10841
|
-
|
|
10842
|
-
|
|
10843
|
-
|
|
10844
|
-
parsed.push(candidate);
|
|
10914
|
+
async function runIndexHealthCheck(projectRoot, host) {
|
|
10915
|
+
try {
|
|
10916
|
+
return { kind: "health", health: await getIndexHealthCheck(projectRoot, host) };
|
|
10917
|
+
} catch (error) {
|
|
10918
|
+
const busyResult = getIndexBusyResult(error);
|
|
10919
|
+
if (!busyResult) throw error;
|
|
10920
|
+
return busyResult;
|
|
10845
10921
|
}
|
|
10846
|
-
return parsed;
|
|
10847
10922
|
}
|
|
10848
|
-
function
|
|
10849
|
-
|
|
10850
|
-
|
|
10851
|
-
|
|
10852
|
-
|
|
10853
|
-
|
|
10854
|
-
|
|
10855
|
-
|
|
10856
|
-
|
|
10857
|
-
|
|
10858
|
-
budgetPath: "benchmarks/budgets/default.json",
|
|
10859
|
-
ciMode: false,
|
|
10860
|
-
reindex: false,
|
|
10861
|
-
sweep: {}
|
|
10862
|
-
};
|
|
10863
|
-
for (let i = 0; i < argv.length; i += 1) {
|
|
10864
|
-
const arg = argv[i];
|
|
10865
|
-
const next = argv[i + 1];
|
|
10866
|
-
if (arg === "--project" && next) {
|
|
10867
|
-
parsed.projectRoot = path16.resolve(cwd, next);
|
|
10868
|
-
i += 1;
|
|
10869
|
-
continue;
|
|
10870
|
-
}
|
|
10871
|
-
if (arg === "--config" && next) {
|
|
10872
|
-
parsed.configPath = path16.resolve(cwd, next);
|
|
10873
|
-
i += 1;
|
|
10874
|
-
continue;
|
|
10875
|
-
}
|
|
10876
|
-
if (arg === "--dataset" && next) {
|
|
10877
|
-
parsed.datasetPath = next;
|
|
10878
|
-
i += 1;
|
|
10879
|
-
continue;
|
|
10880
|
-
}
|
|
10881
|
-
if (arg === "--current" && next) {
|
|
10882
|
-
parsed.currentPath = next;
|
|
10883
|
-
i += 1;
|
|
10884
|
-
continue;
|
|
10885
|
-
}
|
|
10886
|
-
if (arg === "--output" && next) {
|
|
10887
|
-
parsed.outputRoot = next;
|
|
10888
|
-
i += 1;
|
|
10889
|
-
continue;
|
|
10890
|
-
}
|
|
10891
|
-
if (arg === "--against" && next) {
|
|
10892
|
-
parsed.againstPath = next;
|
|
10893
|
-
i += 1;
|
|
10894
|
-
continue;
|
|
10895
|
-
}
|
|
10896
|
-
if (arg === "--budget" && next) {
|
|
10897
|
-
parsed.budgetPath = next;
|
|
10898
|
-
i += 1;
|
|
10899
|
-
continue;
|
|
10900
|
-
}
|
|
10901
|
-
if (arg === "--ci") {
|
|
10902
|
-
parsed.ciMode = true;
|
|
10903
|
-
continue;
|
|
10904
|
-
}
|
|
10905
|
-
if (arg === "--reindex") {
|
|
10906
|
-
parsed.reindex = true;
|
|
10907
|
-
continue;
|
|
10908
|
-
}
|
|
10909
|
-
if (arg === "--fusionStrategy" && next) {
|
|
10910
|
-
if (next !== "rrf" && next !== "weighted") {
|
|
10911
|
-
throw new Error("--fusionStrategy must be rrf or weighted");
|
|
10912
|
-
}
|
|
10913
|
-
parsed.fusionStrategy = next;
|
|
10914
|
-
i += 1;
|
|
10915
|
-
continue;
|
|
10916
|
-
}
|
|
10917
|
-
if (arg === "--hybridWeight" && next) {
|
|
10918
|
-
parsed.hybridWeight = parseNumber(next, "--hybridWeight");
|
|
10919
|
-
i += 1;
|
|
10920
|
-
continue;
|
|
10921
|
-
}
|
|
10922
|
-
if (arg === "--rrfK" && next) {
|
|
10923
|
-
parsed.rrfK = parseNumber(next, "--rrfK");
|
|
10924
|
-
i += 1;
|
|
10925
|
-
continue;
|
|
10926
|
-
}
|
|
10927
|
-
if (arg === "--rerankTopN" && next) {
|
|
10928
|
-
parsed.rerankTopN = parseNumber(next, "--rerankTopN");
|
|
10929
|
-
i += 1;
|
|
10930
|
-
continue;
|
|
10931
|
-
}
|
|
10932
|
-
if (arg === "--sweepFusionStrategy" && next) {
|
|
10933
|
-
parsed.sweep.fusionStrategy = parseCsvFusion(next);
|
|
10934
|
-
i += 1;
|
|
10935
|
-
continue;
|
|
10936
|
-
}
|
|
10937
|
-
if (arg === "--sweepHybridWeight" && next) {
|
|
10938
|
-
parsed.sweep.hybridWeight = parseCsvNumbers(next, "--sweepHybridWeight");
|
|
10939
|
-
i += 1;
|
|
10940
|
-
continue;
|
|
10941
|
-
}
|
|
10942
|
-
if (arg === "--sweepRrfK" && next) {
|
|
10943
|
-
parsed.sweep.rrfK = parseCsvNumbers(next, "--sweepRrfK");
|
|
10944
|
-
i += 1;
|
|
10945
|
-
continue;
|
|
10946
|
-
}
|
|
10947
|
-
if (arg === "--sweepRerankTopN" && next) {
|
|
10948
|
-
parsed.sweep.rerankTopN = parseCsvNumbers(next, "--sweepRerankTopN");
|
|
10949
|
-
i += 1;
|
|
10950
|
-
continue;
|
|
10951
|
-
}
|
|
10952
|
-
}
|
|
10953
|
-
return parsed;
|
|
10923
|
+
async function getPrImpact(projectRoot, host, params) {
|
|
10924
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10925
|
+
return indexer.getPrImpact({
|
|
10926
|
+
pr: params.pr,
|
|
10927
|
+
branch: params.branch,
|
|
10928
|
+
maxDepth: params.maxDepth,
|
|
10929
|
+
hubThreshold: params.hubThreshold,
|
|
10930
|
+
checkConflicts: params.checkConflicts,
|
|
10931
|
+
direction: params.direction
|
|
10932
|
+
});
|
|
10954
10933
|
}
|
|
10955
|
-
function
|
|
10956
|
-
|
|
10957
|
-
const
|
|
10958
|
-
|
|
10959
|
-
|
|
10960
|
-
|
|
10961
|
-
|
|
10962
|
-
|
|
10963
|
-
|
|
10964
|
-
|
|
10965
|
-
|
|
10966
|
-
|
|
10934
|
+
async function getIndexMetrics(projectRoot, host) {
|
|
10935
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10936
|
+
const logger = indexer.getLogger();
|
|
10937
|
+
if (!logger.isEnabled()) {
|
|
10938
|
+
return {
|
|
10939
|
+
enabled: false,
|
|
10940
|
+
metricsEnabled: false,
|
|
10941
|
+
text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true,\n "metrics": true\n }\n}\n```'
|
|
10942
|
+
};
|
|
10943
|
+
}
|
|
10944
|
+
if (!logger.isMetricsEnabled()) {
|
|
10945
|
+
return {
|
|
10946
|
+
enabled: true,
|
|
10947
|
+
metricsEnabled: false,
|
|
10948
|
+
text: 'Metrics collection is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true,\n "metrics": true\n }\n}\n```'
|
|
10949
|
+
};
|
|
10967
10950
|
}
|
|
10968
10951
|
return {
|
|
10969
|
-
|
|
10970
|
-
|
|
10952
|
+
enabled: true,
|
|
10953
|
+
metricsEnabled: true,
|
|
10954
|
+
text: logger.formatMetrics()
|
|
10971
10955
|
};
|
|
10972
10956
|
}
|
|
10973
|
-
function
|
|
10974
|
-
|
|
10975
|
-
|
|
10976
|
-
|
|
10977
|
-
|
|
10978
|
-
|
|
10979
|
-
|
|
10980
|
-
|
|
10981
|
-
|
|
10982
|
-
|
|
10983
|
-
|
|
10984
|
-
|
|
10985
|
-
|
|
10986
|
-
|
|
10987
|
-
|
|
10988
|
-
|
|
10989
|
-
}
|
|
10957
|
+
async function getIndexLogs(projectRoot, host, args) {
|
|
10958
|
+
const indexer = getIndexerForProject(projectRoot, host);
|
|
10959
|
+
const logger = indexer.getLogger();
|
|
10960
|
+
if (!logger.isEnabled()) {
|
|
10961
|
+
return {
|
|
10962
|
+
kind: "disabled",
|
|
10963
|
+
text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true\n }\n}\n```'
|
|
10964
|
+
};
|
|
10965
|
+
}
|
|
10966
|
+
let logs;
|
|
10967
|
+
if (args.category) {
|
|
10968
|
+
logs = logger.getLogsByCategory(args.category, args.limit);
|
|
10969
|
+
} else if (args.level) {
|
|
10970
|
+
logs = logger.getLogsByLevel(args.level, args.limit);
|
|
10971
|
+
} else {
|
|
10972
|
+
logs = logger.getLogs(args.limit);
|
|
10973
|
+
}
|
|
10974
|
+
if (logs.length === 0) {
|
|
10975
|
+
return {
|
|
10976
|
+
kind: "entries",
|
|
10977
|
+
text: "No logs recorded yet. Logs are captured during indexing and search operations."
|
|
10978
|
+
};
|
|
10979
|
+
}
|
|
10980
|
+
const text = logs.map((entry) => {
|
|
10981
|
+
const dataStr = entry.data ? ` ${JSON.stringify(entry.data)}` : "";
|
|
10982
|
+
return `[${entry.timestamp}] [${entry.level.toUpperCase()}] [${entry.category}] ${entry.message}${dataStr}`;
|
|
10983
|
+
}).join("\n");
|
|
10984
|
+
return { kind: "entries", text };
|
|
10990
10985
|
}
|
|
10991
10986
|
|
|
10992
|
-
// src/
|
|
10993
|
-
|
|
10994
|
-
|
|
10995
|
-
|
|
10996
|
-
|
|
10997
|
-
|
|
10987
|
+
// src/tools/symbol-inference.ts
|
|
10988
|
+
var IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g;
|
|
10989
|
+
var QUOTED_BACKTICK_RE = /`([^`]+)`/g;
|
|
10990
|
+
var QUOTED_SINGLE_RE = /'([^'\\]+)'/g;
|
|
10991
|
+
var QUOTED_DOUBLE_RE = /"([^"]+)"/g;
|
|
10992
|
+
var SYMBOL_LIKE_RE = /^(?:[A-Za-z_$][A-Za-z0-9_$]*)$/;
|
|
10993
|
+
var CAMEL_CASE_RE = /^[a-z_][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*$/;
|
|
10994
|
+
var PASCAL_CASE_RE = /^[A-Z][A-Za-z0-9_$]*$/;
|
|
10995
|
+
var SNAKE_CASE_RE = /^[a-z][a-z0-9_]*_[a-z0-9_]+$/;
|
|
10996
|
+
var DEFINITION_INTENT_RE = /\b(where|defined|definition|define|declaration|symbol|function|method|class|interface|type)\b/i;
|
|
10997
|
+
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
10998
|
+
"a",
|
|
10999
|
+
"an",
|
|
11000
|
+
"and",
|
|
11001
|
+
"are",
|
|
11002
|
+
"at",
|
|
11003
|
+
"for",
|
|
11004
|
+
"find",
|
|
11005
|
+
"how",
|
|
11006
|
+
"i",
|
|
11007
|
+
"in",
|
|
11008
|
+
"is",
|
|
11009
|
+
"it",
|
|
11010
|
+
"of",
|
|
11011
|
+
"on",
|
|
11012
|
+
"that",
|
|
11013
|
+
"the",
|
|
11014
|
+
"definition",
|
|
11015
|
+
"show",
|
|
11016
|
+
"to",
|
|
11017
|
+
"where",
|
|
11018
|
+
"which",
|
|
11019
|
+
"what",
|
|
11020
|
+
"you",
|
|
11021
|
+
"your",
|
|
11022
|
+
"with"
|
|
11023
|
+
]);
|
|
11024
|
+
function stripCallSuffix(token) {
|
|
11025
|
+
return token.replace(/\(\s*\)$/, "");
|
|
11026
|
+
}
|
|
11027
|
+
function isLikelySymbolName(token) {
|
|
11028
|
+
if (!SYMBOL_LIKE_RE.test(token)) {
|
|
11029
|
+
return false;
|
|
10998
11030
|
}
|
|
10999
|
-
if (
|
|
11000
|
-
|
|
11001
|
-
if (explicitAgainst) {
|
|
11002
|
-
parsed.againstPath = explicitAgainst;
|
|
11003
|
-
}
|
|
11004
|
-
const runOptions = toRunOptions(parsed);
|
|
11005
|
-
if (hasSweepOptions(parsed.sweep)) {
|
|
11006
|
-
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
11007
|
-
console.log(`Eval sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
11008
|
-
console.log(`Sweep runs: ${sweep.aggregate.runCount}`);
|
|
11009
|
-
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
11010
|
-
console.error(
|
|
11011
|
-
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
11012
|
-
);
|
|
11013
|
-
return 1;
|
|
11014
|
-
}
|
|
11015
|
-
return 0;
|
|
11016
|
-
}
|
|
11017
|
-
const result = await runEvaluation(runOptions);
|
|
11018
|
-
console.log(`Eval run complete. Artifacts: ${result.outputDir}`);
|
|
11019
|
-
console.log(
|
|
11020
|
-
`Hit@5=${(result.summary.metrics.hitAt5 * 100).toFixed(2)}% MRR@10=${result.summary.metrics.mrrAt10.toFixed(4)} p95=${result.summary.metrics.latencyMs.p95.toFixed(3)}ms`
|
|
11021
|
-
);
|
|
11022
|
-
if (result.gate && !result.gate.passed) {
|
|
11023
|
-
for (const violation of result.gate.violations) {
|
|
11024
|
-
console.error(`[CI-GATE] ${violation.metric}: ${violation.message}`);
|
|
11025
|
-
}
|
|
11026
|
-
return 1;
|
|
11027
|
-
}
|
|
11028
|
-
return 0;
|
|
11031
|
+
if (STOP_WORDS.has(token.toLowerCase())) {
|
|
11032
|
+
return false;
|
|
11029
11033
|
}
|
|
11030
|
-
|
|
11031
|
-
|
|
11032
|
-
|
|
11033
|
-
|
|
11034
|
-
|
|
11035
|
-
|
|
11036
|
-
|
|
11037
|
-
|
|
11038
|
-
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
11039
|
-
console.log(`Eval compare sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
11040
|
-
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
11041
|
-
console.error(
|
|
11042
|
-
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
11043
|
-
);
|
|
11044
|
-
return 1;
|
|
11045
|
-
}
|
|
11046
|
-
return 0;
|
|
11034
|
+
return CAMEL_CASE_RE.test(token) || PASCAL_CASE_RE.test(token) || SNAKE_CASE_RE.test(token);
|
|
11035
|
+
}
|
|
11036
|
+
function extractQuotedIdentifiers(query) {
|
|
11037
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
11038
|
+
for (const match of query.matchAll(QUOTED_BACKTICK_RE)) {
|
|
11039
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
11040
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
11041
|
+
identifiers.add(candidate);
|
|
11047
11042
|
}
|
|
11048
|
-
const result = await runEvaluation(runOptions);
|
|
11049
|
-
console.log(`Eval compare complete. Artifacts: ${result.outputDir}`);
|
|
11050
|
-
return 0;
|
|
11051
11043
|
}
|
|
11052
|
-
|
|
11053
|
-
const
|
|
11054
|
-
if (
|
|
11055
|
-
|
|
11056
|
-
}
|
|
11057
|
-
if (!parsed.currentPath) {
|
|
11058
|
-
throw new Error("eval diff requires --current <current summary.json>");
|
|
11044
|
+
for (const match of query.matchAll(QUOTED_SINGLE_RE)) {
|
|
11045
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
11046
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
11047
|
+
identifiers.add(candidate);
|
|
11059
11048
|
}
|
|
11060
|
-
|
|
11061
|
-
|
|
11062
|
-
|
|
11063
|
-
|
|
11049
|
+
}
|
|
11050
|
+
for (const match of query.matchAll(QUOTED_DOUBLE_RE)) {
|
|
11051
|
+
const candidate = stripCallSuffix(match[1].trim());
|
|
11052
|
+
if (candidate && isLikelySymbolName(candidate)) {
|
|
11053
|
+
identifiers.add(candidate);
|
|
11064
11054
|
}
|
|
11065
|
-
|
|
11066
|
-
|
|
11055
|
+
}
|
|
11056
|
+
return [...identifiers];
|
|
11057
|
+
}
|
|
11058
|
+
function extractBareIdentifiers(query) {
|
|
11059
|
+
const unquoted = query.replace(QUOTED_BACKTICK_RE, " ").replace(QUOTED_SINGLE_RE, " ").replace(QUOTED_DOUBLE_RE, " ");
|
|
11060
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
11061
|
+
for (const match of unquoted.matchAll(IDENTIFIER_RE)) {
|
|
11062
|
+
const candidate = stripCallSuffix(match[0]);
|
|
11063
|
+
if (isLikelySymbolName(candidate)) {
|
|
11064
|
+
identifiers.add(candidate);
|
|
11067
11065
|
}
|
|
11068
|
-
const currentSummary = loadSummary(path17.resolve(parsed.projectRoot, currentPath), {
|
|
11069
|
-
allowLegacyDiversityMetrics: true
|
|
11070
|
-
});
|
|
11071
|
-
const baselineSummary = loadSummary(path17.resolve(parsed.projectRoot, parsed.againstPath), {
|
|
11072
|
-
allowLegacyDiversityMetrics: true
|
|
11073
|
-
});
|
|
11074
|
-
const comparison = compareSummaries(
|
|
11075
|
-
currentSummary,
|
|
11076
|
-
baselineSummary,
|
|
11077
|
-
path17.resolve(parsed.projectRoot, parsed.againstPath)
|
|
11078
|
-
);
|
|
11079
|
-
const outputDir = createRunDirectory(path17.resolve(parsed.projectRoot, parsed.outputRoot));
|
|
11080
|
-
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
11081
|
-
writeJson(path17.join(outputDir, "compare.json"), comparison);
|
|
11082
|
-
writeText(path17.join(outputDir, "summary.md"), summaryMd);
|
|
11083
|
-
writeJson(path17.join(outputDir, "summary.json"), currentSummary);
|
|
11084
|
-
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
11085
|
-
return 0;
|
|
11086
11066
|
}
|
|
11087
|
-
|
|
11067
|
+
return [...identifiers];
|
|
11068
|
+
}
|
|
11069
|
+
function isSingleMeaningfulToken(query, symbol) {
|
|
11070
|
+
const tokens = query.replace(/[`'"()]/g, " ").split(/[^A-Za-z0-9_$]+/).map((token) => token.trim().toLowerCase()).filter((token) => token.length > 0).filter((token) => !STOP_WORDS.has(token));
|
|
11071
|
+
return tokens.length === 1 && tokens[0] === symbol.toLowerCase();
|
|
11072
|
+
}
|
|
11073
|
+
function inferExactSymbolFromQuery(query) {
|
|
11074
|
+
const quoted = extractQuotedIdentifiers(query);
|
|
11075
|
+
if (quoted.length === 1) {
|
|
11076
|
+
return quoted[0];
|
|
11077
|
+
}
|
|
11078
|
+
if (quoted.length > 1) {
|
|
11079
|
+
return void 0;
|
|
11080
|
+
}
|
|
11081
|
+
const candidates = extractBareIdentifiers(query);
|
|
11082
|
+
if (candidates.length !== 1) {
|
|
11083
|
+
return void 0;
|
|
11084
|
+
}
|
|
11085
|
+
const candidate = candidates[0];
|
|
11086
|
+
if (DEFINITION_INTENT_RE.test(query)) {
|
|
11087
|
+
return candidate;
|
|
11088
|
+
}
|
|
11089
|
+
if (isSingleMeaningfulToken(query, candidate)) {
|
|
11090
|
+
return candidate;
|
|
11091
|
+
}
|
|
11092
|
+
return void 0;
|
|
11088
11093
|
}
|
|
11089
11094
|
|
|
11090
|
-
// src/
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
|
|
11095
|
-
|
|
11096
|
-
|
|
11097
|
-
|
|
11098
|
-
|
|
11099
|
-
|
|
11100
|
-
|
|
11101
|
-
|
|
11102
|
-
|
|
11103
|
-
|
|
11104
|
-
|
|
11105
|
-
|
|
11106
|
-
|
|
11107
|
-
|
|
11108
|
-
|
|
11109
|
-
|
|
11110
|
-
|
|
11111
|
-
|
|
11112
|
-
|
|
11113
|
-
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11118
|
-
|
|
11119
|
-
|
|
11120
|
-
|
|
11121
|
-
|
|
11122
|
-
|
|
11123
|
-
|
|
11124
|
-
|
|
11125
|
-
|
|
11126
|
-
|
|
11127
|
-
|
|
11128
|
-
|
|
11129
|
-
|
|
11130
|
-
|
|
11131
|
-
|
|
11132
|
-
|
|
11133
|
-
|
|
11134
|
-
|
|
11135
|
-
|
|
11136
|
-
|
|
11137
|
-
|
|
11138
|
-
|
|
11139
|
-
|
|
11140
|
-
|
|
11141
|
-
|
|
11142
|
-
|
|
11143
|
-
}
|
|
11095
|
+
// src/tools/context.ts
|
|
11096
|
+
var MIN_CONTEXT_RESULT_LIMIT = 1;
|
|
11097
|
+
var MAX_CONTEXT_RESULT_LIMIT = 100;
|
|
11098
|
+
var MIN_CONTEXT_PATH_DEPTH = 1;
|
|
11099
|
+
var MAX_CONTEXT_PATH_DEPTH = 100;
|
|
11100
|
+
function locations(results) {
|
|
11101
|
+
return results.map((result) => ({
|
|
11102
|
+
filePath: result.filePath,
|
|
11103
|
+
startLine: result.startLine,
|
|
11104
|
+
endLine: result.endLine,
|
|
11105
|
+
score: result.score,
|
|
11106
|
+
chunkType: result.chunkType,
|
|
11107
|
+
name: result.name
|
|
11108
|
+
}));
|
|
11109
|
+
}
|
|
11110
|
+
function packedResult(route, routedQuery, pack) {
|
|
11111
|
+
return {
|
|
11112
|
+
text: pack.text,
|
|
11113
|
+
details: {
|
|
11114
|
+
route,
|
|
11115
|
+
routedQuery,
|
|
11116
|
+
tokenBudget: pack.tokenBudget,
|
|
11117
|
+
tokenEstimate: pack.tokenEstimate,
|
|
11118
|
+
candidateCount: pack.candidateCount,
|
|
11119
|
+
deduplicatedCount: pack.deduplicatedCount,
|
|
11120
|
+
selectedCount: pack.selectedCount,
|
|
11121
|
+
omittedCount: pack.omittedCount,
|
|
11122
|
+
duplicateCount: pack.duplicateCount,
|
|
11123
|
+
limitOmittedCount: pack.limitOmittedCount,
|
|
11124
|
+
budgetOmittedCount: pack.budgetOmittedCount,
|
|
11125
|
+
results: locations(pack.results)
|
|
11126
|
+
}
|
|
11127
|
+
};
|
|
11128
|
+
}
|
|
11129
|
+
async function resolveSearchContext(input, operations) {
|
|
11130
|
+
const explicitSymbol = input.symbol ?? void 0;
|
|
11131
|
+
const limit = input.limit ?? 10;
|
|
11132
|
+
const tokenBudget = input.tokenBudget ?? void 0;
|
|
11133
|
+
const lookupSymbol = explicitSymbol ?? inferExactSymbolFromQuery(input.query);
|
|
11134
|
+
if (lookupSymbol) {
|
|
11135
|
+
const definitions = await operations.lookup(lookupSymbol, MAX_CONTEXT_RESULT_LIMIT);
|
|
11136
|
+
if (definitions.length > 0) {
|
|
11137
|
+
return packedResult("definition", lookupSymbol, buildContextPack(definitions, {
|
|
11138
|
+
tokenBudget,
|
|
11139
|
+
maxResults: limit,
|
|
11140
|
+
heading: `Definition evidence for ${JSON.stringify(lookupSymbol)}`
|
|
11141
|
+
}));
|
|
11142
|
+
}
|
|
11143
|
+
if (explicitSymbol) {
|
|
11144
|
+
const fitted = fitTextToContextBudget(
|
|
11145
|
+
formatDefinitionLookup(definitions, lookupSymbol),
|
|
11146
|
+
tokenBudget
|
|
11147
|
+
);
|
|
11148
|
+
return { text: fitted.text };
|
|
11149
|
+
}
|
|
11150
|
+
}
|
|
11151
|
+
const results = await operations.search(input.query, MAX_CONTEXT_RESULT_LIMIT);
|
|
11152
|
+
if (results.length === 0) {
|
|
11153
|
+
const fitted = fitTextToContextBudget(
|
|
11154
|
+
"No matching code found. Try a different query or run index_codebase first.",
|
|
11155
|
+
tokenBudget
|
|
11156
|
+
);
|
|
11157
|
+
return { text: fitted.text };
|
|
11158
|
+
}
|
|
11159
|
+
return packedResult("conceptual", input.query, buildContextPack(results, {
|
|
11160
|
+
tokenBudget,
|
|
11161
|
+
maxResults: limit,
|
|
11162
|
+
heading: `Codebase evidence for ${JSON.stringify(input.query)}`
|
|
11163
|
+
}));
|
|
11164
|
+
}
|
|
11165
|
+
function fittedDetails(route, fitted) {
|
|
11166
|
+
return {
|
|
11167
|
+
route,
|
|
11168
|
+
tokenBudget: fitted.tokenBudget,
|
|
11169
|
+
tokenEstimate: fitted.tokenEstimate,
|
|
11170
|
+
truncated: fitted.truncated
|
|
11171
|
+
};
|
|
11172
|
+
}
|
|
11173
|
+
async function resolveCodebaseContext(projectRoot, host, input) {
|
|
11174
|
+
const from = input.from ?? void 0;
|
|
11175
|
+
const to = input.to ?? void 0;
|
|
11176
|
+
const symbol = input.symbol ?? void 0;
|
|
11177
|
+
const limit = input.limit ?? 10;
|
|
11178
|
+
const maxDepth = input.maxDepth ?? 10;
|
|
11179
|
+
const fileType = input.fileType ?? void 0;
|
|
11180
|
+
const directory = input.directory ?? void 0;
|
|
11181
|
+
const tokenBudget = input.tokenBudget ?? void 0;
|
|
11182
|
+
if (from && to) {
|
|
11183
|
+
const path27 = await getCallGraphPath(projectRoot, host, from, to, maxDepth);
|
|
11184
|
+
if (path27.length > 0) {
|
|
11185
|
+
const fitted2 = fitTextToContextBudget(
|
|
11186
|
+
formatCallGraphPath(from, to, path27),
|
|
11187
|
+
tokenBudget
|
|
11188
|
+
);
|
|
11144
11189
|
return {
|
|
11145
|
-
|
|
11146
|
-
|
|
11147
|
-
content: { type: "text", text: instruction }
|
|
11148
|
-
}]
|
|
11190
|
+
text: fitted2.text,
|
|
11191
|
+
details: fittedDetails("path", fitted2)
|
|
11149
11192
|
};
|
|
11150
11193
|
}
|
|
11151
|
-
|
|
11152
|
-
|
|
11153
|
-
|
|
11154
|
-
|
|
11155
|
-
|
|
11156
|
-
()
|
|
11157
|
-
|
|
11158
|
-
|
|
11159
|
-
|
|
11160
|
-
|
|
11161
|
-
|
|
11162
|
-
|
|
11163
|
-
|
|
11164
|
-
|
|
11165
|
-
|
|
11166
|
-
|
|
11167
|
-
|
|
11168
|
-
|
|
11169
|
-
|
|
11170
|
-
|
|
11171
|
-
|
|
11172
|
-
|
|
11173
|
-
|
|
11174
|
-
|
|
11175
|
-
|
|
11176
|
-
|
|
11177
|
-
|
|
11178
|
-
|
|
11179
|
-
|
|
11194
|
+
const { callers } = await getCallGraphData(projectRoot, host, {
|
|
11195
|
+
name: to,
|
|
11196
|
+
direction: "callers"
|
|
11197
|
+
});
|
|
11198
|
+
const directEdge = callers.find((edge) => edge.fromSymbolName === from);
|
|
11199
|
+
if (directEdge) {
|
|
11200
|
+
const location = directEdge.fromSymbolFilePath ? ` at ${directEdge.fromSymbolFilePath}:${directEdge.line}` : "";
|
|
11201
|
+
const fitted2 = fitTextToContextBudget(
|
|
11202
|
+
`Direct path: ${from} --${directEdge.callType}--> ${to}${location} (edge is ${directEdge.isResolved ? "resolved" : "unresolved"}).`,
|
|
11203
|
+
tokenBudget
|
|
11204
|
+
);
|
|
11205
|
+
return {
|
|
11206
|
+
text: fitted2.text,
|
|
11207
|
+
details: fittedDetails("direct-edge", fitted2)
|
|
11208
|
+
};
|
|
11209
|
+
}
|
|
11210
|
+
const fitted = fitTextToContextBudget(
|
|
11211
|
+
formatCallGraphPath(from, to, path27),
|
|
11212
|
+
tokenBudget
|
|
11213
|
+
);
|
|
11214
|
+
return {
|
|
11215
|
+
text: fitted.text,
|
|
11216
|
+
details: fittedDetails("path", fitted)
|
|
11217
|
+
};
|
|
11218
|
+
}
|
|
11219
|
+
return resolveSearchContext({ query: input.query, symbol, limit, tokenBudget }, {
|
|
11220
|
+
lookup: (lookupSymbol, retrievalLimit) => implementationLookup(projectRoot, host, lookupSymbol, {
|
|
11221
|
+
limit: retrievalLimit,
|
|
11222
|
+
fileType,
|
|
11223
|
+
directory
|
|
11224
|
+
}),
|
|
11225
|
+
search: (query, retrievalLimit) => searchCodebase(projectRoot, host, query, {
|
|
11226
|
+
limit: retrievalLimit,
|
|
11227
|
+
fileType,
|
|
11228
|
+
directory,
|
|
11229
|
+
metadataOnly: true
|
|
11180
11230
|
})
|
|
11181
|
-
);
|
|
11231
|
+
});
|
|
11182
11232
|
}
|
|
11183
11233
|
|
|
11184
|
-
// src/
|
|
11185
|
-
|
|
11186
|
-
|
|
11187
|
-
|
|
11188
|
-
|
|
11189
|
-
|
|
11190
|
-
|
|
11191
|
-
|
|
11192
|
-
|
|
11193
|
-
|
|
11194
|
-
}
|
|
11195
|
-
function formatIndexStats(stats, verbose = false) {
|
|
11196
|
-
if (stats.resetCorruptedIndex) {
|
|
11197
|
-
return stats.warning ?? "Detected a corrupted local index and reset it during indexing. Run index_codebase again to rebuild search data.";
|
|
11234
|
+
// src/eval/budget.ts
|
|
11235
|
+
function evaluateBudgetGate(budget, summary, comparison) {
|
|
11236
|
+
const BASELINE_P95_EPSILON_MS = 1e-3;
|
|
11237
|
+
const violations = [];
|
|
11238
|
+
const { thresholds } = budget;
|
|
11239
|
+
if (thresholds.minHitAt5 !== void 0 && summary.metrics.hitAt5 < thresholds.minHitAt5) {
|
|
11240
|
+
violations.push({
|
|
11241
|
+
metric: "minHitAt5",
|
|
11242
|
+
message: `Hit@5 ${summary.metrics.hitAt5.toFixed(4)} is below minimum ${thresholds.minHitAt5.toFixed(4)}`
|
|
11243
|
+
});
|
|
11198
11244
|
}
|
|
11199
|
-
|
|
11200
|
-
|
|
11201
|
-
|
|
11202
|
-
|
|
11203
|
-
|
|
11204
|
-
}
|
|
11205
|
-
lines.push("");
|
|
11245
|
+
if (thresholds.minMrrAt10 !== void 0 && summary.metrics.mrrAt10 < thresholds.minMrrAt10) {
|
|
11246
|
+
violations.push({
|
|
11247
|
+
metric: "minMrrAt10",
|
|
11248
|
+
message: `MRR@10 ${summary.metrics.mrrAt10.toFixed(4)} is below minimum ${thresholds.minMrrAt10.toFixed(4)}`
|
|
11249
|
+
});
|
|
11206
11250
|
}
|
|
11207
|
-
if (
|
|
11208
|
-
|
|
11209
|
-
|
|
11210
|
-
|
|
11211
|
-
|
|
11212
|
-
|
|
11213
|
-
|
|
11214
|
-
|
|
11251
|
+
if (thresholds.minRawDistinctTop3Ratio !== void 0 && summary.metrics.rawDistinctTop3Ratio < thresholds.minRawDistinctTop3Ratio) {
|
|
11252
|
+
violations.push({
|
|
11253
|
+
metric: "minRawDistinctTop3Ratio",
|
|
11254
|
+
message: `Raw Distinct Top@3 ${summary.metrics.rawDistinctTop3Ratio.toFixed(4)} is below minimum ${thresholds.minRawDistinctTop3Ratio.toFixed(4)}`
|
|
11255
|
+
});
|
|
11256
|
+
}
|
|
11257
|
+
if (comparison) {
|
|
11258
|
+
if (thresholds.hitAt5MaxDrop !== void 0 && comparison.deltas.hitAt5.absolute < -thresholds.hitAt5MaxDrop) {
|
|
11259
|
+
violations.push({
|
|
11260
|
+
metric: "hitAt5MaxDrop",
|
|
11261
|
+
message: `Hit@5 drop ${comparison.deltas.hitAt5.absolute.toFixed(4)} exceeds allowed -${thresholds.hitAt5MaxDrop.toFixed(4)}`
|
|
11262
|
+
});
|
|
11215
11263
|
}
|
|
11216
|
-
|
|
11217
|
-
|
|
11218
|
-
|
|
11264
|
+
if (thresholds.mrrAt10MaxDrop !== void 0 && comparison.deltas.mrrAt10.absolute < -thresholds.mrrAt10MaxDrop) {
|
|
11265
|
+
violations.push({
|
|
11266
|
+
metric: "mrrAt10MaxDrop",
|
|
11267
|
+
message: `MRR@10 drop ${comparison.deltas.mrrAt10.absolute.toFixed(4)} exceeds allowed -${thresholds.mrrAt10MaxDrop.toFixed(4)}`
|
|
11268
|
+
});
|
|
11219
11269
|
}
|
|
11220
|
-
if (
|
|
11221
|
-
|
|
11270
|
+
if (thresholds.rawDistinctTop3RatioMaxDrop !== void 0 && comparison.deltas.rawDistinctTop3Ratio.absolute < -thresholds.rawDistinctTop3RatioMaxDrop) {
|
|
11271
|
+
violations.push({
|
|
11272
|
+
metric: "rawDistinctTop3RatioMaxDrop",
|
|
11273
|
+
message: `Raw Distinct Top@3 drop ${comparison.deltas.rawDistinctTop3Ratio.absolute.toFixed(4)} exceeds allowed -${thresholds.rawDistinctTop3RatioMaxDrop.toFixed(4)}`
|
|
11274
|
+
});
|
|
11222
11275
|
}
|
|
11223
|
-
|
|
11224
|
-
|
|
11225
|
-
|
|
11226
|
-
|
|
11227
|
-
|
|
11228
|
-
|
|
11229
|
-
|
|
11230
|
-
|
|
11231
|
-
|
|
11232
|
-
|
|
11233
|
-
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
11234
|
-
}
|
|
11235
|
-
if (excluded.length > 0) {
|
|
11236
|
-
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
11237
|
-
}
|
|
11238
|
-
if (gitignored.length > 0) {
|
|
11239
|
-
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
11276
|
+
if (thresholds.p95LatencyMaxMultiplier !== void 0) {
|
|
11277
|
+
const baselineP95 = comparison.deltas.latencyP95Ms.baseline;
|
|
11278
|
+
if (baselineP95 > BASELINE_P95_EPSILON_MS) {
|
|
11279
|
+
const allowed = baselineP95 * thresholds.p95LatencyMaxMultiplier;
|
|
11280
|
+
if (summary.metrics.latencyMs.p95 > allowed) {
|
|
11281
|
+
violations.push({
|
|
11282
|
+
metric: "p95LatencyMaxMultiplier",
|
|
11283
|
+
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds allowed ${allowed.toFixed(3)}ms (${thresholds.p95LatencyMaxMultiplier.toFixed(2)}x baseline)`
|
|
11284
|
+
});
|
|
11285
|
+
}
|
|
11240
11286
|
}
|
|
11241
11287
|
}
|
|
11242
|
-
|
|
11243
|
-
|
|
11244
|
-
|
|
11288
|
+
}
|
|
11289
|
+
if (thresholds.p95LatencyMaxAbsoluteMs !== void 0 && summary.metrics.latencyMs.p95 > thresholds.p95LatencyMaxAbsoluteMs) {
|
|
11290
|
+
violations.push({
|
|
11291
|
+
metric: "p95LatencyMaxAbsoluteMs",
|
|
11292
|
+
message: `p95 latency ${summary.metrics.latencyMs.p95.toFixed(3)}ms exceeds absolute maximum ${thresholds.p95LatencyMaxAbsoluteMs.toFixed(3)}ms`
|
|
11293
|
+
});
|
|
11294
|
+
}
|
|
11295
|
+
const context = summary.metrics.contextEfficiency;
|
|
11296
|
+
if (context.queryCount > 0) {
|
|
11297
|
+
if (thresholds.maxContextResponseTokensAverage !== void 0 && context.responseTokens.average > thresholds.maxContextResponseTokensAverage) {
|
|
11298
|
+
violations.push({
|
|
11299
|
+
metric: "maxContextResponseTokensAverage",
|
|
11300
|
+
message: `Context response average ${context.responseTokens.average.toFixed(1)} tokens exceeds maximum ${thresholds.maxContextResponseTokensAverage.toFixed(1)}`
|
|
11301
|
+
});
|
|
11302
|
+
}
|
|
11303
|
+
if (thresholds.maxContextResponseTokensP95 !== void 0 && context.responseTokens.p95 > thresholds.maxContextResponseTokensP95) {
|
|
11304
|
+
violations.push({
|
|
11305
|
+
metric: "maxContextResponseTokensP95",
|
|
11306
|
+
message: `Context response p95 ${context.responseTokens.p95.toFixed(1)} tokens exceeds maximum ${thresholds.maxContextResponseTokensP95.toFixed(1)}`
|
|
11307
|
+
});
|
|
11308
|
+
}
|
|
11309
|
+
if (thresholds.maxContextResponseTokensMax !== void 0 && context.responseTokens.max > thresholds.maxContextResponseTokensMax) {
|
|
11310
|
+
violations.push({
|
|
11311
|
+
metric: "maxContextResponseTokensMax",
|
|
11312
|
+
message: `Context response maximum ${context.responseTokens.max.toFixed(1)} tokens exceeds maximum ${thresholds.maxContextResponseTokensMax.toFixed(1)}`
|
|
11313
|
+
});
|
|
11314
|
+
}
|
|
11315
|
+
if (thresholds.maxContextDuplicateCandidateRatio !== void 0 && context.duplicateCandidateRatio > thresholds.maxContextDuplicateCandidateRatio) {
|
|
11316
|
+
violations.push({
|
|
11317
|
+
metric: "maxContextDuplicateCandidateRatio",
|
|
11318
|
+
message: `Context duplicate candidate ratio ${context.duplicateCandidateRatio.toFixed(4)} exceeds maximum ${thresholds.maxContextDuplicateCandidateRatio.toFixed(4)}`
|
|
11319
|
+
});
|
|
11320
|
+
}
|
|
11321
|
+
if (thresholds.minContextSelectedFileRatio !== void 0 && context.selectedFileRatio < thresholds.minContextSelectedFileRatio) {
|
|
11322
|
+
violations.push({
|
|
11323
|
+
metric: "minContextSelectedFileRatio",
|
|
11324
|
+
message: `Context selected-file ratio ${context.selectedFileRatio.toFixed(4)} is below minimum ${thresholds.minContextSelectedFileRatio.toFixed(4)}`
|
|
11325
|
+
});
|
|
11326
|
+
}
|
|
11327
|
+
if (thresholds.minContextHitAt5Per1kResponseTokens !== void 0 && context.hitAt5Per1kResponseTokens < thresholds.minContextHitAt5Per1kResponseTokens) {
|
|
11328
|
+
violations.push({
|
|
11329
|
+
metric: "minContextHitAt5Per1kResponseTokens",
|
|
11330
|
+
message: `Context Hit@5 per 1k response tokens ${context.hitAt5Per1kResponseTokens.toFixed(4)} is below minimum ${thresholds.minContextHitAt5Per1kResponseTokens.toFixed(4)}`
|
|
11331
|
+
});
|
|
11332
|
+
}
|
|
11333
|
+
if (thresholds.minContextMrrAt10Per1kResponseTokens !== void 0 && context.mrrAt10Per1kResponseTokens < thresholds.minContextMrrAt10Per1kResponseTokens) {
|
|
11334
|
+
violations.push({
|
|
11335
|
+
metric: "minContextMrrAt10Per1kResponseTokens",
|
|
11336
|
+
message: `Context MRR@10 per 1k response tokens ${context.mrrAt10Per1kResponseTokens.toFixed(4)} is below minimum ${thresholds.minContextMrrAt10Per1kResponseTokens.toFixed(4)}`
|
|
11337
|
+
});
|
|
11245
11338
|
}
|
|
11246
11339
|
}
|
|
11247
|
-
return
|
|
11340
|
+
return {
|
|
11341
|
+
passed: violations.length === 0,
|
|
11342
|
+
budgetName: budget.name,
|
|
11343
|
+
violations
|
|
11344
|
+
};
|
|
11248
11345
|
}
|
|
11249
|
-
|
|
11250
|
-
|
|
11251
|
-
|
|
11252
|
-
|
|
11253
|
-
|
|
11254
|
-
|
|
11255
|
-
|
|
11256
|
-
|
|
11257
|
-
|
|
11258
|
-
|
|
11259
|
-
|
|
11260
|
-
|
|
11261
|
-
|
|
11262
|
-
|
|
11346
|
+
|
|
11347
|
+
// src/eval/metrics.ts
|
|
11348
|
+
function percentile(values, p) {
|
|
11349
|
+
if (values.length === 0) return 0;
|
|
11350
|
+
if (values.length === 1) return values[0];
|
|
11351
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
11352
|
+
const x = p * (sorted.length - 1);
|
|
11353
|
+
const lowerIndex = Math.floor(x);
|
|
11354
|
+
const upperIndex = Math.ceil(x);
|
|
11355
|
+
if (lowerIndex === upperIndex) {
|
|
11356
|
+
return sorted[lowerIndex];
|
|
11357
|
+
}
|
|
11358
|
+
const fraction = x - lowerIndex;
|
|
11359
|
+
return sorted[lowerIndex] + fraction * (sorted[upperIndex] - sorted[lowerIndex]);
|
|
11360
|
+
}
|
|
11361
|
+
function normalizePath(input) {
|
|
11362
|
+
return normalizePathSeparators(input);
|
|
11363
|
+
}
|
|
11364
|
+
function uniqueResultsByPath(results) {
|
|
11365
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11366
|
+
const unique = [];
|
|
11367
|
+
for (const result of results) {
|
|
11368
|
+
const normalized = normalizePath(result.filePath);
|
|
11369
|
+
if (seen.has(normalized)) continue;
|
|
11370
|
+
seen.add(normalized);
|
|
11371
|
+
unique.push(result);
|
|
11372
|
+
}
|
|
11373
|
+
return unique;
|
|
11374
|
+
}
|
|
11375
|
+
function distinctTopKRatio(results, k) {
|
|
11376
|
+
const top = results.slice(0, k);
|
|
11377
|
+
if (top.length === 0) return 0;
|
|
11378
|
+
const distinct = new Set(top.map((result) => normalizePath(result.filePath))).size;
|
|
11379
|
+
return distinct / top.length;
|
|
11380
|
+
}
|
|
11381
|
+
function pathMatchesExpected(actualPath, expectedPath) {
|
|
11382
|
+
const actual = normalizePath(actualPath);
|
|
11383
|
+
const expected = normalizePath(expectedPath);
|
|
11384
|
+
if (actual === expected) return true;
|
|
11385
|
+
return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);
|
|
11386
|
+
}
|
|
11387
|
+
function getRelevantPaths(query) {
|
|
11388
|
+
const fromExact = query.expected.filePath ? [query.expected.filePath] : [];
|
|
11389
|
+
const fromAcceptable = query.expected.acceptableFiles ?? [];
|
|
11390
|
+
return Array.from(/* @__PURE__ */ new Set([...fromExact, ...fromAcceptable]));
|
|
11391
|
+
}
|
|
11392
|
+
function isRelevantResult(filePath, relevantPaths) {
|
|
11393
|
+
return relevantPaths.some((expected) => pathMatchesExpected(filePath, expected));
|
|
11394
|
+
}
|
|
11395
|
+
function reciprocalRankAtK(results, relevantPaths, k) {
|
|
11396
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
11397
|
+
for (let i = 0; i < top.length; i += 1) {
|
|
11398
|
+
if (isRelevantResult(top[i].filePath, relevantPaths)) {
|
|
11399
|
+
return 1 / (i + 1);
|
|
11263
11400
|
}
|
|
11264
|
-
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
11265
11401
|
}
|
|
11266
|
-
|
|
11267
|
-
|
|
11268
|
-
|
|
11269
|
-
|
|
11270
|
-
|
|
11271
|
-
|
|
11272
|
-
|
|
11273
|
-
|
|
11274
|
-
|
|
11402
|
+
return 0;
|
|
11403
|
+
}
|
|
11404
|
+
function ndcgAtK(results, relevantPaths, k) {
|
|
11405
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
11406
|
+
const dcg = top.reduce((sum, result, i) => {
|
|
11407
|
+
const rel = isRelevantResult(result.filePath, relevantPaths) ? 1 : 0;
|
|
11408
|
+
return sum + rel / Math.log2(i + 2);
|
|
11409
|
+
}, 0);
|
|
11410
|
+
const idealLen = Math.min(k, relevantPaths.length);
|
|
11411
|
+
const idcg = Array.from({ length: idealLen }, (_, i) => 1 / Math.log2(i + 2)).reduce(
|
|
11412
|
+
(sum, value) => sum + value,
|
|
11413
|
+
0
|
|
11414
|
+
);
|
|
11415
|
+
return idcg === 0 ? 0 : dcg / idcg;
|
|
11416
|
+
}
|
|
11417
|
+
function isDocsOrTestsPath(filePath) {
|
|
11418
|
+
const lowered = normalizePath(filePath).toLowerCase();
|
|
11419
|
+
return lowered.includes("/docs/") || lowered.includes("/test/") || lowered.includes("/tests/") || lowered.includes("readme") || lowered.includes("/benchmarks/");
|
|
11420
|
+
}
|
|
11421
|
+
function classifyFailureBucket(query, results, k) {
|
|
11422
|
+
const relevantPaths = getRelevantPaths(query);
|
|
11423
|
+
const top = uniqueResultsByPath(results).slice(0, k);
|
|
11424
|
+
const hasRelevantTopK = top.some((result) => isRelevantResult(result.filePath, relevantPaths));
|
|
11425
|
+
if (!hasRelevantTopK) {
|
|
11426
|
+
return "no-relevant-hit-top-k";
|
|
11275
11427
|
}
|
|
11276
|
-
if (
|
|
11277
|
-
|
|
11278
|
-
|
|
11279
|
-
|
|
11280
|
-
|
|
11281
|
-
}
|
|
11428
|
+
if (query.expected.symbol) {
|
|
11429
|
+
const hasSymbol = top.some(
|
|
11430
|
+
(result) => isRelevantResult(result.filePath, relevantPaths) && result.name === query.expected.symbol
|
|
11431
|
+
);
|
|
11432
|
+
if (!hasSymbol) return "wrong-symbol";
|
|
11282
11433
|
}
|
|
11283
|
-
|
|
11284
|
-
|
|
11285
|
-
|
|
11434
|
+
const top1 = top[0];
|
|
11435
|
+
if (top1 && !isRelevantResult(top1.filePath, relevantPaths) && isDocsOrTestsPath(top1.filePath)) {
|
|
11436
|
+
return "docs-tests-outranking-source";
|
|
11286
11437
|
}
|
|
11287
|
-
if (
|
|
11288
|
-
|
|
11289
|
-
lines.push(`COMPATIBILITY WARNING: ${status.compatibility.reason}`);
|
|
11290
|
-
if (status.compatibility.storedMetadata) {
|
|
11291
|
-
const stored = status.compatibility.storedMetadata;
|
|
11292
|
-
lines.push(`Index was built with: ${stored.embeddingProvider}/${stored.embeddingModel} (${stored.embeddingDimensions}D)`);
|
|
11293
|
-
lines.push(`Current config: ${status.provider}/${status.model}`);
|
|
11294
|
-
}
|
|
11295
|
-
} else if (!status.compatibility) {
|
|
11296
|
-
lines.push(`Compatibility: No compatibility information found. Maybe the index is not initialized yet, try running index_codebase.`);
|
|
11297
|
-
} else {
|
|
11298
|
-
lines.push(`Compatibility: Index is compatible with the current provider and model.`);
|
|
11438
|
+
if (top1 && !isRelevantResult(top1.filePath, relevantPaths)) {
|
|
11439
|
+
return "wrong-file";
|
|
11299
11440
|
}
|
|
11300
|
-
return
|
|
11441
|
+
return void 0;
|
|
11301
11442
|
}
|
|
11302
|
-
function
|
|
11303
|
-
|
|
11304
|
-
|
|
11305
|
-
|
|
11306
|
-
|
|
11307
|
-
|
|
11308
|
-
|
|
11309
|
-
|
|
11310
|
-
|
|
11311
|
-
|
|
11312
|
-
|
|
11313
|
-
|
|
11314
|
-
|
|
11315
|
-
|
|
11316
|
-
|
|
11443
|
+
function buildPerQueryResult(query, results, latencyMs, k, route = {
|
|
11444
|
+
resolvedRoute: "search",
|
|
11445
|
+
routedQuery: query.query
|
|
11446
|
+
}, context) {
|
|
11447
|
+
const relevantPaths = getRelevantPaths(query);
|
|
11448
|
+
const deduped = uniqueResultsByPath(results);
|
|
11449
|
+
const hitAt = (cutoff) => deduped.slice(0, cutoff).some((result) => isRelevantResult(result.filePath, relevantPaths));
|
|
11450
|
+
const perQuery = {
|
|
11451
|
+
id: query.id,
|
|
11452
|
+
query: query.query,
|
|
11453
|
+
queryType: query.queryType,
|
|
11454
|
+
retrievalMode: query.retrievalMode ?? "search",
|
|
11455
|
+
resolvedRoute: route.resolvedRoute,
|
|
11456
|
+
routedQuery: route.routedQuery,
|
|
11457
|
+
latencyMs,
|
|
11458
|
+
hitAt1: hitAt(1),
|
|
11459
|
+
hitAt3: hitAt(3),
|
|
11460
|
+
hitAt5: hitAt(5),
|
|
11461
|
+
hitAt10: hitAt(10),
|
|
11462
|
+
reciprocalRankAt10: reciprocalRankAtK(deduped, relevantPaths, 10),
|
|
11463
|
+
ndcgAt10: ndcgAtK(deduped, relevantPaths, 10),
|
|
11464
|
+
failureBucket: classifyFailureBucket(query, results, k),
|
|
11465
|
+
rawTop3DistinctRatio: distinctTopKRatio(results, 3),
|
|
11466
|
+
tokenBudget: context?.tokenBudget,
|
|
11467
|
+
responseTokens: context?.responseTokens ?? 0,
|
|
11468
|
+
candidateCount: context?.candidateCount ?? results.length,
|
|
11469
|
+
deduplicatedCount: context?.deduplicatedCount ?? results.length,
|
|
11470
|
+
selectedCount: results.length,
|
|
11471
|
+
omittedCount: context?.omittedCount ?? 0,
|
|
11472
|
+
duplicateCandidateRatio: context && context.candidateCount > 0 ? (context.candidateCount - context.deduplicatedCount) / context.candidateCount : 0,
|
|
11473
|
+
selectedFileRatio: results.length === 0 ? 0 : new Set(results.map((result) => normalizePath(result.filePath))).size / results.length,
|
|
11474
|
+
results: deduped
|
|
11475
|
+
};
|
|
11476
|
+
return perQuery;
|
|
11317
11477
|
}
|
|
11318
|
-
function
|
|
11319
|
-
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11326
|
-
|
|
11327
|
-
|
|
11478
|
+
function computeEvalMetrics(queries, perQuery, embeddingCallCount, embeddingTokensUsed, costPer1MTokensUsd) {
|
|
11479
|
+
const count = perQuery.length;
|
|
11480
|
+
const safeDiv = (value) => count === 0 ? 0 : value / count;
|
|
11481
|
+
const sum = {
|
|
11482
|
+
hitAt1: 0,
|
|
11483
|
+
hitAt3: 0,
|
|
11484
|
+
hitAt5: 0,
|
|
11485
|
+
hitAt10: 0,
|
|
11486
|
+
mrrAt10: 0,
|
|
11487
|
+
ndcgAt10: 0,
|
|
11488
|
+
distinctTop3Ratio: 0,
|
|
11489
|
+
rawDistinctTop3Ratio: 0
|
|
11490
|
+
};
|
|
11491
|
+
const failureBuckets = {
|
|
11492
|
+
"wrong-file": 0,
|
|
11493
|
+
"wrong-symbol": 0,
|
|
11494
|
+
"docs-tests-outranking-source": 0,
|
|
11495
|
+
"no-relevant-hit-top-k": 0
|
|
11496
|
+
};
|
|
11497
|
+
const latencies = perQuery.map((item) => item.latencyMs);
|
|
11498
|
+
const contextQueries = perQuery.filter((item) => item.retrievalMode === "context");
|
|
11499
|
+
const contextResponseTokens = contextQueries.map((item) => item.responseTokens);
|
|
11500
|
+
const totalContextResponseTokens = contextResponseTokens.reduce((sum2, value) => sum2 + value, 0);
|
|
11501
|
+
const contextTokenUnits = totalContextResponseTokens / 1e3;
|
|
11502
|
+
for (const query of perQuery) {
|
|
11503
|
+
if (query.hitAt1) sum.hitAt1 += 1;
|
|
11504
|
+
if (query.hitAt3) sum.hitAt3 += 1;
|
|
11505
|
+
if (query.hitAt5) sum.hitAt5 += 1;
|
|
11506
|
+
if (query.hitAt10) sum.hitAt10 += 1;
|
|
11507
|
+
sum.mrrAt10 += query.reciprocalRankAt10;
|
|
11508
|
+
sum.ndcgAt10 += query.ndcgAt10;
|
|
11509
|
+
sum.distinctTop3Ratio += distinctTopKRatio(query.results, 3);
|
|
11510
|
+
sum.rawDistinctTop3Ratio += query.rawTop3DistinctRatio;
|
|
11511
|
+
if (query.failureBucket) {
|
|
11512
|
+
failureBuckets[query.failureBucket] += 1;
|
|
11513
|
+
}
|
|
11328
11514
|
}
|
|
11329
|
-
|
|
11330
|
-
return
|
|
11515
|
+
const queryTokens = queries.reduce((acc, q) => acc + estimateTokens(q.query), 0);
|
|
11516
|
+
return {
|
|
11517
|
+
hitAt1: safeDiv(sum.hitAt1),
|
|
11518
|
+
hitAt3: safeDiv(sum.hitAt3),
|
|
11519
|
+
hitAt5: safeDiv(sum.hitAt5),
|
|
11520
|
+
hitAt10: safeDiv(sum.hitAt10),
|
|
11521
|
+
mrrAt10: safeDiv(sum.mrrAt10),
|
|
11522
|
+
ndcgAt10: safeDiv(sum.ndcgAt10),
|
|
11523
|
+
distinctTop3Ratio: safeDiv(sum.distinctTop3Ratio),
|
|
11524
|
+
rawDistinctTop3Ratio: safeDiv(sum.rawDistinctTop3Ratio),
|
|
11525
|
+
latencyMs: {
|
|
11526
|
+
p50: percentile(latencies, 0.5),
|
|
11527
|
+
p95: percentile(latencies, 0.95),
|
|
11528
|
+
p99: percentile(latencies, 0.99)
|
|
11529
|
+
},
|
|
11530
|
+
tokenEstimate: {
|
|
11531
|
+
queryTokens,
|
|
11532
|
+
embeddingTokensUsed
|
|
11533
|
+
},
|
|
11534
|
+
embedding: {
|
|
11535
|
+
callCount: embeddingCallCount,
|
|
11536
|
+
estimatedCostUsd: embeddingTokensUsed / 1e6 * costPer1MTokensUsd,
|
|
11537
|
+
costPer1MTokensUsd
|
|
11538
|
+
},
|
|
11539
|
+
contextEfficiency: {
|
|
11540
|
+
queryCount: contextQueries.length,
|
|
11541
|
+
responseTokens: {
|
|
11542
|
+
total: totalContextResponseTokens,
|
|
11543
|
+
average: contextQueries.length === 0 ? 0 : totalContextResponseTokens / contextQueries.length,
|
|
11544
|
+
p95: percentile(contextResponseTokens, 0.95),
|
|
11545
|
+
max: contextResponseTokens.length === 0 ? 0 : Math.max(...contextResponseTokens)
|
|
11546
|
+
},
|
|
11547
|
+
duplicateCandidateRatio: contextQueries.length === 0 ? 0 : contextQueries.reduce((sum2, item) => sum2 + item.duplicateCandidateRatio, 0) / contextQueries.length,
|
|
11548
|
+
selectedFileRatio: contextQueries.length === 0 ? 0 : contextQueries.reduce((sum2, item) => sum2 + item.selectedFileRatio, 0) / contextQueries.length,
|
|
11549
|
+
hitAt5Per1kResponseTokens: contextTokenUnits === 0 ? 0 : contextQueries.filter((item) => item.hitAt5).length / contextTokenUnits,
|
|
11550
|
+
mrrAt10Per1kResponseTokens: contextTokenUnits === 0 ? 0 : contextQueries.reduce((sum2, item) => sum2 + item.reciprocalRankAt10, 0) / contextTokenUnits
|
|
11551
|
+
},
|
|
11552
|
+
failureBuckets
|
|
11553
|
+
};
|
|
11331
11554
|
}
|
|
11332
|
-
|
|
11333
|
-
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11339
|
-
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})${formatBlame(r)}`;
|
|
11340
|
-
});
|
|
11341
|
-
return formatted.join("\n");
|
|
11555
|
+
|
|
11556
|
+
// src/eval/runner-config.ts
|
|
11557
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync6, readFileSync as readFileSync9, rmSync as rmSync2, writeFileSync as writeFileSync6 } from "fs";
|
|
11558
|
+
import * as os5 from "os";
|
|
11559
|
+
import * as path18 from "path";
|
|
11560
|
+
function isRecord2(value) {
|
|
11561
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
11342
11562
|
}
|
|
11343
|
-
function
|
|
11344
|
-
|
|
11345
|
-
|
|
11563
|
+
function isStringArray3(value) {
|
|
11564
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
11565
|
+
}
|
|
11566
|
+
function validateEvalConfigShape(rawConfig, filePath) {
|
|
11567
|
+
if (!isRecord2(rawConfig)) {
|
|
11568
|
+
throw new Error(`Eval config at ${filePath} must contain a JSON object at the root.`);
|
|
11346
11569
|
}
|
|
11347
|
-
|
|
11348
|
-
|
|
11570
|
+
const config = rawConfig;
|
|
11571
|
+
if (config.knowledgeBases !== void 0 && !isStringArray3(config.knowledgeBases)) {
|
|
11572
|
+
throw new Error(`Eval config at ${filePath} field 'knowledgeBases' must be an array of strings.`);
|
|
11349
11573
|
}
|
|
11350
|
-
|
|
11351
|
-
|
|
11352
|
-
lines.push(`Removed stale entries: ${result.removed}`);
|
|
11574
|
+
if (config.additionalInclude !== void 0 && !isStringArray3(config.additionalInclude)) {
|
|
11575
|
+
throw new Error(`Eval config at ${filePath} field 'additionalInclude' must be an array of strings.`);
|
|
11353
11576
|
}
|
|
11354
|
-
if (
|
|
11355
|
-
|
|
11577
|
+
if (config.include !== void 0 && !isStringArray3(config.include)) {
|
|
11578
|
+
throw new Error(`Eval config at ${filePath} field 'include' must be an array of strings.`);
|
|
11356
11579
|
}
|
|
11357
|
-
if (
|
|
11358
|
-
|
|
11580
|
+
if (config.exclude !== void 0 && !isStringArray3(config.exclude)) {
|
|
11581
|
+
throw new Error(`Eval config at ${filePath} field 'exclude' must be an array of strings.`);
|
|
11359
11582
|
}
|
|
11360
|
-
|
|
11361
|
-
|
|
11583
|
+
for (const section of ["customProvider", "indexing", "search", "debug", "reranker"]) {
|
|
11584
|
+
const value = config[section];
|
|
11585
|
+
if (value !== void 0 && !isRecord2(value)) {
|
|
11586
|
+
throw new Error(`Eval config at ${filePath} field '${section}' must be an object.`);
|
|
11587
|
+
}
|
|
11362
11588
|
}
|
|
11363
|
-
|
|
11364
|
-
|
|
11589
|
+
return config;
|
|
11590
|
+
}
|
|
11591
|
+
function parseJsonConfigFile(filePath) {
|
|
11592
|
+
try {
|
|
11593
|
+
return validateEvalConfigShape(JSON.parse(readFileSync9(filePath, "utf-8")), filePath);
|
|
11594
|
+
} catch (error) {
|
|
11595
|
+
if (error instanceof Error && error.message.startsWith("Eval config at ")) {
|
|
11596
|
+
throw error;
|
|
11597
|
+
}
|
|
11598
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
11599
|
+
throw new Error(`Failed to parse eval config JSON at ${filePath}: ${message}`);
|
|
11365
11600
|
}
|
|
11366
|
-
|
|
11367
|
-
|
|
11601
|
+
}
|
|
11602
|
+
function toAbsolute(projectRoot, maybeRelative) {
|
|
11603
|
+
return path18.isAbsolute(maybeRelative) ? maybeRelative : path18.join(projectRoot, maybeRelative);
|
|
11604
|
+
}
|
|
11605
|
+
function isProjectScopedConfigPath(configPath) {
|
|
11606
|
+
return path18.basename(configPath) === "codebase-index.json" && path18.basename(path18.dirname(configPath)) === ".opencode";
|
|
11607
|
+
}
|
|
11608
|
+
function normalizeEvalConfigKnowledgeBases(rawConfig, projectRoot, resolvedConfigPath) {
|
|
11609
|
+
const config = rawConfig && typeof rawConfig === "object" ? { ...rawConfig } : {};
|
|
11610
|
+
const rebaseEntries = (values) => isProjectScopedConfigPath(resolvedConfigPath) ? resolveInheritedKnowledgeBaseEntries(
|
|
11611
|
+
values,
|
|
11612
|
+
path18.dirname(path18.dirname(resolvedConfigPath)),
|
|
11613
|
+
projectRoot
|
|
11614
|
+
) : rebasePathEntries(
|
|
11615
|
+
values,
|
|
11616
|
+
path18.dirname(resolvedConfigPath),
|
|
11617
|
+
projectRoot
|
|
11618
|
+
);
|
|
11619
|
+
if (Array.isArray(config.knowledgeBases)) {
|
|
11620
|
+
config.knowledgeBases = rebaseEntries(config.knowledgeBases);
|
|
11621
|
+
}
|
|
11622
|
+
if (Array.isArray(config.additionalInclude)) {
|
|
11623
|
+
config.additionalInclude = rebaseEntries(config.additionalInclude);
|
|
11368
11624
|
}
|
|
11369
|
-
return
|
|
11625
|
+
return config;
|
|
11370
11626
|
}
|
|
11371
|
-
function
|
|
11372
|
-
|
|
11373
|
-
|
|
11627
|
+
function loadRawConfig(projectRoot, configPath) {
|
|
11628
|
+
const fromPath = configPath ? toAbsolute(projectRoot, configPath) : null;
|
|
11629
|
+
if (fromPath && existsSync11(fromPath)) {
|
|
11630
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
11631
|
+
parseJsonConfigFile(fromPath),
|
|
11632
|
+
projectRoot,
|
|
11633
|
+
fromPath
|
|
11634
|
+
);
|
|
11374
11635
|
}
|
|
11375
|
-
const
|
|
11376
|
-
|
|
11377
|
-
return
|
|
11378
|
-
|
|
11379
|
-
|
|
11380
|
-
|
|
11381
|
-
|
|
11382
|
-
}
|
|
11383
|
-
function formatCallGraphCallees(symbolId, callees, relationshipType) {
|
|
11384
|
-
if (callees.length === 0) {
|
|
11385
|
-
return `No callees found for symbol ${symbolId}${relationshipType ? ` with type ${relationshipType}` : ""}. The function may not call any other tracked functions.`;
|
|
11636
|
+
const projectConfig = resolveProjectConfigPath(projectRoot);
|
|
11637
|
+
if (existsSync11(projectConfig)) {
|
|
11638
|
+
return normalizeEvalConfigKnowledgeBases(
|
|
11639
|
+
parseJsonConfigFile(projectConfig),
|
|
11640
|
+
projectRoot,
|
|
11641
|
+
projectConfig
|
|
11642
|
+
);
|
|
11386
11643
|
}
|
|
11387
|
-
|
|
11388
|
-
|
|
11389
|
-
return
|
|
11390
|
-
}).join("\n");
|
|
11391
|
-
}
|
|
11392
|
-
function formatCallGraphPath(from, to, path27) {
|
|
11393
|
-
if (path27.length === 0) {
|
|
11394
|
-
return `No path found between "${from}" and "${to}". They may be in disconnected components, or the call graph index needs updating.`;
|
|
11644
|
+
const globalConfig = path18.join(os5.homedir(), ".config", "opencode", "codebase-index.json");
|
|
11645
|
+
if (existsSync11(globalConfig)) {
|
|
11646
|
+
return parseJsonConfigFile(globalConfig);
|
|
11395
11647
|
}
|
|
11396
|
-
|
|
11397
|
-
const prefix = index === 0 ? "[start]" : `--${hop.callType}-->`;
|
|
11398
|
-
const location = hop.filePath ? ` (${hop.filePath}:${hop.line})` : "";
|
|
11399
|
-
return `${prefix} ${hop.symbolName}${location}`;
|
|
11400
|
-
});
|
|
11401
|
-
return `Path (${path27.length} hops):
|
|
11402
|
-
${formatted.join("\n")}`;
|
|
11648
|
+
return {};
|
|
11403
11649
|
}
|
|
11404
|
-
function
|
|
11405
|
-
return
|
|
11650
|
+
function getIndexRootPath(projectRoot, scope) {
|
|
11651
|
+
return scope === "global" ? getGlobalIndexPath() : resolveProjectIndexPath(projectRoot, scope);
|
|
11406
11652
|
}
|
|
11407
|
-
function
|
|
11408
|
-
|
|
11409
|
-
return "";
|
|
11410
|
-
}
|
|
11411
|
-
const date = new Date(result.blame.committedAt * 1e3).toISOString().slice(0, 10);
|
|
11412
|
-
return `
|
|
11413
|
-
${result.blame.sha.slice(0, 7)} | ${result.blame.author} | ${date} | ${result.blame.summary}`;
|
|
11653
|
+
function getLocalProjectIndexRoot(projectRoot) {
|
|
11654
|
+
return path18.join(projectRoot, ".opencode", "index");
|
|
11414
11655
|
}
|
|
11415
|
-
function
|
|
11416
|
-
|
|
11417
|
-
return `No definition found for "${query}". Try codebase_search for broader discovery, or verify the symbol name.`;
|
|
11418
|
-
}
|
|
11419
|
-
const formatted = results.map((r, idx) => {
|
|
11420
|
-
const header = formatResultHeader(r, idx);
|
|
11421
|
-
return `${header} (score: ${r.score.toFixed(2)})${formatBlame(r)}
|
|
11422
|
-
\`\`\`
|
|
11423
|
-
${truncateContent(r.content)}
|
|
11424
|
-
\`\`\``;
|
|
11425
|
-
});
|
|
11426
|
-
return formatted.join("\n\n");
|
|
11656
|
+
function getLocalProjectConfigPath(projectRoot) {
|
|
11657
|
+
return path18.join(projectRoot, ".opencode", "codebase-index.json");
|
|
11427
11658
|
}
|
|
11428
|
-
function
|
|
11429
|
-
const
|
|
11430
|
-
|
|
11431
|
-
|
|
11432
|
-
|
|
11433
|
-
\`\`\`
|
|
11434
|
-
${truncateContent(r.content)}
|
|
11435
|
-
\`\`\``;
|
|
11436
|
-
});
|
|
11437
|
-
return formatted.join("\n\n");
|
|
11659
|
+
function clearIndexRoot(projectRoot, scope) {
|
|
11660
|
+
const indexRoot = scope === "global" ? getIndexRootPath(projectRoot, scope) : getLocalProjectIndexRoot(projectRoot);
|
|
11661
|
+
if (existsSync11(indexRoot)) {
|
|
11662
|
+
rmSync2(indexRoot, { recursive: true, force: true });
|
|
11663
|
+
}
|
|
11438
11664
|
}
|
|
11439
|
-
|
|
11440
|
-
|
|
11441
|
-
|
|
11442
|
-
|
|
11443
|
-
|
|
11444
|
-
for (const file of result.changedFiles) {
|
|
11445
|
-
lines.push(` - ${file}`);
|
|
11665
|
+
function ensureLocalEvalProjectConfig(projectRoot, configPath) {
|
|
11666
|
+
const localConfigPath = getLocalProjectConfigPath(projectRoot);
|
|
11667
|
+
const resolvedConfigPath = configPath ? toAbsolute(projectRoot, configPath) : resolveProjectConfigPath(projectRoot);
|
|
11668
|
+
if (!configPath && existsSync11(localConfigPath)) {
|
|
11669
|
+
return localConfigPath;
|
|
11446
11670
|
}
|
|
11447
|
-
|
|
11448
|
-
|
|
11449
|
-
const directionLabel = result.direction === "callees" ? "callees" : result.direction === "both" ? "reachable (callers + callees)" : "callers";
|
|
11450
|
-
lines.push(
|
|
11451
|
-
`\u2192 Symbols affected: ${result.totalAffected} (${directCount} direct, ${transitiveCount} transitive ${directionLabel})`
|
|
11452
|
-
);
|
|
11453
|
-
if (directCount > 0) {
|
|
11454
|
-
lines.push(
|
|
11455
|
-
` Direct: ${result.directSymbols.map((s) => `${s.name} (${s.kind})`).join(", ")}`
|
|
11456
|
-
);
|
|
11671
|
+
if (!existsSync11(resolvedConfigPath) || resolvedConfigPath === localConfigPath) {
|
|
11672
|
+
return resolvedConfigPath;
|
|
11457
11673
|
}
|
|
11458
|
-
|
|
11459
|
-
|
|
11460
|
-
|
|
11461
|
-
|
|
11674
|
+
const sourceConfig = normalizeEvalConfigKnowledgeBases(
|
|
11675
|
+
parseJsonConfigFile(resolvedConfigPath),
|
|
11676
|
+
projectRoot,
|
|
11677
|
+
resolvedConfigPath
|
|
11678
|
+
);
|
|
11679
|
+
mkdirSync6(path18.dirname(localConfigPath), { recursive: true });
|
|
11680
|
+
writeFileSync6(localConfigPath, JSON.stringify(sourceConfig, null, 2), "utf-8");
|
|
11681
|
+
return localConfigPath;
|
|
11682
|
+
}
|
|
11683
|
+
function loadParsedConfig(projectRoot, configPath) {
|
|
11684
|
+
const raw = loadRawConfig(projectRoot, configPath);
|
|
11685
|
+
return parseConfig(raw);
|
|
11686
|
+
}
|
|
11687
|
+
function resolveSearchConfig(parsedConfig, overrides) {
|
|
11688
|
+
const nextSearch = {
|
|
11689
|
+
...parsedConfig.search
|
|
11690
|
+
};
|
|
11691
|
+
if (overrides?.fusionStrategy !== void 0) {
|
|
11692
|
+
nextSearch.fusionStrategy = overrides.fusionStrategy;
|
|
11462
11693
|
}
|
|
11463
|
-
if (
|
|
11464
|
-
|
|
11465
|
-
`\u2192 Communities touched: ${result.communities.map((c) => c.label).join(", ")}`
|
|
11466
|
-
);
|
|
11467
|
-
for (const community of result.communities) {
|
|
11468
|
-
lines.push(
|
|
11469
|
-
` - ${community.label}: ${community.symbolCount} symbols`
|
|
11470
|
-
);
|
|
11471
|
-
}
|
|
11472
|
-
} else {
|
|
11473
|
-
lines.push("\u2192 Communities touched: none");
|
|
11694
|
+
if (overrides?.hybridWeight !== void 0) {
|
|
11695
|
+
nextSearch.hybridWeight = overrides.hybridWeight;
|
|
11474
11696
|
}
|
|
11475
|
-
|
|
11476
|
-
|
|
11477
|
-
lines.push(" Hub nodes in change scope:");
|
|
11478
|
-
for (const hub of result.hubNodes) {
|
|
11479
|
-
lines.push(` - ${hub.name} (${hub.callerCount} callers) at ${hub.filePath}`);
|
|
11480
|
-
}
|
|
11697
|
+
if (overrides?.rrfK !== void 0) {
|
|
11698
|
+
nextSearch.rrfK = overrides.rrfK;
|
|
11481
11699
|
}
|
|
11482
|
-
if (
|
|
11483
|
-
|
|
11484
|
-
(c) => `PR #${c.pr} (also touches ${c.overlappingCommunities.join(", ")} community)`
|
|
11485
|
-
);
|
|
11486
|
-
lines.push(`\u2192 Potential conflicts with: ${conflictList.join(", ")}`);
|
|
11700
|
+
if (overrides?.rerankTopN !== void 0) {
|
|
11701
|
+
nextSearch.rerankTopN = overrides.rerankTopN;
|
|
11487
11702
|
}
|
|
11488
|
-
return
|
|
11703
|
+
return {
|
|
11704
|
+
...parsedConfig,
|
|
11705
|
+
search: nextSearch
|
|
11706
|
+
};
|
|
11707
|
+
}
|
|
11708
|
+
function getEmbeddingCostPer1MTokens(embeddingProvider) {
|
|
11709
|
+
return embeddingProvider === "custom" || embeddingProvider === "auto" ? 0 : getDefaultModelForProvider(embeddingProvider).costPer1MTokens;
|
|
11489
11710
|
}
|
|
11490
11711
|
|
|
11491
|
-
// src/
|
|
11492
|
-
import {
|
|
11493
|
-
|
|
11494
|
-
|
|
11495
|
-
|
|
11496
|
-
|
|
11497
|
-
|
|
11498
|
-
|
|
11499
|
-
|
|
11500
|
-
|
|
11501
|
-
|
|
11502
|
-
"reranker",
|
|
11503
|
-
"include",
|
|
11504
|
-
"exclude",
|
|
11505
|
-
"indexing",
|
|
11506
|
-
"search",
|
|
11507
|
-
"debug",
|
|
11508
|
-
"scope"
|
|
11509
|
-
];
|
|
11510
|
-
var MERGE_ARRAY_KEYS = ["knowledgeBases", "additionalInclude"];
|
|
11712
|
+
// src/eval/schema.ts
|
|
11713
|
+
import { readFileSync as readFileSync10 } from "fs";
|
|
11714
|
+
function parseJsonFile(filePath) {
|
|
11715
|
+
const content = readFileSync10(filePath, "utf-8");
|
|
11716
|
+
try {
|
|
11717
|
+
return JSON.parse(content);
|
|
11718
|
+
} catch (error) {
|
|
11719
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
11720
|
+
throw new Error(`Failed to parse JSON from ${filePath}: ${message}`);
|
|
11721
|
+
}
|
|
11722
|
+
}
|
|
11511
11723
|
function isRecord3(value) {
|
|
11512
|
-
return typeof value === "object" && value !== null
|
|
11724
|
+
return typeof value === "object" && value !== null;
|
|
11513
11725
|
}
|
|
11514
11726
|
function isStringArray4(value) {
|
|
11515
11727
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
11516
11728
|
}
|
|
11517
|
-
function
|
|
11518
|
-
if (
|
|
11519
|
-
|
|
11520
|
-
return;
|
|
11729
|
+
function asPositiveNumber(value, path27) {
|
|
11730
|
+
if (typeof value !== "number" || Number.isNaN(value) || value < 0) {
|
|
11731
|
+
throw new Error(`${path27} must be a non-negative number`);
|
|
11521
11732
|
}
|
|
11522
|
-
|
|
11523
|
-
|
|
11733
|
+
return value;
|
|
11734
|
+
}
|
|
11735
|
+
function parseQueryType(value, path27) {
|
|
11736
|
+
if (value === "definition" || value === "implementation-intent" || value === "similarity" || value === "keyword-heavy" || value === "conceptual") {
|
|
11737
|
+
return value;
|
|
11524
11738
|
}
|
|
11739
|
+
throw new Error(
|
|
11740
|
+
`${path27} must be one of: definition, implementation-intent, similarity, keyword-heavy, conceptual`
|
|
11741
|
+
);
|
|
11525
11742
|
}
|
|
11526
|
-
function
|
|
11527
|
-
|
|
11743
|
+
function parseRetrievalMode(value, path27) {
|
|
11744
|
+
if (value === void 0 || value === "search") return "search";
|
|
11745
|
+
if (value === "context") return value;
|
|
11746
|
+
throw new Error(`${path27} must be one of: search, context`);
|
|
11528
11747
|
}
|
|
11529
|
-
function
|
|
11530
|
-
|
|
11531
|
-
|
|
11532
|
-
while (normalized.length > root.length && /[\\/]$/.test(normalized)) {
|
|
11533
|
-
normalized = normalized.slice(0, -1);
|
|
11748
|
+
function parseExpected(input, path27) {
|
|
11749
|
+
if (!isRecord3(input)) {
|
|
11750
|
+
throw new Error(`${path27} must be an object`);
|
|
11534
11751
|
}
|
|
11535
|
-
|
|
11536
|
-
|
|
11537
|
-
|
|
11538
|
-
|
|
11752
|
+
const filePathRaw = input.filePath;
|
|
11753
|
+
const acceptableFilesRaw = input.acceptableFiles;
|
|
11754
|
+
const symbolRaw = input.symbol;
|
|
11755
|
+
const branchRaw = input.branch;
|
|
11756
|
+
const filePath = typeof filePathRaw === "string" ? filePathRaw : void 0;
|
|
11757
|
+
const acceptableFiles = isStringArray4(acceptableFilesRaw) ? acceptableFilesRaw : void 0;
|
|
11758
|
+
if (!filePath && (!acceptableFiles || acceptableFiles.length === 0)) {
|
|
11759
|
+
throw new Error(`${path27} must include either expected.filePath or expected.acceptableFiles`);
|
|
11760
|
+
}
|
|
11761
|
+
if (acceptableFilesRaw !== void 0 && !isStringArray4(acceptableFilesRaw)) {
|
|
11762
|
+
throw new Error(`${path27}.acceptableFiles must be an array of strings`);
|
|
11763
|
+
}
|
|
11764
|
+
if (symbolRaw !== void 0 && typeof symbolRaw !== "string") {
|
|
11765
|
+
throw new Error(`${path27}.symbol must be a string when provided`);
|
|
11766
|
+
}
|
|
11767
|
+
if (branchRaw !== void 0 && typeof branchRaw !== "string") {
|
|
11768
|
+
throw new Error(`${path27}.branch must be a string when provided`);
|
|
11769
|
+
}
|
|
11770
|
+
return {
|
|
11771
|
+
filePath,
|
|
11772
|
+
acceptableFiles,
|
|
11773
|
+
symbol: typeof symbolRaw === "string" ? symbolRaw : void 0,
|
|
11774
|
+
branch: typeof branchRaw === "string" ? branchRaw : void 0
|
|
11775
|
+
};
|
|
11539
11776
|
}
|
|
11540
|
-
function
|
|
11541
|
-
|
|
11542
|
-
|
|
11777
|
+
function parseQuery(input, index) {
|
|
11778
|
+
const path27 = `queries[${index}]`;
|
|
11779
|
+
if (!isRecord3(input)) {
|
|
11780
|
+
throw new Error(`${path27} must be an object`);
|
|
11781
|
+
}
|
|
11782
|
+
const id = input.id;
|
|
11783
|
+
const query = input.query;
|
|
11784
|
+
const queryType = input.queryType;
|
|
11785
|
+
const retrievalMode = input.retrievalMode;
|
|
11786
|
+
const expected = input.expected;
|
|
11787
|
+
if (typeof id !== "string" || id.trim().length === 0) {
|
|
11788
|
+
throw new Error(`${path27}.id must be a non-empty string`);
|
|
11789
|
+
}
|
|
11790
|
+
if (typeof query !== "string" || query.trim().length === 0) {
|
|
11791
|
+
throw new Error(`${path27}.query must be a non-empty string`);
|
|
11543
11792
|
}
|
|
11544
|
-
|
|
11545
|
-
|
|
11793
|
+
return {
|
|
11794
|
+
id,
|
|
11795
|
+
query,
|
|
11796
|
+
queryType: parseQueryType(queryType, `${path27}.queryType`),
|
|
11797
|
+
retrievalMode: parseRetrievalMode(retrievalMode, `${path27}.retrievalMode`),
|
|
11798
|
+
expected: parseExpected(expected, `${path27}.expected`)
|
|
11799
|
+
};
|
|
11800
|
+
}
|
|
11801
|
+
function parseGoldenDataset(raw, sourceLabel) {
|
|
11802
|
+
if (!isRecord3(raw)) {
|
|
11803
|
+
throw new Error(`${sourceLabel} must be a JSON object`);
|
|
11546
11804
|
}
|
|
11547
|
-
|
|
11548
|
-
|
|
11805
|
+
const version = raw.version;
|
|
11806
|
+
const name = raw.name;
|
|
11807
|
+
const description = raw.description;
|
|
11808
|
+
const queriesRaw = raw.queries;
|
|
11809
|
+
if (typeof version !== "string" || version.trim().length === 0) {
|
|
11810
|
+
throw new Error(`${sourceLabel}.version must be a non-empty string`);
|
|
11549
11811
|
}
|
|
11550
|
-
if (
|
|
11551
|
-
throw new Error(
|
|
11812
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
11813
|
+
throw new Error(`${sourceLabel}.name must be a non-empty string`);
|
|
11552
11814
|
}
|
|
11553
|
-
if (
|
|
11554
|
-
throw new Error(
|
|
11815
|
+
if (description !== void 0 && typeof description !== "string") {
|
|
11816
|
+
throw new Error(`${sourceLabel}.description must be a string when provided`);
|
|
11555
11817
|
}
|
|
11556
|
-
|
|
11557
|
-
|
|
11558
|
-
if (value !== void 0 && !isRecord3(value)) {
|
|
11559
|
-
throw new Error(`Config file ${filePath} field '${section}' must be an object.`);
|
|
11560
|
-
}
|
|
11818
|
+
if (!Array.isArray(queriesRaw)) {
|
|
11819
|
+
throw new Error(`${sourceLabel}.queries must be an array`);
|
|
11561
11820
|
}
|
|
11562
|
-
|
|
11563
|
-
}
|
|
11564
|
-
function loadJsonFile(filePath) {
|
|
11565
|
-
if (!existsSync10(filePath)) {
|
|
11566
|
-
return null;
|
|
11821
|
+
if (queriesRaw.length === 0) {
|
|
11822
|
+
throw new Error(`${sourceLabel}.queries must contain at least one query`);
|
|
11567
11823
|
}
|
|
11568
|
-
|
|
11569
|
-
|
|
11570
|
-
|
|
11571
|
-
|
|
11572
|
-
|
|
11573
|
-
throw error;
|
|
11824
|
+
const queries = queriesRaw.map((query, idx) => parseQuery(query, idx));
|
|
11825
|
+
const idSet = /* @__PURE__ */ new Set();
|
|
11826
|
+
for (const query of queries) {
|
|
11827
|
+
if (idSet.has(query.id)) {
|
|
11828
|
+
throw new Error(`${sourceLabel}.queries has duplicate id: ${query.id}`);
|
|
11574
11829
|
}
|
|
11575
|
-
|
|
11576
|
-
throw new Error(`Failed to load config file ${filePath}: ${message}`);
|
|
11830
|
+
idSet.add(query.id);
|
|
11577
11831
|
}
|
|
11832
|
+
return {
|
|
11833
|
+
version,
|
|
11834
|
+
name,
|
|
11835
|
+
description: typeof description === "string" ? description : void 0,
|
|
11836
|
+
queries
|
|
11837
|
+
};
|
|
11578
11838
|
}
|
|
11579
|
-
function
|
|
11580
|
-
|
|
11581
|
-
|
|
11582
|
-
function materializeLocalProjectConfig(projectRoot, config, host = "opencode") {
|
|
11583
|
-
const localConfigPath = resolveWritableProjectConfigPath(projectRoot, host);
|
|
11584
|
-
mkdirSync5(path18.dirname(localConfigPath), { recursive: true });
|
|
11585
|
-
writeFileSync5(localConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
11586
|
-
return localConfigPath;
|
|
11839
|
+
function loadGoldenDataset(datasetPath) {
|
|
11840
|
+
const parsed = parseJsonFile(datasetPath);
|
|
11841
|
+
return parseGoldenDataset(parsed, datasetPath);
|
|
11587
11842
|
}
|
|
11588
|
-
function
|
|
11589
|
-
|
|
11590
|
-
const projectConfig = loadJsonFile(projectConfigPath);
|
|
11591
|
-
if (!projectConfig) {
|
|
11592
|
-
return {};
|
|
11593
|
-
}
|
|
11594
|
-
const normalizedConfig = { ...projectConfig };
|
|
11595
|
-
const projectConfigBaseDir = path18.dirname(path18.dirname(projectConfigPath));
|
|
11596
|
-
if (Array.isArray(normalizedConfig.knowledgeBases)) {
|
|
11597
|
-
normalizedConfig.knowledgeBases = resolveInheritedKnowledgeBaseEntries(
|
|
11598
|
-
normalizedConfig.knowledgeBases,
|
|
11599
|
-
projectConfigBaseDir,
|
|
11600
|
-
projectRoot
|
|
11601
|
-
);
|
|
11602
|
-
}
|
|
11603
|
-
return normalizedConfig;
|
|
11843
|
+
function parseThresholdValue(value, fieldName, sourceLabel) {
|
|
11844
|
+
return value === void 0 ? void 0 : asPositiveNumber(value, `${sourceLabel}.thresholds.${fieldName}`);
|
|
11604
11845
|
}
|
|
11605
|
-
function
|
|
11606
|
-
|
|
11607
|
-
|
|
11608
|
-
let globalConfig = null;
|
|
11609
|
-
let globalConfigError = null;
|
|
11610
|
-
try {
|
|
11611
|
-
globalConfig = loadJsonFile(globalConfigPath);
|
|
11612
|
-
} catch (error) {
|
|
11613
|
-
globalConfigError = error instanceof Error ? error : new Error(String(error));
|
|
11614
|
-
}
|
|
11615
|
-
const projectConfig = loadJsonFile(projectConfigPath);
|
|
11616
|
-
const normalizedProjectConfig = loadProjectConfigLayer(projectRoot, host);
|
|
11617
|
-
if (globalConfigError) {
|
|
11618
|
-
if (!projectConfig) {
|
|
11619
|
-
throw globalConfigError;
|
|
11620
|
-
}
|
|
11621
|
-
globalConfig = null;
|
|
11622
|
-
}
|
|
11623
|
-
if (!globalConfig && !projectConfig) {
|
|
11624
|
-
return {};
|
|
11846
|
+
function parseBudget(raw, sourceLabel) {
|
|
11847
|
+
if (!isRecord3(raw)) {
|
|
11848
|
+
throw new Error(`${sourceLabel} must be a JSON object`);
|
|
11625
11849
|
}
|
|
11626
|
-
|
|
11627
|
-
|
|
11850
|
+
const name = raw.name;
|
|
11851
|
+
const baselinePath = raw.baselinePath;
|
|
11852
|
+
const failOnMissingBaseline = raw.failOnMissingBaseline;
|
|
11853
|
+
const thresholds = raw.thresholds;
|
|
11854
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
11855
|
+
throw new Error(`${sourceLabel}.name must be a non-empty string`);
|
|
11628
11856
|
}
|
|
11629
|
-
if (
|
|
11630
|
-
|
|
11857
|
+
if (baselinePath !== void 0 && typeof baselinePath !== "string") {
|
|
11858
|
+
throw new Error(`${sourceLabel}.baselinePath must be a string when provided`);
|
|
11631
11859
|
}
|
|
11632
|
-
if (!
|
|
11633
|
-
|
|
11860
|
+
if (!isRecord3(thresholds)) {
|
|
11861
|
+
throw new Error(`${sourceLabel}.thresholds must be an object`);
|
|
11634
11862
|
}
|
|
11635
|
-
|
|
11636
|
-
|
|
11637
|
-
|
|
11863
|
+
return {
|
|
11864
|
+
name,
|
|
11865
|
+
baselinePath: typeof baselinePath === "string" ? baselinePath : void 0,
|
|
11866
|
+
failOnMissingBaseline: typeof failOnMissingBaseline === "boolean" ? failOnMissingBaseline : true,
|
|
11867
|
+
thresholds: {
|
|
11868
|
+
hitAt5MaxDrop: parseThresholdValue(
|
|
11869
|
+
thresholds.hitAt5MaxDrop,
|
|
11870
|
+
"hitAt5MaxDrop",
|
|
11871
|
+
sourceLabel
|
|
11872
|
+
),
|
|
11873
|
+
mrrAt10MaxDrop: parseThresholdValue(
|
|
11874
|
+
thresholds.mrrAt10MaxDrop,
|
|
11875
|
+
"mrrAt10MaxDrop",
|
|
11876
|
+
sourceLabel
|
|
11877
|
+
),
|
|
11878
|
+
rawDistinctTop3RatioMaxDrop: parseThresholdValue(
|
|
11879
|
+
thresholds.rawDistinctTop3RatioMaxDrop,
|
|
11880
|
+
"rawDistinctTop3RatioMaxDrop",
|
|
11881
|
+
sourceLabel
|
|
11882
|
+
),
|
|
11883
|
+
p95LatencyMaxMultiplier: parseThresholdValue(
|
|
11884
|
+
thresholds.p95LatencyMaxMultiplier,
|
|
11885
|
+
"p95LatencyMaxMultiplier",
|
|
11886
|
+
sourceLabel
|
|
11887
|
+
),
|
|
11888
|
+
p95LatencyMaxAbsoluteMs: parseThresholdValue(
|
|
11889
|
+
thresholds.p95LatencyMaxAbsoluteMs,
|
|
11890
|
+
"p95LatencyMaxAbsoluteMs",
|
|
11891
|
+
sourceLabel
|
|
11892
|
+
),
|
|
11893
|
+
minHitAt5: parseThresholdValue(
|
|
11894
|
+
thresholds.minHitAt5,
|
|
11895
|
+
"minHitAt5",
|
|
11896
|
+
sourceLabel
|
|
11897
|
+
),
|
|
11898
|
+
minMrrAt10: parseThresholdValue(
|
|
11899
|
+
thresholds.minMrrAt10,
|
|
11900
|
+
"minMrrAt10",
|
|
11901
|
+
sourceLabel
|
|
11902
|
+
),
|
|
11903
|
+
minRawDistinctTop3Ratio: parseThresholdValue(
|
|
11904
|
+
thresholds.minRawDistinctTop3Ratio,
|
|
11905
|
+
"minRawDistinctTop3Ratio",
|
|
11906
|
+
sourceLabel
|
|
11907
|
+
),
|
|
11908
|
+
maxContextResponseTokensAverage: parseThresholdValue(
|
|
11909
|
+
thresholds.maxContextResponseTokensAverage,
|
|
11910
|
+
"maxContextResponseTokensAverage",
|
|
11911
|
+
sourceLabel
|
|
11912
|
+
),
|
|
11913
|
+
maxContextResponseTokensP95: parseThresholdValue(
|
|
11914
|
+
thresholds.maxContextResponseTokensP95,
|
|
11915
|
+
"maxContextResponseTokensP95",
|
|
11916
|
+
sourceLabel
|
|
11917
|
+
),
|
|
11918
|
+
maxContextResponseTokensMax: parseThresholdValue(
|
|
11919
|
+
thresholds.maxContextResponseTokensMax,
|
|
11920
|
+
"maxContextResponseTokensMax",
|
|
11921
|
+
sourceLabel
|
|
11922
|
+
),
|
|
11923
|
+
maxContextDuplicateCandidateRatio: parseThresholdValue(
|
|
11924
|
+
thresholds.maxContextDuplicateCandidateRatio,
|
|
11925
|
+
"maxContextDuplicateCandidateRatio",
|
|
11926
|
+
sourceLabel
|
|
11927
|
+
),
|
|
11928
|
+
minContextSelectedFileRatio: parseThresholdValue(
|
|
11929
|
+
thresholds.minContextSelectedFileRatio,
|
|
11930
|
+
"minContextSelectedFileRatio",
|
|
11931
|
+
sourceLabel
|
|
11932
|
+
),
|
|
11933
|
+
minContextHitAt5Per1kResponseTokens: parseThresholdValue(
|
|
11934
|
+
thresholds.minContextHitAt5Per1kResponseTokens,
|
|
11935
|
+
"minContextHitAt5Per1kResponseTokens",
|
|
11936
|
+
sourceLabel
|
|
11937
|
+
),
|
|
11938
|
+
minContextMrrAt10Per1kResponseTokens: parseThresholdValue(
|
|
11939
|
+
thresholds.minContextMrrAt10Per1kResponseTokens,
|
|
11940
|
+
"minContextMrrAt10Per1kResponseTokens",
|
|
11941
|
+
sourceLabel
|
|
11942
|
+
)
|
|
11943
|
+
}
|
|
11944
|
+
};
|
|
11945
|
+
}
|
|
11946
|
+
function loadBudget(budgetPath) {
|
|
11947
|
+
const parsed = parseJsonFile(budgetPath);
|
|
11948
|
+
return parseBudget(parsed, budgetPath);
|
|
11949
|
+
}
|
|
11950
|
+
|
|
11951
|
+
// src/eval/runner.ts
|
|
11952
|
+
async function runEvaluation(options) {
|
|
11953
|
+
const datasetPath = toAbsolute(options.projectRoot, options.datasetPath);
|
|
11954
|
+
const againstPath = options.againstPath ? toAbsolute(options.projectRoot, options.againstPath) : void 0;
|
|
11955
|
+
const budgetPath = options.budgetPath ? toAbsolute(options.projectRoot, options.budgetPath) : void 0;
|
|
11956
|
+
const dataset = loadGoldenDataset(datasetPath);
|
|
11957
|
+
const resolvedEvalConfigPath = options.reindex ? ensureLocalEvalProjectConfig(options.projectRoot, options.configPath) : options.configPath;
|
|
11958
|
+
const parsedConfig = loadParsedConfig(options.projectRoot, resolvedEvalConfigPath);
|
|
11959
|
+
const effectiveConfig = resolveSearchConfig(parsedConfig, options.searchOverrides);
|
|
11960
|
+
if (options.reindex) {
|
|
11961
|
+
clearIndexRoot(options.projectRoot, effectiveConfig.scope);
|
|
11638
11962
|
}
|
|
11639
|
-
|
|
11640
|
-
|
|
11641
|
-
|
|
11642
|
-
|
|
11963
|
+
const indexer = new Indexer(options.projectRoot, effectiveConfig);
|
|
11964
|
+
try {
|
|
11965
|
+
await indexer.index();
|
|
11966
|
+
const perQuery = [];
|
|
11967
|
+
for (const query of dataset.queries) {
|
|
11968
|
+
if (query.expected.branch && query.expected.branch !== indexer.getCurrentBranch()) {
|
|
11969
|
+
throw new Error(
|
|
11970
|
+
`Query '${query.id}' expects branch '${query.expected.branch}', but current branch is '${indexer.getCurrentBranch()}'. Switch branch before running this dataset.`
|
|
11971
|
+
);
|
|
11972
|
+
}
|
|
11973
|
+
const start = performance3.now();
|
|
11974
|
+
const contextResult = query.retrievalMode === "context" ? await resolveSearchContext({
|
|
11975
|
+
query: query.query,
|
|
11976
|
+
limit: 10,
|
|
11977
|
+
tokenBudget: DEFAULT_CONTEXT_PACK_TOKEN_BUDGET
|
|
11978
|
+
}, {
|
|
11979
|
+
lookup: (symbol, limit) => indexer.search(symbol, limit, {
|
|
11980
|
+
metadataOnly: true,
|
|
11981
|
+
filterByBranch: !!query.expected.branch,
|
|
11982
|
+
definitionIntent: true
|
|
11983
|
+
}),
|
|
11984
|
+
search: (searchQuery, limit) => indexer.search(searchQuery, limit, {
|
|
11985
|
+
metadataOnly: true,
|
|
11986
|
+
filterByBranch: !!query.expected.branch,
|
|
11987
|
+
definitionIntent: false
|
|
11988
|
+
})
|
|
11989
|
+
}) : void 0;
|
|
11990
|
+
const result = contextResult?.details?.results ?? await indexer.search(query.query, 10, {
|
|
11991
|
+
metadataOnly: true,
|
|
11992
|
+
filterByBranch: !!query.expected.branch
|
|
11993
|
+
});
|
|
11994
|
+
const elapsed = performance3.now() - start;
|
|
11995
|
+
const resolvedRoute = contextResult?.details?.route === "definition" ? "definition" : "search";
|
|
11996
|
+
const routedQuery = contextResult?.details?.routedQuery ?? query.query;
|
|
11997
|
+
const materialized = result.map((item) => ({
|
|
11998
|
+
filePath: item.filePath,
|
|
11999
|
+
startLine: item.startLine,
|
|
12000
|
+
endLine: item.endLine,
|
|
12001
|
+
score: item.score,
|
|
12002
|
+
chunkType: item.chunkType,
|
|
12003
|
+
name: item.name
|
|
12004
|
+
}));
|
|
12005
|
+
perQuery.push(buildPerQueryResult(query, materialized, elapsed, 10, {
|
|
12006
|
+
resolvedRoute,
|
|
12007
|
+
routedQuery
|
|
12008
|
+
}, contextResult?.details ? {
|
|
12009
|
+
tokenBudget: contextResult.details.tokenBudget,
|
|
12010
|
+
responseTokens: contextResult.details.tokenEstimate,
|
|
12011
|
+
candidateCount: contextResult.details.candidateCount ?? 0,
|
|
12012
|
+
deduplicatedCount: contextResult.details.deduplicatedCount ?? 0,
|
|
12013
|
+
omittedCount: contextResult.details.omittedCount ?? 0
|
|
12014
|
+
} : void 0));
|
|
12015
|
+
}
|
|
12016
|
+
const logger = indexer.getLogger();
|
|
12017
|
+
const metricSnapshot = logger.getMetrics();
|
|
12018
|
+
const costPer1MTokensUsd = getEmbeddingCostPer1MTokens(effectiveConfig.embeddingProvider);
|
|
12019
|
+
const summary = {
|
|
12020
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12021
|
+
projectRoot: options.projectRoot,
|
|
12022
|
+
datasetPath,
|
|
12023
|
+
datasetName: dataset.name,
|
|
12024
|
+
datasetVersion: dataset.version,
|
|
12025
|
+
queryCount: dataset.queries.length,
|
|
12026
|
+
topK: 10,
|
|
12027
|
+
searchConfig: {
|
|
12028
|
+
fusionStrategy: effectiveConfig.search.fusionStrategy,
|
|
12029
|
+
hybridWeight: effectiveConfig.search.hybridWeight,
|
|
12030
|
+
rrfK: effectiveConfig.search.rrfK,
|
|
12031
|
+
rerankTopN: effectiveConfig.search.rerankTopN
|
|
12032
|
+
},
|
|
12033
|
+
metrics: computeEvalMetrics(
|
|
12034
|
+
dataset.queries,
|
|
12035
|
+
perQuery,
|
|
12036
|
+
metricSnapshot.embeddingApiCalls,
|
|
12037
|
+
metricSnapshot.embeddingTokensUsed,
|
|
12038
|
+
costPer1MTokensUsd
|
|
12039
|
+
)
|
|
12040
|
+
};
|
|
12041
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
12042
|
+
const perQueryArtifact = buildPerQueryArtifact(perQuery);
|
|
12043
|
+
writeJson(path19.join(outputDir, "summary.json"), summary);
|
|
12044
|
+
writeJson(path19.join(outputDir, "per-query.json"), perQueryArtifact);
|
|
12045
|
+
let comparison;
|
|
12046
|
+
if (againstPath) {
|
|
12047
|
+
const baseline = loadSummary(againstPath);
|
|
12048
|
+
comparison = compareSummaries(summary, baseline, againstPath);
|
|
12049
|
+
writeJson(path19.join(outputDir, "compare.json"), comparison);
|
|
12050
|
+
}
|
|
12051
|
+
let gate;
|
|
12052
|
+
if (options.ciMode) {
|
|
12053
|
+
if (!budgetPath) {
|
|
12054
|
+
throw new Error("CI mode requires --budget path");
|
|
11643
12055
|
}
|
|
11644
|
-
|
|
12056
|
+
const budget = loadBudget(budgetPath);
|
|
12057
|
+
if (!comparison && budget.baselinePath) {
|
|
12058
|
+
const resolvedBaseline = toAbsolute(options.projectRoot, budget.baselinePath);
|
|
12059
|
+
if (existsSync12(resolvedBaseline)) {
|
|
12060
|
+
const baselineSummary = loadSummary(resolvedBaseline);
|
|
12061
|
+
comparison = compareSummaries(summary, baselineSummary, resolvedBaseline);
|
|
12062
|
+
writeJson(path19.join(outputDir, "compare.json"), comparison);
|
|
12063
|
+
} else if (budget.failOnMissingBaseline) {
|
|
12064
|
+
throw new Error(
|
|
12065
|
+
`Budget baseline is missing: ${resolvedBaseline}. Set failOnMissingBaseline=false to allow CI run without baseline.`
|
|
12066
|
+
);
|
|
12067
|
+
}
|
|
12068
|
+
}
|
|
12069
|
+
gate = evaluateBudgetGate(budget, summary, comparison);
|
|
11645
12070
|
}
|
|
12071
|
+
const markdown = createSummaryMarkdown(summary, comparison, gate);
|
|
12072
|
+
writeText(path19.join(outputDir, "summary.md"), markdown);
|
|
12073
|
+
return { outputDir, summary, perQuery, comparison, gate };
|
|
12074
|
+
} finally {
|
|
12075
|
+
await indexer.close();
|
|
11646
12076
|
}
|
|
11647
|
-
const globalKbs = globalConfig && Array.isArray(globalConfig.knowledgeBases) ? globalConfig.knowledgeBases : [];
|
|
11648
|
-
const projectKbs = projectConfig ? Array.isArray(normalizedProjectConfig.knowledgeBases) ? normalizedProjectConfig.knowledgeBases : [] : [];
|
|
11649
|
-
const allKbs = [...globalKbs, ...projectKbs];
|
|
11650
|
-
merged.knowledgeBases = mergeKnowledgeBasePaths(allKbs);
|
|
11651
|
-
const globalAdditional = globalConfig && Array.isArray(globalConfig.additionalInclude) ? globalConfig.additionalInclude : [];
|
|
11652
|
-
const projectAdditional = projectConfig && Array.isArray(projectConfig.additionalInclude) ? projectConfig.additionalInclude : [];
|
|
11653
|
-
const allAdditional = [...globalAdditional, ...projectAdditional];
|
|
11654
|
-
merged.additionalInclude = mergeUniqueStringArray(allAdditional);
|
|
11655
|
-
return merged;
|
|
11656
12077
|
}
|
|
11657
|
-
|
|
11658
|
-
|
|
11659
|
-
|
|
11660
|
-
|
|
11661
|
-
const
|
|
11662
|
-
|
|
11663
|
-
|
|
12078
|
+
async function runSweep(options, sweep) {
|
|
12079
|
+
const fusionValues = sweep.fusionStrategy && sweep.fusionStrategy.length > 0 ? [...sweep.fusionStrategy] : [void 0];
|
|
12080
|
+
const weightValues = sweep.hybridWeight && sweep.hybridWeight.length > 0 ? [...sweep.hybridWeight] : [void 0];
|
|
12081
|
+
const rrfValues = sweep.rrfK && sweep.rrfK.length > 0 ? [...sweep.rrfK] : [void 0];
|
|
12082
|
+
const rerankValues = sweep.rerankTopN && sweep.rerankTopN.length > 0 ? [...sweep.rerankTopN] : [void 0];
|
|
12083
|
+
const runs = [];
|
|
12084
|
+
for (const fusion of fusionValues) {
|
|
12085
|
+
for (const hybridWeight of weightValues) {
|
|
12086
|
+
for (const rrfK of rrfValues) {
|
|
12087
|
+
for (const rerankTopN of rerankValues) {
|
|
12088
|
+
const run = await runEvaluation({
|
|
12089
|
+
...options,
|
|
12090
|
+
searchOverrides: {
|
|
12091
|
+
...fusion !== void 0 ? { fusionStrategy: fusion } : {},
|
|
12092
|
+
...hybridWeight !== void 0 ? { hybridWeight } : {},
|
|
12093
|
+
...rrfK !== void 0 ? { rrfK } : {},
|
|
12094
|
+
...rerankTopN !== void 0 ? { rerankTopN } : {}
|
|
12095
|
+
}
|
|
12096
|
+
});
|
|
12097
|
+
runs.push({
|
|
12098
|
+
searchConfig: run.summary.searchConfig,
|
|
12099
|
+
summary: run.summary,
|
|
12100
|
+
comparison: run.comparison,
|
|
12101
|
+
gate: run.gate
|
|
12102
|
+
});
|
|
12103
|
+
}
|
|
12104
|
+
}
|
|
12105
|
+
}
|
|
11664
12106
|
}
|
|
11665
|
-
const
|
|
11666
|
-
|
|
12107
|
+
const bestByHitAt5 = [...runs].sort(
|
|
12108
|
+
(a, b) => b.summary.metrics.hitAt5 - a.summary.metrics.hitAt5
|
|
12109
|
+
)[0];
|
|
12110
|
+
const bestByMrrAt10 = [...runs].sort(
|
|
12111
|
+
(a, b) => b.summary.metrics.mrrAt10 - a.summary.metrics.mrrAt10
|
|
12112
|
+
)[0];
|
|
12113
|
+
const bestByP95Latency = [...runs].sort(
|
|
12114
|
+
(a, b) => a.summary.metrics.latencyMs.p95 - b.summary.metrics.latencyMs.p95
|
|
12115
|
+
)[0];
|
|
12116
|
+
const outputDir = createRunDirectory(toAbsolute(options.projectRoot, options.outputRoot));
|
|
12117
|
+
const failedGateRuns = runs.filter((run) => run.gate && !run.gate.passed).length;
|
|
12118
|
+
const gatePassed = failedGateRuns === 0;
|
|
12119
|
+
const aggregate = {
|
|
12120
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12121
|
+
againstPath: options.againstPath,
|
|
12122
|
+
runCount: runs.length,
|
|
12123
|
+
runs,
|
|
12124
|
+
gatePassed,
|
|
12125
|
+
failedGateRuns,
|
|
12126
|
+
bestByHitAt5,
|
|
12127
|
+
bestByMrrAt10,
|
|
12128
|
+
bestByP95Latency
|
|
12129
|
+
};
|
|
12130
|
+
writeJson(path19.join(outputDir, "compare.json"), aggregate);
|
|
12131
|
+
const md = createSummaryMarkdown(
|
|
12132
|
+
bestByHitAt5?.summary ?? runs[0].summary,
|
|
12133
|
+
bestByHitAt5?.comparison,
|
|
12134
|
+
void 0,
|
|
12135
|
+
aggregate
|
|
12136
|
+
);
|
|
12137
|
+
writeText(path19.join(outputDir, "summary.md"), md);
|
|
12138
|
+
writeJson(path19.join(outputDir, "summary.json"), bestByHitAt5?.summary ?? runs[0].summary);
|
|
12139
|
+
return { outputDir, aggregate };
|
|
11667
12140
|
}
|
|
11668
12141
|
|
|
11669
|
-
// src/
|
|
11670
|
-
import
|
|
12142
|
+
// src/eval/cli.ts
|
|
12143
|
+
import * as path21 from "path";
|
|
12144
|
+
|
|
12145
|
+
// src/eval/cli-parser.ts
|
|
11671
12146
|
import * as path20 from "path";
|
|
11672
|
-
function
|
|
11673
|
-
|
|
11674
|
-
|
|
11675
|
-
|
|
11676
|
-
|
|
11677
|
-
|
|
11678
|
-
|
|
11679
|
-
|
|
12147
|
+
function printUsage() {
|
|
12148
|
+
console.log(`
|
|
12149
|
+
Usage:
|
|
12150
|
+
opencode-codebase-index-mcp eval run [options]
|
|
12151
|
+
opencode-codebase-index-mcp eval compare --against <summary.json> [options]
|
|
12152
|
+
opencode-codebase-index-mcp eval diff --current <summary.json> --against <summary.json> [options]
|
|
12153
|
+
|
|
12154
|
+
Options:
|
|
12155
|
+
--project <path> Project root (default: cwd)
|
|
12156
|
+
--config <path> Config JSON path
|
|
12157
|
+
--dataset <path> Golden dataset path (default: benchmarks/golden/small.json)
|
|
12158
|
+
--current <path> Current summary.json path (required for eval diff)
|
|
12159
|
+
--output <path> Output root dir (default: benchmarks/results)
|
|
12160
|
+
--against <path> Baseline summary.json to compare against
|
|
12161
|
+
--budget <path> Budget file for CI mode (default: benchmarks/budgets/default.json)
|
|
12162
|
+
--ci Enable CI gate mode
|
|
12163
|
+
--reindex Force reindex before eval
|
|
12164
|
+
|
|
12165
|
+
Search overrides:
|
|
12166
|
+
--fusionStrategy <rrf|weighted>
|
|
12167
|
+
--hybridWeight <0-1>
|
|
12168
|
+
--rrfK <number>
|
|
12169
|
+
--rerankTopN <number>
|
|
12170
|
+
|
|
12171
|
+
Sweep options (comma-separated values):
|
|
12172
|
+
--sweepFusionStrategy <rrf,weighted>
|
|
12173
|
+
--sweepHybridWeight <0.3,0.5,0.7>
|
|
12174
|
+
--sweepRrfK <30,60,90>
|
|
12175
|
+
--sweepRerankTopN <10,20,40>
|
|
12176
|
+
`);
|
|
11680
12177
|
}
|
|
11681
|
-
function
|
|
11682
|
-
|
|
11683
|
-
|
|
12178
|
+
function parseNumber(value, flag) {
|
|
12179
|
+
const parsed = Number(value);
|
|
12180
|
+
if (Number.isNaN(parsed)) {
|
|
12181
|
+
throw new Error(`${flag} must be a number`);
|
|
11684
12182
|
}
|
|
11685
|
-
return
|
|
12183
|
+
return parsed;
|
|
11686
12184
|
}
|
|
11687
|
-
function
|
|
11688
|
-
return
|
|
12185
|
+
function parseCsvNumbers(value, flag) {
|
|
12186
|
+
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0).map((item) => parseNumber(item, flag));
|
|
11689
12187
|
}
|
|
11690
|
-
|
|
11691
|
-
|
|
11692
|
-
|
|
11693
|
-
|
|
11694
|
-
|
|
11695
|
-
|
|
11696
|
-
|
|
11697
|
-
|
|
11698
|
-
const ownerText = owner ? `PID ${owner.pid}, operation ${owner.operation}, since ${owner.startedAt}` : "unreadable owner";
|
|
11699
|
-
if (error.reason === "legacy-lock") {
|
|
11700
|
-
return {
|
|
11701
|
-
kind: "busy",
|
|
11702
|
-
text: `INDEX_BUSY: legacy lock format detected (${ownerText}). Verify the PID and remove this lock manually only if it is stale.`
|
|
11703
|
-
};
|
|
12188
|
+
function parseCsvFusion(value) {
|
|
12189
|
+
const values = value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
12190
|
+
const parsed = [];
|
|
12191
|
+
for (const candidate of values) {
|
|
12192
|
+
if (candidate !== "rrf" && candidate !== "weighted") {
|
|
12193
|
+
throw new Error("--sweepFusionStrategy accepts only rrf,weighted");
|
|
12194
|
+
}
|
|
12195
|
+
parsed.push(candidate);
|
|
11704
12196
|
}
|
|
11705
|
-
|
|
11706
|
-
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
12197
|
+
return parsed;
|
|
12198
|
+
}
|
|
12199
|
+
function hasSweepOptions(sweep) {
|
|
12200
|
+
return Boolean(
|
|
12201
|
+
sweep.fusionStrategy && sweep.fusionStrategy.length > 0 || sweep.hybridWeight && sweep.hybridWeight.length > 0 || sweep.rrfK && sweep.rrfK.length > 0 || sweep.rerankTopN && sweep.rerankTopN.length > 0
|
|
12202
|
+
);
|
|
12203
|
+
}
|
|
12204
|
+
function parseEvalArgs(argv, cwd) {
|
|
12205
|
+
const parsed = {
|
|
12206
|
+
projectRoot: cwd,
|
|
12207
|
+
datasetPath: "benchmarks/golden/small.json",
|
|
12208
|
+
outputRoot: "benchmarks/results",
|
|
12209
|
+
budgetPath: "benchmarks/budgets/default.json",
|
|
12210
|
+
ciMode: false,
|
|
12211
|
+
reindex: false,
|
|
12212
|
+
sweep: {}
|
|
12213
|
+
};
|
|
12214
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
12215
|
+
const arg = argv[i];
|
|
12216
|
+
const next = argv[i + 1];
|
|
12217
|
+
if (arg === "--project" && next) {
|
|
12218
|
+
parsed.projectRoot = path20.resolve(cwd, next);
|
|
12219
|
+
i += 1;
|
|
12220
|
+
continue;
|
|
12221
|
+
}
|
|
12222
|
+
if (arg === "--config" && next) {
|
|
12223
|
+
parsed.configPath = path20.resolve(cwd, next);
|
|
12224
|
+
i += 1;
|
|
12225
|
+
continue;
|
|
12226
|
+
}
|
|
12227
|
+
if (arg === "--dataset" && next) {
|
|
12228
|
+
parsed.datasetPath = next;
|
|
12229
|
+
i += 1;
|
|
12230
|
+
continue;
|
|
12231
|
+
}
|
|
12232
|
+
if (arg === "--current" && next) {
|
|
12233
|
+
parsed.currentPath = next;
|
|
12234
|
+
i += 1;
|
|
12235
|
+
continue;
|
|
12236
|
+
}
|
|
12237
|
+
if (arg === "--output" && next) {
|
|
12238
|
+
parsed.outputRoot = next;
|
|
12239
|
+
i += 1;
|
|
12240
|
+
continue;
|
|
12241
|
+
}
|
|
12242
|
+
if (arg === "--against" && next) {
|
|
12243
|
+
parsed.againstPath = next;
|
|
12244
|
+
i += 1;
|
|
12245
|
+
continue;
|
|
12246
|
+
}
|
|
12247
|
+
if (arg === "--budget" && next) {
|
|
12248
|
+
parsed.budgetPath = next;
|
|
12249
|
+
i += 1;
|
|
12250
|
+
continue;
|
|
12251
|
+
}
|
|
12252
|
+
if (arg === "--ci") {
|
|
12253
|
+
parsed.ciMode = true;
|
|
12254
|
+
continue;
|
|
12255
|
+
}
|
|
12256
|
+
if (arg === "--reindex") {
|
|
12257
|
+
parsed.reindex = true;
|
|
12258
|
+
continue;
|
|
12259
|
+
}
|
|
12260
|
+
if (arg === "--fusionStrategy" && next) {
|
|
12261
|
+
if (next !== "rrf" && next !== "weighted") {
|
|
12262
|
+
throw new Error("--fusionStrategy must be rrf or weighted");
|
|
12263
|
+
}
|
|
12264
|
+
parsed.fusionStrategy = next;
|
|
12265
|
+
i += 1;
|
|
12266
|
+
continue;
|
|
12267
|
+
}
|
|
12268
|
+
if (arg === "--hybridWeight" && next) {
|
|
12269
|
+
parsed.hybridWeight = parseNumber(next, "--hybridWeight");
|
|
12270
|
+
i += 1;
|
|
12271
|
+
continue;
|
|
12272
|
+
}
|
|
12273
|
+
if (arg === "--rrfK" && next) {
|
|
12274
|
+
parsed.rrfK = parseNumber(next, "--rrfK");
|
|
12275
|
+
i += 1;
|
|
12276
|
+
continue;
|
|
12277
|
+
}
|
|
12278
|
+
if (arg === "--rerankTopN" && next) {
|
|
12279
|
+
parsed.rerankTopN = parseNumber(next, "--rerankTopN");
|
|
12280
|
+
i += 1;
|
|
12281
|
+
continue;
|
|
12282
|
+
}
|
|
12283
|
+
if (arg === "--sweepFusionStrategy" && next) {
|
|
12284
|
+
parsed.sweep.fusionStrategy = parseCsvFusion(next);
|
|
12285
|
+
i += 1;
|
|
12286
|
+
continue;
|
|
12287
|
+
}
|
|
12288
|
+
if (arg === "--sweepHybridWeight" && next) {
|
|
12289
|
+
parsed.sweep.hybridWeight = parseCsvNumbers(next, "--sweepHybridWeight");
|
|
12290
|
+
i += 1;
|
|
12291
|
+
continue;
|
|
12292
|
+
}
|
|
12293
|
+
if (arg === "--sweepRrfK" && next) {
|
|
12294
|
+
parsed.sweep.rrfK = parseCsvNumbers(next, "--sweepRrfK");
|
|
12295
|
+
i += 1;
|
|
12296
|
+
continue;
|
|
12297
|
+
}
|
|
12298
|
+
if (arg === "--sweepRerankTopN" && next) {
|
|
12299
|
+
parsed.sweep.rerankTopN = parseCsvNumbers(next, "--sweepRerankTopN");
|
|
12300
|
+
i += 1;
|
|
12301
|
+
continue;
|
|
12302
|
+
}
|
|
11710
12303
|
}
|
|
11711
|
-
return
|
|
12304
|
+
return parsed;
|
|
11712
12305
|
}
|
|
11713
|
-
function
|
|
11714
|
-
|
|
11715
|
-
|
|
11716
|
-
|
|
11717
|
-
|
|
11718
|
-
|
|
11719
|
-
|
|
12306
|
+
function parseEvalSubcommandOptions(argv, cwd) {
|
|
12307
|
+
let explicitAgainst;
|
|
12308
|
+
const filtered = [];
|
|
12309
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
12310
|
+
const current = argv[i];
|
|
12311
|
+
const next = argv[i + 1];
|
|
12312
|
+
if (current === "--against" && next) {
|
|
12313
|
+
explicitAgainst = next;
|
|
12314
|
+
i += 1;
|
|
12315
|
+
continue;
|
|
12316
|
+
}
|
|
12317
|
+
filtered.push(current);
|
|
11720
12318
|
}
|
|
11721
|
-
return
|
|
12319
|
+
return {
|
|
12320
|
+
parsed: parseEvalArgs(filtered, cwd),
|
|
12321
|
+
explicitAgainst
|
|
12322
|
+
};
|
|
11722
12323
|
}
|
|
11723
|
-
function
|
|
11724
|
-
return
|
|
12324
|
+
function toRunOptions(parsed) {
|
|
12325
|
+
return {
|
|
12326
|
+
projectRoot: parsed.projectRoot,
|
|
12327
|
+
configPath: parsed.configPath,
|
|
12328
|
+
datasetPath: parsed.datasetPath,
|
|
12329
|
+
outputRoot: parsed.outputRoot,
|
|
12330
|
+
againstPath: parsed.againstPath,
|
|
12331
|
+
budgetPath: parsed.budgetPath,
|
|
12332
|
+
ciMode: parsed.ciMode,
|
|
12333
|
+
reindex: parsed.reindex,
|
|
12334
|
+
searchOverrides: {
|
|
12335
|
+
...parsed.fusionStrategy !== void 0 ? { fusionStrategy: parsed.fusionStrategy } : {},
|
|
12336
|
+
...parsed.hybridWeight !== void 0 ? { hybridWeight: parsed.hybridWeight } : {},
|
|
12337
|
+
...parsed.rrfK !== void 0 ? { rrfK: parsed.rrfK } : {},
|
|
12338
|
+
...parsed.rerankTopN !== void 0 ? { rerankTopN: parsed.rerankTopN } : {}
|
|
12339
|
+
}
|
|
12340
|
+
};
|
|
11725
12341
|
}
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
12342
|
+
|
|
12343
|
+
// src/eval/cli.ts
|
|
12344
|
+
async function handleEvalCommand(args, cwd) {
|
|
12345
|
+
const subcommand = args[0];
|
|
12346
|
+
if (!subcommand || subcommand === "--help" || subcommand === "-h") {
|
|
12347
|
+
printUsage();
|
|
12348
|
+
return 0;
|
|
11731
12349
|
}
|
|
11732
|
-
|
|
11733
|
-
|
|
11734
|
-
|
|
11735
|
-
|
|
11736
|
-
|
|
11737
|
-
|
|
11738
|
-
|
|
11739
|
-
|
|
11740
|
-
|
|
11741
|
-
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
}
|
|
11745
|
-
|
|
11746
|
-
|
|
11747
|
-
|
|
11748
|
-
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
11752
|
-
|
|
11753
|
-
|
|
11754
|
-
|
|
11755
|
-
|
|
11756
|
-
|
|
11757
|
-
|
|
11758
|
-
|
|
11759
|
-
|
|
12350
|
+
if (subcommand === "run") {
|
|
12351
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
12352
|
+
if (explicitAgainst) {
|
|
12353
|
+
parsed.againstPath = explicitAgainst;
|
|
12354
|
+
}
|
|
12355
|
+
const runOptions = toRunOptions(parsed);
|
|
12356
|
+
if (hasSweepOptions(parsed.sweep)) {
|
|
12357
|
+
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
12358
|
+
console.log(`Eval sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
12359
|
+
console.log(`Sweep runs: ${sweep.aggregate.runCount}`);
|
|
12360
|
+
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
12361
|
+
console.error(
|
|
12362
|
+
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
12363
|
+
);
|
|
12364
|
+
return 1;
|
|
12365
|
+
}
|
|
12366
|
+
return 0;
|
|
12367
|
+
}
|
|
12368
|
+
const result = await runEvaluation(runOptions);
|
|
12369
|
+
console.log(`Eval run complete. Artifacts: ${result.outputDir}`);
|
|
12370
|
+
console.log(
|
|
12371
|
+
`Hit@5=${(result.summary.metrics.hitAt5 * 100).toFixed(2)}% MRR@10=${result.summary.metrics.mrrAt10.toFixed(4)} p95=${result.summary.metrics.latencyMs.p95.toFixed(3)}ms`
|
|
12372
|
+
);
|
|
12373
|
+
if (result.gate && !result.gate.passed) {
|
|
12374
|
+
for (const violation of result.gate.violations) {
|
|
12375
|
+
console.error(`[CI-GATE] ${violation.metric}: ${violation.message}`);
|
|
12376
|
+
}
|
|
12377
|
+
return 1;
|
|
12378
|
+
}
|
|
12379
|
+
return 0;
|
|
11760
12380
|
}
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11766
|
-
return indexer.search(query, options.limit, {
|
|
11767
|
-
fileType: options.fileType,
|
|
11768
|
-
directory: options.directory,
|
|
11769
|
-
chunkType: options.chunkType,
|
|
11770
|
-
contextLines: options.contextLines,
|
|
11771
|
-
metadataOnly: options.metadataOnly,
|
|
11772
|
-
definitionIntent: options.definitionIntent,
|
|
11773
|
-
blameAuthor: options.blameAuthor,
|
|
11774
|
-
blameSha: options.blameSha,
|
|
11775
|
-
blameSince: options.blameSince
|
|
11776
|
-
});
|
|
11777
|
-
}
|
|
11778
|
-
async function findSimilarCode(projectRoot, host, code, options = {}) {
|
|
11779
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11780
|
-
return indexer.findSimilar(code, options.limit, {
|
|
11781
|
-
fileType: options.fileType,
|
|
11782
|
-
directory: options.directory,
|
|
11783
|
-
chunkType: options.chunkType,
|
|
11784
|
-
excludeFile: options.excludeFile
|
|
11785
|
-
});
|
|
11786
|
-
}
|
|
11787
|
-
async function implementationLookup(projectRoot, host, query, options = {}) {
|
|
11788
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11789
|
-
return indexer.search(query, options.limit, {
|
|
11790
|
-
fileType: options.fileType,
|
|
11791
|
-
directory: options.directory,
|
|
11792
|
-
definitionIntent: true
|
|
11793
|
-
});
|
|
11794
|
-
}
|
|
11795
|
-
async function getCallGraphData(projectRoot, host, params) {
|
|
11796
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11797
|
-
if (params.direction === "callees") {
|
|
11798
|
-
if (!params.symbolId) {
|
|
11799
|
-
return { direction: "callees", callees: [], callers: [] };
|
|
12381
|
+
if (subcommand === "compare") {
|
|
12382
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
12383
|
+
if (!explicitAgainst) {
|
|
12384
|
+
throw new Error("eval compare requires --against <baseline summary.json>");
|
|
11800
12385
|
}
|
|
11801
|
-
|
|
11802
|
-
|
|
12386
|
+
parsed.againstPath = explicitAgainst;
|
|
12387
|
+
const runOptions = toRunOptions(parsed);
|
|
12388
|
+
if (hasSweepOptions(parsed.sweep)) {
|
|
12389
|
+
const sweep = await runSweep(runOptions, parsed.sweep);
|
|
12390
|
+
console.log(`Eval compare sweep complete. Artifacts: ${sweep.outputDir}`);
|
|
12391
|
+
if (parsed.ciMode && sweep.aggregate.gatePassed === false) {
|
|
12392
|
+
console.error(
|
|
12393
|
+
`[CI-GATE] Sweep failed: ${sweep.aggregate.failedGateRuns ?? 0} run(s) violated budget/baseline gates`
|
|
12394
|
+
);
|
|
12395
|
+
return 1;
|
|
12396
|
+
}
|
|
12397
|
+
return 0;
|
|
12398
|
+
}
|
|
12399
|
+
const result = await runEvaluation(runOptions);
|
|
12400
|
+
console.log(`Eval compare complete. Artifacts: ${result.outputDir}`);
|
|
12401
|
+
return 0;
|
|
11803
12402
|
}
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11809
|
-
return indexer.findCallPath(from, to, maxDepth);
|
|
11810
|
-
}
|
|
11811
|
-
async function runIndexCodebase(projectRoot, host, args, onProgress) {
|
|
11812
|
-
const root = getProjectRoot(projectRoot, host);
|
|
11813
|
-
const key = getIndexerCacheKey(root, host);
|
|
11814
|
-
let indexer = getIndexerForProject(root, host);
|
|
11815
|
-
const runtimeConfig = configCache.get(key);
|
|
11816
|
-
try {
|
|
11817
|
-
if (args.estimateOnly) {
|
|
11818
|
-
return { kind: "estimate", estimate: await indexer.estimateCost() };
|
|
12403
|
+
if (subcommand === "diff") {
|
|
12404
|
+
const { parsed, explicitAgainst } = parseEvalSubcommandOptions(args.slice(1), cwd);
|
|
12405
|
+
if (!explicitAgainst) {
|
|
12406
|
+
throw new Error("eval diff requires --against <baseline summary.json>");
|
|
11819
12407
|
}
|
|
11820
|
-
|
|
11821
|
-
|
|
11822
|
-
|
|
11823
|
-
|
|
11824
|
-
|
|
11825
|
-
|
|
11826
|
-
|
|
11827
|
-
totalFiles: progress.totalFiles,
|
|
11828
|
-
chunksProcessed: progress.chunksProcessed,
|
|
11829
|
-
totalChunks: progress.totalChunks,
|
|
11830
|
-
percentage: calculatePercentage(progress)
|
|
11831
|
-
});
|
|
11832
|
-
}
|
|
11833
|
-
return Promise.resolve();
|
|
11834
|
-
});
|
|
11835
|
-
};
|
|
11836
|
-
let stats;
|
|
11837
|
-
if (args.force && shouldForceLocalizeProjectIndex(root, host)) {
|
|
11838
|
-
const inheritedIndexPath = resolveProjectIndexPath(root, runtimeConfig.scope, host);
|
|
11839
|
-
stats = await withIndexLock(inheritedIndexPath, "force-index", async () => {
|
|
11840
|
-
materializeLocalProjectConfig(root, loadProjectConfigLayer(root, host), host);
|
|
11841
|
-
refreshIndexerForDirectory(root, host, runtimeConfig);
|
|
11842
|
-
indexer = getIndexerForProject(root, host);
|
|
11843
|
-
return runIndex(indexer);
|
|
11844
|
-
}, { completeRecoveries: false });
|
|
11845
|
-
} else {
|
|
11846
|
-
stats = await runIndex(indexer);
|
|
12408
|
+
if (!parsed.currentPath) {
|
|
12409
|
+
throw new Error("eval diff requires --current <current summary.json>");
|
|
12410
|
+
}
|
|
12411
|
+
parsed.againstPath = explicitAgainst;
|
|
12412
|
+
const currentPath = parsed.currentPath;
|
|
12413
|
+
if (!currentPath.endsWith(".json")) {
|
|
12414
|
+
throw new Error("eval diff --current must point to a summary JSON file");
|
|
11847
12415
|
}
|
|
11848
|
-
|
|
11849
|
-
|
|
11850
|
-
|
|
11851
|
-
|
|
11852
|
-
|
|
11853
|
-
|
|
11854
|
-
|
|
11855
|
-
|
|
11856
|
-
|
|
11857
|
-
|
|
11858
|
-
|
|
11859
|
-
|
|
11860
|
-
|
|
11861
|
-
|
|
11862
|
-
|
|
11863
|
-
|
|
11864
|
-
|
|
11865
|
-
|
|
11866
|
-
|
|
11867
|
-
|
|
11868
|
-
|
|
11869
|
-
return busyResult;
|
|
12416
|
+
if (!parsed.againstPath.endsWith(".json")) {
|
|
12417
|
+
throw new Error("eval diff --against must point to a summary JSON file");
|
|
12418
|
+
}
|
|
12419
|
+
const currentSummary = loadSummary(path21.resolve(parsed.projectRoot, currentPath), {
|
|
12420
|
+
allowLegacyDiversityMetrics: true
|
|
12421
|
+
});
|
|
12422
|
+
const baselineSummary = loadSummary(path21.resolve(parsed.projectRoot, parsed.againstPath), {
|
|
12423
|
+
allowLegacyDiversityMetrics: true
|
|
12424
|
+
});
|
|
12425
|
+
const comparison = compareSummaries(
|
|
12426
|
+
currentSummary,
|
|
12427
|
+
baselineSummary,
|
|
12428
|
+
path21.resolve(parsed.projectRoot, parsed.againstPath)
|
|
12429
|
+
);
|
|
12430
|
+
const outputDir = createRunDirectory(path21.resolve(parsed.projectRoot, parsed.outputRoot));
|
|
12431
|
+
const summaryMd = createSummaryMarkdown(currentSummary, comparison);
|
|
12432
|
+
writeJson(path21.join(outputDir, "compare.json"), comparison);
|
|
12433
|
+
writeText(path21.join(outputDir, "summary.md"), summaryMd);
|
|
12434
|
+
writeJson(path21.join(outputDir, "summary.json"), currentSummary);
|
|
12435
|
+
console.log(`Eval diff complete. Artifacts: ${outputDir}`);
|
|
12436
|
+
return 0;
|
|
11870
12437
|
}
|
|
12438
|
+
throw new Error(`Unknown eval subcommand: ${subcommand}`);
|
|
11871
12439
|
}
|
|
11872
|
-
|
|
11873
|
-
|
|
11874
|
-
|
|
11875
|
-
|
|
11876
|
-
|
|
11877
|
-
|
|
11878
|
-
|
|
11879
|
-
|
|
11880
|
-
|
|
11881
|
-
|
|
12440
|
+
|
|
12441
|
+
// src/mcp-server.ts
|
|
12442
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
12443
|
+
import { readFileSync as readFileSync11 } from "fs";
|
|
12444
|
+
|
|
12445
|
+
// src/mcp-server/register-prompts.ts
|
|
12446
|
+
import { z } from "zod";
|
|
12447
|
+
function registerMcpPrompts(server) {
|
|
12448
|
+
server.prompt(
|
|
12449
|
+
"search",
|
|
12450
|
+
"Search codebase by meaning using semantic search",
|
|
12451
|
+
{ query: z.string().describe("What to search for in the codebase") },
|
|
12452
|
+
(args) => ({
|
|
12453
|
+
messages: [{
|
|
12454
|
+
role: "user",
|
|
12455
|
+
content: {
|
|
12456
|
+
type: "text",
|
|
12457
|
+
text: `Search the codebase for: "${args.query}"
|
|
12458
|
+
|
|
12459
|
+
Use the codebase_search tool with this query. If you need just locations first, use codebase_peek instead to save tokens.`
|
|
12460
|
+
}
|
|
12461
|
+
}]
|
|
12462
|
+
})
|
|
12463
|
+
);
|
|
12464
|
+
server.prompt(
|
|
12465
|
+
"find",
|
|
12466
|
+
"Find code using hybrid approach (semantic + grep)",
|
|
12467
|
+
{ query: z.string().describe("What to find in the codebase") },
|
|
12468
|
+
(args) => ({
|
|
12469
|
+
messages: [{
|
|
12470
|
+
role: "user",
|
|
12471
|
+
content: {
|
|
12472
|
+
type: "text",
|
|
12473
|
+
text: `Find code related to: "${args.query}"
|
|
12474
|
+
|
|
12475
|
+
Use a hybrid approach:
|
|
12476
|
+
1. First use codebase_peek to find semantic matches by meaning
|
|
12477
|
+
2. Then use grep for exact identifier matches
|
|
12478
|
+
3. Combine results for comprehensive coverage`
|
|
12479
|
+
}
|
|
12480
|
+
}]
|
|
12481
|
+
})
|
|
12482
|
+
);
|
|
12483
|
+
server.prompt(
|
|
12484
|
+
"index",
|
|
12485
|
+
"Index the codebase for semantic search",
|
|
12486
|
+
{ options: z.string().optional().describe("Options: 'force' to rebuild, 'estimate' to check costs") },
|
|
12487
|
+
(args) => {
|
|
12488
|
+
const opts = args.options?.toLowerCase() ?? "";
|
|
12489
|
+
let instruction = "Use the index_codebase tool to index the codebase for semantic search.";
|
|
12490
|
+
if (opts.includes("force")) {
|
|
12491
|
+
instruction = "Use the index_codebase tool with force=true to rebuild the entire index from scratch.";
|
|
12492
|
+
} else if (opts.includes("estimate")) {
|
|
12493
|
+
instruction = "Use the index_codebase tool with estimateOnly=true to check the cost estimate before indexing.";
|
|
12494
|
+
}
|
|
12495
|
+
return {
|
|
12496
|
+
messages: [{
|
|
12497
|
+
role: "user",
|
|
12498
|
+
content: { type: "text", text: instruction }
|
|
12499
|
+
}]
|
|
12500
|
+
};
|
|
12501
|
+
}
|
|
12502
|
+
);
|
|
12503
|
+
server.prompt(
|
|
12504
|
+
"status",
|
|
12505
|
+
"Check if the codebase is indexed and ready",
|
|
12506
|
+
{},
|
|
12507
|
+
() => ({
|
|
12508
|
+
messages: [{
|
|
12509
|
+
role: "user",
|
|
12510
|
+
content: {
|
|
12511
|
+
type: "text",
|
|
12512
|
+
text: "Use the index_status tool to check if the codebase index is ready and show its current state."
|
|
12513
|
+
}
|
|
12514
|
+
}]
|
|
12515
|
+
})
|
|
12516
|
+
);
|
|
12517
|
+
server.prompt(
|
|
12518
|
+
"definition",
|
|
12519
|
+
"Find where a symbol is defined in the codebase",
|
|
12520
|
+
{ query: z.string().describe("Symbol name or description to find the definition of") },
|
|
12521
|
+
(args) => ({
|
|
12522
|
+
messages: [{
|
|
12523
|
+
role: "user",
|
|
12524
|
+
content: {
|
|
12525
|
+
type: "text",
|
|
12526
|
+
text: `Find the definition of: "${args.query}"
|
|
12527
|
+
|
|
12528
|
+
Use the implementation_lookup tool to find where this symbol is defined. This prioritizes real implementation files over tests, docs, and examples. If no definition is found, fall back to codebase_search for broader discovery.`
|
|
12529
|
+
}
|
|
12530
|
+
}]
|
|
12531
|
+
})
|
|
12532
|
+
);
|
|
11882
12533
|
}
|
|
11883
|
-
|
|
11884
|
-
|
|
11885
|
-
|
|
11886
|
-
|
|
11887
|
-
|
|
11888
|
-
|
|
11889
|
-
|
|
11890
|
-
|
|
11891
|
-
|
|
12534
|
+
|
|
12535
|
+
// src/mcp-server/register-tools.ts
|
|
12536
|
+
import { z as z2 } from "zod";
|
|
12537
|
+
|
|
12538
|
+
// src/tools/format-pr-impact.ts
|
|
12539
|
+
function formatPrImpact(result) {
|
|
12540
|
+
const lines = [];
|
|
12541
|
+
lines.push(`\u2192 Files changed: ${result.changedFiles.length}`);
|
|
12542
|
+
for (const file of result.changedFiles) {
|
|
12543
|
+
lines.push(` - ${file}`);
|
|
11892
12544
|
}
|
|
11893
|
-
|
|
11894
|
-
|
|
11895
|
-
|
|
11896
|
-
|
|
11897
|
-
|
|
11898
|
-
|
|
12545
|
+
const directCount = result.directSymbols.length;
|
|
12546
|
+
const transitiveCount = result.transitiveCallers.length;
|
|
12547
|
+
const directionLabel = result.direction === "callees" ? "callees" : result.direction === "both" ? "reachable (callers + callees)" : "callers";
|
|
12548
|
+
lines.push(
|
|
12549
|
+
`\u2192 Symbols affected: ${result.totalAffected} (${directCount} direct, ${transitiveCount} transitive ${directionLabel})`
|
|
12550
|
+
);
|
|
12551
|
+
if (directCount > 0) {
|
|
12552
|
+
lines.push(
|
|
12553
|
+
` Direct: ${result.directSymbols.map((s) => `${s.name} (${s.kind})`).join(", ")}`
|
|
12554
|
+
);
|
|
11899
12555
|
}
|
|
11900
|
-
|
|
11901
|
-
|
|
11902
|
-
|
|
11903
|
-
|
|
11904
|
-
};
|
|
11905
|
-
}
|
|
11906
|
-
async function getIndexLogs(projectRoot, host, args) {
|
|
11907
|
-
const indexer = getIndexerForProject(projectRoot, host);
|
|
11908
|
-
const logger = indexer.getLogger();
|
|
11909
|
-
if (!logger.isEnabled()) {
|
|
11910
|
-
return {
|
|
11911
|
-
kind: "disabled",
|
|
11912
|
-
text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true\n }\n}\n```'
|
|
11913
|
-
};
|
|
12556
|
+
if (transitiveCount > 0) {
|
|
12557
|
+
lines.push(
|
|
12558
|
+
` Transitive ${directionLabel}: ${result.transitiveCallers.map((s) => s.name).join(", ")}`
|
|
12559
|
+
);
|
|
11914
12560
|
}
|
|
11915
|
-
|
|
11916
|
-
|
|
11917
|
-
|
|
11918
|
-
|
|
11919
|
-
|
|
12561
|
+
if (result.communities.length > 0) {
|
|
12562
|
+
lines.push(
|
|
12563
|
+
`\u2192 Communities touched: ${result.communities.map((c) => c.label).join(", ")}`
|
|
12564
|
+
);
|
|
12565
|
+
for (const community of result.communities) {
|
|
12566
|
+
lines.push(
|
|
12567
|
+
` - ${community.label}: ${community.symbolCount} symbols`
|
|
12568
|
+
);
|
|
12569
|
+
}
|
|
11920
12570
|
} else {
|
|
11921
|
-
|
|
12571
|
+
lines.push("\u2192 Communities touched: none");
|
|
11922
12572
|
}
|
|
11923
|
-
|
|
11924
|
-
|
|
11925
|
-
|
|
11926
|
-
|
|
11927
|
-
};
|
|
12573
|
+
lines.push(`\u2192 Risk: ${result.riskLevel} \u2014 ${result.riskReason}`);
|
|
12574
|
+
if (result.hubNodes.length > 0) {
|
|
12575
|
+
lines.push(" Hub nodes in change scope:");
|
|
12576
|
+
for (const hub of result.hubNodes) {
|
|
12577
|
+
lines.push(` - ${hub.name} (${hub.callerCount} callers) at ${hub.filePath}`);
|
|
12578
|
+
}
|
|
11928
12579
|
}
|
|
11929
|
-
|
|
11930
|
-
const
|
|
11931
|
-
|
|
11932
|
-
|
|
11933
|
-
|
|
12580
|
+
if (result.conflictingPRs && result.conflictingPRs.length > 0) {
|
|
12581
|
+
const conflictList = result.conflictingPRs.map(
|
|
12582
|
+
(c) => `PR #${c.pr} (also touches ${c.overlappingCommunities.join(", ")} community)`
|
|
12583
|
+
);
|
|
12584
|
+
lines.push(`\u2192 Potential conflicts with: ${conflictList.join(", ")}`);
|
|
12585
|
+
}
|
|
12586
|
+
return lines.join("\n");
|
|
11934
12587
|
}
|
|
11935
12588
|
|
|
11936
12589
|
// src/mcp-server/shared.ts
|
|
@@ -11955,66 +12608,27 @@ function allowNullAsUndefined(schema) {
|
|
|
11955
12608
|
function registerMcpTools(server, runtime) {
|
|
11956
12609
|
server.tool(
|
|
11957
12610
|
"codebase_context",
|
|
11958
|
-
"PREFERRED FIRST TOOL for any question about this repository. Use before built-in code search, grep, shell search, or broad file reads. Provide from+to for a dependency path, symbol for a definition, or only query for low-token conceptual discovery. Use call_graph directly for callers or callees.",
|
|
12611
|
+
"PREFERRED FIRST TOOL for any question about this repository. Returns a deduplicated, file-diverse evidence pack within tokenBudget. Use before built-in code search, grep, shell search, or broad file reads. Provide from+to for a dependency path, symbol for a definition, or only query for low-token conceptual discovery. Use call_graph directly for callers or callees.",
|
|
11959
12612
|
{
|
|
11960
12613
|
query: z2.string().describe("The codebase question or behavior to locate. Always provide the user's repository question here."),
|
|
11961
12614
|
from: allowNullAsUndefined(z2.string().optional()).describe("Source symbol. For dependency-path questions, extract the first endpoint and provide it here."),
|
|
11962
12615
|
to: allowNullAsUndefined(z2.string().optional()).describe("Target symbol. For dependency-path questions, extract the second endpoint and provide it here."),
|
|
11963
12616
|
symbol: allowNullAsUndefined(z2.string().optional()).describe("Exact symbol for an authoritative definition lookup. Omit when from and to are supplied."),
|
|
11964
|
-
limit: allowNullAsUndefined(
|
|
11965
|
-
|
|
12617
|
+
limit: allowNullAsUndefined(
|
|
12618
|
+
z2.number().int().min(MIN_CONTEXT_RESULT_LIMIT).max(MAX_CONTEXT_RESULT_LIMIT).optional().default(10)
|
|
12619
|
+
).describe(`Maximum number of search or definition results (${MIN_CONTEXT_RESULT_LIMIT}-${MAX_CONTEXT_RESULT_LIMIT})`),
|
|
12620
|
+
maxDepth: allowNullAsUndefined(
|
|
12621
|
+
z2.number().int().min(MIN_CONTEXT_PATH_DEPTH).max(MAX_CONTEXT_PATH_DEPTH).optional().default(10)
|
|
12622
|
+
).describe(`Maximum call-graph traversal depth for from/to path lookup (${MIN_CONTEXT_PATH_DEPTH}-${MAX_CONTEXT_PATH_DEPTH})`),
|
|
11966
12623
|
fileType: allowNullAsUndefined(z2.string().optional()).describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
11967
|
-
directory: allowNullAsUndefined(z2.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')")
|
|
12624
|
+
directory: allowNullAsUndefined(z2.string().optional()).describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
12625
|
+
tokenBudget: allowNullAsUndefined(
|
|
12626
|
+
z2.number().int().min(MIN_CONTEXT_PACK_TOKEN_BUDGET).max(MAX_CONTEXT_PACK_TOKEN_BUDGET).optional().default(DEFAULT_CONTEXT_PACK_TOKEN_BUDGET)
|
|
12627
|
+
).describe(`Maximum response tokens for this context pack (${MIN_CONTEXT_PACK_TOKEN_BUDGET}-${MAX_CONTEXT_PACK_TOKEN_BUDGET})`)
|
|
11968
12628
|
},
|
|
11969
12629
|
async (args) => {
|
|
11970
|
-
|
|
11971
|
-
|
|
11972
|
-
if (path27.length > 0) {
|
|
11973
|
-
return { content: [{ type: "text", text: formatCallGraphPath(args.from, args.to, path27) }] };
|
|
11974
|
-
}
|
|
11975
|
-
const { callers } = await getCallGraphData(runtime.projectRoot, runtime.host, {
|
|
11976
|
-
name: args.to,
|
|
11977
|
-
direction: "callers"
|
|
11978
|
-
});
|
|
11979
|
-
const directEdge = callers.find((edge) => edge.fromSymbolName === args.from);
|
|
11980
|
-
if (directEdge) {
|
|
11981
|
-
const location = directEdge.fromSymbolFilePath ? ` at ${directEdge.fromSymbolFilePath}:${directEdge.line}` : "";
|
|
11982
|
-
return {
|
|
11983
|
-
content: [{
|
|
11984
|
-
type: "text",
|
|
11985
|
-
text: `Direct path: ${args.from} --${directEdge.callType}--> ${args.to}${location} (edge is ${directEdge.isResolved ? "resolved" : "unresolved"}).`
|
|
11986
|
-
}]
|
|
11987
|
-
};
|
|
11988
|
-
}
|
|
11989
|
-
return { content: [{ type: "text", text: formatCallGraphPath(args.from, args.to, path27) }] };
|
|
11990
|
-
}
|
|
11991
|
-
if (args.symbol) {
|
|
11992
|
-
const results2 = await implementationLookup(runtime.projectRoot, runtime.host, args.symbol, {
|
|
11993
|
-
limit: args.limit ?? 10,
|
|
11994
|
-
fileType: args.fileType,
|
|
11995
|
-
directory: args.directory
|
|
11996
|
-
});
|
|
11997
|
-
return { content: [{ type: "text", text: formatDefinitionLookup(results2, args.symbol) }] };
|
|
11998
|
-
}
|
|
11999
|
-
const results = await searchCodebase(runtime.projectRoot, runtime.host, args.query, {
|
|
12000
|
-
limit: args.limit ?? 10,
|
|
12001
|
-
fileType: args.fileType,
|
|
12002
|
-
directory: args.directory,
|
|
12003
|
-
metadataOnly: true
|
|
12004
|
-
});
|
|
12005
|
-
if (results.length === 0) {
|
|
12006
|
-
return { content: [{ type: "text", text: "No matching code found. Try a different query or run index_codebase first." }] };
|
|
12007
|
-
}
|
|
12008
|
-
return {
|
|
12009
|
-
content: [{
|
|
12010
|
-
type: "text",
|
|
12011
|
-
text: `Found ${results.length} locations for "${args.query}":
|
|
12012
|
-
|
|
12013
|
-
${formatCodebasePeek(results)}
|
|
12014
|
-
|
|
12015
|
-
Use implementation_lookup for an authoritative definition, or call call_graph/call_graph_path once you have a symbol.`
|
|
12016
|
-
}]
|
|
12017
|
-
};
|
|
12630
|
+
const result = await resolveCodebaseContext(runtime.projectRoot, runtime.host, args);
|
|
12631
|
+
return { content: [{ type: "text", text: result.text }] };
|
|
12018
12632
|
}
|
|
12019
12633
|
);
|
|
12020
12634
|
server.tool(
|
|
@@ -12268,7 +12882,7 @@ ${formatSearchResults(results)}` }] };
|
|
|
12268
12882
|
// src/mcp-server.ts
|
|
12269
12883
|
function getServerInstructions(host) {
|
|
12270
12884
|
const hostText = `host ${host}`;
|
|
12271
|
-
return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns
|
|
12885
|
+
return `This MCP server is the preferred codebase-understanding path for ${hostText}. Start a repository task with index_status when index readiness or freshness is unknown. Use codebase_context as the preferred first entry point because it returns a token-budgeted location pack and routes to definitions or call-graph helpers when symbol intent is present. Keep the default tokenBudget for normal discovery, then use implementation_lookup, codebase_search, or a targeted file read only for selected locations that need source content. Use codebase_peek for direct conceptual location lookup. For exact identifiers or exhaustive matches, use grep. After identifying symbols, use call_graph or call_graph_path to trace dependencies. If the index is unavailable, run index_codebase, then retry the retrieval tool.`;
|
|
12272
12886
|
}
|
|
12273
12887
|
function getPackageVersion() {
|
|
12274
12888
|
const raw = JSON.parse(readFileSync11(new URL("../package.json", import.meta.url), "utf-8"));
|