codesearch-mcp 0.2.3-alpha → 0.3.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -1
- package/README.md +46 -26
- package/dist/app.js +5 -19
- package/dist/src/chunking/AstChunker.d.ts +12 -0
- package/dist/src/chunking/AstChunker.js +30 -0
- package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
- package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
- package/dist/src/chunking/FragmentMerger.d.ts +11 -0
- package/dist/src/chunking/FragmentMerger.js +61 -0
- package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
- package/dist/src/chunking/ImportOnlyFilter.js +14 -0
- package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
- package/dist/src/chunking/LangchainTextSplitter.js +133 -0
- package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
- package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
- package/dist/src/chunking/RunPacker.d.ts +12 -0
- package/dist/src/chunking/RunPacker.js +84 -0
- package/dist/src/chunking/create-chunker.d.ts +3 -0
- package/dist/src/chunking/create-chunker.js +15 -0
- package/dist/src/chunking/interfaces.d.ts +27 -0
- package/dist/src/chunking/interfaces.js +2 -0
- package/dist/src/commands/ConfigCommand.js +1 -9
- package/dist/src/commands/EvalCommand.d.ts +4 -3
- package/dist/src/commands/EvalCommand.js +85 -52
- package/dist/src/commands/IndexCommand.js +0 -22
- package/dist/src/commands/McpCommand.js +2 -8
- package/dist/src/commands/WebCommand.js +2 -5
- package/dist/src/config-error.d.ts +0 -4
- package/dist/src/config-error.js +0 -4
- package/dist/src/config-store.d.ts +1 -54
- package/dist/src/config-store.js +8 -69
- package/dist/src/config.d.ts +8 -17
- package/dist/src/config.js +35 -34
- package/dist/src/constants.d.ts +2 -28
- package/dist/src/constants.js +6 -28
- package/dist/src/container.d.ts +6 -0
- package/dist/src/container.js +12 -3
- package/dist/src/db/schema.d.ts +17 -0
- package/dist/src/db/schema.js +1 -6
- package/dist/src/declarations/DeclarationTree.d.ts +19 -0
- package/dist/src/declarations/DeclarationTree.js +66 -0
- package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
- package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
- package/dist/src/declarations/SourceLines.d.ts +37 -0
- package/dist/src/declarations/SourceLines.js +415 -0
- package/dist/src/dependency-traversal.d.ts +0 -6
- package/dist/src/dependency-traversal.js +0 -6
- package/dist/src/enriched-prompt.d.ts +0 -6
- package/dist/src/enriched-prompt.js +0 -6
- package/dist/src/error-record.d.ts +12 -0
- package/dist/src/error-record.js +248 -0
- package/dist/src/eval-corpus.d.ts +16 -0
- package/dist/src/eval-corpus.js +169 -0
- package/dist/src/eval-results.d.ts +6 -3
- package/dist/src/eval-results.js +309 -26
- package/dist/src/file-utils.d.ts +0 -5
- package/dist/src/file-utils.js +0 -5
- package/dist/src/graph.d.ts +0 -25
- package/dist/src/graph.js +0 -28
- package/dist/src/hnsw-settings.d.ts +3 -0
- package/dist/src/hnsw-settings.js +24 -0
- package/dist/src/interfaces/IAstParser.d.ts +2 -39
- package/dist/src/interfaces/IChunker.d.ts +3 -12
- package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
- package/dist/src/interfaces/ICorpusInspector.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +2 -22
- package/dist/src/interfaces/IEmbedder.d.ts +0 -9
- package/dist/src/interfaces/ILogger.d.ts +1 -11
- package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
- package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
- package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +134 -12
- package/dist/src/interfaces/ISearcher.d.ts +16 -5
- package/dist/src/interrupt.js +0 -2
- package/dist/src/owner-path.d.ts +9 -10
- package/dist/src/owner-path.js +11 -23
- package/dist/src/project-scope.d.ts +0 -12
- package/dist/src/project-scope.js +0 -12
- package/dist/src/project-selector.d.ts +0 -5
- package/dist/src/project-selector.js +0 -11
- package/dist/src/retrieval-eval.d.ts +14 -6
- package/dist/src/retrieval-eval.js +345 -59
- package/dist/src/search/result-shaping.d.ts +4 -0
- package/dist/src/search/result-shaping.js +19 -0
- package/dist/src/services/AstParserService.d.ts +2 -62
- package/dist/src/services/AstParserService.js +25 -137
- package/dist/src/services/DatabaseService.d.ts +2 -7
- package/dist/src/services/DatabaseService.js +42 -29
- package/dist/src/services/EmbedderFactory.d.ts +0 -4
- package/dist/src/services/EmbedderFactory.js +0 -5
- package/dist/src/services/IndexerService.d.ts +0 -16
- package/dist/src/services/IndexerService.js +16 -45
- package/dist/src/services/InfinityEmbedder.d.ts +0 -21
- package/dist/src/services/InfinityEmbedder.js +0 -15
- package/dist/src/services/LoggerService.d.ts +7 -5
- package/dist/src/services/LoggerService.js +37 -41
- package/dist/src/services/McpServer.d.ts +22 -14
- package/dist/src/services/McpServer.js +70 -37
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
- package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
- package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
- package/dist/src/services/NxMonorepoResolver.js +5 -56
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
- package/dist/src/services/OpenAICompatibleEmbedder.js +0 -14
- package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
- package/dist/src/services/ProjectDiscoveryService.js +0 -4
- package/dist/src/services/SearcherService.d.ts +9 -116
- package/dist/src/services/SearcherService.js +95 -176
- package/dist/src/services/VoyageEmbedder.d.ts +0 -6
- package/dist/src/services/VoyageEmbedder.js +0 -6
- package/dist/src/services/WebServerService.d.ts +1 -53
- package/dist/src/services/WebServerService.js +11 -57
- package/dist/src/services/http-retry.d.ts +0 -7
- package/dist/src/services/http-retry.js +1 -10
- package/dist/src/services/infinity-provisioning.d.ts +0 -74
- package/dist/src/services/infinity-provisioning.js +5 -88
- package/dist/src/services/legacy-ollama-index.d.ts +0 -16
- package/dist/src/services/legacy-ollama-index.js +0 -23
- package/dist/src/services/reranker.d.ts +0 -9
- package/dist/src/services/reranker.js +46 -12
- package/dist/src/setup.d.ts +11 -23
- package/dist/src/setup.js +98 -71
- package/dist/src/symbol-name.d.ts +0 -12
- package/dist/src/symbol-name.js +0 -13
- package/dist/src/symbol-rows.d.ts +0 -14
- package/dist/src/symbol-rows.js +0 -14
- package/dist/src/tool-call-log.d.ts +2 -0
- package/dist/src/tool-call-log.js +24 -0
- package/dist/src/tools/GetDependenciesTool.js +1 -1
- package/dist/src/tools/GetDependentsTool.js +1 -1
- package/dist/src/tools/SearchCodebaseTool.js +6 -3
- package/dist/src/tools/format-helpers.d.ts +0 -6
- package/dist/src/tools/format-helpers.js +87 -14
- package/dist/src/web/dashboard.d.ts +0 -8
- package/dist/src/web/dashboard.js +7 -10
- package/package.json +2 -2
- package/dist/src/services/AstChunker.d.ts +0 -42
- package/dist/src/services/AstChunker.js +0 -158
- package/docs/web-dashboard.md +0 -206
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RetrievalEvaluator = exports.EvalSuiteError = void 0;
|
|
3
|
+
exports.RetrievalEvaluator = exports.LIVE_GROUP = exports.EvalSuiteError = exports.STAGE_ORDER = void 0;
|
|
4
4
|
exports.parseSuite = parseSuite;
|
|
5
|
+
exports.corporaOf = corporaOf;
|
|
6
|
+
exports.isUnderRoot = isUnderRoot;
|
|
7
|
+
exports.checkCaseCorpora = checkCaseCorpora;
|
|
5
8
|
exports.mergeSuites = mergeSuites;
|
|
9
|
+
exports.signTest = signTest;
|
|
10
|
+
exports.pairedChange = pairedChange;
|
|
6
11
|
exports.compare = compare;
|
|
7
12
|
const zod_1 = require("zod");
|
|
8
|
-
|
|
13
|
+
const CATEGORIES = ['identifier', 'behaviour', 'concept'];
|
|
14
|
+
exports.STAGE_ORDER = ['vector', 'reranked'];
|
|
15
|
+
const STAGE_SOURCE = { reranked: 'vector' };
|
|
16
|
+
const UNRANKED = 11;
|
|
9
17
|
class EvalSuiteError extends Error {
|
|
10
18
|
caseId;
|
|
11
19
|
field;
|
|
@@ -29,11 +37,15 @@ const CaseSchema = zod_1.z.object({
|
|
|
29
37
|
expect: zod_1.z.array(ExpectedAnswerSchema).min(1),
|
|
30
38
|
threshold: zod_1.z.number().min(0).max(1).optional(),
|
|
31
39
|
});
|
|
40
|
+
const CorpusSchema = zod_1.z.object({
|
|
41
|
+
root: zod_1.z.string().min(1).refine(r => !r.startsWith('/') && !r.split('/').includes('..') && !r.endsWith('/'), 'must be a workspace-relative directory'),
|
|
42
|
+
commit: zod_1.z.string().regex(/^[0-9a-f]{40}$/, 'must be a full 40-hex commit SHA'),
|
|
43
|
+
}).strict();
|
|
32
44
|
const SuiteSchema = zod_1.z.object({
|
|
33
45
|
version: zod_1.z.literal(1),
|
|
46
|
+
corpus: CorpusSchema.optional(),
|
|
34
47
|
cases: zod_1.z.array(CaseSchema),
|
|
35
48
|
});
|
|
36
|
-
/** Best-effort id of the case at `cases[index]` in a raw (possibly invalid) object, for error messages. */
|
|
37
49
|
function rawCaseId(raw, index) {
|
|
38
50
|
if (typeof raw !== 'object' || raw === null)
|
|
39
51
|
return null;
|
|
@@ -46,7 +58,6 @@ function rawCaseId(raw, index) {
|
|
|
46
58
|
const id = entry.id;
|
|
47
59
|
return typeof id === 'string' ? id : null;
|
|
48
60
|
}
|
|
49
|
-
/** Validates a loaded suite object against the schema; rejects before any search runs. */
|
|
50
61
|
function parseSuite(raw) {
|
|
51
62
|
const result = SuiteSchema.safeParse(raw);
|
|
52
63
|
if (!result.success) {
|
|
@@ -64,10 +75,40 @@ function parseSuite(raw) {
|
|
|
64
75
|
}
|
|
65
76
|
seen.add(c.id);
|
|
66
77
|
}
|
|
67
|
-
|
|
78
|
+
const corpus = result.data.corpus?.root ?? null;
|
|
79
|
+
return { ...result.data, cases: result.data.cases.map(c => ({ ...c, corpus })) };
|
|
80
|
+
}
|
|
81
|
+
function corporaOf(suites) {
|
|
82
|
+
const byRoot = new Map();
|
|
83
|
+
for (const suite of suites) {
|
|
84
|
+
if (!suite.corpus)
|
|
85
|
+
continue;
|
|
86
|
+
const prev = byRoot.get(suite.corpus.root);
|
|
87
|
+
if (prev && prev.commit !== suite.corpus.commit) {
|
|
88
|
+
throw new EvalSuiteError(`Corpus ${suite.corpus.root} is declared at two commits: ${prev.commit} and ${suite.corpus.commit}`, null, 'corpus');
|
|
89
|
+
}
|
|
90
|
+
byRoot.set(suite.corpus.root, { root: suite.corpus.root, commit: suite.corpus.commit });
|
|
91
|
+
}
|
|
92
|
+
return [...byRoot.values()].sort((a, b) => (a.root < b.root ? -1 : a.root > b.root ? 1 : 0));
|
|
93
|
+
}
|
|
94
|
+
function isUnderRoot(dir, root) {
|
|
95
|
+
return dir === root || dir.startsWith(root + '/');
|
|
96
|
+
}
|
|
97
|
+
async function checkCaseCorpora(cases, projectDir) {
|
|
98
|
+
for (const c of cases) {
|
|
99
|
+
if (!c.corpus)
|
|
100
|
+
continue;
|
|
101
|
+
const dir = await projectDir(c.project);
|
|
102
|
+
if (dir === null) {
|
|
103
|
+
throw new EvalSuiteError(`Project ${c.project} is not indexed, so it cannot be checked against corpus ${c.corpus} (case ${c.id}, field project)`, c.id, 'project');
|
|
104
|
+
}
|
|
105
|
+
if (!isUnderRoot(dir, c.corpus)) {
|
|
106
|
+
throw new EvalSuiteError(`Project ${c.project} (${dir}) is outside corpus ${c.corpus} (case ${c.id}, field project)`, c.id, 'project');
|
|
107
|
+
}
|
|
108
|
+
}
|
|
68
109
|
}
|
|
69
|
-
/** Merges several suites (already parsed) into one; case ids must be unique across all of them. */
|
|
70
110
|
function mergeSuites(suites) {
|
|
111
|
+
corporaOf(suites);
|
|
71
112
|
const cases = [];
|
|
72
113
|
const seen = new Set();
|
|
73
114
|
for (const suite of suites) {
|
|
@@ -80,7 +121,6 @@ function mergeSuites(suites) {
|
|
|
80
121
|
}
|
|
81
122
|
return { version: 1, cases };
|
|
82
123
|
}
|
|
83
|
-
/** `owner` matches exactly, or as a prefix at a `.` boundary (`RepairsService` accepts any member of the class). */
|
|
84
124
|
function ownerMatches(ownerPath, expectedOwner) {
|
|
85
125
|
return ownerPath === expectedOwner || ownerPath.startsWith(expectedOwner + '.');
|
|
86
126
|
}
|
|
@@ -117,45 +157,209 @@ function rankOf(hits, valid) {
|
|
|
117
157
|
function emptyMetrics() {
|
|
118
158
|
return { count: 0, hit1: 0, hit5: 0, mrr10: 0 };
|
|
119
159
|
}
|
|
120
|
-
function accumulate(
|
|
121
|
-
const n =
|
|
160
|
+
function accumulate(ranks) {
|
|
161
|
+
const n = ranks.length;
|
|
122
162
|
if (n === 0)
|
|
123
163
|
return emptyMetrics();
|
|
124
164
|
let hit1 = 0, hit5 = 0, mrrSum = 0;
|
|
125
|
-
for (const
|
|
126
|
-
if (
|
|
165
|
+
for (const rank of ranks) {
|
|
166
|
+
if (rank === 1)
|
|
127
167
|
hit1++;
|
|
128
|
-
if (
|
|
168
|
+
if (rank !== null && rank <= 5)
|
|
129
169
|
hit5++;
|
|
130
|
-
mrrSum +=
|
|
170
|
+
mrrSum += rank !== null ? 1 / rank : 0;
|
|
131
171
|
}
|
|
132
172
|
return { count: n, hit1: hit1 / n, hit5: hit5 / n, mrr10: mrrSum / n };
|
|
133
173
|
}
|
|
174
|
+
function stageRank(c, stage) {
|
|
175
|
+
return c.stages[stage]?.rank ?? null;
|
|
176
|
+
}
|
|
177
|
+
function stageMetrics(cases, stage) {
|
|
178
|
+
const byCategory = {};
|
|
179
|
+
for (const category of CATEGORIES) {
|
|
180
|
+
const subset = cases.filter(c => c.category === category);
|
|
181
|
+
if (subset.length > 0)
|
|
182
|
+
byCategory[category] = accumulate(subset.map(c => stageRank(c, stage)));
|
|
183
|
+
}
|
|
184
|
+
return { overall: accumulate(cases.map(c => stageRank(c, stage))), byCategory };
|
|
185
|
+
}
|
|
186
|
+
function poolMetricsOf(cases) {
|
|
187
|
+
const count = cases.length;
|
|
188
|
+
return {
|
|
189
|
+
count,
|
|
190
|
+
recall: count === 0 ? 0 : cases.filter(c => c.poolRank !== null).length / count,
|
|
191
|
+
short: cases.filter(c => c.short).length,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
function binomialCdfHalf(k, n) {
|
|
195
|
+
let pmf = Math.pow(0.5, n);
|
|
196
|
+
let cdf = pmf;
|
|
197
|
+
for (let i = 1; i <= k; i++) {
|
|
198
|
+
pmf = pmf * (n - i + 1) / i;
|
|
199
|
+
cdf += pmf;
|
|
200
|
+
}
|
|
201
|
+
return cdf;
|
|
202
|
+
}
|
|
203
|
+
function signTest(improved, worsened) {
|
|
204
|
+
const n = improved + worsened;
|
|
205
|
+
if (n === 0)
|
|
206
|
+
return 1;
|
|
207
|
+
return Math.min(1, 2 * binomialCdfHalf(Math.min(improved, worsened), n));
|
|
208
|
+
}
|
|
209
|
+
function pairCounts(pairs) {
|
|
210
|
+
let improved = 0, worsened = 0, unchanged = 0;
|
|
211
|
+
for (const p of pairs) {
|
|
212
|
+
if (p.to < p.from)
|
|
213
|
+
improved++;
|
|
214
|
+
else if (p.to > p.from)
|
|
215
|
+
worsened++;
|
|
216
|
+
else
|
|
217
|
+
unchanged++;
|
|
218
|
+
}
|
|
219
|
+
return { improved, worsened, unchanged, signTestP: signTest(improved, worsened) };
|
|
220
|
+
}
|
|
221
|
+
function pairedChange(from, to) {
|
|
222
|
+
const fromById = new Map(from.map(c => [c.id, c.rank]));
|
|
223
|
+
const pairs = [];
|
|
224
|
+
for (const c of to) {
|
|
225
|
+
if (!fromById.has(c.id))
|
|
226
|
+
continue;
|
|
227
|
+
pairs.push({ id: c.id, category: c.category, from: fromById.get(c.id) ?? UNRANKED, to: c.rank ?? UNRANKED });
|
|
228
|
+
}
|
|
229
|
+
const byCategory = {};
|
|
230
|
+
for (const category of CATEGORIES) {
|
|
231
|
+
const subset = pairs.filter(p => p.category === category);
|
|
232
|
+
if (subset.length > 0)
|
|
233
|
+
byCategory[category] = pairCounts(subset);
|
|
234
|
+
}
|
|
235
|
+
return {
|
|
236
|
+
overall: pairCounts(pairs),
|
|
237
|
+
byCategory,
|
|
238
|
+
hit5Gained: pairs.filter(p => p.from > 5 && p.to <= 5).map(p => p.id),
|
|
239
|
+
hit5Lost: pairs.filter(p => p.from <= 5 && p.to > 5).map(p => p.id),
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
function rankedCases(cases, stage) {
|
|
243
|
+
return cases.map(c => ({ id: c.id, category: c.category, rank: stageRank(c, stage) }));
|
|
244
|
+
}
|
|
245
|
+
function stageChangesOf(cases, stages) {
|
|
246
|
+
const changes = [];
|
|
247
|
+
for (const to of stages) {
|
|
248
|
+
const from = STAGE_SOURCE[to];
|
|
249
|
+
if (from && stages.includes(from)) {
|
|
250
|
+
changes.push({ from, to, change: pairedChange(rankedCases(cases, from), rankedCases(cases, to)) });
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return changes;
|
|
254
|
+
}
|
|
255
|
+
function reportedStages(stages) {
|
|
256
|
+
return exports.STAGE_ORDER.filter(s => stages[s] !== undefined);
|
|
257
|
+
}
|
|
258
|
+
function sortedDistinct(values) {
|
|
259
|
+
return [...new Set(values)].sort((a, b) => {
|
|
260
|
+
if (a === b)
|
|
261
|
+
return 0;
|
|
262
|
+
if (a === null)
|
|
263
|
+
return -1;
|
|
264
|
+
if (b === null)
|
|
265
|
+
return 1;
|
|
266
|
+
return a < b ? -1 : 1;
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
exports.LIVE_GROUP = 'live';
|
|
270
|
+
function runFigures(cases, stages, shortPool) {
|
|
271
|
+
const metrics = {};
|
|
272
|
+
for (const stage of stages)
|
|
273
|
+
metrics[stage] = stageMetrics(cases, stage);
|
|
274
|
+
const poolInput = (subset) => subset.map(c => ({ poolRank: c.poolRank, short: shortPool.get(c.id) }));
|
|
275
|
+
const poolByCategory = {};
|
|
276
|
+
for (const category of CATEGORIES) {
|
|
277
|
+
const subset = cases.filter(c => c.category === category);
|
|
278
|
+
if (subset.length > 0)
|
|
279
|
+
poolByCategory[category] = poolMetricsOf(poolInput(subset));
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
cases: cases.length,
|
|
283
|
+
metrics,
|
|
284
|
+
pool: { overall: poolMetricsOf(poolInput(cases)), byCategory: poolByCategory },
|
|
285
|
+
stageChanges: stageChangesOf(cases, stages),
|
|
286
|
+
};
|
|
287
|
+
}
|
|
134
288
|
class RetrievalEvaluator {
|
|
135
|
-
async evaluate(suite, search, inspect) {
|
|
289
|
+
async evaluate(suite, search, inspect, onCase) {
|
|
290
|
+
await checkCaseCorpora(suite.cases, p => inspect.projectDir(p));
|
|
136
291
|
const cases = [];
|
|
292
|
+
const shortPool = new Map();
|
|
137
293
|
const invalid = [];
|
|
138
294
|
const warnings = [];
|
|
139
|
-
|
|
295
|
+
const fetchLimits = [];
|
|
296
|
+
const efSearches = [];
|
|
297
|
+
const iterativeScans = [];
|
|
298
|
+
let stages = null;
|
|
299
|
+
const total = suite.cases.length;
|
|
300
|
+
for (const [i, c] of suite.cases.entries()) {
|
|
301
|
+
const started = Date.now();
|
|
302
|
+
const report = (result, inv) => onCase?.({ index: i + 1, total, id: c.id, result, invalid: inv, ms: Date.now() - started });
|
|
140
303
|
const { valid, invalid: invalidExpects } = await validateCase(c, inspect);
|
|
141
304
|
if (valid.length === 0) {
|
|
142
|
-
|
|
305
|
+
const inv = { id: c.id, reason: 'no valid expected answer', invalidExpects };
|
|
306
|
+
invalid.push(inv);
|
|
307
|
+
report(null, inv);
|
|
143
308
|
continue;
|
|
144
309
|
}
|
|
145
310
|
if (invalidExpects.length > 0) {
|
|
146
311
|
warnings.push({ id: c.id, droppedExpects: invalidExpects });
|
|
147
312
|
}
|
|
148
313
|
const hits = await search(c.query, 10, c.project, c.threshold);
|
|
149
|
-
const
|
|
150
|
-
|
|
314
|
+
const caseStages = reportedStages(hits.stages);
|
|
315
|
+
if (stages === null) {
|
|
316
|
+
stages = caseStages;
|
|
317
|
+
}
|
|
318
|
+
else if (caseStages.join(',') !== stages.join(',')) {
|
|
319
|
+
throw new Error(`Case ${c.id} reported stages [${caseStages.join(', ')}], but the run's stages are [${stages.join(', ')}]`);
|
|
320
|
+
}
|
|
321
|
+
const ranks = {};
|
|
322
|
+
for (const stage of caseStages)
|
|
323
|
+
ranks[stage] = rankOf(hits.stages[stage], valid);
|
|
324
|
+
const result = {
|
|
325
|
+
id: c.id, project: c.project, category: c.category, query: c.query,
|
|
326
|
+
stages: ranks,
|
|
327
|
+
poolRank: rankOf(hits.pool, valid).rank,
|
|
328
|
+
poolSize: hits.pool.length,
|
|
329
|
+
rerankFailed: hits.rerankFailed,
|
|
330
|
+
corpus: c.corpus ?? null,
|
|
331
|
+
};
|
|
332
|
+
cases.push(result);
|
|
333
|
+
shortPool.set(c.id, hits.pool.length < hits.fetchLimit);
|
|
334
|
+
fetchLimits.push(hits.fetchLimit);
|
|
335
|
+
efSearches.push(hits.indexParams.efSearch);
|
|
336
|
+
iterativeScans.push(hits.indexParams.iterativeScan);
|
|
337
|
+
report(result, null);
|
|
151
338
|
}
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
339
|
+
const runStages = stages ?? ['vector'];
|
|
340
|
+
const figures = (subset) => runFigures(subset, runStages, shortPool);
|
|
341
|
+
const byCorpus = {};
|
|
342
|
+
for (const key of [...new Set(cases.map(c => c.corpus ?? exports.LIVE_GROUP))].sort()) {
|
|
343
|
+
byCorpus[key] = figures(cases.filter(c => (c.corpus ?? exports.LIVE_GROUP) === key));
|
|
157
344
|
}
|
|
158
|
-
|
|
345
|
+
const all = figures(cases);
|
|
346
|
+
const observed = {
|
|
347
|
+
fetchLimit: sortedDistinct(fetchLimits),
|
|
348
|
+
efSearch: sortedDistinct(efSearches),
|
|
349
|
+
iterativeScan: sortedDistinct(iterativeScans),
|
|
350
|
+
};
|
|
351
|
+
return {
|
|
352
|
+
run: {
|
|
353
|
+
cases, invalid, warnings,
|
|
354
|
+
stages: runStages,
|
|
355
|
+
metrics: all.metrics,
|
|
356
|
+
pool: all.pool,
|
|
357
|
+
stageChanges: all.stageChanges,
|
|
358
|
+
rerankFailures: cases.filter(c => c.rerankFailed).length,
|
|
359
|
+
byCorpus,
|
|
360
|
+
},
|
|
361
|
+
observed,
|
|
362
|
+
};
|
|
159
363
|
}
|
|
160
364
|
}
|
|
161
365
|
exports.RetrievalEvaluator = RetrievalEvaluator;
|
|
@@ -170,45 +374,127 @@ function metricsDelta(baseline, current) {
|
|
|
170
374
|
mrr10: metricDelta(baseline.mrr10, current.mrr10),
|
|
171
375
|
};
|
|
172
376
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
377
|
+
function metricsDeltaByCategory(baseline, current) {
|
|
378
|
+
const out = {};
|
|
379
|
+
for (const category of CATEGORIES) {
|
|
380
|
+
const b = baseline[category];
|
|
381
|
+
const cur = current[category];
|
|
382
|
+
if (b && cur)
|
|
383
|
+
out[category] = metricsDelta(b, cur);
|
|
178
384
|
}
|
|
179
|
-
|
|
180
|
-
|
|
385
|
+
return out;
|
|
386
|
+
}
|
|
387
|
+
function poolValueDelta(baseline, current) {
|
|
388
|
+
return { baseline, current, delta: baseline === null ? null : current - baseline };
|
|
389
|
+
}
|
|
390
|
+
function poolDelta(baseline, current) {
|
|
391
|
+
return {
|
|
392
|
+
recall: poolValueDelta(baseline?.recall ?? null, current.recall),
|
|
393
|
+
short: poolValueDelta(baseline?.short ?? null, current.short),
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
function formatCondition(value) {
|
|
397
|
+
return Array.isArray(value) ? `[${value.map(v => (v === null ? 'not sent' : String(v))).join(', ')}]` : String(value);
|
|
398
|
+
}
|
|
399
|
+
function corpusWarningsOf(b, cur) {
|
|
400
|
+
const warnings = [];
|
|
401
|
+
if (b.corpora.length === 0)
|
|
402
|
+
warnings.push('baseline records no corpus');
|
|
403
|
+
const baseByRoot = new Map(b.corpora.map(c => [c.root, c]));
|
|
404
|
+
const curByRoot = new Map(cur.corpora.map(c => [c.root, c]));
|
|
405
|
+
for (const root of [...new Set([...baseByRoot.keys(), ...curByRoot.keys()])].sort()) {
|
|
406
|
+
const bc = baseByRoot.get(root), cc = curByRoot.get(root);
|
|
407
|
+
if (!bc) {
|
|
408
|
+
warnings.push(`corpus ${root} only in the current run`);
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
if (!cc) {
|
|
412
|
+
warnings.push(`corpus ${root} only in the baseline`);
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
if (bc.commit !== cc.commit) {
|
|
416
|
+
warnings.push(`corpus ${root} commit differs: baseline ${bc.commit} vs current ${cc.commit}`);
|
|
417
|
+
}
|
|
418
|
+
else if (bc.fingerprint !== cc.fingerprint) {
|
|
419
|
+
warnings.push(`corpus ${root} was searched on a different index build: fingerprint ${bc.fingerprint} (${bc.chunks} chunks) vs ${cc.fingerprint} (${cc.chunks} chunks)`);
|
|
420
|
+
}
|
|
181
421
|
}
|
|
182
|
-
|
|
183
|
-
|
|
422
|
+
const live = (side, n) => `${side} includes ${n} live-corpus case(s); differences may come from code changes`;
|
|
423
|
+
if (b.liveCases > 0)
|
|
424
|
+
warnings.push(live('baseline', b.liveCases));
|
|
425
|
+
if (cur.liveCases > 0)
|
|
426
|
+
warnings.push(live('current run', cur.liveCases));
|
|
427
|
+
return warnings;
|
|
428
|
+
}
|
|
429
|
+
function conditionWarningsOf(baseline, current) {
|
|
430
|
+
const warnings = [];
|
|
431
|
+
const b = baseline.metadata, cur = current.metadata;
|
|
432
|
+
if (b.suiteHash !== cur.suiteHash) {
|
|
433
|
+
warnings.push(`suite differs: baseline ${b.suiteHash} vs current ${cur.suiteHash}`);
|
|
184
434
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
435
|
+
if (b.embeddingModel !== cur.embeddingModel) {
|
|
436
|
+
warnings.push(`embedding model differs: baseline ${b.embeddingModel} vs current ${cur.embeddingModel}`);
|
|
437
|
+
}
|
|
438
|
+
if (b.rerankerModel !== cur.rerankerModel) {
|
|
439
|
+
warnings.push(`reranker differs: baseline ${b.rerankerModel ?? 'none'} vs current ${cur.rerankerModel ?? 'none'}`);
|
|
440
|
+
}
|
|
441
|
+
if (!b.conditions)
|
|
442
|
+
warnings.push('baseline records no search conditions');
|
|
443
|
+
if (!cur.conditions)
|
|
444
|
+
warnings.push('current run records no search conditions');
|
|
445
|
+
if (b.conditions && cur.conditions) {
|
|
446
|
+
const bc = b.conditions;
|
|
447
|
+
const cc = cur.conditions;
|
|
448
|
+
for (const key of [...new Set([...Object.keys(bc), ...Object.keys(cc)])]) {
|
|
449
|
+
if (!(key in bc))
|
|
450
|
+
warnings.push(`search condition ${key} recorded only by the current run (${formatCondition(cc[key])})`);
|
|
451
|
+
else if (!(key in cc))
|
|
452
|
+
warnings.push(`search condition ${key} recorded only by the baseline (${formatCondition(bc[key])})`);
|
|
453
|
+
else if (JSON.stringify(bc[key]) !== JSON.stringify(cc[key])) {
|
|
454
|
+
warnings.push(`${key} differs: baseline ${formatCondition(bc[key])} vs current ${formatCondition(cc[key])}`);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
warnings.push(...corpusWarningsOf(b, cur));
|
|
459
|
+
if (baseline.run.rerankFailures > 0)
|
|
460
|
+
warnings.push(`baseline has ${baseline.run.rerankFailures} rerank failure(s)`);
|
|
461
|
+
if (current.run.rerankFailures > 0)
|
|
462
|
+
warnings.push(`current run has ${current.run.rerankFailures} rerank failure(s)`);
|
|
463
|
+
return warnings;
|
|
464
|
+
}
|
|
465
|
+
function compare(baseline, current) {
|
|
466
|
+
const stages = [];
|
|
467
|
+
for (const stage of current.run.stages) {
|
|
468
|
+
const b = baseline.run.metrics[stage];
|
|
469
|
+
const cur = current.run.metrics[stage];
|
|
470
|
+
if (!baseline.run.stages.includes(stage) || !b || !cur)
|
|
471
|
+
continue;
|
|
472
|
+
stages.push({
|
|
473
|
+
stage,
|
|
474
|
+
overall: metricsDelta(b.overall, cur.overall),
|
|
475
|
+
byCategory: metricsDeltaByCategory(b.byCategory, cur.byCategory),
|
|
476
|
+
change: pairedChange(rankedCases(baseline.run.cases, stage), rankedCases(current.run.cases, stage)),
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
const unmatchedStages = [
|
|
480
|
+
...baseline.run.stages.filter(s => !current.run.stages.includes(s)).map(stage => ({ stage, side: 'baseline' })),
|
|
481
|
+
...current.run.stages.filter(s => !baseline.run.stages.includes(s)).map(stage => ({ stage, side: 'current' })),
|
|
482
|
+
];
|
|
483
|
+
const basePool = baseline.legacy ? null : baseline.run.pool;
|
|
484
|
+
const poolByCategory = {};
|
|
485
|
+
for (const category of CATEGORIES) {
|
|
486
|
+
const cur = current.run.pool.byCategory[category];
|
|
487
|
+
if (!cur)
|
|
488
|
+
continue;
|
|
489
|
+
if (basePool === null)
|
|
490
|
+
poolByCategory[category] = poolDelta(null, cur);
|
|
491
|
+
else if (basePool.byCategory[category])
|
|
492
|
+
poolByCategory[category] = poolDelta(basePool.byCategory[category], cur);
|
|
206
493
|
}
|
|
207
494
|
return {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
conditionWarnings,
|
|
495
|
+
stages,
|
|
496
|
+
unmatchedStages,
|
|
497
|
+
pool: { overall: poolDelta(basePool?.overall ?? null, current.run.pool.overall), byCategory: poolByCategory },
|
|
498
|
+
conditionWarnings: conditionWarningsOf(baseline, current),
|
|
213
499
|
};
|
|
214
500
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_PER_FILE = void 0;
|
|
4
|
+
exports.limitPerFile = limitPerFile;
|
|
5
|
+
exports.DEFAULT_PER_FILE = 2;
|
|
6
|
+
function limitPerFile(ranked, limit, perFile) {
|
|
7
|
+
const taken = [];
|
|
8
|
+
const perFileCount = new Map();
|
|
9
|
+
for (const entry of ranked) {
|
|
10
|
+
if (taken.length >= limit)
|
|
11
|
+
break;
|
|
12
|
+
const count = perFileCount.get(entry.filePath) ?? 0;
|
|
13
|
+
if (count >= perFile)
|
|
14
|
+
continue;
|
|
15
|
+
perFileCount.set(entry.filePath, count + 1);
|
|
16
|
+
taken.push(entry);
|
|
17
|
+
}
|
|
18
|
+
return taken;
|
|
19
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { Node } from 'ts-morph';
|
|
2
2
|
import type { IAstParser, DepEdge, FileAnalysis, AnalyzeProjectOpts } from '../interfaces/IAstParser';
|
|
3
3
|
import type { ILogger } from '../interfaces';
|
|
4
|
-
/** Test seam for `graphql-symbols`' resolve-field parent-type resolution —
|
|
5
|
-
* defaults to the alias-following type-checker lookup `extractCalls` already uses. */
|
|
6
4
|
export interface AstParserDeps {
|
|
7
5
|
resolveDeclaration?(identifier: Node): Node | undefined;
|
|
8
6
|
}
|
|
@@ -10,6 +8,7 @@ export declare class AstParserService implements IAstParser {
|
|
|
10
8
|
private logger;
|
|
11
9
|
private deps;
|
|
12
10
|
private morphProjects;
|
|
11
|
+
private readonly declarationBuilder;
|
|
13
12
|
constructor(logger?: ILogger, deps?: AstParserDeps);
|
|
14
13
|
analyzeProject(filePaths: string[], serviceName: string, projectRoot?: string, opts?: AnalyzeProjectOpts): Promise<Map<string, FileAnalysis>>;
|
|
15
14
|
clearCache(projectRoot?: string): void;
|
|
@@ -19,77 +18,32 @@ export declare class AstParserService implements IAstParser {
|
|
|
19
18
|
private createMorphProject;
|
|
20
19
|
private getMorphProject;
|
|
21
20
|
private extractEdges;
|
|
22
|
-
/** Manual fallback for a bindingless import specifier (design.md decision 2) —
|
|
23
|
-
* ts-morph's `getModuleSpecifierSourceFile()` is `null` for these even when
|
|
24
|
-
* the target is a real project file, since nothing is bound to resolve through. */
|
|
25
21
|
private resolveSideEffectModuleSpecifier;
|
|
26
22
|
private resolveFromSymbol;
|
|
27
23
|
private buildUsageMap;
|
|
28
24
|
private getPos;
|
|
29
|
-
/** Trimmed text of the source line containing `node`'s start — the signature
|
|
30
|
-
* rule for every kind (name node for members, the position node for
|
|
31
|
-
* constructors/anonymous default classes). */
|
|
32
25
|
private lineTextAt;
|
|
33
26
|
private extractSymbols;
|
|
34
27
|
private extractMembers;
|
|
35
|
-
|
|
36
|
-
* members) except `method_signature`. Each range spans the declaration node's
|
|
37
|
-
* own first-to-last line — decorator-inclusive, comment-exclusive, and (for a
|
|
38
|
-
* variable) the declarator alone, never the whole statement. */
|
|
39
|
-
private buildOwnerRanges;
|
|
40
|
-
/** Local name → the name exported by `@nestjs/graphql`, for every named import
|
|
41
|
-
* of that module specifier (aliases followed). Recognition is import-based,
|
|
42
|
-
* not type-checker-based, so `skipFileDependencyResolution` batches agree. */
|
|
28
|
+
private buildDeclarations;
|
|
43
29
|
private buildGraphqlImportMap;
|
|
44
|
-
/** The identifier naming a decorator's expression (`@Foo` or the callee of `@Foo(...)`). */
|
|
45
30
|
private static decoratorIdentifier;
|
|
46
|
-
/** `dec` recognised against `importMap` — its exported `@nestjs/graphql` name and its call arguments (`[]` for a bare `@Foo`). */
|
|
47
31
|
private recognizedDecorator;
|
|
48
32
|
private static literalStringValue;
|
|
49
33
|
private static checkGraphqlGrammar;
|
|
50
|
-
/** `true` when any decorator argument is an (unwrapped) options object
|
|
51
|
-
* literal with `isAbstract: true` — an abstract base type (`PaginatedType`
|
|
52
|
-
* pattern) that NestJS never registers in the actual GraphQL schema. */
|
|
53
34
|
private static isAbstractGraphqlType;
|
|
54
|
-
/** `obj`'s own `name` property (`PropertyAssignment` or `ShorthandPropertyAssignment`), if any. */
|
|
55
35
|
private static findNameProperty;
|
|
56
|
-
/** Strips parens/`as`/`satisfies`/non-null wrappers (any nesting) to find the
|
|
57
|
-
* underlying node — used only to classify an argument's *shape* (type
|
|
58
|
-
* function vs. options object vs. literal vs. other), never to read a name
|
|
59
|
-
* value through the wrapper (decision 4 of `graphql-symbols` forbids that). */
|
|
60
36
|
private static peekThroughWrapper;
|
|
61
|
-
/**
|
|
62
|
-
* GraphQL name from a decorator's arguments (design decision 4 of
|
|
63
|
-
* `graphql-symbols`) — syntactic, no wrapper looked through at any step.
|
|
64
|
-
* `allowOptionsName` gates step 2 (the `name` option — `Query`/`Mutation`/
|
|
65
|
-
* `Subscription`/`ResolveField` only); `fallbackName` is the TS name used at
|
|
66
|
-
* step 3.
|
|
67
|
-
*/
|
|
68
37
|
private deriveGraphqlName;
|
|
69
38
|
private logGraphqlSkip;
|
|
70
|
-
/** `declNode`'s qualified TypeScript name for diagnostics, independent of
|
|
71
|
-
* whether its own `code_symbol` row was actually written. */
|
|
72
39
|
private declarationNameFor;
|
|
73
|
-
/** The identifier named by a `@Resolver` argument: directly, or as an arrow
|
|
74
|
-
* function's expression body; or the sole identifier of a single-element
|
|
75
|
-
* array literal in either position (list-returning resolver). */
|
|
76
40
|
private static extractResolverIdentifier;
|
|
77
41
|
private defaultResolveDeclaration;
|
|
78
|
-
/** The GraphQL type name of a `@ResolveField`'s resolved parent class
|
|
79
|
-
* (design decision 5, second half): its recognised `@ObjectType` (else
|
|
80
|
-
* `@InterfaceType`) decorator, or its class name when it carries no
|
|
81
|
-
* decorator spelled `ObjectType`/`InterfaceType` at all, recognised or not. */
|
|
82
42
|
private classGraphqlTypeName;
|
|
83
|
-
/** The parent type of a `@ResolveField` — the enclosing class's recognised
|
|
84
|
-
* `@Resolver` argument (design decision 5, first half). */
|
|
85
43
|
private resolveResolverParentType;
|
|
86
44
|
private extractClassTypeRows;
|
|
87
45
|
private extractResolveFieldRow;
|
|
88
46
|
private extractMethodOperationRows;
|
|
89
|
-
/** GraphQL-named rows for every recognised `@nestjs/graphql` decorator in
|
|
90
|
-
* the file (`graphql-symbols`). `ownRows` is `symbols ∪ members` — the
|
|
91
|
-
* declarations' own `code_symbol` candidates, checked so a GraphQL row is
|
|
92
|
-
* never written without its decorated declaration's own row. */
|
|
93
47
|
private extractGraphqlNames;
|
|
94
48
|
private static isValidSegment;
|
|
95
49
|
private static memberName;
|
|
@@ -99,24 +53,10 @@ export declare class AstParserService implements IAstParser {
|
|
|
99
53
|
private classExpressionRoot;
|
|
100
54
|
private objectLiteralRootValid;
|
|
101
55
|
private isTopLevelOrNamespaceDirect;
|
|
102
|
-
/** One step of the ancestor walk used by `containerSegments`: does `node`
|
|
103
|
-
* contribute a named segment, pass through transparently, or disqualify
|
|
104
|
-
* (exclude) everything nested inside it? */
|
|
105
56
|
private walkStep;
|
|
106
|
-
/** The dotted container path of `node`'s enclosing named scopes, outermost
|
|
107
|
-
* first, or null when any contributing ancestor is invalid (a computed /
|
|
108
|
-
* string-literal segment, a string-named module, `declare global`, or an
|
|
109
|
-
* unrooted class expression / object literal). */
|
|
110
57
|
private containerSegments;
|
|
111
|
-
/** If `node` is a declaration this service indexes (top-level symbol, member,
|
|
112
|
-
* or collected class), its own name + container. Otherwise null. Shared by
|
|
113
|
-
* member extraction, ownership ranges and the call graph so all three name a
|
|
114
|
-
* declaration identically. */
|
|
115
58
|
private describeIndexedNode;
|
|
116
59
|
private qualifiedName;
|
|
117
|
-
/** The qualified name of the nearest ancestor of `node` that is itself an
|
|
118
|
-
* indexed declaration (method, accessor, constructor, function property,
|
|
119
|
-
* namespace/top-level function, or module-/namespace-level variable). */
|
|
120
60
|
private findFromSymbol;
|
|
121
61
|
private extractCalls;
|
|
122
62
|
}
|