codesearch-mcp 0.2.3-alpha → 0.3.0-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/CHANGELOG.md +31 -1
  2. package/README.md +46 -26
  3. package/dist/app.js +5 -19
  4. package/dist/src/chunking/AstChunker.d.ts +12 -0
  5. package/dist/src/chunking/AstChunker.js +30 -0
  6. package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
  7. package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
  8. package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
  9. package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
  10. package/dist/src/chunking/FragmentMerger.d.ts +11 -0
  11. package/dist/src/chunking/FragmentMerger.js +61 -0
  12. package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
  13. package/dist/src/chunking/ImportOnlyFilter.js +14 -0
  14. package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
  15. package/dist/src/chunking/LangchainTextSplitter.js +133 -0
  16. package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
  17. package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
  18. package/dist/src/chunking/RunPacker.d.ts +12 -0
  19. package/dist/src/chunking/RunPacker.js +84 -0
  20. package/dist/src/chunking/create-chunker.d.ts +3 -0
  21. package/dist/src/chunking/create-chunker.js +15 -0
  22. package/dist/src/chunking/interfaces.d.ts +27 -0
  23. package/dist/src/chunking/interfaces.js +2 -0
  24. package/dist/src/commands/ConfigCommand.js +1 -9
  25. package/dist/src/commands/EvalCommand.d.ts +4 -3
  26. package/dist/src/commands/EvalCommand.js +85 -52
  27. package/dist/src/commands/IndexCommand.js +0 -22
  28. package/dist/src/commands/McpCommand.js +2 -8
  29. package/dist/src/commands/WebCommand.js +2 -5
  30. package/dist/src/config-error.d.ts +0 -4
  31. package/dist/src/config-error.js +0 -4
  32. package/dist/src/config-store.d.ts +1 -54
  33. package/dist/src/config-store.js +8 -69
  34. package/dist/src/config.d.ts +8 -17
  35. package/dist/src/config.js +35 -34
  36. package/dist/src/constants.d.ts +2 -28
  37. package/dist/src/constants.js +6 -28
  38. package/dist/src/container.d.ts +6 -0
  39. package/dist/src/container.js +12 -3
  40. package/dist/src/db/schema.d.ts +17 -0
  41. package/dist/src/db/schema.js +1 -6
  42. package/dist/src/declarations/DeclarationTree.d.ts +19 -0
  43. package/dist/src/declarations/DeclarationTree.js +66 -0
  44. package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
  45. package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
  46. package/dist/src/declarations/SourceLines.d.ts +37 -0
  47. package/dist/src/declarations/SourceLines.js +415 -0
  48. package/dist/src/dependency-traversal.d.ts +0 -6
  49. package/dist/src/dependency-traversal.js +0 -6
  50. package/dist/src/enriched-prompt.d.ts +0 -6
  51. package/dist/src/enriched-prompt.js +0 -6
  52. package/dist/src/error-record.d.ts +12 -0
  53. package/dist/src/error-record.js +248 -0
  54. package/dist/src/eval-corpus.d.ts +16 -0
  55. package/dist/src/eval-corpus.js +169 -0
  56. package/dist/src/eval-results.d.ts +6 -3
  57. package/dist/src/eval-results.js +309 -26
  58. package/dist/src/file-utils.d.ts +0 -5
  59. package/dist/src/file-utils.js +0 -5
  60. package/dist/src/graph.d.ts +0 -25
  61. package/dist/src/graph.js +0 -28
  62. package/dist/src/hnsw-settings.d.ts +3 -0
  63. package/dist/src/hnsw-settings.js +24 -0
  64. package/dist/src/interfaces/IAstParser.d.ts +2 -39
  65. package/dist/src/interfaces/IChunker.d.ts +3 -12
  66. package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
  67. package/dist/src/interfaces/ICorpusInspector.js +2 -0
  68. package/dist/src/interfaces/IDatabase.d.ts +2 -22
  69. package/dist/src/interfaces/IEmbedder.d.ts +0 -9
  70. package/dist/src/interfaces/ILogger.d.ts +1 -11
  71. package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
  72. package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
  73. package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
  74. package/dist/src/interfaces/IRetrievalEvaluator.d.ts +134 -12
  75. package/dist/src/interfaces/ISearcher.d.ts +16 -5
  76. package/dist/src/interrupt.js +0 -2
  77. package/dist/src/owner-path.d.ts +9 -10
  78. package/dist/src/owner-path.js +11 -23
  79. package/dist/src/project-scope.d.ts +0 -12
  80. package/dist/src/project-scope.js +0 -12
  81. package/dist/src/project-selector.d.ts +0 -5
  82. package/dist/src/project-selector.js +0 -11
  83. package/dist/src/retrieval-eval.d.ts +14 -6
  84. package/dist/src/retrieval-eval.js +345 -59
  85. package/dist/src/search/result-shaping.d.ts +4 -0
  86. package/dist/src/search/result-shaping.js +19 -0
  87. package/dist/src/services/AstParserService.d.ts +2 -62
  88. package/dist/src/services/AstParserService.js +25 -137
  89. package/dist/src/services/DatabaseService.d.ts +2 -7
  90. package/dist/src/services/DatabaseService.js +42 -29
  91. package/dist/src/services/EmbedderFactory.d.ts +0 -4
  92. package/dist/src/services/EmbedderFactory.js +0 -5
  93. package/dist/src/services/IndexerService.d.ts +0 -16
  94. package/dist/src/services/IndexerService.js +16 -45
  95. package/dist/src/services/InfinityEmbedder.d.ts +0 -21
  96. package/dist/src/services/InfinityEmbedder.js +0 -15
  97. package/dist/src/services/LoggerService.d.ts +7 -5
  98. package/dist/src/services/LoggerService.js +37 -41
  99. package/dist/src/services/McpServer.d.ts +22 -14
  100. package/dist/src/services/McpServer.js +70 -37
  101. package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
  102. package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
  103. package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
  104. package/dist/src/services/NxMonorepoResolver.js +5 -56
  105. package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
  106. package/dist/src/services/OpenAICompatibleEmbedder.js +0 -14
  107. package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
  108. package/dist/src/services/ProjectDiscoveryService.js +0 -4
  109. package/dist/src/services/SearcherService.d.ts +9 -116
  110. package/dist/src/services/SearcherService.js +95 -176
  111. package/dist/src/services/VoyageEmbedder.d.ts +0 -6
  112. package/dist/src/services/VoyageEmbedder.js +0 -6
  113. package/dist/src/services/WebServerService.d.ts +1 -53
  114. package/dist/src/services/WebServerService.js +11 -57
  115. package/dist/src/services/http-retry.d.ts +0 -7
  116. package/dist/src/services/http-retry.js +1 -10
  117. package/dist/src/services/infinity-provisioning.d.ts +0 -74
  118. package/dist/src/services/infinity-provisioning.js +5 -88
  119. package/dist/src/services/legacy-ollama-index.d.ts +0 -16
  120. package/dist/src/services/legacy-ollama-index.js +0 -23
  121. package/dist/src/services/reranker.d.ts +0 -9
  122. package/dist/src/services/reranker.js +46 -12
  123. package/dist/src/setup.d.ts +11 -23
  124. package/dist/src/setup.js +98 -71
  125. package/dist/src/symbol-name.d.ts +0 -12
  126. package/dist/src/symbol-name.js +0 -13
  127. package/dist/src/symbol-rows.d.ts +0 -14
  128. package/dist/src/symbol-rows.js +0 -14
  129. package/dist/src/tool-call-log.d.ts +2 -0
  130. package/dist/src/tool-call-log.js +24 -0
  131. package/dist/src/tools/GetDependenciesTool.js +1 -1
  132. package/dist/src/tools/GetDependentsTool.js +1 -1
  133. package/dist/src/tools/SearchCodebaseTool.js +6 -3
  134. package/dist/src/tools/format-helpers.d.ts +0 -6
  135. package/dist/src/tools/format-helpers.js +87 -14
  136. package/dist/src/web/dashboard.d.ts +0 -8
  137. package/dist/src/web/dashboard.js +7 -10
  138. package/package.json +2 -2
  139. package/dist/src/services/AstChunker.d.ts +0 -42
  140. package/dist/src/services/AstChunker.js +0 -158
  141. package/docs/web-dashboard.md +0 -206
@@ -1,11 +1,19 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.RetrievalEvaluator = exports.EvalSuiteError = void 0;
3
+ exports.RetrievalEvaluator = exports.LIVE_GROUP = exports.EvalSuiteError = exports.STAGE_ORDER = void 0;
4
4
  exports.parseSuite = parseSuite;
5
+ exports.corporaOf = corporaOf;
6
+ exports.isUnderRoot = isUnderRoot;
7
+ exports.checkCaseCorpora = checkCaseCorpora;
5
8
  exports.mergeSuites = mergeSuites;
9
+ exports.signTest = signTest;
10
+ exports.pairedChange = pairedChange;
6
11
  exports.compare = compare;
7
12
  const zod_1 = require("zod");
8
- /** Thrown by `parseSuite` before any search runs — names the offending case and field. */
13
+ const CATEGORIES = ['identifier', 'behaviour', 'concept'];
14
+ exports.STAGE_ORDER = ['vector', 'reranked'];
15
+ const STAGE_SOURCE = { reranked: 'vector' };
16
+ const UNRANKED = 11;
9
17
  class EvalSuiteError extends Error {
10
18
  caseId;
11
19
  field;
@@ -29,11 +37,15 @@ const CaseSchema = zod_1.z.object({
29
37
  expect: zod_1.z.array(ExpectedAnswerSchema).min(1),
30
38
  threshold: zod_1.z.number().min(0).max(1).optional(),
31
39
  });
40
+ const CorpusSchema = zod_1.z.object({
41
+ root: zod_1.z.string().min(1).refine(r => !r.startsWith('/') && !r.split('/').includes('..') && !r.endsWith('/'), 'must be a workspace-relative directory'),
42
+ commit: zod_1.z.string().regex(/^[0-9a-f]{40}$/, 'must be a full 40-hex commit SHA'),
43
+ }).strict();
32
44
  const SuiteSchema = zod_1.z.object({
33
45
  version: zod_1.z.literal(1),
46
+ corpus: CorpusSchema.optional(),
34
47
  cases: zod_1.z.array(CaseSchema),
35
48
  });
36
- /** Best-effort id of the case at `cases[index]` in a raw (possibly invalid) object, for error messages. */
37
49
  function rawCaseId(raw, index) {
38
50
  if (typeof raw !== 'object' || raw === null)
39
51
  return null;
@@ -46,7 +58,6 @@ function rawCaseId(raw, index) {
46
58
  const id = entry.id;
47
59
  return typeof id === 'string' ? id : null;
48
60
  }
49
- /** Validates a loaded suite object against the schema; rejects before any search runs. */
50
61
  function parseSuite(raw) {
51
62
  const result = SuiteSchema.safeParse(raw);
52
63
  if (!result.success) {
@@ -64,10 +75,40 @@ function parseSuite(raw) {
64
75
  }
65
76
  seen.add(c.id);
66
77
  }
67
- return result.data;
78
+ const corpus = result.data.corpus?.root ?? null;
79
+ return { ...result.data, cases: result.data.cases.map(c => ({ ...c, corpus })) };
80
+ }
81
+ function corporaOf(suites) {
82
+ const byRoot = new Map();
83
+ for (const suite of suites) {
84
+ if (!suite.corpus)
85
+ continue;
86
+ const prev = byRoot.get(suite.corpus.root);
87
+ if (prev && prev.commit !== suite.corpus.commit) {
88
+ throw new EvalSuiteError(`Corpus ${suite.corpus.root} is declared at two commits: ${prev.commit} and ${suite.corpus.commit}`, null, 'corpus');
89
+ }
90
+ byRoot.set(suite.corpus.root, { root: suite.corpus.root, commit: suite.corpus.commit });
91
+ }
92
+ return [...byRoot.values()].sort((a, b) => (a.root < b.root ? -1 : a.root > b.root ? 1 : 0));
93
+ }
94
+ function isUnderRoot(dir, root) {
95
+ return dir === root || dir.startsWith(root + '/');
96
+ }
97
+ async function checkCaseCorpora(cases, projectDir) {
98
+ for (const c of cases) {
99
+ if (!c.corpus)
100
+ continue;
101
+ const dir = await projectDir(c.project);
102
+ if (dir === null) {
103
+ throw new EvalSuiteError(`Project ${c.project} is not indexed, so it cannot be checked against corpus ${c.corpus} (case ${c.id}, field project)`, c.id, 'project');
104
+ }
105
+ if (!isUnderRoot(dir, c.corpus)) {
106
+ throw new EvalSuiteError(`Project ${c.project} (${dir}) is outside corpus ${c.corpus} (case ${c.id}, field project)`, c.id, 'project');
107
+ }
108
+ }
68
109
  }
69
- /** Merges several suites (already parsed) into one; case ids must be unique across all of them. */
70
110
  function mergeSuites(suites) {
111
+ corporaOf(suites);
71
112
  const cases = [];
72
113
  const seen = new Set();
73
114
  for (const suite of suites) {
@@ -80,7 +121,6 @@ function mergeSuites(suites) {
80
121
  }
81
122
  return { version: 1, cases };
82
123
  }
83
- /** `owner` matches exactly, or as a prefix at a `.` boundary (`RepairsService` accepts any member of the class). */
84
124
  function ownerMatches(ownerPath, expectedOwner) {
85
125
  return ownerPath === expectedOwner || ownerPath.startsWith(expectedOwner + '.');
86
126
  }
@@ -117,45 +157,209 @@ function rankOf(hits, valid) {
117
157
  function emptyMetrics() {
118
158
  return { count: 0, hit1: 0, hit5: 0, mrr10: 0 };
119
159
  }
120
- function accumulate(cases) {
121
- const n = cases.length;
160
+ function accumulate(ranks) {
161
+ const n = ranks.length;
122
162
  if (n === 0)
123
163
  return emptyMetrics();
124
164
  let hit1 = 0, hit5 = 0, mrrSum = 0;
125
- for (const c of cases) {
126
- if (c.rank === 1)
165
+ for (const rank of ranks) {
166
+ if (rank === 1)
127
167
  hit1++;
128
- if (c.rank !== null && c.rank <= 5)
168
+ if (rank !== null && rank <= 5)
129
169
  hit5++;
130
- mrrSum += c.rank !== null ? 1 / c.rank : 0;
170
+ mrrSum += rank !== null ? 1 / rank : 0;
131
171
  }
132
172
  return { count: n, hit1: hit1 / n, hit5: hit5 / n, mrr10: mrrSum / n };
133
173
  }
174
+ function stageRank(c, stage) {
175
+ return c.stages[stage]?.rank ?? null;
176
+ }
177
+ function stageMetrics(cases, stage) {
178
+ const byCategory = {};
179
+ for (const category of CATEGORIES) {
180
+ const subset = cases.filter(c => c.category === category);
181
+ if (subset.length > 0)
182
+ byCategory[category] = accumulate(subset.map(c => stageRank(c, stage)));
183
+ }
184
+ return { overall: accumulate(cases.map(c => stageRank(c, stage))), byCategory };
185
+ }
186
+ function poolMetricsOf(cases) {
187
+ const count = cases.length;
188
+ return {
189
+ count,
190
+ recall: count === 0 ? 0 : cases.filter(c => c.poolRank !== null).length / count,
191
+ short: cases.filter(c => c.short).length,
192
+ };
193
+ }
194
+ function binomialCdfHalf(k, n) {
195
+ let pmf = Math.pow(0.5, n);
196
+ let cdf = pmf;
197
+ for (let i = 1; i <= k; i++) {
198
+ pmf = pmf * (n - i + 1) / i;
199
+ cdf += pmf;
200
+ }
201
+ return cdf;
202
+ }
203
+ function signTest(improved, worsened) {
204
+ const n = improved + worsened;
205
+ if (n === 0)
206
+ return 1;
207
+ return Math.min(1, 2 * binomialCdfHalf(Math.min(improved, worsened), n));
208
+ }
209
+ function pairCounts(pairs) {
210
+ let improved = 0, worsened = 0, unchanged = 0;
211
+ for (const p of pairs) {
212
+ if (p.to < p.from)
213
+ improved++;
214
+ else if (p.to > p.from)
215
+ worsened++;
216
+ else
217
+ unchanged++;
218
+ }
219
+ return { improved, worsened, unchanged, signTestP: signTest(improved, worsened) };
220
+ }
221
+ function pairedChange(from, to) {
222
+ const fromById = new Map(from.map(c => [c.id, c.rank]));
223
+ const pairs = [];
224
+ for (const c of to) {
225
+ if (!fromById.has(c.id))
226
+ continue;
227
+ pairs.push({ id: c.id, category: c.category, from: fromById.get(c.id) ?? UNRANKED, to: c.rank ?? UNRANKED });
228
+ }
229
+ const byCategory = {};
230
+ for (const category of CATEGORIES) {
231
+ const subset = pairs.filter(p => p.category === category);
232
+ if (subset.length > 0)
233
+ byCategory[category] = pairCounts(subset);
234
+ }
235
+ return {
236
+ overall: pairCounts(pairs),
237
+ byCategory,
238
+ hit5Gained: pairs.filter(p => p.from > 5 && p.to <= 5).map(p => p.id),
239
+ hit5Lost: pairs.filter(p => p.from <= 5 && p.to > 5).map(p => p.id),
240
+ };
241
+ }
242
+ function rankedCases(cases, stage) {
243
+ return cases.map(c => ({ id: c.id, category: c.category, rank: stageRank(c, stage) }));
244
+ }
245
+ function stageChangesOf(cases, stages) {
246
+ const changes = [];
247
+ for (const to of stages) {
248
+ const from = STAGE_SOURCE[to];
249
+ if (from && stages.includes(from)) {
250
+ changes.push({ from, to, change: pairedChange(rankedCases(cases, from), rankedCases(cases, to)) });
251
+ }
252
+ }
253
+ return changes;
254
+ }
255
+ function reportedStages(stages) {
256
+ return exports.STAGE_ORDER.filter(s => stages[s] !== undefined);
257
+ }
258
+ function sortedDistinct(values) {
259
+ return [...new Set(values)].sort((a, b) => {
260
+ if (a === b)
261
+ return 0;
262
+ if (a === null)
263
+ return -1;
264
+ if (b === null)
265
+ return 1;
266
+ return a < b ? -1 : 1;
267
+ });
268
+ }
269
+ exports.LIVE_GROUP = 'live';
270
+ function runFigures(cases, stages, shortPool) {
271
+ const metrics = {};
272
+ for (const stage of stages)
273
+ metrics[stage] = stageMetrics(cases, stage);
274
+ const poolInput = (subset) => subset.map(c => ({ poolRank: c.poolRank, short: shortPool.get(c.id) }));
275
+ const poolByCategory = {};
276
+ for (const category of CATEGORIES) {
277
+ const subset = cases.filter(c => c.category === category);
278
+ if (subset.length > 0)
279
+ poolByCategory[category] = poolMetricsOf(poolInput(subset));
280
+ }
281
+ return {
282
+ cases: cases.length,
283
+ metrics,
284
+ pool: { overall: poolMetricsOf(poolInput(cases)), byCategory: poolByCategory },
285
+ stageChanges: stageChangesOf(cases, stages),
286
+ };
287
+ }
134
288
  class RetrievalEvaluator {
135
- async evaluate(suite, search, inspect) {
289
+ async evaluate(suite, search, inspect, onCase) {
290
+ await checkCaseCorpora(suite.cases, p => inspect.projectDir(p));
136
291
  const cases = [];
292
+ const shortPool = new Map();
137
293
  const invalid = [];
138
294
  const warnings = [];
139
- for (const c of suite.cases) {
295
+ const fetchLimits = [];
296
+ const efSearches = [];
297
+ const iterativeScans = [];
298
+ let stages = null;
299
+ const total = suite.cases.length;
300
+ for (const [i, c] of suite.cases.entries()) {
301
+ const started = Date.now();
302
+ const report = (result, inv) => onCase?.({ index: i + 1, total, id: c.id, result, invalid: inv, ms: Date.now() - started });
140
303
  const { valid, invalid: invalidExpects } = await validateCase(c, inspect);
141
304
  if (valid.length === 0) {
142
- invalid.push({ id: c.id, reason: 'no valid expected answer', invalidExpects });
305
+ const inv = { id: c.id, reason: 'no valid expected answer', invalidExpects };
306
+ invalid.push(inv);
307
+ report(null, inv);
143
308
  continue;
144
309
  }
145
310
  if (invalidExpects.length > 0) {
146
311
  warnings.push({ id: c.id, droppedExpects: invalidExpects });
147
312
  }
148
313
  const hits = await search(c.query, 10, c.project, c.threshold);
149
- const { rank, matched } = rankOf(hits, valid);
150
- cases.push({ id: c.id, project: c.project, category: c.category, query: c.query, rank, matched });
314
+ const caseStages = reportedStages(hits.stages);
315
+ if (stages === null) {
316
+ stages = caseStages;
317
+ }
318
+ else if (caseStages.join(',') !== stages.join(',')) {
319
+ throw new Error(`Case ${c.id} reported stages [${caseStages.join(', ')}], but the run's stages are [${stages.join(', ')}]`);
320
+ }
321
+ const ranks = {};
322
+ for (const stage of caseStages)
323
+ ranks[stage] = rankOf(hits.stages[stage], valid);
324
+ const result = {
325
+ id: c.id, project: c.project, category: c.category, query: c.query,
326
+ stages: ranks,
327
+ poolRank: rankOf(hits.pool, valid).rank,
328
+ poolSize: hits.pool.length,
329
+ rerankFailed: hits.rerankFailed,
330
+ corpus: c.corpus ?? null,
331
+ };
332
+ cases.push(result);
333
+ shortPool.set(c.id, hits.pool.length < hits.fetchLimit);
334
+ fetchLimits.push(hits.fetchLimit);
335
+ efSearches.push(hits.indexParams.efSearch);
336
+ iterativeScans.push(hits.indexParams.iterativeScan);
337
+ report(result, null);
151
338
  }
152
- const byCategory = {};
153
- for (const category of ['identifier', 'behaviour', 'concept']) {
154
- const subset = cases.filter(c => c.category === category);
155
- if (subset.length > 0)
156
- byCategory[category] = accumulate(subset);
339
+ const runStages = stages ?? ['vector'];
340
+ const figures = (subset) => runFigures(subset, runStages, shortPool);
341
+ const byCorpus = {};
342
+ for (const key of [...new Set(cases.map(c => c.corpus ?? exports.LIVE_GROUP))].sort()) {
343
+ byCorpus[key] = figures(cases.filter(c => (c.corpus ?? exports.LIVE_GROUP) === key));
157
344
  }
158
- return { cases, invalid, warnings, metrics: { overall: accumulate(cases), byCategory } };
345
+ const all = figures(cases);
346
+ const observed = {
347
+ fetchLimit: sortedDistinct(fetchLimits),
348
+ efSearch: sortedDistinct(efSearches),
349
+ iterativeScan: sortedDistinct(iterativeScans),
350
+ };
351
+ return {
352
+ run: {
353
+ cases, invalid, warnings,
354
+ stages: runStages,
355
+ metrics: all.metrics,
356
+ pool: all.pool,
357
+ stageChanges: all.stageChanges,
358
+ rerankFailures: cases.filter(c => c.rerankFailed).length,
359
+ byCorpus,
360
+ },
361
+ observed,
362
+ };
159
363
  }
160
364
  }
161
365
  exports.RetrievalEvaluator = RetrievalEvaluator;
@@ -170,45 +374,127 @@ function metricsDelta(baseline, current) {
170
374
  mrr10: metricDelta(baseline.mrr10, current.mrr10),
171
375
  };
172
376
  }
173
- /** Pure comparison of two result files; prints a warning (never refuses) on mismatched conditions. */
174
- function compare(baseline, current) {
175
- const conditionWarnings = [];
176
- if (baseline.metadata.suiteHash !== current.metadata.suiteHash) {
177
- conditionWarnings.push(`suite differs: baseline ${baseline.metadata.suiteHash} vs current ${current.metadata.suiteHash}`);
377
+ function metricsDeltaByCategory(baseline, current) {
378
+ const out = {};
379
+ for (const category of CATEGORIES) {
380
+ const b = baseline[category];
381
+ const cur = current[category];
382
+ if (b && cur)
383
+ out[category] = metricsDelta(b, cur);
178
384
  }
179
- if (baseline.metadata.embeddingModel !== current.metadata.embeddingModel) {
180
- conditionWarnings.push(`embedding model differs: baseline ${baseline.metadata.embeddingModel} vs current ${current.metadata.embeddingModel}`);
385
+ return out;
386
+ }
387
+ function poolValueDelta(baseline, current) {
388
+ return { baseline, current, delta: baseline === null ? null : current - baseline };
389
+ }
390
+ function poolDelta(baseline, current) {
391
+ return {
392
+ recall: poolValueDelta(baseline?.recall ?? null, current.recall),
393
+ short: poolValueDelta(baseline?.short ?? null, current.short),
394
+ };
395
+ }
396
+ function formatCondition(value) {
397
+ return Array.isArray(value) ? `[${value.map(v => (v === null ? 'not sent' : String(v))).join(', ')}]` : String(value);
398
+ }
399
+ function corpusWarningsOf(b, cur) {
400
+ const warnings = [];
401
+ if (b.corpora.length === 0)
402
+ warnings.push('baseline records no corpus');
403
+ const baseByRoot = new Map(b.corpora.map(c => [c.root, c]));
404
+ const curByRoot = new Map(cur.corpora.map(c => [c.root, c]));
405
+ for (const root of [...new Set([...baseByRoot.keys(), ...curByRoot.keys()])].sort()) {
406
+ const bc = baseByRoot.get(root), cc = curByRoot.get(root);
407
+ if (!bc) {
408
+ warnings.push(`corpus ${root} only in the current run`);
409
+ continue;
410
+ }
411
+ if (!cc) {
412
+ warnings.push(`corpus ${root} only in the baseline`);
413
+ continue;
414
+ }
415
+ if (bc.commit !== cc.commit) {
416
+ warnings.push(`corpus ${root} commit differs: baseline ${bc.commit} vs current ${cc.commit}`);
417
+ }
418
+ else if (bc.fingerprint !== cc.fingerprint) {
419
+ warnings.push(`corpus ${root} was searched on a different index build: fingerprint ${bc.fingerprint} (${bc.chunks} chunks) vs ${cc.fingerprint} (${cc.chunks} chunks)`);
420
+ }
181
421
  }
182
- if (baseline.metadata.rerankerModel !== current.metadata.rerankerModel) {
183
- conditionWarnings.push(`reranker differs: baseline ${baseline.metadata.rerankerModel ?? 'none'} vs current ${current.metadata.rerankerModel ?? 'none'}`);
422
+ const live = (side, n) => `${side} includes ${n} live-corpus case(s); differences may come from code changes`;
423
+ if (b.liveCases > 0)
424
+ warnings.push(live('baseline', b.liveCases));
425
+ if (cur.liveCases > 0)
426
+ warnings.push(live('current run', cur.liveCases));
427
+ return warnings;
428
+ }
429
+ function conditionWarningsOf(baseline, current) {
430
+ const warnings = [];
431
+ const b = baseline.metadata, cur = current.metadata;
432
+ if (b.suiteHash !== cur.suiteHash) {
433
+ warnings.push(`suite differs: baseline ${b.suiteHash} vs current ${cur.suiteHash}`);
184
434
  }
185
- const byCategory = {};
186
- for (const category of ['identifier', 'behaviour', 'concept']) {
187
- const b = baseline.run.metrics.byCategory[category];
188
- const cur = current.run.metrics.byCategory[category];
189
- if (b && cur)
190
- byCategory[category] = metricsDelta(b, cur);
191
- }
192
- const baselineRanked = new Map(baseline.run.cases.map(c => [c.id, c.rank]));
193
- const currentRanked = new Map(current.run.cases.map(c => [c.id, c.rank]));
194
- const hit5Gained = [];
195
- const hit5Lost = [];
196
- const allIds = new Set([...baselineRanked.keys(), ...currentRanked.keys()]);
197
- for (const id of allIds) {
198
- const b = baselineRanked.get(id) ?? null;
199
- const cur = currentRanked.get(id) ?? null;
200
- const bHit5 = b !== null && b <= 5;
201
- const curHit5 = cur !== null && cur <= 5;
202
- if (!bHit5 && curHit5)
203
- hit5Gained.push(id);
204
- if (bHit5 && !curHit5)
205
- hit5Lost.push(id);
435
+ if (b.embeddingModel !== cur.embeddingModel) {
436
+ warnings.push(`embedding model differs: baseline ${b.embeddingModel} vs current ${cur.embeddingModel}`);
437
+ }
438
+ if (b.rerankerModel !== cur.rerankerModel) {
439
+ warnings.push(`reranker differs: baseline ${b.rerankerModel ?? 'none'} vs current ${cur.rerankerModel ?? 'none'}`);
440
+ }
441
+ if (!b.conditions)
442
+ warnings.push('baseline records no search conditions');
443
+ if (!cur.conditions)
444
+ warnings.push('current run records no search conditions');
445
+ if (b.conditions && cur.conditions) {
446
+ const bc = b.conditions;
447
+ const cc = cur.conditions;
448
+ for (const key of [...new Set([...Object.keys(bc), ...Object.keys(cc)])]) {
449
+ if (!(key in bc))
450
+ warnings.push(`search condition ${key} recorded only by the current run (${formatCondition(cc[key])})`);
451
+ else if (!(key in cc))
452
+ warnings.push(`search condition ${key} recorded only by the baseline (${formatCondition(bc[key])})`);
453
+ else if (JSON.stringify(bc[key]) !== JSON.stringify(cc[key])) {
454
+ warnings.push(`${key} differs: baseline ${formatCondition(bc[key])} vs current ${formatCondition(cc[key])}`);
455
+ }
456
+ }
457
+ }
458
+ warnings.push(...corpusWarningsOf(b, cur));
459
+ if (baseline.run.rerankFailures > 0)
460
+ warnings.push(`baseline has ${baseline.run.rerankFailures} rerank failure(s)`);
461
+ if (current.run.rerankFailures > 0)
462
+ warnings.push(`current run has ${current.run.rerankFailures} rerank failure(s)`);
463
+ return warnings;
464
+ }
465
+ function compare(baseline, current) {
466
+ const stages = [];
467
+ for (const stage of current.run.stages) {
468
+ const b = baseline.run.metrics[stage];
469
+ const cur = current.run.metrics[stage];
470
+ if (!baseline.run.stages.includes(stage) || !b || !cur)
471
+ continue;
472
+ stages.push({
473
+ stage,
474
+ overall: metricsDelta(b.overall, cur.overall),
475
+ byCategory: metricsDeltaByCategory(b.byCategory, cur.byCategory),
476
+ change: pairedChange(rankedCases(baseline.run.cases, stage), rankedCases(current.run.cases, stage)),
477
+ });
478
+ }
479
+ const unmatchedStages = [
480
+ ...baseline.run.stages.filter(s => !current.run.stages.includes(s)).map(stage => ({ stage, side: 'baseline' })),
481
+ ...current.run.stages.filter(s => !baseline.run.stages.includes(s)).map(stage => ({ stage, side: 'current' })),
482
+ ];
483
+ const basePool = baseline.legacy ? null : baseline.run.pool;
484
+ const poolByCategory = {};
485
+ for (const category of CATEGORIES) {
486
+ const cur = current.run.pool.byCategory[category];
487
+ if (!cur)
488
+ continue;
489
+ if (basePool === null)
490
+ poolByCategory[category] = poolDelta(null, cur);
491
+ else if (basePool.byCategory[category])
492
+ poolByCategory[category] = poolDelta(basePool.byCategory[category], cur);
206
493
  }
207
494
  return {
208
- overall: metricsDelta(baseline.run.metrics.overall, current.run.metrics.overall),
209
- byCategory,
210
- hit5Gained,
211
- hit5Lost,
212
- conditionWarnings,
495
+ stages,
496
+ unmatchedStages,
497
+ pool: { overall: poolDelta(basePool?.overall ?? null, current.run.pool.overall), byCategory: poolByCategory },
498
+ conditionWarnings: conditionWarningsOf(baseline, current),
213
499
  };
214
500
  }
@@ -0,0 +1,4 @@
1
+ export declare const DEFAULT_PER_FILE = 2;
2
+ export declare function limitPerFile<T extends {
3
+ filePath: string;
4
+ }>(ranked: T[], limit: number, perFile: number): T[];
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.DEFAULT_PER_FILE = void 0;
4
+ exports.limitPerFile = limitPerFile;
5
+ exports.DEFAULT_PER_FILE = 2;
6
+ function limitPerFile(ranked, limit, perFile) {
7
+ const taken = [];
8
+ const perFileCount = new Map();
9
+ for (const entry of ranked) {
10
+ if (taken.length >= limit)
11
+ break;
12
+ const count = perFileCount.get(entry.filePath) ?? 0;
13
+ if (count >= perFile)
14
+ continue;
15
+ perFileCount.set(entry.filePath, count + 1);
16
+ taken.push(entry);
17
+ }
18
+ return taken;
19
+ }
@@ -1,8 +1,6 @@
1
1
  import { Node } from 'ts-morph';
2
2
  import type { IAstParser, DepEdge, FileAnalysis, AnalyzeProjectOpts } from '../interfaces/IAstParser';
3
3
  import type { ILogger } from '../interfaces';
4
- /** Test seam for `graphql-symbols`' resolve-field parent-type resolution —
5
- * defaults to the alias-following type-checker lookup `extractCalls` already uses. */
6
4
  export interface AstParserDeps {
7
5
  resolveDeclaration?(identifier: Node): Node | undefined;
8
6
  }
@@ -10,6 +8,7 @@ export declare class AstParserService implements IAstParser {
10
8
  private logger;
11
9
  private deps;
12
10
  private morphProjects;
11
+ private readonly declarationBuilder;
13
12
  constructor(logger?: ILogger, deps?: AstParserDeps);
14
13
  analyzeProject(filePaths: string[], serviceName: string, projectRoot?: string, opts?: AnalyzeProjectOpts): Promise<Map<string, FileAnalysis>>;
15
14
  clearCache(projectRoot?: string): void;
@@ -19,77 +18,32 @@ export declare class AstParserService implements IAstParser {
19
18
  private createMorphProject;
20
19
  private getMorphProject;
21
20
  private extractEdges;
22
- /** Manual fallback for a bindingless import specifier (design.md decision 2) —
23
- * ts-morph's `getModuleSpecifierSourceFile()` is `null` for these even when
24
- * the target is a real project file, since nothing is bound to resolve through. */
25
21
  private resolveSideEffectModuleSpecifier;
26
22
  private resolveFromSymbol;
27
23
  private buildUsageMap;
28
24
  private getPos;
29
- /** Trimmed text of the source line containing `node`'s start — the signature
30
- * rule for every kind (name node for members, the position node for
31
- * constructors/anonymous default classes). */
32
25
  private lineTextAt;
33
26
  private extractSymbols;
34
27
  private extractMembers;
35
- /** Chunk-ownership ranges from every indexed declaration (top-level symbols +
36
- * members) except `method_signature`. Each range spans the declaration node's
37
- * own first-to-last line — decorator-inclusive, comment-exclusive, and (for a
38
- * variable) the declarator alone, never the whole statement. */
39
- private buildOwnerRanges;
40
- /** Local name → the name exported by `@nestjs/graphql`, for every named import
41
- * of that module specifier (aliases followed). Recognition is import-based,
42
- * not type-checker-based, so `skipFileDependencyResolution` batches agree. */
28
+ private buildDeclarations;
43
29
  private buildGraphqlImportMap;
44
- /** The identifier naming a decorator's expression (`@Foo` or the callee of `@Foo(...)`). */
45
30
  private static decoratorIdentifier;
46
- /** `dec` recognised against `importMap` — its exported `@nestjs/graphql` name and its call arguments (`[]` for a bare `@Foo`). */
47
31
  private recognizedDecorator;
48
32
  private static literalStringValue;
49
33
  private static checkGraphqlGrammar;
50
- /** `true` when any decorator argument is an (unwrapped) options object
51
- * literal with `isAbstract: true` — an abstract base type (`PaginatedType`
52
- * pattern) that NestJS never registers in the actual GraphQL schema. */
53
34
  private static isAbstractGraphqlType;
54
- /** `obj`'s own `name` property (`PropertyAssignment` or `ShorthandPropertyAssignment`), if any. */
55
35
  private static findNameProperty;
56
- /** Strips parens/`as`/`satisfies`/non-null wrappers (any nesting) to find the
57
- * underlying node — used only to classify an argument's *shape* (type
58
- * function vs. options object vs. literal vs. other), never to read a name
59
- * value through the wrapper (decision 4 of `graphql-symbols` forbids that). */
60
36
  private static peekThroughWrapper;
61
- /**
62
- * GraphQL name from a decorator's arguments (design decision 4 of
63
- * `graphql-symbols`) — syntactic, no wrapper looked through at any step.
64
- * `allowOptionsName` gates step 2 (the `name` option — `Query`/`Mutation`/
65
- * `Subscription`/`ResolveField` only); `fallbackName` is the TS name used at
66
- * step 3.
67
- */
68
37
  private deriveGraphqlName;
69
38
  private logGraphqlSkip;
70
- /** `declNode`'s qualified TypeScript name for diagnostics, independent of
71
- * whether its own `code_symbol` row was actually written. */
72
39
  private declarationNameFor;
73
- /** The identifier named by a `@Resolver` argument: directly, or as an arrow
74
- * function's expression body; or the sole identifier of a single-element
75
- * array literal in either position (list-returning resolver). */
76
40
  private static extractResolverIdentifier;
77
41
  private defaultResolveDeclaration;
78
- /** The GraphQL type name of a `@ResolveField`'s resolved parent class
79
- * (design decision 5, second half): its recognised `@ObjectType` (else
80
- * `@InterfaceType`) decorator, or its class name when it carries no
81
- * decorator spelled `ObjectType`/`InterfaceType` at all, recognised or not. */
82
42
  private classGraphqlTypeName;
83
- /** The parent type of a `@ResolveField` — the enclosing class's recognised
84
- * `@Resolver` argument (design decision 5, first half). */
85
43
  private resolveResolverParentType;
86
44
  private extractClassTypeRows;
87
45
  private extractResolveFieldRow;
88
46
  private extractMethodOperationRows;
89
- /** GraphQL-named rows for every recognised `@nestjs/graphql` decorator in
90
- * the file (`graphql-symbols`). `ownRows` is `symbols ∪ members` — the
91
- * declarations' own `code_symbol` candidates, checked so a GraphQL row is
92
- * never written without its decorated declaration's own row. */
93
47
  private extractGraphqlNames;
94
48
  private static isValidSegment;
95
49
  private static memberName;
@@ -99,24 +53,10 @@ export declare class AstParserService implements IAstParser {
99
53
  private classExpressionRoot;
100
54
  private objectLiteralRootValid;
101
55
  private isTopLevelOrNamespaceDirect;
102
- /** One step of the ancestor walk used by `containerSegments`: does `node`
103
- * contribute a named segment, pass through transparently, or disqualify
104
- * (exclude) everything nested inside it? */
105
56
  private walkStep;
106
- /** The dotted container path of `node`'s enclosing named scopes, outermost
107
- * first, or null when any contributing ancestor is invalid (a computed /
108
- * string-literal segment, a string-named module, `declare global`, or an
109
- * unrooted class expression / object literal). */
110
57
  private containerSegments;
111
- /** If `node` is a declaration this service indexes (top-level symbol, member,
112
- * or collected class), its own name + container. Otherwise null. Shared by
113
- * member extraction, ownership ranges and the call graph so all three name a
114
- * declaration identically. */
115
58
  private describeIndexedNode;
116
59
  private qualifiedName;
117
- /** The qualified name of the nearest ancestor of `node` that is itself an
118
- * indexed declaration (method, accessor, constructor, function property,
119
- * namespace/top-level function, or module-/namespace-level variable). */
120
60
  private findFromSymbol;
121
61
  private extractCalls;
122
62
  }