codesearch-mcp 0.2.3-alpha → 0.3.0-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/CHANGELOG.md +31 -1
  2. package/README.md +46 -26
  3. package/dist/app.js +5 -19
  4. package/dist/src/chunking/AstChunker.d.ts +12 -0
  5. package/dist/src/chunking/AstChunker.js +30 -0
  6. package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
  7. package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
  8. package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
  9. package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
  10. package/dist/src/chunking/FragmentMerger.d.ts +11 -0
  11. package/dist/src/chunking/FragmentMerger.js +61 -0
  12. package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
  13. package/dist/src/chunking/ImportOnlyFilter.js +14 -0
  14. package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
  15. package/dist/src/chunking/LangchainTextSplitter.js +133 -0
  16. package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
  17. package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
  18. package/dist/src/chunking/RunPacker.d.ts +12 -0
  19. package/dist/src/chunking/RunPacker.js +84 -0
  20. package/dist/src/chunking/create-chunker.d.ts +3 -0
  21. package/dist/src/chunking/create-chunker.js +15 -0
  22. package/dist/src/chunking/interfaces.d.ts +27 -0
  23. package/dist/src/chunking/interfaces.js +2 -0
  24. package/dist/src/commands/ConfigCommand.js +1 -9
  25. package/dist/src/commands/EvalCommand.d.ts +4 -3
  26. package/dist/src/commands/EvalCommand.js +85 -52
  27. package/dist/src/commands/IndexCommand.js +0 -22
  28. package/dist/src/commands/McpCommand.js +2 -8
  29. package/dist/src/commands/WebCommand.js +2 -5
  30. package/dist/src/config-error.d.ts +0 -4
  31. package/dist/src/config-error.js +0 -4
  32. package/dist/src/config-store.d.ts +1 -54
  33. package/dist/src/config-store.js +8 -69
  34. package/dist/src/config.d.ts +8 -17
  35. package/dist/src/config.js +35 -34
  36. package/dist/src/constants.d.ts +2 -28
  37. package/dist/src/constants.js +6 -28
  38. package/dist/src/container.d.ts +6 -0
  39. package/dist/src/container.js +12 -3
  40. package/dist/src/db/schema.d.ts +17 -0
  41. package/dist/src/db/schema.js +1 -6
  42. package/dist/src/declarations/DeclarationTree.d.ts +19 -0
  43. package/dist/src/declarations/DeclarationTree.js +66 -0
  44. package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
  45. package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
  46. package/dist/src/declarations/SourceLines.d.ts +37 -0
  47. package/dist/src/declarations/SourceLines.js +415 -0
  48. package/dist/src/dependency-traversal.d.ts +0 -6
  49. package/dist/src/dependency-traversal.js +0 -6
  50. package/dist/src/enriched-prompt.d.ts +0 -6
  51. package/dist/src/enriched-prompt.js +0 -6
  52. package/dist/src/error-record.d.ts +12 -0
  53. package/dist/src/error-record.js +248 -0
  54. package/dist/src/eval-corpus.d.ts +16 -0
  55. package/dist/src/eval-corpus.js +169 -0
  56. package/dist/src/eval-results.d.ts +6 -3
  57. package/dist/src/eval-results.js +309 -26
  58. package/dist/src/file-utils.d.ts +0 -5
  59. package/dist/src/file-utils.js +0 -5
  60. package/dist/src/graph.d.ts +0 -25
  61. package/dist/src/graph.js +0 -28
  62. package/dist/src/hnsw-settings.d.ts +3 -0
  63. package/dist/src/hnsw-settings.js +24 -0
  64. package/dist/src/interfaces/IAstParser.d.ts +2 -39
  65. package/dist/src/interfaces/IChunker.d.ts +3 -12
  66. package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
  67. package/dist/src/interfaces/ICorpusInspector.js +2 -0
  68. package/dist/src/interfaces/IDatabase.d.ts +2 -22
  69. package/dist/src/interfaces/IEmbedder.d.ts +0 -9
  70. package/dist/src/interfaces/ILogger.d.ts +1 -11
  71. package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
  72. package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
  73. package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
  74. package/dist/src/interfaces/IRetrievalEvaluator.d.ts +134 -12
  75. package/dist/src/interfaces/ISearcher.d.ts +16 -5
  76. package/dist/src/interrupt.js +0 -2
  77. package/dist/src/owner-path.d.ts +9 -10
  78. package/dist/src/owner-path.js +11 -23
  79. package/dist/src/project-scope.d.ts +0 -12
  80. package/dist/src/project-scope.js +0 -12
  81. package/dist/src/project-selector.d.ts +0 -5
  82. package/dist/src/project-selector.js +0 -11
  83. package/dist/src/retrieval-eval.d.ts +14 -6
  84. package/dist/src/retrieval-eval.js +345 -59
  85. package/dist/src/search/result-shaping.d.ts +4 -0
  86. package/dist/src/search/result-shaping.js +19 -0
  87. package/dist/src/services/AstParserService.d.ts +2 -62
  88. package/dist/src/services/AstParserService.js +25 -137
  89. package/dist/src/services/DatabaseService.d.ts +2 -7
  90. package/dist/src/services/DatabaseService.js +42 -29
  91. package/dist/src/services/EmbedderFactory.d.ts +0 -4
  92. package/dist/src/services/EmbedderFactory.js +0 -5
  93. package/dist/src/services/IndexerService.d.ts +0 -16
  94. package/dist/src/services/IndexerService.js +16 -45
  95. package/dist/src/services/InfinityEmbedder.d.ts +0 -21
  96. package/dist/src/services/InfinityEmbedder.js +0 -15
  97. package/dist/src/services/LoggerService.d.ts +7 -5
  98. package/dist/src/services/LoggerService.js +37 -41
  99. package/dist/src/services/McpServer.d.ts +22 -14
  100. package/dist/src/services/McpServer.js +70 -37
  101. package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
  102. package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
  103. package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
  104. package/dist/src/services/NxMonorepoResolver.js +5 -56
  105. package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
  106. package/dist/src/services/OpenAICompatibleEmbedder.js +0 -14
  107. package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
  108. package/dist/src/services/ProjectDiscoveryService.js +0 -4
  109. package/dist/src/services/SearcherService.d.ts +9 -116
  110. package/dist/src/services/SearcherService.js +95 -176
  111. package/dist/src/services/VoyageEmbedder.d.ts +0 -6
  112. package/dist/src/services/VoyageEmbedder.js +0 -6
  113. package/dist/src/services/WebServerService.d.ts +1 -53
  114. package/dist/src/services/WebServerService.js +11 -57
  115. package/dist/src/services/http-retry.d.ts +0 -7
  116. package/dist/src/services/http-retry.js +1 -10
  117. package/dist/src/services/infinity-provisioning.d.ts +0 -74
  118. package/dist/src/services/infinity-provisioning.js +5 -88
  119. package/dist/src/services/legacy-ollama-index.d.ts +0 -16
  120. package/dist/src/services/legacy-ollama-index.js +0 -23
  121. package/dist/src/services/reranker.d.ts +0 -9
  122. package/dist/src/services/reranker.js +46 -12
  123. package/dist/src/setup.d.ts +11 -23
  124. package/dist/src/setup.js +98 -71
  125. package/dist/src/symbol-name.d.ts +0 -12
  126. package/dist/src/symbol-name.js +0 -13
  127. package/dist/src/symbol-rows.d.ts +0 -14
  128. package/dist/src/symbol-rows.js +0 -14
  129. package/dist/src/tool-call-log.d.ts +2 -0
  130. package/dist/src/tool-call-log.js +24 -0
  131. package/dist/src/tools/GetDependenciesTool.js +1 -1
  132. package/dist/src/tools/GetDependentsTool.js +1 -1
  133. package/dist/src/tools/SearchCodebaseTool.js +6 -3
  134. package/dist/src/tools/format-helpers.d.ts +0 -6
  135. package/dist/src/tools/format-helpers.js +87 -14
  136. package/dist/src/web/dashboard.d.ts +0 -8
  137. package/dist/src/web/dashboard.js +7 -10
  138. package/package.json +2 -2
  139. package/dist/src/services/AstChunker.d.ts +0 -42
  140. package/dist/src/services/AstChunker.js +0 -158
  141. package/docs/web-dashboard.md +0 -206
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.SearcherService = void 0;
37
+ exports.rerankDocument = rerankDocument;
37
38
  const fs = __importStar(require("fs"));
38
39
  const path = __importStar(require("path"));
39
40
  const readline = __importStar(require("readline"));
@@ -43,12 +44,14 @@ const MonorepoAwareDiscoveryService_1 = require("./MonorepoAwareDiscoveryService
43
44
  const legacy_ollama_index_1 = require("./legacy-ollama-index");
44
45
  const symbol_name_1 = require("../symbol-name");
45
46
  const dependency_traversal_1 = require("../dependency-traversal");
47
+ const result_shaping_1 = require("../search/result-shaping");
48
+ const hnsw_settings_1 = require("../hnsw-settings");
46
49
  const SILENT_LOGGER = {
47
50
  debug() { },
48
51
  info() { },
49
52
  warn() { },
50
53
  error() { },
51
- metric() { },
54
+ withContext(_, fn) { return fn(); },
52
55
  };
53
56
  const NODE_FS = {
54
57
  realpath: p => fs.promises.realpath(p),
@@ -66,7 +69,6 @@ const GREP_EXCLUDE_DIRS = [...file_utils_1.IGNORE_DIRS].map(d => `--exclude-dir=
66
69
  const FIND_SYMBOL_LIMIT = 50;
67
70
  const FIND_SYMBOL_RAW_LIMIT = FIND_SYMBOL_LIMIT * 10;
68
71
  const PATH_CANDIDATE_LIMIT = 3;
69
- /** Every path column of the index; `content` marks those whose file the indexer read. */
70
72
  const INDEX_PATH_COLUMNS = [
71
73
  { table: 'code_chunk', column: '"filePath"', content: true },
72
74
  { table: 'code_symbol', column: '"filePath"', content: true },
@@ -79,7 +81,6 @@ const OUTSIDE_WORKSPACE_VALUE = (col) => `(${col} = '..' OR ${col} LIKE '../%')`
79
81
  function stripTrailingSlash(p) {
80
82
  return p.length > 1 && p.endsWith('/') ? p.slice(0, -1) : p;
81
83
  }
82
- /** Segment-boundary containment: `/ws-other` is not within `/ws`; `.` contains every relative path. */
83
84
  function isWithin(p, root) {
84
85
  if (root === '.')
85
86
  return true;
@@ -90,6 +91,13 @@ function isWithin(p, root) {
90
91
  function errorCode(err) {
91
92
  return err?.code ?? 'EIO';
92
93
  }
94
+ function rerankDocument(row) {
95
+ const symbol = row.ownerPath ? ` | Symbol: ${oneLine(row.ownerPath)}` : '';
96
+ return `File: ${oneLine(row.filePath)}${symbol}\n${row.content}`;
97
+ }
98
+ function oneLine(text) {
99
+ return text.replace(/\s+/g, ' ').trim();
100
+ }
93
101
  class SearcherService {
94
102
  db;
95
103
  embedder;
@@ -105,38 +113,47 @@ class SearcherService {
105
113
  this.logger = logger;
106
114
  this.fs = { ...NODE_FS, ...fsDeps };
107
115
  }
108
- /**
109
- * Whether `code_symbol.target` exists — `true` once a check finds it (never
110
- * re-checked again in this process), `false` until then (re-checked on
111
- * every lookup, so a `rag-index` run that adds the column takes effect
112
- * without restarting `rag-mcp`). See "find_symbol works before the target
113
- * column exists" (`graphql-symbols`).
114
- */
115
116
  targetColumnPresent = false;
116
- /** Real path of `workspaceRoot`; cached only once resolved, so a transient failure is retried. */
117
117
  realRootCache;
118
- /** Lazily-built, cached map of project name → real filesystem root. */
119
118
  rootMap;
120
- /** Cached, lazily-built map of leaf project name → real filesystem root (via discovery). */
119
+ pgvectorVersionRead;
120
+ degradedScanWarned = false;
121
+ async iterativeScanEnabled() {
122
+ const read = this.pgvectorVersionRead ??= this.db.pgvectorVersion().then(v => {
123
+ if (v === null)
124
+ throw new Error('vector extension not found in pg_extension');
125
+ return v;
126
+ });
127
+ let version;
128
+ try {
129
+ version = await read;
130
+ }
131
+ catch (err) {
132
+ if (this.pgvectorVersionRead === read)
133
+ this.pgvectorVersionRead = undefined;
134
+ this.warnDegradedScan('unknown', err);
135
+ return false;
136
+ }
137
+ if ((0, hnsw_settings_1.supportsIterativeScan)(version))
138
+ return true;
139
+ this.warnDegradedScan(version);
140
+ return false;
141
+ }
142
+ warnDegradedScan(version, err) {
143
+ if (this.degradedScanWarned)
144
+ return;
145
+ this.degradedScanWarned = true;
146
+ this.logger.warn(`SearcherService: pgvector ${version} lacks hnsw.iterative_scan — filtered searches on large projects may return fewer candidates than requested; upgrade to pgvector ≥ 0.8`, err ? { err } : undefined);
147
+ }
121
148
  projectRootMap() {
122
149
  if (!this.rootMap) {
123
150
  this.rootMap = new Map((0, MonorepoAwareDiscoveryService_1.toLeafProjects)(this.discovery.discover(this.config.workspaceRoot)).map(p => [p.name, p.path]));
124
151
  }
125
152
  return this.rootMap;
126
153
  }
127
- /**
128
- * Resolve a project name to its real filesystem root. Uses discovery (leaf
129
- * projects) so nested-monorepo names like `repo/app` map to their true path;
130
- * falls back to `join(workspaceRoot, name)` when the name is unknown (e.g. a
131
- * project since removed from disk, or a flat project not surfaced by discovery).
132
- */
133
154
  resolveProjectRoot(name) {
134
155
  let root = this.projectRootMap().get(name);
135
156
  if (root === undefined) {
136
- // Cache miss — the project may have been added/renamed on disk since the
137
- // map was built (mcp/web servers are long-lived). Rebuild once and retry
138
- // before the flat fallback, which is wrong for nested-monorepo names
139
- // (`repo/app` lives at `repo/apps/app`, not `repo/app`).
140
157
  this.rootMap = undefined;
141
158
  root = this.projectRootMap().get(name);
142
159
  }
@@ -163,12 +180,6 @@ class SearcherService {
163
180
  return null;
164
181
  }
165
182
  }
166
- /**
167
- * Lexical validation of a tool path argument: refusals for an empty value,
168
- * `\`, an absolute path and a `..` escape; otherwise the canonical
169
- * workspace-relative form. The only filesystem access is the cached real path
170
- * of `workspaceRoot`.
171
- */
172
183
  async resolveToolPath(input) {
173
184
  if (input == null || input === '')
174
185
  return { ok: false, reason: 'invalid', missing: true };
@@ -190,11 +201,6 @@ class SearcherService {
190
201
  return { ok: false, reason: 'outside_workspace' };
191
202
  return { ok: true, relPath, absPath: this.toAbs(relPath) };
192
203
  }
193
- /**
194
- * Whether the real location of `absPath` — of the target, else of its nearest
195
- * existing ancestor plus the remaining segments — lies under the real
196
- * workspace root.
197
- */
198
204
  async realLocation(absPath) {
199
205
  const root = (await this.realWorkspaceRoot()) ?? this.workspaceRoot();
200
206
  let existing = absPath;
@@ -215,11 +221,6 @@ class SearcherService {
215
221
  }
216
222
  }
217
223
  }
218
- /**
219
- * Workspace boundary of the disk tools. A real location outside the workspace
220
- * is allowed only for content the indexer already read (a symlinked share):
221
- * the file itself for `file`, any file below it for `dir`.
222
- */
223
224
  async checkRealLocation(relPath, absPath, kind) {
224
225
  const loc = await this.realLocation(absPath);
225
226
  if (!loc.ok)
@@ -234,11 +235,9 @@ class SearcherService {
234
235
  const [{ exists }] = await this.db.query(`SELECT EXISTS (${branches.join(' UNION ALL ')}) AS exists`, [param]);
235
236
  return exists;
236
237
  }
237
- /** Whether any table of the index holds `relPath` (a canonical path, never `../…`). */
238
238
  isIndexedPath(relPath) {
239
239
  return this.pathExists(INDEX_PATH_COLUMNS, '$col = $1', relPath);
240
240
  }
241
- /** Whether the indexer read the file at `relPath`. */
242
241
  isContentPath(relPath) {
243
242
  return this.pathExists(INDEX_PATH_COLUMNS.filter(c => c.content), '$col = $1', relPath);
244
243
  }
@@ -246,11 +245,6 @@ class SearcherService {
246
245
  const prefix = relPath === '.' ? '' : (0, symbol_name_1.escapeLike)(relPath) + '/';
247
246
  return this.pathExists(INDEX_PATH_COLUMNS.filter(c => c.content), `$col LIKE $1 ESCAPE '\\'`, prefix + '%');
248
247
  }
249
- /**
250
- * Up to three index paths ending in `/<relPath>` (`file`) or directories
251
- * ending in `/<relPath>` that hold one (`dir`), in byte order. For `dir`, the
252
- * root of a project named `relPath` comes first.
253
- */
254
248
  async findPathCandidates(relPath, kind) {
255
249
  const union = INDEX_PATH_COLUMNS.map(c => `SELECT ${c.column} AS p FROM ${c.table}`).join(' UNION ALL ');
256
250
  const notOutside = `NOT ${OUTSIDE_WORKSPACE_VALUE('p')}`;
@@ -273,21 +267,15 @@ class SearcherService {
273
267
  const rest = rows.map(r => r.p).filter(p => !first.includes(p));
274
268
  return [...first, ...rest].slice(0, PATH_CANDIDATE_LIMIT);
275
269
  }
276
- /** `findPathCandidates`, with a failed lookup logged and answered by no candidates. */
277
270
  async candidatesOrNone(relPath, kind) {
278
271
  try {
279
272
  return await this.findPathCandidates(relPath, kind);
280
273
  }
281
274
  catch (err) {
282
- this.logger.error('SearcherService: path candidate lookup failed', { relPath, reason: err.message });
275
+ this.logger.error('SearcherService: path candidate lookup failed', { relPath, err });
283
276
  return [];
284
277
  }
285
278
  }
286
- /**
287
- * `relPath` with a leading project name replaced by that project's root
288
- * (`repo/app/src/x.ts` → `repo/apps/app/src/x.ts`), or null when no name
289
- * prefixes it, the root equals the name, or the result names nothing.
290
- */
291
279
  async substituteProjectName(relPath) {
292
280
  let named = [];
293
281
  for (let attempt = 0; attempt < 2 && named.length === 0; attempt++) {
@@ -320,7 +308,6 @@ class SearcherService {
320
308
  }
321
309
  return root === undefined ? null : this.toRel(root);
322
310
  }
323
- /** Longest workspace-relative project root accepted by `match`; discovery is re-run once before giving up. */
324
311
  findProjectRoot(match) {
325
312
  for (let attempt = 0; attempt < 2; attempt++) {
326
313
  if (attempt === 1)
@@ -334,15 +321,9 @@ class SearcherService {
334
321
  }
335
322
  return null;
336
323
  }
337
- /** Root directory (workspace-relative) of the project the text-search fallback for `relPath` runs in. */
338
324
  fallbackRoot(relPath) {
339
325
  return this.findProjectRoot(r => isWithin(relPath, r))?.rel ?? null;
340
326
  }
341
- /**
342
- * Why a graph tool has no data for `relPath`, or null when the path is in the
343
- * index path set. Touches the disk only for an unknown path, and answers a
344
- * real location outside the workspace as `not_found` without looking at it.
345
- */
346
327
  async checkGraphPath(relPath) {
347
328
  if (await this.isIndexedPath(relPath))
348
329
  return null;
@@ -381,17 +362,8 @@ class SearcherService {
381
362
  ) AS exists`, [owner.name]);
382
363
  return exists ? { kind: 'unknown' } : { kind: 'project_not_indexed', project: owner.name };
383
364
  }
384
- /**
385
- * Project-level dependency edges aggregated from `code_dependency`: an edge
386
- * `A → B` means some file in project A imports a file owned by project B.
387
- * Used by the graph to draw links between a monorepo's sub-projects. Only
388
- * edges whose target file belongs to an indexed project are returned (targets
389
- * outside any indexed project — e.g. unindexed libs — are skipped).
390
- */
391
365
  async getProjectEdges() {
392
366
  const rows = await this.db.query(`SELECT DISTINCT project, "toFile" FROM code_dependency`);
393
- // Reverse map: workspace-relative project root → name, longest first so the
394
- // most specific owner wins (e.g. repo/apps/account before repo).
395
367
  const byRelPath = [...this.projectRootMap().entries()]
396
368
  .map(([name, abs]) => ({ name, rel: path.relative(this.config.workspaceRoot, abs) }))
397
369
  .sort((a, b) => b.rel.length - a.rel.length);
@@ -416,29 +388,15 @@ class SearcherService {
416
388
  }
417
389
  return edges;
418
390
  }
419
- // Query translation was removed (legacy). The query is embedded as-is; the
420
- // method is kept so the tool layer needs no change and `translated` is always
421
- // false.
422
391
  async prepareQuery(raw) {
423
392
  return { query: raw, translated: false };
424
393
  }
425
- /**
426
- * Refuse when the configured embedding model disagrees with the one that built
427
- * the index — embedding a query into a foreign vector space returns noise. A
428
- * missing `index_meta` row with an empty (or altogether absent) `code_chunk`
429
- * does not block — but a *populated* `code_chunk` with no `index_meta` row is a
430
- * pre-`index_meta` install, treated as the legacy `ollama`/`bge-m3` index (same
431
- * interpretation as the `rag-index` legacy-Ollama guard), read-only — this check
432
- * never writes `index_meta`.
433
- */
434
394
  async assertIndexMatches() {
435
395
  const meta = await this.db.readIndexMeta();
436
396
  if (meta) {
437
397
  this.compareIndexed(meta.provider, meta.model, meta.dimensions);
438
398
  return;
439
399
  }
440
- // No row — a missing code_chunk table counts as empty; any other DB error
441
- // while counting propagates rather than being read as "not indexed".
442
400
  if (await (0, legacy_ollama_index_1.hasIndexedChunks)(this.db)) {
443
401
  this.compareIndexed(legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.provider, legacy_ollama_index_1.LEGACY_OLLAMA_IDENTITY.model, null);
444
402
  }
@@ -449,14 +407,6 @@ class SearcherService {
449
407
  throw new Error(`Index was built with ${indexedProvider}/${indexedModel} but the config selects ${provider}/${model}. ` +
450
408
  `Run \`rag-index --reindex\` to rebuild the index for the new model.`);
451
409
  }
452
- // Same model but a changed dimension (e.g. a Matryoshka `embedding.dimensions`
453
- // override) would embed the query at a width the column can't hold — pgvector
454
- // would throw an opaque "different vector dimensions" error mid-query. Catch
455
- // it here with the friendly hint. `embedder.dimensions` is 0 when the width is
456
- // only knowable by probing, in which case we skip (can't compare cheaply).
457
- // `indexedDimensions` is null for the legacy-inferred identity, whose provider
458
- // can never match a configured provider (ollama is no longer selectable), so
459
- // this branch is unreached in that case.
460
410
  const dim = this.embedder.dimensions;
461
411
  if (indexedDimensions != null && dim > 0 && dim !== indexedDimensions) {
462
412
  throw new Error(`Index was built at ${indexedDimensions} dimensions but ${provider}/${model} now resolves to ${dim}. ` +
@@ -484,22 +434,20 @@ class SearcherService {
484
434
  };
485
435
  }));
486
436
  }
487
- async searchChunks(query, limit, project, threshold, rerankOverride) {
437
+ async searchChunks(query, limit, project, threshold, rerankOverride, perFile = result_shaping_1.DEFAULT_PER_FILE) {
438
+ const { stages } = await this.searchChunksWithStages(query, limit, project, threshold, rerankOverride, perFile);
439
+ return stages.reranked ?? stages.vector;
440
+ }
441
+ async searchChunksWithStages(query, limit, project, threshold, rerankOverride, perFile = result_shaping_1.DEFAULT_PER_FILE) {
488
442
  const t0 = Date.now();
489
443
  await this.assertIndexMatches();
490
444
  const effectiveThreshold = threshold ?? this.config.search.similarityThreshold;
491
445
  const reranking = rerankOverride === false ? false : !!this.embedder.rerank;
492
- // Candidate floor: decouple the pool size from `limit` so a small `limit`
493
- // (e.g. the default 5) still gives the reranker a large-enough menu. Without
494
- // a reranker the extra candidates are never reordered, so keep `limit`.
446
+ const perFileActive = perFile < limit;
495
447
  const fetchLimit = reranking
496
448
  ? Math.min(Math.max(limit * this.config.reranker.overfetch, this.config.reranker.candidateFloor), 200)
497
- : limit;
498
- // With metrics on, fetch one extra row to detect a saturated pool (more
499
- // under-threshold candidates existed than the window took). The extra row is
500
- // dropped before reranking so it never affects results.
501
- const metricsOn = this.config.logging.searchMetrics;
502
- const dbLimit = metricsOn ? fetchLimit + 1 : fetchLimit;
449
+ : perFileActive ? Math.min(limit * 3, 200) : limit;
450
+ const dbLimit = fetchLimit + 1;
503
451
  this.logger.debug('SearcherService: searchChunks', {
504
452
  query: query.slice(0, 120), limit, fetchLimit, project, threshold: effectiveThreshold,
505
453
  });
@@ -512,8 +460,14 @@ class SearcherService {
512
460
  params.push(project);
513
461
  projectFilter = `AND "serviceName" = $${params.length}`;
514
462
  }
463
+ const iterative = await this.iterativeScanEnabled();
464
+ const settings = (0, hnsw_settings_1.hnswSettings)(dbLimit, iterative);
465
+ const indexParams = {
466
+ efSearch: settings['hnsw.ef_search'],
467
+ iterativeScan: iterative ? hnsw_settings_1.ITERATIVE_SCAN_MODE : 'off',
468
+ };
515
469
  const tDb0 = Date.now();
516
- const fetched = await this.db.query(`SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
470
+ const fetched = await this.db.queryWithLocalSettings(settings, `SELECT "serviceName", "filePath", "startLine", content, "ownerPath",
517
471
  ROUND((embedding <=> $1)::numeric, 4)::float8 AS score
518
472
  FROM code_chunk
519
473
  WHERE embedding <=> $1 < $2
@@ -521,9 +475,8 @@ class SearcherService {
521
475
  ORDER BY embedding <=> $1
522
476
  LIMIT $3`, params);
523
477
  const dbMs = Date.now() - tDb0;
524
- // Saturated iff the extra probe row came back — more candidates were under
525
- // the threshold than `fetchLimit`. Trim the probe row before reranking.
526
- const poolSaturated = metricsOn && fetched.length > fetchLimit;
478
+ fetched.sort((a, b) => a.score - b.score);
479
+ const poolSaturated = fetched.length > fetchLimit;
527
480
  const candidates = fetched.slice(0, fetchLimit);
528
481
  this.logger.debug('SearcherService: searchChunks candidates', {
529
482
  count: candidates.length,
@@ -531,45 +484,45 @@ class SearcherService {
531
484
  files: candidates.map(r => r.filePath),
532
485
  ms: Date.now() - t0,
533
486
  });
534
- let results;
487
+ const vector = (0, result_shaping_1.limitPerFile)(candidates, limit, perFile);
488
+ let reranked;
489
+ let rerankFailed = false;
535
490
  let rerankMs = 0;
536
- if (!reranking || candidates.length === 0) {
537
- results = candidates.slice(0, limit);
491
+ if (reranking && candidates.length === 0) {
492
+ reranked = [];
538
493
  }
539
- else {
494
+ else if (reranking) {
540
495
  const tRerank0 = Date.now();
541
496
  try {
542
- const indices = await this.embedder.rerank(query, candidates.map(r => r.content), limit);
497
+ const topN = perFileActive ? candidates.length : limit;
498
+ const indices = await this.embedder.rerank(query, candidates.map(rerankDocument), topN);
543
499
  rerankMs = Date.now() - tRerank0;
544
500
  this.logger.debug('SearcherService: searchChunks rerank done', { indices, ms: Date.now() - t0 });
545
- // Cap at `limit`: some rerank servers ignore `top_n` and echo back every
546
- // candidate, so slicing here keeps the result count within the request.
547
- results = indices.filter(i => i >= 0 && i < candidates.length).slice(0, limit).map(i => candidates[i]);
501
+ const order = indices.filter(i => i >= 0 && i < candidates.length).map(i => candidates[i]);
502
+ reranked = (0, result_shaping_1.limitPerFile)(order, limit, perFile);
548
503
  }
549
504
  catch (err) {
550
505
  rerankMs = Date.now() - tRerank0;
551
506
  this.logger.warn('SearcherService: rerank failed, falling back to vector order', {
552
- reason: err instanceof Error ? err.message : String(err),
507
+ err,
553
508
  ms: Date.now() - t0,
554
509
  });
555
- results = candidates.slice(0, limit);
510
+ reranked = vector;
511
+ rerankFailed = true;
556
512
  }
557
513
  }
558
- if (metricsOn) {
559
- this.emitSearchMetrics(query, {
560
- fetchLimit, candidates, results, poolSaturated, embedMs, dbMs, rerankMs, totalMs: Date.now() - t0,
561
- });
562
- }
563
- return results;
514
+ this.emitSearchMetrics(query, {
515
+ fetchLimit, candidates, results: reranked ?? vector, poolSaturated, indexParams,
516
+ embedMs, dbMs, rerankMs, totalMs: Date.now() - t0,
517
+ });
518
+ return {
519
+ pool: candidates,
520
+ stages: reranked ? { vector, reranked } : { vector },
521
+ rerankFailed,
522
+ fetchLimit,
523
+ indexParams,
524
+ };
564
525
  }
565
- /**
566
- * Emit one structured search-tuning line (via `logger.metric`, at `info`
567
- * regardless of `logging.level`). `candidates` are in cosine order, so a
568
- * candidate's array index is its cosine rank — `maxCosineRankSurvived` is the
569
- * deepest cosine position that survived reranking into the returned set. Read
570
- * it alongside `poolSaturated` to tune `candidateFloor`/`overfetch`/threshold
571
- * per hardware. Never logs chunk content, embeddings, or credentials.
572
- */
573
526
  emitSearchMetrics(query, m) {
574
527
  const { candidates, results } = m;
575
528
  const contentIndex = new Map(candidates.map((c, i) => [c, i]));
@@ -578,7 +531,7 @@ class SearcherService {
578
531
  for (const r of results)
579
532
  fileCounts.set(r.filePath, (fileCounts.get(r.filePath) ?? 0) + 1);
580
533
  const topFileCount = results.length ? Math.max(...fileCounts.values()) : 0;
581
- this.logger.metric('search metrics', {
534
+ this.logger.info('search metrics', {
582
535
  query: query.slice(0, 120),
583
536
  fetchLimit: m.fetchLimit,
584
537
  candidatesReturned: candidates.length,
@@ -589,6 +542,8 @@ class SearcherService {
589
542
  topFileShare: results.length ? +(topFileCount / results.length).toFixed(3) : 0,
590
543
  bestResultScore: results.length ? Math.min(...results.map(r => r.score)) : null,
591
544
  worstResultScore: results.length ? Math.max(...results.map(r => r.score)) : null,
545
+ efSearch: m.indexParams.efSearch,
546
+ iterativeScan: m.indexParams.iterativeScan,
592
547
  embedMs: m.embedMs,
593
548
  rerankMs: m.rerankMs,
594
549
  dbMs: m.dbMs,
@@ -635,7 +590,6 @@ class SearcherService {
635
590
  this.logger.debug('SearcherService: fetchGraphContext', { files: files.length });
636
591
  if (files.length === 0)
637
592
  return new Map();
638
- // Use proportional limit: 50 imports + 50 usedBy per file
639
593
  const proportionalLimit = Math.max(2000, files.length * 100);
640
594
  const values = files.map((_, i) => `($${i + 1}::text)`).join(', ');
641
595
  const rows = await this.db.query(`SELECT r.fp AS "filePath", 'import' AS direction, d."toFile" AS related
@@ -679,11 +633,12 @@ class SearcherService {
679
633
  params.push(symbol);
680
634
  symbolFilter = `AND "toSymbol" = $${params.length}`;
681
635
  }
682
- const direct = await this.db.query(`SELECT DISTINCT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
636
+ const direct = await this.db.query(`SELECT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType", min("id"::text) AS "id"
683
637
  FROM code_dependency
684
638
  WHERE "fromFile" = $1 AND NOT ${OUTSIDE_WORKSPACE_VALUE('"toFile"')}
685
639
  ${symbolFilter}
686
- ORDER BY "toFile"`, params);
640
+ GROUP BY "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
641
+ ORDER BY "toFile", min("id"::text)`, params);
687
642
  if (!transitive) {
688
643
  this.logger.debug('SearcherService: getDependencies result', { direct: direct.length, ms: Date.now() - t0 });
689
644
  return direct;
@@ -709,11 +664,12 @@ class SearcherService {
709
664
  params.push(symbol);
710
665
  symbolFilter = `AND "toSymbol" = $${params.length}`;
711
666
  }
712
- const direct = await this.db.query(`SELECT DISTINCT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
667
+ const direct = await this.db.query(`SELECT "fromFile", "fromSymbol", "toFile", "toSymbol", "depType", min("id"::text) AS "id"
713
668
  FROM code_dependency
714
669
  WHERE "toFile" = $1 AND NOT ${OUTSIDE_WORKSPACE_VALUE('"fromFile"')}
715
670
  ${symbolFilter}
716
- ORDER BY "fromFile"`, params);
671
+ GROUP BY "fromFile", "fromSymbol", "toFile", "toSymbol", "depType"
672
+ ORDER BY "fromFile", min("id"::text)`, params);
717
673
  if (!transitive) {
718
674
  this.logger.debug('SearcherService: getDependents result', { direct: direct.length, ms: Date.now() - t0 });
719
675
  return direct;
@@ -763,11 +719,6 @@ class SearcherService {
763
719
  this.logger.debug('SearcherService: getCallers result', { count: rows.length, ms: Date.now() - t0 });
764
720
  return rows;
765
721
  }
766
- /**
767
- * Whole-word text search for `symbol` under `rootDir` (workspace-relative).
768
- * grep exiting ≥ 2 after printing matches is a partial result; without
769
- * matches, or when grep cannot be spawned, the search failed.
770
- */
771
722
  async grepSymbol(symbol, rootDir, excludeFile) {
772
723
  const t0 = Date.now();
773
724
  this.logger.debug('SearcherService: grepSymbol', { symbol, rootDir, excludeFile });
@@ -795,19 +746,12 @@ class SearcherService {
795
746
  this.logger.debug('SearcherService: grepSymbol result', { matches: Math.min(all.length, GREP_MAX_RESULTS), truncated, partial: !result.ok, ms: Date.now() - t0 });
796
747
  return { ok: true, matches: all.slice(0, GREP_MAX_RESULTS), truncated, partial: !result.ok };
797
748
  }
798
- // Sync wrapper: validates name (throws synchronously), then returns Promise.
799
- // Every kind, including 'method', is looked up in the index first.
800
749
  findSymbol(name, project, kind) {
801
750
  const parsed = (0, symbol_name_1.parseFindSymbolName)(name);
802
751
  if (!parsed)
803
752
  throw new Error('Invalid symbol name');
804
753
  return this.findSymbolFromIndex(parsed, project, kind);
805
754
  }
806
- /**
807
- * Whether `code_symbol.target` exists. A present result is cached for the
808
- * process; an absent one is re-checked on every call. Propagates on a
809
- * failing check, same as any other failed index query.
810
- */
811
755
  async hasTargetColumn() {
812
756
  if (this.targetColumnPresent)
813
757
  return true;
@@ -870,7 +814,6 @@ class SearcherService {
870
814
  this.logger.debug('SearcherService: findSymbol result', { source: 'index', hits: hits.length, truncated, ms: Date.now() - t0 });
871
815
  if (hits.length > 0)
872
816
  return { hits, truncated, grepFailed: false };
873
- // GraphQL kinds never grep — an empty result just means no row yet (stale index).
874
817
  if (isGraphqlKind)
875
818
  return { hits: [], truncated: false, grepFailed: false };
876
819
  if (parsed.form === 'quoted' || parsed.form === 'computed')
@@ -878,8 +821,7 @@ class SearcherService {
878
821
  return this.grepFindSymbol(parsed, project, kind);
879
822
  }
880
823
  catch (err) {
881
- this.logger.warn('SearcherService: symbol index unavailable', { reason: err.message, ms: Date.now() - t0 });
882
- // GraphQL kinds never grep — a failing index query propagates as an error response instead.
824
+ this.logger.warn('SearcherService: symbol index unavailable', { err, ms: Date.now() - t0 });
883
825
  if (isGraphqlKind)
884
826
  throw err;
885
827
  if (parsed.form === 'quoted' || parsed.form === 'computed')
@@ -889,7 +831,7 @@ class SearcherService {
889
831
  }
890
832
  async grepFindSymbol(parsed, project, kind) {
891
833
  const t0 = Date.now();
892
- const grepName = parsed.name; // qualified names are grepped by their last segment, no container filter
834
+ const grepName = parsed.name;
893
835
  this.logger.debug('SearcherService: findSymbol', { grepName, project, kind, source: 'grep' });
894
836
  const pattern = this.buildFindSymbolPattern(grepName, kind);
895
837
  if (pattern === null)
@@ -971,10 +913,6 @@ class SearcherService {
971
913
  }
972
914
  return { ok: true, tree: lines.join('\n') };
973
915
  }
974
- /**
975
- * The read tools' common front: workspace boundary, then existence and type.
976
- * A missing file still in the index is `deleted`; otherwise `not_found`.
977
- */
978
916
  async openableFile(relPath, absPath) {
979
917
  const loc = await this.checkRealLocation(relPath, absPath, 'file');
980
918
  if (!loc.ok)
@@ -990,13 +928,8 @@ class SearcherService {
990
928
  return { ok: false, reason: 'deleted' };
991
929
  return { ok: false, reason: 'not_found', candidates: await this.candidatesOrNone(relPath, 'file') };
992
930
  }
993
- // Checked before opening: a FIFO or device would block the reader.
994
931
  return stat.isFile() ? null : { ok: false, reason: 'not_a_file' };
995
932
  }
996
- /**
997
- * Streams the file's lines (as `readline` yields them) to `onLine` until it
998
- * returns false. An open or read error discards everything read so far.
999
- */
1000
933
  async readLines(absPath, onLine) {
1001
934
  const stream = this.fs.createReadStream(absPath);
1002
935
  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
@@ -1069,7 +1002,6 @@ class SearcherService {
1069
1002
  this.logger.debug('SearcherService: readFileRange result', { linesRead: lines.length, hasMore, ms: Date.now() - t0 });
1070
1003
  return { ok: true, filePath: relPath, startLine, endLine: Math.min(endLine, lastRead), lines, hasMore };
1071
1004
  }
1072
- /** Throws when `absDir` itself cannot be read; an unreadable subdirectory is marked in place. */
1073
1005
  async walkDir(absDir, currentDepth, maxDepth, pattern, lines) {
1074
1006
  const entries = await this.fs.readdir(absDir);
1075
1007
  entries.sort((a, b) => {
@@ -1101,11 +1033,6 @@ class SearcherService {
1101
1033
  }
1102
1034
  }
1103
1035
  }
1104
- /**
1105
- * ERE pattern for a grep-fallback kind, or null when that kind is never
1106
- * grepped (`function_property`/`method_signature` are index-only; a bare
1107
- * `constructor` search only makes sense when the name itself is `constructor`).
1108
- */
1109
1036
  buildFindSymbolPattern(name, kind) {
1110
1037
  if (kind === 'function_property' || kind === 'method_signature')
1111
1038
  return null;
@@ -1113,7 +1040,7 @@ class SearcherService {
1113
1040
  return null;
1114
1041
  if (kind === 'constructor')
1115
1042
  return name === 'constructor' ? `(^|[[:space:]])constructor[[:space:]]*\\(` : null;
1116
- const esc = name.replace(/\$/g, '\\$'); // ERE end-of-line metachar
1043
+ const esc = name.replace(/\$/g, '\\$');
1117
1044
  const MODIFIER = '(public|private|protected|static|abstract|async|override)';
1118
1045
  const BOUNDARY = '(^|[^[:alnum:]_$])';
1119
1046
  const patterns = {
@@ -1130,7 +1057,6 @@ class SearcherService {
1130
1057
  return Object.values(patterns).join('|');
1131
1058
  return patterns[kind];
1132
1059
  }
1133
- /** 1-based index of the first whole-word occurrence of `name` in `line`, or -1. */
1134
1060
  static wholeWordIndex(line, name) {
1135
1061
  const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
1136
1062
  const re = new RegExp(`(?<![\\w$#])${escaped}(?![\\w$#])`);
@@ -1160,12 +1086,6 @@ class SearcherService {
1160
1086
  return 'method';
1161
1087
  return 'variable';
1162
1088
  }
1163
- /**
1164
- * Async wrapper around grep to avoid blocking the event loop. `ok: false`
1165
- * means grep could not be spawned (`output: null`) or exited with status ≥ 2,
1166
- * keeping whatever it printed — distinguishable from a clean "no matches",
1167
- * which is `ok: true, output: null`.
1168
- */
1169
1089
  spawnGrep(args) {
1170
1090
  return new Promise((resolve) => {
1171
1091
  const proc = (0, child_process_1.spawn)('grep', args, { stdio: ['ignore', 'pipe', 'pipe'] });
@@ -1180,12 +1100,11 @@ class SearcherService {
1180
1100
  });
1181
1101
  proc.stderr.on('data', () => { });
1182
1102
  proc.on('error', (err) => {
1183
- this.logger.error('SearcherService: grep spawn failed', { reason: err.message });
1103
+ this.logger.error('SearcherService: grep spawn failed', { err });
1184
1104
  resolve({ ok: false, output: null });
1185
1105
  });
1186
1106
  proc.on('close', (code) => {
1187
1107
  const output = chunks.length === 0 ? null : Buffer.concat(chunks).toString('utf-8');
1188
- // grep returns 1 for "no matches" — not an error; 2+ is a real failure
1189
1108
  if (code !== null && code >= 2) {
1190
1109
  this.logger.warn('SearcherService: grep exited with code', { code });
1191
1110
  resolve({ ok: false, output });
@@ -2,12 +2,6 @@ import type { Config } from '../config';
2
2
  import type { IEmbedder, InputType } from '../interfaces/IEmbedder';
3
3
  import type { ILogger } from '../interfaces/ILogger';
4
4
  import type { EmbedderDeps } from './OpenAICompatibleEmbedder';
5
- /**
6
- * Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
7
- * of strings, `input_type` (`query` | `document`) is required for best quality,
8
- * and the optional `output_dimension` requests Matryoshka truncation. Response
9
- * is `{ data: [{ embedding, index }] }`.
10
- */
11
5
  export declare class VoyageEmbedder implements IEmbedder {
12
6
  private config;
13
7
  private logger;
@@ -6,12 +6,6 @@ const config_1 = require("../config");
6
6
  const reranker_1 = require("./reranker");
7
7
  const http_retry_1 = require("./http-retry");
8
8
  const constants_1 = require("../constants");
9
- /**
10
- * Voyage AI embedder. Voyage's schema differs from OpenAI's: `input` is an array
11
- * of strings, `input_type` (`query` | `document`) is required for best quality,
12
- * and the optional `output_dimension` requests Matryoshka truncation. Response
13
- * is `{ data: [{ embedding, index }] }`.
14
- */
15
9
  class VoyageEmbedder {
16
10
  config;
17
11
  logger;