codesearch-mcp 0.1.0-alpha → 0.2.1-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/README.md +146 -41
  3. package/dist/app.js +4 -19
  4. package/dist/src/commands/IndexCommand.js +68 -7
  5. package/dist/src/config-error.d.ts +7 -0
  6. package/dist/src/config-error.js +14 -0
  7. package/dist/src/config-store.d.ts +30 -12
  8. package/dist/src/config-store.js +81 -26
  9. package/dist/src/config.d.ts +5 -4
  10. package/dist/src/config.js +14 -16
  11. package/dist/src/constants.d.ts +8 -2
  12. package/dist/src/constants.js +8 -5
  13. package/dist/src/db/index.d.ts +1 -1
  14. package/dist/src/db/index.js +2 -1
  15. package/dist/src/db/schema.d.ts +75 -0
  16. package/dist/src/db/schema.js +10 -1
  17. package/dist/src/enriched-prompt.d.ts +8 -0
  18. package/dist/src/enriched-prompt.js +87 -0
  19. package/dist/src/help.js +1 -1
  20. package/dist/src/interfaces/IAstParser.d.ts +18 -2
  21. package/dist/src/interfaces/IDatabase.d.ts +31 -0
  22. package/dist/src/interfaces/ILogger.d.ts +7 -0
  23. package/dist/src/interfaces/ISearcher.d.ts +3 -0
  24. package/dist/src/services/AstChunker.d.ts +11 -0
  25. package/dist/src/services/AstChunker.js +44 -0
  26. package/dist/src/services/AstParserService.d.ts +39 -7
  27. package/dist/src/services/AstParserService.js +504 -104
  28. package/dist/src/services/DatabaseService.d.ts +17 -0
  29. package/dist/src/services/DatabaseService.js +130 -2
  30. package/dist/src/services/EmbedderFactory.d.ts +1 -1
  31. package/dist/src/services/EmbedderFactory.js +2 -5
  32. package/dist/src/services/IndexerService.d.ts +4 -14
  33. package/dist/src/services/IndexerService.js +54 -156
  34. package/dist/src/services/InfinityEmbedder.d.ts +14 -55
  35. package/dist/src/services/InfinityEmbedder.js +30 -141
  36. package/dist/src/services/LoggerService.d.ts +3 -1
  37. package/dist/src/services/LoggerService.js +17 -3
  38. package/dist/src/services/McpServer.js +3 -3
  39. package/dist/src/services/SearcherService.d.ts +39 -4
  40. package/dist/src/services/SearcherService.js +254 -81
  41. package/dist/src/services/WebServerService.d.ts +41 -3
  42. package/dist/src/services/WebServerService.js +90 -2
  43. package/dist/src/services/infinity-provisioning.d.ts +115 -0
  44. package/dist/src/services/infinity-provisioning.js +516 -0
  45. package/dist/src/services/legacy-ollama-index.d.ts +24 -0
  46. package/dist/src/services/legacy-ollama-index.js +61 -0
  47. package/dist/src/services/reranker.d.ts +3 -3
  48. package/dist/src/services/reranker.js +3 -3
  49. package/dist/src/setup.d.ts +37 -0
  50. package/dist/src/setup.js +111 -157
  51. package/dist/src/symbol-name.d.ts +44 -0
  52. package/dist/src/symbol-name.js +68 -0
  53. package/dist/src/symbol-rows.d.ts +12 -0
  54. package/dist/src/symbol-rows.js +38 -0
  55. package/dist/src/tools/FindSymbolTool.js +30 -7
  56. package/dist/src/tools/GetCallersTool.js +20 -5
  57. package/docker/docker-compose.yml +5 -15
  58. package/docs/architecture.svg +103 -0
  59. package/docs/web-dashboard.md +36 -17
  60. package/package.json +1 -6
  61. package/dist/src/commands/BackfillCommand.d.ts +0 -7
  62. package/dist/src/commands/BackfillCommand.js +0 -37
  63. package/dist/src/services/OllamaEmbedder.d.ts +0 -25
  64. package/dist/src/services/OllamaEmbedder.js +0 -111
@@ -6,6 +6,7 @@ import * as schema from '../db/schema';
6
6
  type Schema = typeof schema;
7
7
  export type DrizzleDb = NodePgDatabase<Schema>;
8
8
  export type DrizzleTx = NodePgTransaction<Schema, ExtractTablesWithRelations<Schema>>;
9
+ export declare const CURRENT_SYMBOL_INDEX_VERSION = 2;
9
10
  export declare class DatabaseService implements IDatabase {
10
11
  private config;
11
12
  private pool;
@@ -27,8 +28,24 @@ export declare class DatabaseService implements IDatabase {
27
28
  recreateChunkTable(dimensions: number): Promise<void>;
28
29
  readIndexMeta(): Promise<IndexMeta | null>;
29
30
  writeIndexMeta(meta: IndexMeta): Promise<void>;
31
+ listKnownFiles(project: string): Promise<string[]>;
32
+ deleteFiles(project: string, filePaths: string[]): Promise<void>;
33
+ listIndexedProjects(): Promise<string[]>;
34
+ deleteProject(name: string): Promise<void>;
35
+ listProjectsWithIndexRows(): Promise<string[]>;
36
+ /**
37
+ * Among `projectNames` (or every project with rows when null), those with at
38
+ * least one row in `code_chunk`/`code_symbol` and a `symbol_index_version`
39
+ * below `CURRENT_SYMBOL_INDEX_VERSION` (a missing row counts as version 1).
40
+ * A missing `symbol_index_version` table (un-migrated DB, `rag-mcp`/`rag-web`
41
+ * bootstrapped with `initDb`) counts as "no marker rows" — every project with
42
+ * rows is then outdated. Any other database error propagates.
43
+ */
44
+ listOutdatedProjects(projectNames: string[] | null): Promise<string[]>;
45
+ upsertSymbolIndexVersion(project: string): Promise<void>;
30
46
  /** Guard against SQL injection — the dimension is interpolated into DDL. */
31
47
  private assertDim;
48
+ private assertMinPostgresVersion;
32
49
  private ensureDatabaseExists;
33
50
  }
34
51
  export {};
@@ -33,10 +33,14 @@ var __importStar = (this && this.__importStar) || (function () {
33
33
  };
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.DatabaseService = void 0;
36
+ exports.DatabaseService = exports.CURRENT_SYMBOL_INDEX_VERSION = void 0;
37
37
  const node_postgres_1 = require("drizzle-orm/node-postgres");
38
38
  const pg_1 = require("pg");
39
+ const drizzle_orm_1 = require("drizzle-orm");
39
40
  const schema = __importStar(require("../db/schema"));
41
+ const db_1 = require("../db");
42
+ const MIN_PG_VERSION_NUM = 150000;
43
+ exports.CURRENT_SYMBOL_INDEX_VERSION = 2;
40
44
  class DatabaseService {
41
45
  config;
42
46
  pool;
@@ -66,6 +70,12 @@ class DatabaseService {
66
70
  async initDbWithSchema() {
67
71
  await this.ensureDatabaseExists();
68
72
  await this.initDb();
73
+ // Read as the very first statement on the application database, before any
74
+ // CREATE TABLE / CREATE INDEX / ALTER TABLE / DROP — `NULLS NOT DISTINCT`
75
+ // (used below) requires PostgreSQL 15+, and the DROP/CREATE pair must never
76
+ // run against an older server (it would fail after the DROP and leave
77
+ // code_symbol without a unique index).
78
+ await this.assertMinPostgresVersion();
69
79
  // code_chunk table — created at the legacy default width (1024) so search
70
80
  // tools always have a table. IndexerService recreates it at the resolved
71
81
  // width before indexing when the active model's dimension differs.
@@ -128,7 +138,23 @@ class DatabaseService {
128
138
  }
129
139
  await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_name ON code_symbol (name, project)`);
130
140
  await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_file ON code_symbol (project, "filePath")`);
131
- await this.query(`CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique ON code_symbol (project, "filePath", name, kind)`);
141
+ await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS container TEXT`);
142
+ await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS "isStatic" BOOLEAN NOT NULL DEFAULT false`);
143
+ // DDL is transactional in PostgreSQL — either both statements apply or
144
+ // neither does, so a failure here never leaves code_symbol without a
145
+ // unique index (the old idx_symbol_unique is dropped only once the new
146
+ // one is confirmed to exist).
147
+ await this.transaction(async (tx) => {
148
+ await tx.execute('DROP INDEX IF EXISTS idx_symbol_unique');
149
+ await tx.execute('CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique_v2 ' +
150
+ 'ON code_symbol (project, "filePath", container, name, kind, line) NULLS NOT DISTINCT');
151
+ });
152
+ await this.query(`
153
+ CREATE TABLE IF NOT EXISTS symbol_index_version (
154
+ project TEXT PRIMARY KEY,
155
+ version INT NOT NULL
156
+ )
157
+ `);
132
158
  // code_call table
133
159
  await this.query(`
134
160
  CREATE TABLE IF NOT EXISTS code_call (
@@ -198,6 +224,94 @@ class DatabaseService {
198
224
  dimensions = EXCLUDED.dimensions,
199
225
  "indexedAt" = now()`, [meta.provider, meta.model, meta.dimensions]);
200
226
  }
227
+ async listKnownFiles(project) {
228
+ const rows = await this.query(`SELECT "filePath" AS fp FROM code_chunk WHERE "serviceName" = $1
229
+ UNION
230
+ SELECT "filePath" AS fp FROM code_symbol WHERE project = $1
231
+ UNION
232
+ SELECT "fromFile" AS fp FROM code_call WHERE project = $1
233
+ UNION
234
+ SELECT "fromFile" AS fp FROM code_dependency WHERE project = $1`, [project]);
235
+ return rows.map(r => r.fp);
236
+ }
237
+ async deleteFiles(project, filePaths) {
238
+ if (filePaths.length === 0)
239
+ return;
240
+ // PostgreSQL limits ~32 767 bind params per statement; OR queries bind the
241
+ // array twice (fromFile + calleeFile), so cap each batch at 10 000 entries.
242
+ const BATCH = 10_000;
243
+ await this.transaction(async (tx) => {
244
+ for (let i = 0; i < filePaths.length; i += BATCH) {
245
+ const batch = filePaths.slice(i, i + BATCH);
246
+ await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, project), (0, drizzle_orm_1.inArray)(db_1.codeChunk.filePath, batch)));
247
+ await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, project), (0, drizzle_orm_1.inArray)(db_1.codeSymbol.filePath, batch)));
248
+ await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, project), (0, drizzle_orm_1.or)((0, drizzle_orm_1.inArray)(db_1.codeCall.fromFile, batch), (0, drizzle_orm_1.inArray)(db_1.codeCall.calleeFile, batch))));
249
+ await tx.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, project), (0, drizzle_orm_1.or)((0, drizzle_orm_1.inArray)(db_1.codeDependency.fromFile, batch), (0, drizzle_orm_1.inArray)(db_1.codeDependency.toFile, batch))));
250
+ }
251
+ });
252
+ }
253
+ async listIndexedProjects() {
254
+ const rows = await this.query(`SELECT "serviceName" AS name FROM code_chunk
255
+ UNION
256
+ SELECT project AS name FROM code_symbol
257
+ UNION
258
+ SELECT project AS name FROM code_call
259
+ UNION
260
+ SELECT project AS name FROM code_dependency`);
261
+ return rows.map(r => r.name);
262
+ }
263
+ async deleteProject(name) {
264
+ await this.transaction(async (tx) => {
265
+ await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, name));
266
+ await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, name));
267
+ await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, name));
268
+ await tx.delete(db_1.codeDependency).where((0, drizzle_orm_1.eq)(db_1.codeDependency.project, name));
269
+ });
270
+ }
271
+ async listProjectsWithIndexRows() {
272
+ const rows = await this.query(`SELECT DISTINCT name FROM (
273
+ SELECT "serviceName" AS name FROM code_chunk
274
+ UNION
275
+ SELECT project AS name FROM code_symbol
276
+ ) t`);
277
+ return rows.map(r => r.name);
278
+ }
279
+ /**
280
+ * Among `projectNames` (or every project with rows when null), those with at
281
+ * least one row in `code_chunk`/`code_symbol` and a `symbol_index_version`
282
+ * below `CURRENT_SYMBOL_INDEX_VERSION` (a missing row counts as version 1).
283
+ * A missing `symbol_index_version` table (un-migrated DB, `rag-mcp`/`rag-web`
284
+ * bootstrapped with `initDb`) counts as "no marker rows" — every project with
285
+ * rows is then outdated. Any other database error propagates.
286
+ */
287
+ async listOutdatedProjects(projectNames) {
288
+ const rowsUnion = `
289
+ SELECT "serviceName" AS name FROM code_chunk
290
+ UNION
291
+ SELECT project AS name FROM code_symbol
292
+ `;
293
+ const filterSql = projectNames ? 'AND t.name = ANY($1)' : '';
294
+ const params = projectNames ? [projectNames] : [];
295
+ try {
296
+ const rows = await this.query(`SELECT DISTINCT t.name FROM (${rowsUnion}) t
297
+ WHERE NOT EXISTS (
298
+ SELECT 1 FROM symbol_index_version v
299
+ WHERE v.project = t.name AND v.version >= ${exports.CURRENT_SYMBOL_INDEX_VERSION}
300
+ )
301
+ ${filterSql}`, params);
302
+ return rows.map(r => r.name);
303
+ }
304
+ catch (err) {
305
+ if (!/relation "symbol_index_version" does not exist/i.test(err.message))
306
+ throw err;
307
+ const rows = await this.query(`SELECT DISTINCT t.name FROM (${rowsUnion}) t ${projectNames ? 'WHERE t.name = ANY($1)' : ''}`, params);
308
+ return rows.map(r => r.name);
309
+ }
310
+ }
311
+ async upsertSymbolIndexVersion(project) {
312
+ await this.query(`INSERT INTO symbol_index_version (project, version) VALUES ($1, $2)
313
+ ON CONFLICT (project) DO UPDATE SET version = EXCLUDED.version`, [project, exports.CURRENT_SYMBOL_INDEX_VERSION]);
314
+ }
201
315
  /** Guard against SQL injection — the dimension is interpolated into DDL. */
202
316
  assertDim(dimensions) {
203
317
  if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > 2000) {
@@ -205,6 +319,20 @@ class DatabaseService {
205
319
  }
206
320
  return dimensions;
207
321
  }
322
+ async assertMinPostgresVersion() {
323
+ let versionNum;
324
+ try {
325
+ const rows = await this.query(`SELECT current_setting('server_version_num') AS v`);
326
+ versionNum = Number(rows[0]?.v);
327
+ }
328
+ catch (err) {
329
+ throw new Error(`Failed to read PostgreSQL server_version_num (requires PostgreSQL 15+): ${err.message}`);
330
+ }
331
+ if (!Number.isFinite(versionNum) || versionNum < MIN_PG_VERSION_NUM) {
332
+ throw new Error(`PostgreSQL 15+ is required (server_version_num=${versionNum}) — ` +
333
+ `the symbol index's unique index relies on NULLS NOT DISTINCT, added in PostgreSQL 15.`);
334
+ }
335
+ }
208
336
  async ensureDatabaseExists() {
209
337
  const { host, port, username, password, database } = this.config.db;
210
338
  const client = new pg_1.Client({ host, port, user: username, password, database: 'postgres' });
@@ -4,6 +4,6 @@ import type { ILogger } from '../interfaces/ILogger';
4
4
  import { type EmbedderDeps } from './OpenAICompatibleEmbedder';
5
5
  /**
6
6
  * Construct the `IEmbedder` implementation selected by `config.embedding.provider`.
7
- * The default (`ollama`) keeps existing installs working unchanged.
7
+ * The default (`infinity`) is the recommended local backend.
8
8
  */
9
9
  export declare function createEmbedder(config: Config, logger?: ILogger, deps?: EmbedderDeps): IEmbedder;
@@ -2,13 +2,12 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.createEmbedder = createEmbedder;
4
4
  const LoggerService_1 = require("./LoggerService");
5
- const OllamaEmbedder_1 = require("./OllamaEmbedder");
6
5
  const OpenAICompatibleEmbedder_1 = require("./OpenAICompatibleEmbedder");
7
6
  const InfinityEmbedder_1 = require("./InfinityEmbedder");
8
7
  const VoyageEmbedder_1 = require("./VoyageEmbedder");
9
8
  /**
10
9
  * Construct the `IEmbedder` implementation selected by `config.embedding.provider`.
11
- * The default (`ollama`) keeps existing installs working unchanged.
10
+ * The default (`infinity`) is the recommended local backend.
12
11
  */
13
12
  function createEmbedder(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
14
13
  switch (config.embedding.provider) {
@@ -18,12 +17,10 @@ function createEmbedder(config, logger = LoggerService_1.LoggerService.silent(),
18
17
  return new InfinityEmbedder_1.InfinityEmbedder(config, logger, deps);
19
18
  case 'voyage':
20
19
  return new VoyageEmbedder_1.VoyageEmbedder(config, logger, deps);
21
- case 'ollama':
22
- return new OllamaEmbedder_1.OllamaEmbedder(config, logger, deps);
23
20
  default: {
24
21
  // The Zod enum should already reject this; guard anyway for a clear message.
25
22
  const p = config.embedding.provider;
26
- throw new Error(`Unknown embedding provider: "${p}". Expected one of: ollama, openai-compatible, voyage, infinity.`);
23
+ throw new Error(`Unknown embedding provider: "${p}". Expected one of: openai-compatible, voyage, infinity.`);
27
24
  }
28
25
  }
29
26
  }
@@ -35,24 +35,14 @@ export declare class IndexerService {
35
35
  prepareEmbeddingSchema(): Promise<IndexMeta>;
36
36
  private reconcileIndex;
37
37
  private countChunks;
38
- backfillSymbols(signal?: AbortSignal): Promise<{
39
- total: number;
40
- processed: number;
41
- skipped: number;
42
- failed: number;
43
- }>;
44
- backfillCalls(signal?: AbortSignal): Promise<{
45
- total: number;
46
- processed: number;
47
- skipped: number;
48
- failed: number;
49
- }>;
50
38
  private writeDepEdges;
51
39
  scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[], opts?: {
52
40
  rootTsConfigPath?: string;
53
41
  }): Promise<IndexStats>;
54
- private static inferRole;
55
- private getEnrichedPrompt;
42
+ pruneProjects(keep: string[]): Promise<{
43
+ removed: string[];
44
+ failed: string[];
45
+ }>;
56
46
  private static hashContent;
57
47
  private ingestFile;
58
48
  }
@@ -40,6 +40,8 @@ const crypto_1 = require("crypto");
40
40
  const cli_progress_1 = require("cli-progress");
41
41
  const drizzle_orm_1 = require("drizzle-orm");
42
42
  const owner_path_1 = require("../owner-path");
43
+ const enriched_prompt_1 = require("../enriched-prompt");
44
+ const symbol_rows_1 = require("../symbol-rows");
43
45
  const db_1 = require("../db");
44
46
  const file_utils_1 = require("../file-utils");
45
47
  const LoggerService_1 = require("./LoggerService");
@@ -124,97 +126,6 @@ class IndexerService {
124
126
  throw err;
125
127
  }
126
128
  }
127
- async backfillSymbols(signal) {
128
- const t0 = Date.now();
129
- const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
130
- this.logger.debug('IndexerService: backfillSymbols start', { total: rows.length });
131
- let processed = 0, skipped = 0, failed = 0;
132
- for (const { serviceName, filePath } of rows) {
133
- if (signal?.aborted)
134
- break;
135
- const absPath = path.join(this.config.workspaceRoot, filePath);
136
- const projectRoot = path.join(this.config.workspaceRoot, serviceName);
137
- if (!fs.existsSync(absPath)) {
138
- skipped++;
139
- continue;
140
- }
141
- try {
142
- const { symbols: rawSymbols } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
143
- const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
144
- await this.db.transaction(async (tx) => {
145
- await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, filePath)));
146
- if (symbols.length > 0) {
147
- await tx.insert(db_1.codeSymbol).values(symbols);
148
- }
149
- });
150
- processed++;
151
- }
152
- catch (err) {
153
- const reason = err.message;
154
- console.error(` Failed: ${filePath} — ${reason}`);
155
- this.logger.error('IndexerService: symbol backfill failed', { file: filePath, reason });
156
- failed++;
157
- }
158
- const count = processed + failed;
159
- if (count > 0 && count % 50 === 0) {
160
- const msg = ` ${count}/${rows.length} files...`;
161
- process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
162
- }
163
- }
164
- this.logger.debug('IndexerService: backfillSymbols done', { total: rows.length, processed, skipped, failed, ms: Date.now() - t0 });
165
- return { total: rows.length, processed, skipped, failed };
166
- }
167
- async backfillCalls(signal) {
168
- const t0 = Date.now();
169
- const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
170
- this.logger.debug('IndexerService: backfillCalls start', { total: rows.length });
171
- let processed = 0, skipped = 0, failed = 0;
172
- for (const { serviceName, filePath } of rows) {
173
- if (signal?.aborted)
174
- break;
175
- const absPath = path.join(this.config.workspaceRoot, filePath);
176
- const projectRoot = path.join(this.config.workspaceRoot, serviceName);
177
- if (!fs.existsSync(absPath)) {
178
- skipped++;
179
- continue;
180
- }
181
- try {
182
- const { calls: rawCalls } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
183
- const calls = rawCalls
184
- .filter(c => c.calleeFile.startsWith(this.config.workspaceRoot))
185
- .map(c => ({
186
- project: serviceName,
187
- fromFile: filePath,
188
- fromSymbol: c.fromSymbol,
189
- calleeName: c.calleeName,
190
- calleeFile: path.relative(this.config.workspaceRoot, c.calleeFile),
191
- calleeSymbol: c.calleeSymbol,
192
- line: c.line,
193
- col: c.col,
194
- }));
195
- await this.db.transaction(async (tx) => {
196
- await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, filePath)));
197
- if (calls.length > 0) {
198
- await tx.insert(db_1.codeCall).values(calls);
199
- }
200
- });
201
- processed++;
202
- }
203
- catch (err) {
204
- const reason = err.message;
205
- console.error(` Failed: ${filePath} — ${reason}`);
206
- this.logger.error('IndexerService: call backfill failed', { file: filePath, reason });
207
- failed++;
208
- }
209
- const count = processed + failed;
210
- if (count > 0 && count % 50 === 0) {
211
- const msg = ` ${count}/${rows.length} files...`;
212
- process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
213
- }
214
- }
215
- this.logger.debug('IndexerService: backfillCalls done', { total: rows.length, processed, skipped, failed, ms: Date.now() - t0 });
216
- return { total: rows.length, processed, skipped, failed };
217
- }
218
129
  async writeDepEdges(serviceName, relFilePath, edges, tx) {
219
130
  const run = async (t) => {
220
131
  await t.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeDependency.fromFile, relFilePath)));
@@ -231,18 +142,26 @@ class IndexerService {
231
142
  const startTime = Date.now();
232
143
  this.logger.debug('IndexerService: scanAndIndex start', { serviceName, dir, force });
233
144
  if (force) {
234
- await this.db.query(`UPDATE code_chunk SET "lastModified" = NULL, "contentHash" = NULL WHERE "serviceName" = $1`, [serviceName]);
145
+ // Reset incremental state and clear the project's symbol/call rows in one
146
+ // transaction: a file whose re-ingestion later fails must hold no rows at
147
+ // all, never rows in the previous (pre-member-extraction) format.
148
+ await this.db.transaction(async (tx) => {
149
+ await tx.update(db_1.codeChunk)
150
+ .set({ lastModified: null, contentHash: null })
151
+ .where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName));
152
+ await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName));
153
+ await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName));
154
+ });
235
155
  }
236
156
  const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
237
157
  this.logger.debug('IndexerService: scanAndIndex files collected', { serviceName, count: files.length, force });
238
158
  const collectedRelPaths = new Set(files.map(f => path.relative(this.config.workspaceRoot, f)));
239
- const dbPaths = await this.db.query(`SELECT DISTINCT "filePath" FROM code_chunk WHERE "serviceName" = $1`, [serviceName]);
240
- for (const { filePath } of dbPaths) {
241
- if (!collectedRelPaths.has(filePath)) {
242
- await this.db.transaction(tx => tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, filePath))));
243
- await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "fromFile" = $2`, [serviceName, filePath]);
244
- await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "toFile" = $2`, [serviceName, filePath]);
245
- console.log(` Removed stale: ${path.basename(filePath)}`);
159
+ const knownFiles = await this.db.listKnownFiles(serviceName);
160
+ const staleFiles = knownFiles.filter(f => !collectedRelPaths.has(f));
161
+ if (staleFiles.length > 0) {
162
+ await this.db.deleteFiles(serviceName, staleFiles);
163
+ for (const f of staleFiles) {
164
+ console.log(` Removed stale: ${path.basename(f)}`);
246
165
  }
247
166
  }
248
167
  const bar = new cli_progress_1.SingleBar({
@@ -335,6 +254,12 @@ class IndexerService {
335
254
  }
336
255
  }
337
256
  this.ast.clearCache(dir);
257
+ // The marker is written exactly when the scan reaches here without having
258
+ // been interrupted — never on the early-return abort path above, and never
259
+ // when anything in this method throws (the error propagates instead).
260
+ if (!signal?.aborted) {
261
+ await this.db.upsertSymbolIndexVersion(serviceName);
262
+ }
338
263
  stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
339
264
  this.logger.debug('IndexerService: project indexed', {
340
265
  project: serviceName,
@@ -346,50 +271,26 @@ class IndexerService {
346
271
  });
347
272
  return stats;
348
273
  }
349
- static inferRole(relPath) {
350
- const p = relPath.toLowerCase();
351
- if (p.includes('/services/') || p.endsWith('.service.ts'))
352
- return 'Service';
353
- if (p.includes('/controllers/') || p.endsWith('.controller.ts'))
354
- return 'Controller';
355
- if (p.includes('/entities/') || p.endsWith('.entity.ts'))
356
- return 'Entity';
357
- if (p.includes('/guards/') || p.endsWith('.guard.ts'))
358
- return 'Guard';
359
- if (p.includes('/commands/'))
360
- return 'Command';
361
- if (p.includes('/interfaces/'))
362
- return 'Interface';
363
- if (p.includes('/middleware/'))
364
- return 'Middleware';
365
- if (p.includes('/interceptors/'))
366
- return 'Interceptor';
367
- if (p.endsWith('.test.ts') || p.endsWith('.spec.ts') || p.includes('/tests/'))
368
- return 'Test';
369
- if (p.includes('/utils/') || p.includes('/helpers/') || p.endsWith('-utils.ts'))
370
- return 'Utility';
371
- return null;
372
- }
373
- getEnrichedPrompt(piece, relPath, serviceName, _meta, edges, symbols) {
374
- const chunkStart = piece.startLine;
375
- const chunkEnd = piece.endLine;
376
- const lines = [];
377
- lines.push(`Project: ${serviceName} | File: ${relPath}`);
378
- const role = IndexerService.inferRole(relPath);
379
- if (role)
380
- lines.push(`Role: ${role}`);
381
- const overlapping = symbols.filter(s => s.line != null &&
382
- s.line <= chunkEnd &&
383
- (s.endLine ?? s.line) >= chunkStart);
384
- if (overlapping.length > 0) {
385
- lines.push(`Symbols: ${overlapping.map(s => `${s.kind} ${s.name}`).join(', ')}`);
386
- }
387
- if (edges.length > 0) {
388
- const depNames = [...new Set(edges.map(e => path.basename(e.toFile, path.extname(e.toFile)).split('.')[0]))];
389
- lines.push(`Dependencies: ${depNames.join(', ')}`);
274
+ async pruneProjects(keep) {
275
+ if (keep.length === 0)
276
+ return { removed: [], failed: [] };
277
+ const indexed = await this.db.listIndexedProjects();
278
+ const keepSet = new Set(keep);
279
+ const toRemove = indexed.filter(name => !keepSet.has(name));
280
+ const removed = [];
281
+ const failed = [];
282
+ for (const name of toRemove) {
283
+ try {
284
+ await this.db.deleteProject(name);
285
+ removed.push(name);
286
+ }
287
+ catch (err) {
288
+ console.error(`Failed to prune project ${name}: ${err.message}`);
289
+ this.logger.error('IndexerService: pruneProject failed', { name, reason: err.message });
290
+ failed.push(name);
291
+ }
390
292
  }
391
- lines.push(`Code: ${piece.content}`);
392
- return lines.join('\n');
293
+ return { removed, failed };
393
294
  }
394
295
  static hashContent(content) {
395
296
  return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
@@ -414,7 +315,7 @@ class IndexerService {
414
315
  return { status: 'skipped', chunks: 0, pendingMtimeRefresh: relFilePath };
415
316
  }
416
317
  const tAst = Date.now();
417
- const { edges: rawEdges, symbols: rawSymbols, calls: rawCalls, ownerRanges: rawOwnerRanges = [] } = analysis ?? { edges: [], symbols: [], calls: [] };
318
+ const { edges: rawEdges, symbols: rawSymbols, members: rawMembers = [], calls: rawCalls, ownerRanges: rawOwnerRanges = [], } = analysis ?? { edges: [], symbols: [], calls: [] };
418
319
  // astMs is near-zero — actual AST time is spent upfront in analyzeProject()
419
320
  // Normalize edges to workspace-relative paths
420
321
  const edges = rawEdges
@@ -437,8 +338,9 @@ class IndexerService {
437
338
  line: c.line,
438
339
  col: c.col,
439
340
  }));
440
- // Normalize symbols: replace absolute filePath with workspace-relative, strip endLine (not stored)
441
- const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath: relFilePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
341
+ // symbols ∪ members → code_symbol rows: overload sets collapse to their
342
+ // representative row, then a final dedup by (container, name, kind, line).
343
+ const symbolRows = (0, symbol_rows_1.buildSymbolRows)(rawSymbols, rawMembers).map(s => ({ ...s, filePath: relFilePath }));
442
344
  const astMs = Date.now() - tAst;
443
345
  const chunks = await this.chunker.chunk(code, {
444
346
  strategy: this.config.indexer.chunkStrategy,
@@ -446,20 +348,16 @@ class IndexerService {
446
348
  symbols: rawSymbols,
447
349
  });
448
350
  const relPath = path.relative(projectRoot, filePath);
449
- // Owner-path ranges: top-level symbols (container: null) + method ranges.
450
- // Deepest range containing a chunk's start line wins → Class.method.
451
- const ownerRanges = [
452
- ...rawSymbols
453
- .filter(s => s.line != null)
454
- .map(s => ({ name: s.name, container: null, line: s.line, endLine: s.endLine ?? s.line })),
455
- ...rawOwnerRanges,
456
- ];
351
+ // Ownership ranges (decorator-inclusive, symbols ∪ members minus
352
+ // method_signature) are built by AstParserService — the deepest range
353
+ // containing a chunk's start line wins.
354
+ const ownerRanges = rawOwnerRanges;
457
355
  const t2 = Date.now();
458
356
  const records = [];
459
357
  const embedBatchSize = this.config.embedding.batchSize;
460
358
  for (let i = 0; i < chunks.length; i += embedBatchSize) {
461
359
  const batch = chunks.slice(i, i + embedBatchSize);
462
- const prompts = batch.map(piece => this.getEnrichedPrompt(piece, relPath, serviceName, meta ?? 'missing', edges, rawSymbols));
360
+ const prompts = batch.map(piece => (0, enriched_prompt_1.getEnrichedPrompt)(piece, relPath, serviceName, edges, rawSymbols));
463
361
  const vectors = await this.embedder.embedBatch(prompts, 'document');
464
362
  for (let j = 0; j < batch.length; j++) {
465
363
  const piece = batch[j];
@@ -482,8 +380,8 @@ class IndexerService {
482
380
  }
483
381
  await this.writeDepEdges(serviceName, relFilePath, edges, tx);
484
382
  await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, relFilePath)));
485
- if (symbols.length > 0) {
486
- await tx.insert(db_1.codeSymbol).values(symbols);
383
+ if (symbolRows.length > 0) {
384
+ await tx.insert(db_1.codeSymbol).values(symbolRows);
487
385
  }
488
386
  await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, relFilePath)));
489
387
  if (calls.length > 0) {
@@ -1,79 +1,38 @@
1
1
  import type { Config } from '../config';
2
2
  import type { ILogger } from '../interfaces/ILogger';
3
3
  import { OpenAICompatibleEmbedder, type EmbedderDeps } from './OpenAICompatibleEmbedder';
4
- /** Result of a shell command run, mirroring the parts of `spawnSync` we use. */
5
- export interface CommandResult {
6
- status: number | null;
7
- stdout?: string;
8
- }
9
- /**
10
- * Extra injection seams for `InfinityEmbedder` beyond the shared `EmbedderDeps`.
11
- * All optional and defaulted to the real implementations; tests inject fakes so
12
- * `ensureModelAvailable()` runs without a live Infinity server or Docker.
13
- */
14
- export interface InfinityDeps extends EmbedderDeps {
15
- /** Runs a shell command (docker), optionally with extra env. Defaults to `spawnSync`. */
16
- runCommand?: (cmd: string, args: string[], env?: Record<string, string>) => CommandResult;
4
+ import { serverRoot, probeInfinityHealth, listInfinityModels, type ProvisioningDeps } from './infinity-provisioning';
5
+ export { serverRoot, probeInfinityHealth, listInfinityModels };
6
+ export type { CommandResult } from './infinity-provisioning';
7
+ /** Extra injection seams for `InfinityEmbedder` beyond the shared `EmbedderDeps`. */
8
+ export interface InfinityDeps extends EmbedderDeps, ProvisioningDeps {
17
9
  /**
18
10
  * Path to the managed docker-compose.yml. Defaults to the file `rag-setup`
19
11
  * writes into `~/.config/codesearch-mcp/`. A missing file (or one without an
20
12
  * `infinity` service) means "external backend" — the Docker path is skipped.
21
13
  */
22
14
  composePath?: string;
23
- /** Sleep between readiness polls (ms). Defaults to 2000; tests pass 0. */
24
- pollIntervalMs?: number;
25
15
  }
26
- /**
27
- * Strip trailing slashes and a trailing `/embeddings` or `/v1` to get the server
28
- * root — Infinity serves `/health`, `/models`, `/rerank`, `/embeddings` from the
29
- * root, so a baseURL written with an OpenAI-style suffix must be normalized here
30
- * to keep the probe/models URLs aligned with the embeddings URL.
31
- */
32
- export declare function serverRoot(baseURL: string): string;
33
- /**
34
- * Probe an Infinity-style `/health` endpoint. Resolves `true` when the server
35
- * answers with a 2xx, `false` on any error or non-2xx. Reused by setup's
36
- * `waitForInfinity`.
37
- */
38
- export declare function probeInfinityHealth(baseURL: string, fetchImpl?: typeof fetch, timeoutMs?: number): Promise<boolean>;
39
- /**
40
- * List the model ids an Infinity server currently serves via its OpenAI-style
41
- * `/models` listing (`{ data: [{ id }] }`). Returns `[]` on any error.
42
- */
43
- export declare function listInfinityModels(baseURL: string, fetchImpl?: typeof fetch, timeoutMs?: number): Promise<string[]>;
44
16
  /**
45
17
  * Embedder for a local Infinity server. Inherits the OpenAI-style `/embeddings`
46
18
  * HTTP path, batching, retry, and reranker wiring from `OpenAICompatibleEmbedder`
47
- * unchanged — it adds ONLY container-readiness behavior (`ensureModelAvailable`),
19
+ * unchanged — it adds ONLY container-readiness behavior (`ensureModelAvailable`,
20
+ * `listModels`), delegating to the standalone `infinity-provisioning` primitive
48
21
  * because Infinity binds its model set at container startup and exposes no
49
22
  * runtime model-pull API.
50
23
  */
51
24
  export declare class InfinityEmbedder extends OpenAICompatibleEmbedder {
52
25
  private readonly base;
53
- private readonly doFetchImpl;
54
- private readonly runCommand;
55
26
  private readonly composePath;
56
- private readonly pollIntervalMs;
57
- private readonly log;
27
+ private readonly provisioningDeps;
58
28
  constructor(config: Config, logger?: ILogger, deps?: InfinityDeps);
59
- private isHealthy;
60
- private servedModels;
61
- /** True when a managed compose file exists AND declares an `infinity` service. */
62
- private hasManagedService;
63
29
  /**
64
- * Bring the Infinity backend to a serving state per the D4 decision tree:
65
- * healthy + model served → OK
66
- * healthy + model NOT served → throw (restart container with correct --model-id)
67
- * unreachable + managed → `docker compose up -d infinity`, wait /health
68
- * unreachable + external → print manual instructions (no Docker touched)
69
- *
70
- * Called by `IndexCommand` before `prepareEmbeddingSchema()`.
30
+ * Index-time embed readiness (see `local-backend-provisioning`): verifies only
31
+ * the embedding model — indexing does not use the reranker, so a missing
32
+ * reranker model never blocks or restarts indexing. Whenever a recreate is
33
+ * needed, the full configured set is written so an already-served reranker
34
+ * model is never dropped. Called by `IndexCommand` before indexing.
71
35
  */
72
36
  ensureModelAvailable(): Promise<void>;
73
- /**
74
- * Poll `/health` until ready or the timeout elapses. Generous default (30 min)
75
- * because a CPU first boot downloads the model set and runs a lengthy per-model
76
- * warmup before `/health` flips to 200.
77
- */
78
- private waitForHealthy;
37
+ listModels(): Promise<string[]>;
79
38
  }