codesearch-mcp 0.1.0-alpha → 0.2.1-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +146 -41
- package/dist/app.js +4 -19
- package/dist/src/commands/IndexCommand.js +68 -7
- package/dist/src/config-error.d.ts +7 -0
- package/dist/src/config-error.js +14 -0
- package/dist/src/config-store.d.ts +30 -12
- package/dist/src/config-store.js +81 -26
- package/dist/src/config.d.ts +5 -4
- package/dist/src/config.js +14 -16
- package/dist/src/constants.d.ts +8 -2
- package/dist/src/constants.js +8 -5
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +75 -0
- package/dist/src/db/schema.js +10 -1
- package/dist/src/enriched-prompt.d.ts +8 -0
- package/dist/src/enriched-prompt.js +87 -0
- package/dist/src/help.js +1 -1
- package/dist/src/interfaces/IAstParser.d.ts +18 -2
- package/dist/src/interfaces/IDatabase.d.ts +31 -0
- package/dist/src/interfaces/ILogger.d.ts +7 -0
- package/dist/src/interfaces/ISearcher.d.ts +3 -0
- package/dist/src/services/AstChunker.d.ts +11 -0
- package/dist/src/services/AstChunker.js +44 -0
- package/dist/src/services/AstParserService.d.ts +39 -7
- package/dist/src/services/AstParserService.js +504 -104
- package/dist/src/services/DatabaseService.d.ts +17 -0
- package/dist/src/services/DatabaseService.js +130 -2
- package/dist/src/services/EmbedderFactory.d.ts +1 -1
- package/dist/src/services/EmbedderFactory.js +2 -5
- package/dist/src/services/IndexerService.d.ts +4 -14
- package/dist/src/services/IndexerService.js +54 -156
- package/dist/src/services/InfinityEmbedder.d.ts +14 -55
- package/dist/src/services/InfinityEmbedder.js +30 -141
- package/dist/src/services/LoggerService.d.ts +3 -1
- package/dist/src/services/LoggerService.js +17 -3
- package/dist/src/services/McpServer.js +3 -3
- package/dist/src/services/SearcherService.d.ts +39 -4
- package/dist/src/services/SearcherService.js +254 -81
- package/dist/src/services/WebServerService.d.ts +41 -3
- package/dist/src/services/WebServerService.js +90 -2
- package/dist/src/services/infinity-provisioning.d.ts +115 -0
- package/dist/src/services/infinity-provisioning.js +516 -0
- package/dist/src/services/legacy-ollama-index.d.ts +24 -0
- package/dist/src/services/legacy-ollama-index.js +61 -0
- package/dist/src/services/reranker.d.ts +3 -3
- package/dist/src/services/reranker.js +3 -3
- package/dist/src/setup.d.ts +37 -0
- package/dist/src/setup.js +111 -157
- package/dist/src/symbol-name.d.ts +44 -0
- package/dist/src/symbol-name.js +68 -0
- package/dist/src/symbol-rows.d.ts +12 -0
- package/dist/src/symbol-rows.js +38 -0
- package/dist/src/tools/FindSymbolTool.js +30 -7
- package/dist/src/tools/GetCallersTool.js +20 -5
- package/docker/docker-compose.yml +5 -15
- package/docs/architecture.svg +103 -0
- package/docs/web-dashboard.md +36 -17
- package/package.json +1 -6
- package/dist/src/commands/BackfillCommand.d.ts +0 -7
- package/dist/src/commands/BackfillCommand.js +0 -37
- package/dist/src/services/OllamaEmbedder.d.ts +0 -25
- package/dist/src/services/OllamaEmbedder.js +0 -111
|
@@ -6,6 +6,7 @@ import * as schema from '../db/schema';
|
|
|
6
6
|
type Schema = typeof schema;
|
|
7
7
|
export type DrizzleDb = NodePgDatabase<Schema>;
|
|
8
8
|
export type DrizzleTx = NodePgTransaction<Schema, ExtractTablesWithRelations<Schema>>;
|
|
9
|
+
export declare const CURRENT_SYMBOL_INDEX_VERSION = 2;
|
|
9
10
|
export declare class DatabaseService implements IDatabase {
|
|
10
11
|
private config;
|
|
11
12
|
private pool;
|
|
@@ -27,8 +28,24 @@ export declare class DatabaseService implements IDatabase {
|
|
|
27
28
|
recreateChunkTable(dimensions: number): Promise<void>;
|
|
28
29
|
readIndexMeta(): Promise<IndexMeta | null>;
|
|
29
30
|
writeIndexMeta(meta: IndexMeta): Promise<void>;
|
|
31
|
+
listKnownFiles(project: string): Promise<string[]>;
|
|
32
|
+
deleteFiles(project: string, filePaths: string[]): Promise<void>;
|
|
33
|
+
listIndexedProjects(): Promise<string[]>;
|
|
34
|
+
deleteProject(name: string): Promise<void>;
|
|
35
|
+
listProjectsWithIndexRows(): Promise<string[]>;
|
|
36
|
+
/**
|
|
37
|
+
* Among `projectNames` (or every project with rows when null), those with at
|
|
38
|
+
* least one row in `code_chunk`/`code_symbol` and a `symbol_index_version`
|
|
39
|
+
* below `CURRENT_SYMBOL_INDEX_VERSION` (a missing row counts as version 1).
|
|
40
|
+
* A missing `symbol_index_version` table (un-migrated DB, `rag-mcp`/`rag-web`
|
|
41
|
+
* bootstrapped with `initDb`) counts as "no marker rows" — every project with
|
|
42
|
+
* rows is then outdated. Any other database error propagates.
|
|
43
|
+
*/
|
|
44
|
+
listOutdatedProjects(projectNames: string[] | null): Promise<string[]>;
|
|
45
|
+
upsertSymbolIndexVersion(project: string): Promise<void>;
|
|
30
46
|
/** Guard against SQL injection — the dimension is interpolated into DDL. */
|
|
31
47
|
private assertDim;
|
|
48
|
+
private assertMinPostgresVersion;
|
|
32
49
|
private ensureDatabaseExists;
|
|
33
50
|
}
|
|
34
51
|
export {};
|
|
@@ -33,10 +33,14 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.DatabaseService = void 0;
|
|
36
|
+
exports.DatabaseService = exports.CURRENT_SYMBOL_INDEX_VERSION = void 0;
|
|
37
37
|
const node_postgres_1 = require("drizzle-orm/node-postgres");
|
|
38
38
|
const pg_1 = require("pg");
|
|
39
|
+
const drizzle_orm_1 = require("drizzle-orm");
|
|
39
40
|
const schema = __importStar(require("../db/schema"));
|
|
41
|
+
const db_1 = require("../db");
|
|
42
|
+
const MIN_PG_VERSION_NUM = 150000;
|
|
43
|
+
exports.CURRENT_SYMBOL_INDEX_VERSION = 2;
|
|
40
44
|
class DatabaseService {
|
|
41
45
|
config;
|
|
42
46
|
pool;
|
|
@@ -66,6 +70,12 @@ class DatabaseService {
|
|
|
66
70
|
async initDbWithSchema() {
|
|
67
71
|
await this.ensureDatabaseExists();
|
|
68
72
|
await this.initDb();
|
|
73
|
+
// Read as the very first statement on the application database, before any
|
|
74
|
+
// CREATE TABLE / CREATE INDEX / ALTER TABLE / DROP — `NULLS NOT DISTINCT`
|
|
75
|
+
// (used below) requires PostgreSQL 15+, and the DROP/CREATE pair must never
|
|
76
|
+
// run against an older server (it would fail after the DROP and leave
|
|
77
|
+
// code_symbol without a unique index).
|
|
78
|
+
await this.assertMinPostgresVersion();
|
|
69
79
|
// code_chunk table — created at the legacy default width (1024) so search
|
|
70
80
|
// tools always have a table. IndexerService recreates it at the resolved
|
|
71
81
|
// width before indexing when the active model's dimension differs.
|
|
@@ -128,7 +138,23 @@ class DatabaseService {
|
|
|
128
138
|
}
|
|
129
139
|
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_name ON code_symbol (name, project)`);
|
|
130
140
|
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_file ON code_symbol (project, "filePath")`);
|
|
131
|
-
await this.query(`
|
|
141
|
+
await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS container TEXT`);
|
|
142
|
+
await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS "isStatic" BOOLEAN NOT NULL DEFAULT false`);
|
|
143
|
+
// DDL is transactional in PostgreSQL — either both statements apply or
|
|
144
|
+
// neither does, so a failure here never leaves code_symbol without a
|
|
145
|
+
// unique index (the old idx_symbol_unique is dropped only once the new
|
|
146
|
+
// one is confirmed to exist).
|
|
147
|
+
await this.transaction(async (tx) => {
|
|
148
|
+
await tx.execute('DROP INDEX IF EXISTS idx_symbol_unique');
|
|
149
|
+
await tx.execute('CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique_v2 ' +
|
|
150
|
+
'ON code_symbol (project, "filePath", container, name, kind, line) NULLS NOT DISTINCT');
|
|
151
|
+
});
|
|
152
|
+
await this.query(`
|
|
153
|
+
CREATE TABLE IF NOT EXISTS symbol_index_version (
|
|
154
|
+
project TEXT PRIMARY KEY,
|
|
155
|
+
version INT NOT NULL
|
|
156
|
+
)
|
|
157
|
+
`);
|
|
132
158
|
// code_call table
|
|
133
159
|
await this.query(`
|
|
134
160
|
CREATE TABLE IF NOT EXISTS code_call (
|
|
@@ -198,6 +224,94 @@ class DatabaseService {
|
|
|
198
224
|
dimensions = EXCLUDED.dimensions,
|
|
199
225
|
"indexedAt" = now()`, [meta.provider, meta.model, meta.dimensions]);
|
|
200
226
|
}
|
|
227
|
+
async listKnownFiles(project) {
|
|
228
|
+
const rows = await this.query(`SELECT "filePath" AS fp FROM code_chunk WHERE "serviceName" = $1
|
|
229
|
+
UNION
|
|
230
|
+
SELECT "filePath" AS fp FROM code_symbol WHERE project = $1
|
|
231
|
+
UNION
|
|
232
|
+
SELECT "fromFile" AS fp FROM code_call WHERE project = $1
|
|
233
|
+
UNION
|
|
234
|
+
SELECT "fromFile" AS fp FROM code_dependency WHERE project = $1`, [project]);
|
|
235
|
+
return rows.map(r => r.fp);
|
|
236
|
+
}
|
|
237
|
+
async deleteFiles(project, filePaths) {
|
|
238
|
+
if (filePaths.length === 0)
|
|
239
|
+
return;
|
|
240
|
+
// PostgreSQL limits ~32 767 bind params per statement; OR queries bind the
|
|
241
|
+
// array twice (fromFile + calleeFile), so cap each batch at 10 000 entries.
|
|
242
|
+
const BATCH = 10_000;
|
|
243
|
+
await this.transaction(async (tx) => {
|
|
244
|
+
for (let i = 0; i < filePaths.length; i += BATCH) {
|
|
245
|
+
const batch = filePaths.slice(i, i + BATCH);
|
|
246
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, project), (0, drizzle_orm_1.inArray)(db_1.codeChunk.filePath, batch)));
|
|
247
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, project), (0, drizzle_orm_1.inArray)(db_1.codeSymbol.filePath, batch)));
|
|
248
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, project), (0, drizzle_orm_1.or)((0, drizzle_orm_1.inArray)(db_1.codeCall.fromFile, batch), (0, drizzle_orm_1.inArray)(db_1.codeCall.calleeFile, batch))));
|
|
249
|
+
await tx.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, project), (0, drizzle_orm_1.or)((0, drizzle_orm_1.inArray)(db_1.codeDependency.fromFile, batch), (0, drizzle_orm_1.inArray)(db_1.codeDependency.toFile, batch))));
|
|
250
|
+
}
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
async listIndexedProjects() {
|
|
254
|
+
const rows = await this.query(`SELECT "serviceName" AS name FROM code_chunk
|
|
255
|
+
UNION
|
|
256
|
+
SELECT project AS name FROM code_symbol
|
|
257
|
+
UNION
|
|
258
|
+
SELECT project AS name FROM code_call
|
|
259
|
+
UNION
|
|
260
|
+
SELECT project AS name FROM code_dependency`);
|
|
261
|
+
return rows.map(r => r.name);
|
|
262
|
+
}
|
|
263
|
+
async deleteProject(name) {
|
|
264
|
+
await this.transaction(async (tx) => {
|
|
265
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, name));
|
|
266
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, name));
|
|
267
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, name));
|
|
268
|
+
await tx.delete(db_1.codeDependency).where((0, drizzle_orm_1.eq)(db_1.codeDependency.project, name));
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
async listProjectsWithIndexRows() {
|
|
272
|
+
const rows = await this.query(`SELECT DISTINCT name FROM (
|
|
273
|
+
SELECT "serviceName" AS name FROM code_chunk
|
|
274
|
+
UNION
|
|
275
|
+
SELECT project AS name FROM code_symbol
|
|
276
|
+
) t`);
|
|
277
|
+
return rows.map(r => r.name);
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Among `projectNames` (or every project with rows when null), those with at
|
|
281
|
+
* least one row in `code_chunk`/`code_symbol` and a `symbol_index_version`
|
|
282
|
+
* below `CURRENT_SYMBOL_INDEX_VERSION` (a missing row counts as version 1).
|
|
283
|
+
* A missing `symbol_index_version` table (un-migrated DB, `rag-mcp`/`rag-web`
|
|
284
|
+
* bootstrapped with `initDb`) counts as "no marker rows" — every project with
|
|
285
|
+
* rows is then outdated. Any other database error propagates.
|
|
286
|
+
*/
|
|
287
|
+
async listOutdatedProjects(projectNames) {
|
|
288
|
+
const rowsUnion = `
|
|
289
|
+
SELECT "serviceName" AS name FROM code_chunk
|
|
290
|
+
UNION
|
|
291
|
+
SELECT project AS name FROM code_symbol
|
|
292
|
+
`;
|
|
293
|
+
const filterSql = projectNames ? 'AND t.name = ANY($1)' : '';
|
|
294
|
+
const params = projectNames ? [projectNames] : [];
|
|
295
|
+
try {
|
|
296
|
+
const rows = await this.query(`SELECT DISTINCT t.name FROM (${rowsUnion}) t
|
|
297
|
+
WHERE NOT EXISTS (
|
|
298
|
+
SELECT 1 FROM symbol_index_version v
|
|
299
|
+
WHERE v.project = t.name AND v.version >= ${exports.CURRENT_SYMBOL_INDEX_VERSION}
|
|
300
|
+
)
|
|
301
|
+
${filterSql}`, params);
|
|
302
|
+
return rows.map(r => r.name);
|
|
303
|
+
}
|
|
304
|
+
catch (err) {
|
|
305
|
+
if (!/relation "symbol_index_version" does not exist/i.test(err.message))
|
|
306
|
+
throw err;
|
|
307
|
+
const rows = await this.query(`SELECT DISTINCT t.name FROM (${rowsUnion}) t ${projectNames ? 'WHERE t.name = ANY($1)' : ''}`, params);
|
|
308
|
+
return rows.map(r => r.name);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
async upsertSymbolIndexVersion(project) {
|
|
312
|
+
await this.query(`INSERT INTO symbol_index_version (project, version) VALUES ($1, $2)
|
|
313
|
+
ON CONFLICT (project) DO UPDATE SET version = EXCLUDED.version`, [project, exports.CURRENT_SYMBOL_INDEX_VERSION]);
|
|
314
|
+
}
|
|
201
315
|
/** Guard against SQL injection — the dimension is interpolated into DDL. */
|
|
202
316
|
assertDim(dimensions) {
|
|
203
317
|
if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > 2000) {
|
|
@@ -205,6 +319,20 @@ class DatabaseService {
|
|
|
205
319
|
}
|
|
206
320
|
return dimensions;
|
|
207
321
|
}
|
|
322
|
+
async assertMinPostgresVersion() {
|
|
323
|
+
let versionNum;
|
|
324
|
+
try {
|
|
325
|
+
const rows = await this.query(`SELECT current_setting('server_version_num') AS v`);
|
|
326
|
+
versionNum = Number(rows[0]?.v);
|
|
327
|
+
}
|
|
328
|
+
catch (err) {
|
|
329
|
+
throw new Error(`Failed to read PostgreSQL server_version_num (requires PostgreSQL 15+): ${err.message}`);
|
|
330
|
+
}
|
|
331
|
+
if (!Number.isFinite(versionNum) || versionNum < MIN_PG_VERSION_NUM) {
|
|
332
|
+
throw new Error(`PostgreSQL 15+ is required (server_version_num=${versionNum}) — ` +
|
|
333
|
+
`the symbol index's unique index relies on NULLS NOT DISTINCT, added in PostgreSQL 15.`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
208
336
|
async ensureDatabaseExists() {
|
|
209
337
|
const { host, port, username, password, database } = this.config.db;
|
|
210
338
|
const client = new pg_1.Client({ host, port, user: username, password, database: 'postgres' });
|
|
@@ -4,6 +4,6 @@ import type { ILogger } from '../interfaces/ILogger';
|
|
|
4
4
|
import { type EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
5
5
|
/**
|
|
6
6
|
* Construct the `IEmbedder` implementation selected by `config.embedding.provider`.
|
|
7
|
-
* The default (`
|
|
7
|
+
* The default (`infinity`) is the recommended local backend.
|
|
8
8
|
*/
|
|
9
9
|
export declare function createEmbedder(config: Config, logger?: ILogger, deps?: EmbedderDeps): IEmbedder;
|
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createEmbedder = createEmbedder;
|
|
4
4
|
const LoggerService_1 = require("./LoggerService");
|
|
5
|
-
const OllamaEmbedder_1 = require("./OllamaEmbedder");
|
|
6
5
|
const OpenAICompatibleEmbedder_1 = require("./OpenAICompatibleEmbedder");
|
|
7
6
|
const InfinityEmbedder_1 = require("./InfinityEmbedder");
|
|
8
7
|
const VoyageEmbedder_1 = require("./VoyageEmbedder");
|
|
9
8
|
/**
|
|
10
9
|
* Construct the `IEmbedder` implementation selected by `config.embedding.provider`.
|
|
11
|
-
* The default (`
|
|
10
|
+
* The default (`infinity`) is the recommended local backend.
|
|
12
11
|
*/
|
|
13
12
|
function createEmbedder(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
14
13
|
switch (config.embedding.provider) {
|
|
@@ -18,12 +17,10 @@ function createEmbedder(config, logger = LoggerService_1.LoggerService.silent(),
|
|
|
18
17
|
return new InfinityEmbedder_1.InfinityEmbedder(config, logger, deps);
|
|
19
18
|
case 'voyage':
|
|
20
19
|
return new VoyageEmbedder_1.VoyageEmbedder(config, logger, deps);
|
|
21
|
-
case 'ollama':
|
|
22
|
-
return new OllamaEmbedder_1.OllamaEmbedder(config, logger, deps);
|
|
23
20
|
default: {
|
|
24
21
|
// The Zod enum should already reject this; guard anyway for a clear message.
|
|
25
22
|
const p = config.embedding.provider;
|
|
26
|
-
throw new Error(`Unknown embedding provider: "${p}". Expected one of:
|
|
23
|
+
throw new Error(`Unknown embedding provider: "${p}". Expected one of: openai-compatible, voyage, infinity.`);
|
|
27
24
|
}
|
|
28
25
|
}
|
|
29
26
|
}
|
|
@@ -35,24 +35,14 @@ export declare class IndexerService {
|
|
|
35
35
|
prepareEmbeddingSchema(): Promise<IndexMeta>;
|
|
36
36
|
private reconcileIndex;
|
|
37
37
|
private countChunks;
|
|
38
|
-
backfillSymbols(signal?: AbortSignal): Promise<{
|
|
39
|
-
total: number;
|
|
40
|
-
processed: number;
|
|
41
|
-
skipped: number;
|
|
42
|
-
failed: number;
|
|
43
|
-
}>;
|
|
44
|
-
backfillCalls(signal?: AbortSignal): Promise<{
|
|
45
|
-
total: number;
|
|
46
|
-
processed: number;
|
|
47
|
-
skipped: number;
|
|
48
|
-
failed: number;
|
|
49
|
-
}>;
|
|
50
38
|
private writeDepEdges;
|
|
51
39
|
scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[], opts?: {
|
|
52
40
|
rootTsConfigPath?: string;
|
|
53
41
|
}): Promise<IndexStats>;
|
|
54
|
-
|
|
55
|
-
|
|
42
|
+
pruneProjects(keep: string[]): Promise<{
|
|
43
|
+
removed: string[];
|
|
44
|
+
failed: string[];
|
|
45
|
+
}>;
|
|
56
46
|
private static hashContent;
|
|
57
47
|
private ingestFile;
|
|
58
48
|
}
|
|
@@ -40,6 +40,8 @@ const crypto_1 = require("crypto");
|
|
|
40
40
|
const cli_progress_1 = require("cli-progress");
|
|
41
41
|
const drizzle_orm_1 = require("drizzle-orm");
|
|
42
42
|
const owner_path_1 = require("../owner-path");
|
|
43
|
+
const enriched_prompt_1 = require("../enriched-prompt");
|
|
44
|
+
const symbol_rows_1 = require("../symbol-rows");
|
|
43
45
|
const db_1 = require("../db");
|
|
44
46
|
const file_utils_1 = require("../file-utils");
|
|
45
47
|
const LoggerService_1 = require("./LoggerService");
|
|
@@ -124,97 +126,6 @@ class IndexerService {
|
|
|
124
126
|
throw err;
|
|
125
127
|
}
|
|
126
128
|
}
|
|
127
|
-
async backfillSymbols(signal) {
|
|
128
|
-
const t0 = Date.now();
|
|
129
|
-
const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
|
|
130
|
-
this.logger.debug('IndexerService: backfillSymbols start', { total: rows.length });
|
|
131
|
-
let processed = 0, skipped = 0, failed = 0;
|
|
132
|
-
for (const { serviceName, filePath } of rows) {
|
|
133
|
-
if (signal?.aborted)
|
|
134
|
-
break;
|
|
135
|
-
const absPath = path.join(this.config.workspaceRoot, filePath);
|
|
136
|
-
const projectRoot = path.join(this.config.workspaceRoot, serviceName);
|
|
137
|
-
if (!fs.existsSync(absPath)) {
|
|
138
|
-
skipped++;
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
try {
|
|
142
|
-
const { symbols: rawSymbols } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
|
|
143
|
-
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
|
|
144
|
-
await this.db.transaction(async (tx) => {
|
|
145
|
-
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, filePath)));
|
|
146
|
-
if (symbols.length > 0) {
|
|
147
|
-
await tx.insert(db_1.codeSymbol).values(symbols);
|
|
148
|
-
}
|
|
149
|
-
});
|
|
150
|
-
processed++;
|
|
151
|
-
}
|
|
152
|
-
catch (err) {
|
|
153
|
-
const reason = err.message;
|
|
154
|
-
console.error(` Failed: ${filePath} — ${reason}`);
|
|
155
|
-
this.logger.error('IndexerService: symbol backfill failed', { file: filePath, reason });
|
|
156
|
-
failed++;
|
|
157
|
-
}
|
|
158
|
-
const count = processed + failed;
|
|
159
|
-
if (count > 0 && count % 50 === 0) {
|
|
160
|
-
const msg = ` ${count}/${rows.length} files...`;
|
|
161
|
-
process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
this.logger.debug('IndexerService: backfillSymbols done', { total: rows.length, processed, skipped, failed, ms: Date.now() - t0 });
|
|
165
|
-
return { total: rows.length, processed, skipped, failed };
|
|
166
|
-
}
|
|
167
|
-
async backfillCalls(signal) {
|
|
168
|
-
const t0 = Date.now();
|
|
169
|
-
const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
|
|
170
|
-
this.logger.debug('IndexerService: backfillCalls start', { total: rows.length });
|
|
171
|
-
let processed = 0, skipped = 0, failed = 0;
|
|
172
|
-
for (const { serviceName, filePath } of rows) {
|
|
173
|
-
if (signal?.aborted)
|
|
174
|
-
break;
|
|
175
|
-
const absPath = path.join(this.config.workspaceRoot, filePath);
|
|
176
|
-
const projectRoot = path.join(this.config.workspaceRoot, serviceName);
|
|
177
|
-
if (!fs.existsSync(absPath)) {
|
|
178
|
-
skipped++;
|
|
179
|
-
continue;
|
|
180
|
-
}
|
|
181
|
-
try {
|
|
182
|
-
const { calls: rawCalls } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
|
|
183
|
-
const calls = rawCalls
|
|
184
|
-
.filter(c => c.calleeFile.startsWith(this.config.workspaceRoot))
|
|
185
|
-
.map(c => ({
|
|
186
|
-
project: serviceName,
|
|
187
|
-
fromFile: filePath,
|
|
188
|
-
fromSymbol: c.fromSymbol,
|
|
189
|
-
calleeName: c.calleeName,
|
|
190
|
-
calleeFile: path.relative(this.config.workspaceRoot, c.calleeFile),
|
|
191
|
-
calleeSymbol: c.calleeSymbol,
|
|
192
|
-
line: c.line,
|
|
193
|
-
col: c.col,
|
|
194
|
-
}));
|
|
195
|
-
await this.db.transaction(async (tx) => {
|
|
196
|
-
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, filePath)));
|
|
197
|
-
if (calls.length > 0) {
|
|
198
|
-
await tx.insert(db_1.codeCall).values(calls);
|
|
199
|
-
}
|
|
200
|
-
});
|
|
201
|
-
processed++;
|
|
202
|
-
}
|
|
203
|
-
catch (err) {
|
|
204
|
-
const reason = err.message;
|
|
205
|
-
console.error(` Failed: ${filePath} — ${reason}`);
|
|
206
|
-
this.logger.error('IndexerService: call backfill failed', { file: filePath, reason });
|
|
207
|
-
failed++;
|
|
208
|
-
}
|
|
209
|
-
const count = processed + failed;
|
|
210
|
-
if (count > 0 && count % 50 === 0) {
|
|
211
|
-
const msg = ` ${count}/${rows.length} files...`;
|
|
212
|
-
process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
this.logger.debug('IndexerService: backfillCalls done', { total: rows.length, processed, skipped, failed, ms: Date.now() - t0 });
|
|
216
|
-
return { total: rows.length, processed, skipped, failed };
|
|
217
|
-
}
|
|
218
129
|
async writeDepEdges(serviceName, relFilePath, edges, tx) {
|
|
219
130
|
const run = async (t) => {
|
|
220
131
|
await t.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeDependency.fromFile, relFilePath)));
|
|
@@ -231,18 +142,26 @@ class IndexerService {
|
|
|
231
142
|
const startTime = Date.now();
|
|
232
143
|
this.logger.debug('IndexerService: scanAndIndex start', { serviceName, dir, force });
|
|
233
144
|
if (force) {
|
|
234
|
-
|
|
145
|
+
// Reset incremental state and clear the project's symbol/call rows in one
|
|
146
|
+
// transaction: a file whose re-ingestion later fails must hold no rows at
|
|
147
|
+
// all, never rows in the previous (pre-member-extraction) format.
|
|
148
|
+
await this.db.transaction(async (tx) => {
|
|
149
|
+
await tx.update(db_1.codeChunk)
|
|
150
|
+
.set({ lastModified: null, contentHash: null })
|
|
151
|
+
.where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName));
|
|
152
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName));
|
|
153
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName));
|
|
154
|
+
});
|
|
235
155
|
}
|
|
236
156
|
const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
|
|
237
157
|
this.logger.debug('IndexerService: scanAndIndex files collected', { serviceName, count: files.length, force });
|
|
238
158
|
const collectedRelPaths = new Set(files.map(f => path.relative(this.config.workspaceRoot, f)));
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
console.log(` Removed stale: ${path.basename(filePath)}`);
|
|
159
|
+
const knownFiles = await this.db.listKnownFiles(serviceName);
|
|
160
|
+
const staleFiles = knownFiles.filter(f => !collectedRelPaths.has(f));
|
|
161
|
+
if (staleFiles.length > 0) {
|
|
162
|
+
await this.db.deleteFiles(serviceName, staleFiles);
|
|
163
|
+
for (const f of staleFiles) {
|
|
164
|
+
console.log(` Removed stale: ${path.basename(f)}`);
|
|
246
165
|
}
|
|
247
166
|
}
|
|
248
167
|
const bar = new cli_progress_1.SingleBar({
|
|
@@ -335,6 +254,12 @@ class IndexerService {
|
|
|
335
254
|
}
|
|
336
255
|
}
|
|
337
256
|
this.ast.clearCache(dir);
|
|
257
|
+
// The marker is written exactly when the scan reaches here without having
|
|
258
|
+
// been interrupted — never on the early-return abort path above, and never
|
|
259
|
+
// when anything in this method throws (the error propagates instead).
|
|
260
|
+
if (!signal?.aborted) {
|
|
261
|
+
await this.db.upsertSymbolIndexVersion(serviceName);
|
|
262
|
+
}
|
|
338
263
|
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
339
264
|
this.logger.debug('IndexerService: project indexed', {
|
|
340
265
|
project: serviceName,
|
|
@@ -346,50 +271,26 @@ class IndexerService {
|
|
|
346
271
|
});
|
|
347
272
|
return stats;
|
|
348
273
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
if (p.endsWith('.test.ts') || p.endsWith('.spec.ts') || p.includes('/tests/'))
|
|
368
|
-
return 'Test';
|
|
369
|
-
if (p.includes('/utils/') || p.includes('/helpers/') || p.endsWith('-utils.ts'))
|
|
370
|
-
return 'Utility';
|
|
371
|
-
return null;
|
|
372
|
-
}
|
|
373
|
-
getEnrichedPrompt(piece, relPath, serviceName, _meta, edges, symbols) {
|
|
374
|
-
const chunkStart = piece.startLine;
|
|
375
|
-
const chunkEnd = piece.endLine;
|
|
376
|
-
const lines = [];
|
|
377
|
-
lines.push(`Project: ${serviceName} | File: ${relPath}`);
|
|
378
|
-
const role = IndexerService.inferRole(relPath);
|
|
379
|
-
if (role)
|
|
380
|
-
lines.push(`Role: ${role}`);
|
|
381
|
-
const overlapping = symbols.filter(s => s.line != null &&
|
|
382
|
-
s.line <= chunkEnd &&
|
|
383
|
-
(s.endLine ?? s.line) >= chunkStart);
|
|
384
|
-
if (overlapping.length > 0) {
|
|
385
|
-
lines.push(`Symbols: ${overlapping.map(s => `${s.kind} ${s.name}`).join(', ')}`);
|
|
386
|
-
}
|
|
387
|
-
if (edges.length > 0) {
|
|
388
|
-
const depNames = [...new Set(edges.map(e => path.basename(e.toFile, path.extname(e.toFile)).split('.')[0]))];
|
|
389
|
-
lines.push(`Dependencies: ${depNames.join(', ')}`);
|
|
274
|
+
async pruneProjects(keep) {
|
|
275
|
+
if (keep.length === 0)
|
|
276
|
+
return { removed: [], failed: [] };
|
|
277
|
+
const indexed = await this.db.listIndexedProjects();
|
|
278
|
+
const keepSet = new Set(keep);
|
|
279
|
+
const toRemove = indexed.filter(name => !keepSet.has(name));
|
|
280
|
+
const removed = [];
|
|
281
|
+
const failed = [];
|
|
282
|
+
for (const name of toRemove) {
|
|
283
|
+
try {
|
|
284
|
+
await this.db.deleteProject(name);
|
|
285
|
+
removed.push(name);
|
|
286
|
+
}
|
|
287
|
+
catch (err) {
|
|
288
|
+
console.error(`Failed to prune project ${name}: ${err.message}`);
|
|
289
|
+
this.logger.error('IndexerService: pruneProject failed', { name, reason: err.message });
|
|
290
|
+
failed.push(name);
|
|
291
|
+
}
|
|
390
292
|
}
|
|
391
|
-
|
|
392
|
-
return lines.join('\n');
|
|
293
|
+
return { removed, failed };
|
|
393
294
|
}
|
|
394
295
|
static hashContent(content) {
|
|
395
296
|
return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
@@ -414,7 +315,7 @@ class IndexerService {
|
|
|
414
315
|
return { status: 'skipped', chunks: 0, pendingMtimeRefresh: relFilePath };
|
|
415
316
|
}
|
|
416
317
|
const tAst = Date.now();
|
|
417
|
-
const { edges: rawEdges, symbols: rawSymbols, calls: rawCalls, ownerRanges: rawOwnerRanges = [] } = analysis ?? { edges: [], symbols: [], calls: [] };
|
|
318
|
+
const { edges: rawEdges, symbols: rawSymbols, members: rawMembers = [], calls: rawCalls, ownerRanges: rawOwnerRanges = [], } = analysis ?? { edges: [], symbols: [], calls: [] };
|
|
418
319
|
// astMs is near-zero — actual AST time is spent upfront in analyzeProject()
|
|
419
320
|
// Normalize edges to workspace-relative paths
|
|
420
321
|
const edges = rawEdges
|
|
@@ -437,8 +338,9 @@ class IndexerService {
|
|
|
437
338
|
line: c.line,
|
|
438
339
|
col: c.col,
|
|
439
340
|
}));
|
|
440
|
-
//
|
|
441
|
-
|
|
341
|
+
// symbols ∪ members → code_symbol rows: overload sets collapse to their
|
|
342
|
+
// representative row, then a final dedup by (container, name, kind, line).
|
|
343
|
+
const symbolRows = (0, symbol_rows_1.buildSymbolRows)(rawSymbols, rawMembers).map(s => ({ ...s, filePath: relFilePath }));
|
|
442
344
|
const astMs = Date.now() - tAst;
|
|
443
345
|
const chunks = await this.chunker.chunk(code, {
|
|
444
346
|
strategy: this.config.indexer.chunkStrategy,
|
|
@@ -446,20 +348,16 @@ class IndexerService {
|
|
|
446
348
|
symbols: rawSymbols,
|
|
447
349
|
});
|
|
448
350
|
const relPath = path.relative(projectRoot, filePath);
|
|
449
|
-
//
|
|
450
|
-
//
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
.filter(s => s.line != null)
|
|
454
|
-
.map(s => ({ name: s.name, container: null, line: s.line, endLine: s.endLine ?? s.line })),
|
|
455
|
-
...rawOwnerRanges,
|
|
456
|
-
];
|
|
351
|
+
// Ownership ranges (decorator-inclusive, symbols ∪ members minus
|
|
352
|
+
// method_signature) are built by AstParserService — the deepest range
|
|
353
|
+
// containing a chunk's start line wins.
|
|
354
|
+
const ownerRanges = rawOwnerRanges;
|
|
457
355
|
const t2 = Date.now();
|
|
458
356
|
const records = [];
|
|
459
357
|
const embedBatchSize = this.config.embedding.batchSize;
|
|
460
358
|
for (let i = 0; i < chunks.length; i += embedBatchSize) {
|
|
461
359
|
const batch = chunks.slice(i, i + embedBatchSize);
|
|
462
|
-
const prompts = batch.map(piece =>
|
|
360
|
+
const prompts = batch.map(piece => (0, enriched_prompt_1.getEnrichedPrompt)(piece, relPath, serviceName, edges, rawSymbols));
|
|
463
361
|
const vectors = await this.embedder.embedBatch(prompts, 'document');
|
|
464
362
|
for (let j = 0; j < batch.length; j++) {
|
|
465
363
|
const piece = batch[j];
|
|
@@ -482,8 +380,8 @@ class IndexerService {
|
|
|
482
380
|
}
|
|
483
381
|
await this.writeDepEdges(serviceName, relFilePath, edges, tx);
|
|
484
382
|
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, relFilePath)));
|
|
485
|
-
if (
|
|
486
|
-
await tx.insert(db_1.codeSymbol).values(
|
|
383
|
+
if (symbolRows.length > 0) {
|
|
384
|
+
await tx.insert(db_1.codeSymbol).values(symbolRows);
|
|
487
385
|
}
|
|
488
386
|
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, relFilePath)));
|
|
489
387
|
if (calls.length > 0) {
|
|
@@ -1,79 +1,38 @@
|
|
|
1
1
|
import type { Config } from '../config';
|
|
2
2
|
import type { ILogger } from '../interfaces/ILogger';
|
|
3
3
|
import { OpenAICompatibleEmbedder, type EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
4
|
-
|
|
5
|
-
export
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Extra injection seams for `InfinityEmbedder` beyond the shared `EmbedderDeps`.
|
|
11
|
-
* All optional and defaulted to the real implementations; tests inject fakes so
|
|
12
|
-
* `ensureModelAvailable()` runs without a live Infinity server or Docker.
|
|
13
|
-
*/
|
|
14
|
-
export interface InfinityDeps extends EmbedderDeps {
|
|
15
|
-
/** Runs a shell command (docker), optionally with extra env. Defaults to `spawnSync`. */
|
|
16
|
-
runCommand?: (cmd: string, args: string[], env?: Record<string, string>) => CommandResult;
|
|
4
|
+
import { serverRoot, probeInfinityHealth, listInfinityModels, type ProvisioningDeps } from './infinity-provisioning';
|
|
5
|
+
export { serverRoot, probeInfinityHealth, listInfinityModels };
|
|
6
|
+
export type { CommandResult } from './infinity-provisioning';
|
|
7
|
+
/** Extra injection seams for `InfinityEmbedder` beyond the shared `EmbedderDeps`. */
|
|
8
|
+
export interface InfinityDeps extends EmbedderDeps, ProvisioningDeps {
|
|
17
9
|
/**
|
|
18
10
|
* Path to the managed docker-compose.yml. Defaults to the file `rag-setup`
|
|
19
11
|
* writes into `~/.config/codesearch-mcp/`. A missing file (or one without an
|
|
20
12
|
* `infinity` service) means "external backend" — the Docker path is skipped.
|
|
21
13
|
*/
|
|
22
14
|
composePath?: string;
|
|
23
|
-
/** Sleep between readiness polls (ms). Defaults to 2000; tests pass 0. */
|
|
24
|
-
pollIntervalMs?: number;
|
|
25
15
|
}
|
|
26
|
-
/**
|
|
27
|
-
* Strip trailing slashes and a trailing `/embeddings` or `/v1` to get the server
|
|
28
|
-
* root — Infinity serves `/health`, `/models`, `/rerank`, `/embeddings` from the
|
|
29
|
-
* root, so a baseURL written with an OpenAI-style suffix must be normalized here
|
|
30
|
-
* to keep the probe/models URLs aligned with the embeddings URL.
|
|
31
|
-
*/
|
|
32
|
-
export declare function serverRoot(baseURL: string): string;
|
|
33
|
-
/**
|
|
34
|
-
* Probe an Infinity-style `/health` endpoint. Resolves `true` when the server
|
|
35
|
-
* answers with a 2xx, `false` on any error or non-2xx. Reused by setup's
|
|
36
|
-
* `waitForInfinity`.
|
|
37
|
-
*/
|
|
38
|
-
export declare function probeInfinityHealth(baseURL: string, fetchImpl?: typeof fetch, timeoutMs?: number): Promise<boolean>;
|
|
39
|
-
/**
|
|
40
|
-
* List the model ids an Infinity server currently serves via its OpenAI-style
|
|
41
|
-
* `/models` listing (`{ data: [{ id }] }`). Returns `[]` on any error.
|
|
42
|
-
*/
|
|
43
|
-
export declare function listInfinityModels(baseURL: string, fetchImpl?: typeof fetch, timeoutMs?: number): Promise<string[]>;
|
|
44
16
|
/**
|
|
45
17
|
* Embedder for a local Infinity server. Inherits the OpenAI-style `/embeddings`
|
|
46
18
|
* HTTP path, batching, retry, and reranker wiring from `OpenAICompatibleEmbedder`
|
|
47
|
-
* unchanged — it adds ONLY container-readiness behavior (`ensureModelAvailable
|
|
19
|
+
* unchanged — it adds ONLY container-readiness behavior (`ensureModelAvailable`,
|
|
20
|
+
* `listModels`), delegating to the standalone `infinity-provisioning` primitive
|
|
48
21
|
* because Infinity binds its model set at container startup and exposes no
|
|
49
22
|
* runtime model-pull API.
|
|
50
23
|
*/
|
|
51
24
|
export declare class InfinityEmbedder extends OpenAICompatibleEmbedder {
|
|
52
25
|
private readonly base;
|
|
53
|
-
private readonly doFetchImpl;
|
|
54
|
-
private readonly runCommand;
|
|
55
26
|
private readonly composePath;
|
|
56
|
-
private readonly
|
|
57
|
-
private readonly log;
|
|
27
|
+
private readonly provisioningDeps;
|
|
58
28
|
constructor(config: Config, logger?: ILogger, deps?: InfinityDeps);
|
|
59
|
-
private isHealthy;
|
|
60
|
-
private servedModels;
|
|
61
|
-
/** True when a managed compose file exists AND declares an `infinity` service. */
|
|
62
|
-
private hasManagedService;
|
|
63
29
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
* Called by `IndexCommand` before `prepareEmbeddingSchema()`.
|
|
30
|
+
* Index-time embed readiness (see `local-backend-provisioning`): verifies only
|
|
31
|
+
* the embedding model — indexing does not use the reranker, so a missing
|
|
32
|
+
* reranker model never blocks or restarts indexing. Whenever a recreate is
|
|
33
|
+
* needed, the full configured set is written so an already-served reranker
|
|
34
|
+
* model is never dropped. Called by `IndexCommand` before indexing.
|
|
71
35
|
*/
|
|
72
36
|
ensureModelAvailable(): Promise<void>;
|
|
73
|
-
|
|
74
|
-
* Poll `/health` until ready or the timeout elapses. Generous default (30 min)
|
|
75
|
-
* because a CPU first boot downloads the model set and runs a lengthy per-model
|
|
76
|
-
* warmup before `/health` flips to 200.
|
|
77
|
-
*/
|
|
78
|
-
private waitForHealthy;
|
|
37
|
+
listModels(): Promise<string[]>;
|
|
79
38
|
}
|