codesearch-mcp 0.0.15-alpha → 0.0.17-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +74 -33
- package/dist/app.d.ts +1 -1
- package/dist/app.js +67 -16
- package/dist/src/commands/BackfillCommand.js +15 -5
- package/dist/src/commands/ConfigCommand.d.ts +12 -0
- package/dist/src/commands/ConfigCommand.js +98 -0
- package/dist/src/commands/IndexCommand.js +1 -1
- package/dist/src/config-store.d.ts +53 -0
- package/dist/src/config-store.js +197 -0
- package/dist/src/config.d.ts +22 -8
- package/dist/src/config.js +103 -58
- package/dist/src/container.d.ts +1 -1
- package/dist/src/container.js +8 -7
- package/dist/src/db/index.d.ts +2 -3
- package/dist/src/db/index.js +6 -7
- package/dist/src/db/schema.d.ts +577 -0
- package/dist/src/db/schema.js +47 -0
- package/dist/src/help.js +28 -12
- package/dist/src/interfaces/IAstParser.d.ts +19 -0
- package/dist/src/interfaces/IEmbedder.d.ts +1 -0
- package/dist/src/project-selector.d.ts +14 -5
- package/dist/src/project-selector.js +74 -60
- package/dist/src/services/AstParserService.d.ts +7 -0
- package/dist/src/services/AstParserService.js +191 -28
- package/dist/src/services/DatabaseService.d.ts +11 -4
- package/dist/src/services/DatabaseService.js +107 -48
- package/dist/src/services/IndexerService.d.ts +9 -0
- package/dist/src/services/IndexerService.js +224 -60
- package/dist/src/services/LoggerService.d.ts +5 -1
- package/dist/src/services/LoggerService.js +8 -5
- package/dist/src/services/McpServer.d.ts +1 -0
- package/dist/src/services/McpServer.js +74 -4
- package/dist/src/services/OllamaEmbedder.d.ts +2 -0
- package/dist/src/services/OllamaEmbedder.js +29 -0
- package/dist/src/services/SearcherService.d.ts +13 -0
- package/dist/src/services/SearcherService.js +118 -22
- package/dist/src/setup.js +57 -62
- package/package.json +4 -5
|
@@ -1,105 +1,164 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.DatabaseService = void 0;
|
|
4
|
-
const
|
|
37
|
+
const node_postgres_1 = require("drizzle-orm/node-postgres");
|
|
5
38
|
const pg_1 = require("pg");
|
|
6
|
-
const
|
|
39
|
+
const schema = __importStar(require("../db/schema"));
|
|
7
40
|
class DatabaseService {
|
|
8
41
|
config;
|
|
9
|
-
|
|
42
|
+
pool;
|
|
43
|
+
_db;
|
|
10
44
|
constructor(config) {
|
|
11
45
|
this.config = config;
|
|
12
|
-
this.dataSource = new typeorm_1.DataSource({
|
|
13
|
-
type: 'postgres',
|
|
14
|
-
...config.db,
|
|
15
|
-
entities: [db_1.CodeChunk, db_1.CodeDependency, db_1.CodeSymbol],
|
|
16
|
-
synchronize: false,
|
|
17
|
-
});
|
|
18
46
|
}
|
|
19
|
-
|
|
20
|
-
|
|
47
|
+
get orm() { return this._db; }
|
|
48
|
+
async query(sql, params) {
|
|
49
|
+
const result = await this.pool.query(sql, params);
|
|
50
|
+
return result.rows;
|
|
21
51
|
}
|
|
22
52
|
transaction(fn) {
|
|
23
|
-
return this.
|
|
24
|
-
}
|
|
25
|
-
getRepository(entity) {
|
|
26
|
-
return this.dataSource.getRepository(entity);
|
|
53
|
+
return this._db.transaction(fn);
|
|
27
54
|
}
|
|
28
55
|
destroy() {
|
|
29
|
-
if (!this.
|
|
56
|
+
if (!this.pool)
|
|
30
57
|
return Promise.resolve();
|
|
31
|
-
return this.
|
|
58
|
+
return this.pool.end();
|
|
32
59
|
}
|
|
33
60
|
async initDb() {
|
|
34
|
-
|
|
61
|
+
const { host, port, username, password, database } = this.config.db;
|
|
62
|
+
this.pool = new pg_1.Pool({ host, port, user: username, password, database });
|
|
63
|
+
this._db = (0, node_postgres_1.drizzle)(this.pool, { schema });
|
|
64
|
+
await this.pool.query('SELECT 1');
|
|
35
65
|
}
|
|
36
66
|
async initDbWithSchema() {
|
|
37
67
|
await this.ensureDatabaseExists();
|
|
38
|
-
await this.
|
|
68
|
+
await this.initDb();
|
|
39
69
|
// code_chunk table
|
|
40
|
-
const [{ exists }] = await this.
|
|
70
|
+
const [{ exists }] = await this.query(`
|
|
41
71
|
SELECT EXISTS (
|
|
42
72
|
SELECT 1 FROM information_schema.tables
|
|
43
73
|
WHERE table_schema = 'public' AND table_name = 'code_chunk'
|
|
44
74
|
)
|
|
45
75
|
`);
|
|
46
76
|
if (!exists) {
|
|
47
|
-
await this.
|
|
48
|
-
|
|
77
|
+
await this.query(`
|
|
78
|
+
CREATE TABLE code_chunk (
|
|
79
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
80
|
+
"serviceName" TEXT NOT NULL,
|
|
81
|
+
"filePath" TEXT NOT NULL,
|
|
82
|
+
"startLine" INT,
|
|
83
|
+
content TEXT NOT NULL,
|
|
84
|
+
embedding vector(1024) NOT NULL,
|
|
85
|
+
"lastModified" DOUBLE PRECISION,
|
|
86
|
+
"contentHash" VARCHAR(64)
|
|
87
|
+
)
|
|
88
|
+
`);
|
|
89
|
+
await this.query(`
|
|
49
90
|
CREATE INDEX idx_code_chunk_embedding
|
|
50
91
|
ON code_chunk USING hnsw (embedding vector_cosine_ops)
|
|
51
92
|
`);
|
|
52
93
|
}
|
|
53
|
-
await this.
|
|
94
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
95
|
+
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "contentHash" varchar(64) NULL`);
|
|
54
96
|
// code_dependency table
|
|
55
|
-
const [{ existsDep }] = await this.
|
|
97
|
+
const [{ existsDep }] = await this.query(`
|
|
56
98
|
SELECT EXISTS (
|
|
57
99
|
SELECT 1 FROM information_schema.tables
|
|
58
100
|
WHERE table_schema = 'public' AND table_name = 'code_dependency'
|
|
59
101
|
) AS "existsDep"
|
|
60
102
|
`);
|
|
61
103
|
if (!existsDep) {
|
|
62
|
-
await this.
|
|
104
|
+
await this.query(`
|
|
63
105
|
CREATE TABLE code_dependency (
|
|
64
|
-
id
|
|
65
|
-
project
|
|
66
|
-
"fromFile"
|
|
106
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
107
|
+
project TEXT NOT NULL,
|
|
108
|
+
"fromFile" TEXT NOT NULL,
|
|
67
109
|
"fromSymbol" TEXT,
|
|
68
|
-
"toFile"
|
|
69
|
-
"toSymbol"
|
|
70
|
-
"depType"
|
|
110
|
+
"toFile" TEXT NOT NULL,
|
|
111
|
+
"toSymbol" TEXT,
|
|
112
|
+
"depType" TEXT NOT NULL
|
|
71
113
|
)
|
|
72
114
|
`);
|
|
73
115
|
}
|
|
74
|
-
await this.
|
|
75
|
-
await this.
|
|
76
|
-
await this.
|
|
77
|
-
await this.
|
|
78
|
-
await this.
|
|
116
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_from ON code_dependency (project, "fromFile")`);
|
|
117
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_to ON code_dependency (project, "toFile")`);
|
|
118
|
+
await this.query(`DROP INDEX IF EXISTS idx_dep_from`);
|
|
119
|
+
await this.query(`DROP INDEX IF EXISTS idx_dep_to`);
|
|
120
|
+
await this.query(`DROP INDEX IF EXISTS idx_dep_project`);
|
|
79
121
|
// code_symbol table
|
|
80
|
-
const [{ existsSym }] = await this.
|
|
122
|
+
const [{ existsSym }] = await this.query(`
|
|
81
123
|
SELECT EXISTS (
|
|
82
124
|
SELECT 1 FROM information_schema.tables
|
|
83
125
|
WHERE table_schema = 'public' AND table_name = 'code_symbol'
|
|
84
126
|
) AS "existsSym"
|
|
85
127
|
`);
|
|
86
128
|
if (!existsSym) {
|
|
87
|
-
await this.
|
|
129
|
+
await this.query(`
|
|
88
130
|
CREATE TABLE code_symbol (
|
|
89
|
-
id
|
|
90
|
-
project
|
|
131
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
132
|
+
project TEXT NOT NULL,
|
|
91
133
|
"filePath" TEXT NOT NULL,
|
|
92
|
-
name
|
|
93
|
-
kind
|
|
94
|
-
signature
|
|
95
|
-
line
|
|
96
|
-
col
|
|
134
|
+
name TEXT NOT NULL,
|
|
135
|
+
kind TEXT NOT NULL,
|
|
136
|
+
signature TEXT,
|
|
137
|
+
line INT,
|
|
138
|
+
col INT
|
|
97
139
|
)
|
|
98
140
|
`);
|
|
99
141
|
}
|
|
100
|
-
await this.
|
|
101
|
-
await this.
|
|
102
|
-
await this.
|
|
142
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_name ON code_symbol (name, project)`);
|
|
143
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_file ON code_symbol (project, "filePath")`);
|
|
144
|
+
await this.query(`CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique ON code_symbol (project, "filePath", name, kind)`);
|
|
145
|
+
// code_call table
|
|
146
|
+
await this.query(`
|
|
147
|
+
CREATE TABLE IF NOT EXISTS code_call (
|
|
148
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
149
|
+
project TEXT NOT NULL,
|
|
150
|
+
"fromFile" TEXT NOT NULL,
|
|
151
|
+
"fromSymbol" TEXT,
|
|
152
|
+
"calleeName" TEXT NOT NULL,
|
|
153
|
+
"calleeFile" TEXT NOT NULL,
|
|
154
|
+
"calleeSymbol" TEXT,
|
|
155
|
+
line INT,
|
|
156
|
+
col INT
|
|
157
|
+
)
|
|
158
|
+
`);
|
|
159
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_callee ON code_call (project, "calleeName")`);
|
|
160
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_callee_file ON code_call (project, "calleeFile")`);
|
|
161
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_from ON code_call (project, "fromFile")`);
|
|
103
162
|
}
|
|
104
163
|
async ensureDatabaseExists() {
|
|
105
164
|
const { host, port, username, password, database } = this.config.db;
|
|
@@ -27,6 +27,12 @@ export declare class IndexerService {
|
|
|
27
27
|
skipped: number;
|
|
28
28
|
failed: number;
|
|
29
29
|
}>;
|
|
30
|
+
backfillCalls(signal?: AbortSignal): Promise<{
|
|
31
|
+
total: number;
|
|
32
|
+
processed: number;
|
|
33
|
+
skipped: number;
|
|
34
|
+
failed: number;
|
|
35
|
+
}>;
|
|
30
36
|
discoverProjects(rootDir: string): {
|
|
31
37
|
name: string;
|
|
32
38
|
path: string;
|
|
@@ -34,6 +40,9 @@ export declare class IndexerService {
|
|
|
34
40
|
private writeDepEdges;
|
|
35
41
|
scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[]): Promise<IndexStats>;
|
|
36
42
|
private getSplitter;
|
|
43
|
+
private static inferRole;
|
|
44
|
+
private getEnrichedPrompt;
|
|
37
45
|
private embedText;
|
|
46
|
+
private static hashContent;
|
|
38
47
|
private ingestFile;
|
|
39
48
|
}
|
|
@@ -36,8 +36,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.IndexerService = void 0;
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
|
+
const crypto_1 = require("crypto");
|
|
39
40
|
const textsplitters_1 = require("@langchain/textsplitters");
|
|
40
41
|
const cli_progress_1 = require("cli-progress");
|
|
42
|
+
const drizzle_orm_1 = require("drizzle-orm");
|
|
41
43
|
const db_1 = require("../db");
|
|
42
44
|
const file_utils_1 = require("../file-utils");
|
|
43
45
|
const LoggerService_1 = require("./LoggerService");
|
|
@@ -67,7 +69,9 @@ class IndexerService {
|
|
|
67
69
|
});
|
|
68
70
|
}
|
|
69
71
|
async backfillSymbols(signal) {
|
|
72
|
+
const t0 = Date.now();
|
|
70
73
|
const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
|
|
74
|
+
this.logger.debug('IndexerService: backfillSymbols start', { total: rows.length });
|
|
71
75
|
let processed = 0, skipped = 0, failed = 0;
|
|
72
76
|
for (const { serviceName, filePath } of rows) {
|
|
73
77
|
if (signal?.aborted)
|
|
@@ -80,11 +84,11 @@ class IndexerService {
|
|
|
80
84
|
}
|
|
81
85
|
try {
|
|
82
86
|
const { symbols: rawSymbols } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
|
|
83
|
-
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath })).map(s => [`${s.name}:${s.kind}`, s])).values()];
|
|
84
|
-
await this.db.transaction(async (
|
|
85
|
-
await
|
|
87
|
+
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
|
|
88
|
+
await this.db.transaction(async (tx) => {
|
|
89
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, filePath)));
|
|
86
90
|
if (symbols.length > 0) {
|
|
87
|
-
await
|
|
91
|
+
await tx.insert(db_1.codeSymbol).values(symbols);
|
|
88
92
|
}
|
|
89
93
|
});
|
|
90
94
|
processed++;
|
|
@@ -101,6 +105,58 @@ class IndexerService {
|
|
|
101
105
|
process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
|
|
102
106
|
}
|
|
103
107
|
}
|
|
108
|
+
this.logger.debug('IndexerService: backfillSymbols done', { total: rows.length, processed, skipped, failed, ms: Date.now() - t0 });
|
|
109
|
+
return { total: rows.length, processed, skipped, failed };
|
|
110
|
+
}
|
|
111
|
+
async backfillCalls(signal) {
|
|
112
|
+
const t0 = Date.now();
|
|
113
|
+
const rows = await this.db.query(`SELECT DISTINCT "serviceName", "filePath" FROM code_chunk ORDER BY "serviceName", "filePath"`);
|
|
114
|
+
this.logger.debug('IndexerService: backfillCalls start', { total: rows.length });
|
|
115
|
+
let processed = 0, skipped = 0, failed = 0;
|
|
116
|
+
for (const { serviceName, filePath } of rows) {
|
|
117
|
+
if (signal?.aborted)
|
|
118
|
+
break;
|
|
119
|
+
const absPath = path.join(this.config.workspaceRoot, filePath);
|
|
120
|
+
const projectRoot = path.join(this.config.workspaceRoot, serviceName);
|
|
121
|
+
if (!fs.existsSync(absPath)) {
|
|
122
|
+
skipped++;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
const { calls: rawCalls } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
|
|
127
|
+
const calls = rawCalls
|
|
128
|
+
.filter(c => c.calleeFile.startsWith(this.config.workspaceRoot))
|
|
129
|
+
.map(c => ({
|
|
130
|
+
project: serviceName,
|
|
131
|
+
fromFile: filePath,
|
|
132
|
+
fromSymbol: c.fromSymbol,
|
|
133
|
+
calleeName: c.calleeName,
|
|
134
|
+
calleeFile: path.relative(this.config.workspaceRoot, c.calleeFile),
|
|
135
|
+
calleeSymbol: c.calleeSymbol,
|
|
136
|
+
line: c.line,
|
|
137
|
+
col: c.col,
|
|
138
|
+
}));
|
|
139
|
+
await this.db.transaction(async (tx) => {
|
|
140
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, filePath)));
|
|
141
|
+
if (calls.length > 0) {
|
|
142
|
+
await tx.insert(db_1.codeCall).values(calls);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
processed++;
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
const reason = err.message;
|
|
149
|
+
console.error(` Failed: ${filePath} — ${reason}`);
|
|
150
|
+
this.logger.error('IndexerService: call backfill failed', { file: filePath, reason });
|
|
151
|
+
failed++;
|
|
152
|
+
}
|
|
153
|
+
const count = processed + failed;
|
|
154
|
+
if (count > 0 && count % 50 === 0) {
|
|
155
|
+
const msg = ` ${count}/${rows.length} files...`;
|
|
156
|
+
process.stdout.isTTY ? process.stdout.write(msg + '\r') : console.log(msg);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
this.logger.debug('IndexerService: backfillCalls done', { total: rows.length, processed, skipped, failed, ms: Date.now() - t0 });
|
|
104
160
|
return { total: rows.length, processed, skipped, failed };
|
|
105
161
|
}
|
|
106
162
|
discoverProjects(rootDir) {
|
|
@@ -109,30 +165,31 @@ class IndexerService {
|
|
|
109
165
|
.map(d => ({ name: d.name, path: path.join(rootDir, d.name) }))
|
|
110
166
|
.filter(p => fs.existsSync(path.join(p.path, 'package.json')));
|
|
111
167
|
}
|
|
112
|
-
async writeDepEdges(serviceName, relFilePath, edges,
|
|
113
|
-
const run = async (
|
|
114
|
-
await
|
|
168
|
+
async writeDepEdges(serviceName, relFilePath, edges, tx) {
|
|
169
|
+
const run = async (t) => {
|
|
170
|
+
await t.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeDependency.fromFile, relFilePath)));
|
|
115
171
|
if (edges.length > 0) {
|
|
116
|
-
await
|
|
172
|
+
await t.insert(db_1.codeDependency).values(edges);
|
|
117
173
|
}
|
|
118
174
|
};
|
|
119
|
-
if (
|
|
120
|
-
await run(
|
|
175
|
+
if (tx)
|
|
176
|
+
await run(tx);
|
|
121
177
|
else
|
|
122
178
|
await this.db.transaction(run);
|
|
123
179
|
}
|
|
124
180
|
async scanAndIndex(dir, serviceName, force = false, signal, preCollectedFiles) {
|
|
125
181
|
const startTime = Date.now();
|
|
126
|
-
|
|
182
|
+
this.logger.debug('IndexerService: scanAndIndex start', { serviceName, dir, force });
|
|
127
183
|
if (force) {
|
|
128
|
-
await this.db.query(`UPDATE code_chunk SET "lastModified" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
184
|
+
await this.db.query(`UPDATE code_chunk SET "lastModified" = NULL, "contentHash" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
129
185
|
}
|
|
130
186
|
const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
|
|
187
|
+
this.logger.debug('IndexerService: scanAndIndex files collected', { serviceName, count: files.length, force });
|
|
131
188
|
const collectedRelPaths = new Set(files.map(f => path.relative(this.config.workspaceRoot, f)));
|
|
132
189
|
const dbPaths = await this.db.query(`SELECT DISTINCT "filePath" FROM code_chunk WHERE "serviceName" = $1`, [serviceName]);
|
|
133
190
|
for (const { filePath } of dbPaths) {
|
|
134
191
|
if (!collectedRelPaths.has(filePath)) {
|
|
135
|
-
await
|
|
192
|
+
await this.db.transaction(tx => tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, filePath))));
|
|
136
193
|
await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "fromFile" = $2`, [serviceName, filePath]);
|
|
137
194
|
await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "toFile" = $2`, [serviceName, filePath]);
|
|
138
195
|
console.log(` Removed stale: ${path.basename(filePath)}`);
|
|
@@ -167,54 +224,152 @@ class IndexerService {
|
|
|
167
224
|
console.log(` ⚠ ${msg}`);
|
|
168
225
|
warnings.push(msg);
|
|
169
226
|
}
|
|
227
|
+
if (files.length > this.config.indexer.astBatchSize) {
|
|
228
|
+
console.log(` ⚠ Large project (${files.length} files) — AST analysis uses batches of ${this.config.indexer.astBatchSize}; cross-batch dependency edges may be incomplete`);
|
|
229
|
+
}
|
|
170
230
|
bar.start(files.length, 0, { serviceName, filename: '' });
|
|
171
231
|
const stats = {
|
|
172
232
|
serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0, warnings,
|
|
173
233
|
};
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
234
|
+
const mtimeRefreshBatch = [];
|
|
235
|
+
const isLargeProject = files.length > this.config.indexer.astBatchSize;
|
|
236
|
+
for (let batchStart = 0; batchStart < files.length; batchStart += this.config.indexer.astBatchSize) {
|
|
237
|
+
if (signal?.aborted)
|
|
238
|
+
break;
|
|
239
|
+
const batch = files.slice(batchStart, batchStart + this.config.indexer.astBatchSize);
|
|
240
|
+
const analysisMap = await this.ast.analyzeProject(batch, serviceName, dir, { skipDependencyResolution: isLargeProject });
|
|
241
|
+
for (const filePath of batch) {
|
|
242
|
+
if (signal?.aborted) {
|
|
243
|
+
bar.stop();
|
|
244
|
+
console.log(`\n Interrupted — ${serviceName}`);
|
|
245
|
+
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
246
|
+
return stats;
|
|
247
|
+
}
|
|
248
|
+
bar.update({ filename: path.relative(dir, filePath) });
|
|
249
|
+
const mtimeMs = fs.statSync(filePath).mtimeMs;
|
|
250
|
+
const analysis = analysisMap.get(filePath);
|
|
251
|
+
if (analysis === undefined && /\.(ts|tsx|js|jsx)$/.test(filePath) && !/\.d\.ts$/.test(filePath)) {
|
|
252
|
+
this.logger.warn('IndexerService: no pre-computed analysis for TS file', { file: filePath });
|
|
253
|
+
}
|
|
254
|
+
const result = await this.ingestFile(filePath, serviceName, mtimeMs, dir, analysis, meta);
|
|
255
|
+
if (result.pendingMtimeRefresh) {
|
|
256
|
+
mtimeRefreshBatch.push({ relFilePath: result.pendingMtimeRefresh, mtimeMs });
|
|
257
|
+
}
|
|
258
|
+
if (result.status === 'indexed')
|
|
259
|
+
stats.indexed++;
|
|
260
|
+
else if (result.status === 'skipped')
|
|
261
|
+
stats.skipped++;
|
|
262
|
+
else
|
|
263
|
+
stats.failed++;
|
|
264
|
+
stats.chunks += result.chunks;
|
|
265
|
+
bar.increment();
|
|
180
266
|
}
|
|
181
|
-
bar.update({ filename: path.relative(dir, filePath) });
|
|
182
|
-
const mtimeMs = fs.statSync(filePath).mtimeMs;
|
|
183
|
-
const result = await this.ingestFile(filePath, serviceName, mtimeMs, dir);
|
|
184
|
-
if (result.status === 'indexed')
|
|
185
|
-
stats.indexed++;
|
|
186
|
-
else if (result.status === 'skipped')
|
|
187
|
-
stats.skipped++;
|
|
188
|
-
else
|
|
189
|
-
stats.failed++;
|
|
190
|
-
stats.chunks += result.chunks;
|
|
191
|
-
bar.increment();
|
|
192
267
|
}
|
|
193
268
|
bar.stop();
|
|
194
269
|
process.stdout.write('\x1B[J');
|
|
270
|
+
if (mtimeRefreshBatch.length > 0) {
|
|
271
|
+
try {
|
|
272
|
+
await this.db.query(`UPDATE code_chunk SET "lastModified" = t.mtime
|
|
273
|
+
FROM unnest($1::text[], $2::float8[]) AS t(fp, mtime)
|
|
274
|
+
WHERE "serviceName" = $3 AND "filePath" = t.fp`, [
|
|
275
|
+
mtimeRefreshBatch.map(r => r.relFilePath),
|
|
276
|
+
mtimeRefreshBatch.map(r => r.mtimeMs),
|
|
277
|
+
serviceName,
|
|
278
|
+
]);
|
|
279
|
+
}
|
|
280
|
+
catch (err) {
|
|
281
|
+
this.logger.error('IndexerService: mtime refresh batch failed', { reason: err.message });
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
this.ast.clearCache(dir);
|
|
195
285
|
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
286
|
+
this.logger.debug('IndexerService: project indexed', {
|
|
287
|
+
project: serviceName,
|
|
288
|
+
indexed: stats.indexed,
|
|
289
|
+
skipped: stats.skipped,
|
|
290
|
+
failed: stats.failed,
|
|
291
|
+
chunks: stats.chunks,
|
|
292
|
+
elapsedSec: stats.elapsedSec,
|
|
293
|
+
});
|
|
196
294
|
return stats;
|
|
197
295
|
}
|
|
198
296
|
getSplitter(filePath) {
|
|
199
297
|
return SCHEMA_EXTENSIONS.test(filePath) ? this.schemaSplitter : this.jsSplitter;
|
|
200
298
|
}
|
|
299
|
+
static inferRole(relPath) {
|
|
300
|
+
const p = relPath.toLowerCase();
|
|
301
|
+
if (p.includes('/services/') || p.endsWith('.service.ts'))
|
|
302
|
+
return 'Service';
|
|
303
|
+
if (p.includes('/controllers/') || p.endsWith('.controller.ts'))
|
|
304
|
+
return 'Controller';
|
|
305
|
+
if (p.includes('/entities/') || p.endsWith('.entity.ts'))
|
|
306
|
+
return 'Entity';
|
|
307
|
+
if (p.includes('/guards/') || p.endsWith('.guard.ts'))
|
|
308
|
+
return 'Guard';
|
|
309
|
+
if (p.includes('/commands/'))
|
|
310
|
+
return 'Command';
|
|
311
|
+
if (p.includes('/interfaces/'))
|
|
312
|
+
return 'Interface';
|
|
313
|
+
if (p.includes('/middleware/'))
|
|
314
|
+
return 'Middleware';
|
|
315
|
+
if (p.includes('/interceptors/'))
|
|
316
|
+
return 'Interceptor';
|
|
317
|
+
if (p.endsWith('.test.ts') || p.endsWith('.spec.ts') || p.includes('/tests/'))
|
|
318
|
+
return 'Test';
|
|
319
|
+
if (p.includes('/utils/') || p.includes('/helpers/') || p.endsWith('-utils.ts'))
|
|
320
|
+
return 'Utility';
|
|
321
|
+
return null;
|
|
322
|
+
}
|
|
323
|
+
getEnrichedPrompt(chunk, relPath, serviceName, _meta, edges, symbols) {
|
|
324
|
+
const chunkStart = (chunk.metadata?.loc?.lines?.from ?? 1);
|
|
325
|
+
const chunkEnd = (chunk.metadata?.loc?.lines?.to ?? chunkStart);
|
|
326
|
+
const lines = [];
|
|
327
|
+
lines.push(`Project: ${serviceName} | File: ${relPath}`);
|
|
328
|
+
const role = IndexerService.inferRole(relPath);
|
|
329
|
+
if (role)
|
|
330
|
+
lines.push(`Role: ${role}`);
|
|
331
|
+
const overlapping = symbols.filter(s => s.line != null &&
|
|
332
|
+
s.line <= chunkEnd &&
|
|
333
|
+
(s.endLine ?? s.line) >= chunkStart);
|
|
334
|
+
if (overlapping.length > 0) {
|
|
335
|
+
lines.push(`Symbols: ${overlapping.map(s => `${s.kind} ${s.name}`).join(', ')}`);
|
|
336
|
+
}
|
|
337
|
+
if (edges.length > 0) {
|
|
338
|
+
const depNames = [...new Set(edges.map(e => path.basename(e.toFile, path.extname(e.toFile)).split('.')[0]))];
|
|
339
|
+
lines.push(`Dependencies: ${depNames.join(', ')}`);
|
|
340
|
+
}
|
|
341
|
+
lines.push(`Code: ${chunk.pageContent}`);
|
|
342
|
+
return lines.join('\n');
|
|
343
|
+
}
|
|
201
344
|
async embedText(prompt) {
|
|
202
345
|
return this.embedder.embed(prompt);
|
|
203
346
|
}
|
|
204
|
-
|
|
347
|
+
static hashContent(content) {
|
|
348
|
+
return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
349
|
+
}
|
|
350
|
+
async ingestFile(filePath, serviceName, mtimeMs, projectRoot, analysis, meta) {
|
|
205
351
|
try {
|
|
206
|
-
const repo = this.db.getRepository(db_1.CodeChunk);
|
|
207
352
|
const relFilePath = path.relative(this.config.workspaceRoot, filePath);
|
|
208
|
-
const existing = await
|
|
209
|
-
select:
|
|
210
|
-
|
|
211
|
-
|
|
353
|
+
const [existing] = await this.db.orm
|
|
354
|
+
.select({ id: db_1.codeChunk.id, lastModified: db_1.codeChunk.lastModified, contentHash: db_1.codeChunk.contentHash })
|
|
355
|
+
.from(db_1.codeChunk)
|
|
356
|
+
.where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)))
|
|
357
|
+
.limit(1);
|
|
358
|
+
// Fast path: mtime unchanged — skip without reading file
|
|
212
359
|
if (existing?.lastModified === mtimeMs) {
|
|
213
360
|
return { status: 'skipped', chunks: 0 };
|
|
214
361
|
}
|
|
215
362
|
const t0 = Date.now();
|
|
216
|
-
const
|
|
217
|
-
|
|
363
|
+
const code = fs.readFileSync(filePath, 'utf-8');
|
|
364
|
+
const hash = IndexerService.hashContent(code);
|
|
365
|
+
// Content unchanged (mtime drifted — DB transfer, git clone, touch)
|
|
366
|
+
if (existing !== undefined && existing.contentHash !== null && existing.contentHash === hash) {
|
|
367
|
+
return { status: 'skipped', chunks: 0, pendingMtimeRefresh: relFilePath };
|
|
368
|
+
}
|
|
369
|
+
const tAst = Date.now();
|
|
370
|
+
const { edges: rawEdges, symbols: rawSymbols, calls: rawCalls } = analysis ?? { edges: [], symbols: [], calls: [] };
|
|
371
|
+
// astMs is near-zero — actual AST time is spent upfront in analyzeProject()
|
|
372
|
+
// Normalize edges to workspace-relative paths
|
|
218
373
|
const edges = rawEdges
|
|
219
374
|
.filter(e => e.toFile.startsWith(this.config.workspaceRoot))
|
|
220
375
|
.map(e => ({
|
|
@@ -222,10 +377,22 @@ class IndexerService {
|
|
|
222
377
|
fromFile: path.relative(this.config.workspaceRoot, e.fromFile),
|
|
223
378
|
toFile: path.relative(this.config.workspaceRoot, e.toFile),
|
|
224
379
|
}));
|
|
225
|
-
//
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
380
|
+
// Normalize call edges to workspace-relative paths; keep only resolved internal calls
|
|
381
|
+
const calls = rawCalls
|
|
382
|
+
.filter(c => c.calleeFile.startsWith(this.config.workspaceRoot))
|
|
383
|
+
.map(c => ({
|
|
384
|
+
project: serviceName,
|
|
385
|
+
fromFile: relFilePath,
|
|
386
|
+
fromSymbol: c.fromSymbol,
|
|
387
|
+
calleeName: c.calleeName,
|
|
388
|
+
calleeFile: path.relative(this.config.workspaceRoot, c.calleeFile),
|
|
389
|
+
calleeSymbol: c.calleeSymbol,
|
|
390
|
+
line: c.line,
|
|
391
|
+
col: c.col,
|
|
392
|
+
}));
|
|
393
|
+
// Normalize symbols: replace absolute filePath with workspace-relative, strip endLine (not stored)
|
|
394
|
+
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath: relFilePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
|
|
395
|
+
const astMs = Date.now() - tAst;
|
|
229
396
|
const chunks = await this.getSplitter(filePath).createDocuments([code]);
|
|
230
397
|
const relPath = path.relative(projectRoot, filePath);
|
|
231
398
|
const t2 = Date.now();
|
|
@@ -234,43 +401,40 @@ class IndexerService {
|
|
|
234
401
|
const batch = chunks.slice(i, i + this.config.indexer.batchSize);
|
|
235
402
|
const embedded = await Promise.all(batch.map(async (chunk) => {
|
|
236
403
|
const startLine = (chunk.metadata?.loc?.lines?.from ?? 1);
|
|
237
|
-
const prompt =
|
|
404
|
+
const prompt = this.getEnrichedPrompt(chunk, relPath, serviceName, meta ?? 'missing', edges, rawSymbols);
|
|
238
405
|
const embedding = await this.embedText(prompt);
|
|
239
|
-
return
|
|
406
|
+
return {
|
|
240
407
|
serviceName, filePath: relFilePath, startLine,
|
|
241
408
|
content: chunk.pageContent,
|
|
242
409
|
embedding: `[${embedding}]`,
|
|
243
410
|
lastModified: mtimeMs,
|
|
244
|
-
|
|
411
|
+
contentHash: hash,
|
|
412
|
+
};
|
|
245
413
|
}));
|
|
246
414
|
records.push(...embedded);
|
|
247
415
|
}
|
|
248
416
|
const embedMs = Date.now() - t2;
|
|
249
417
|
const t3 = Date.now();
|
|
250
|
-
await this.db.transaction(async (
|
|
251
|
-
await
|
|
418
|
+
await this.db.transaction(async (tx) => {
|
|
419
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)));
|
|
252
420
|
for (let i = 0; i < records.length; i += this.config.indexer.batchSize) {
|
|
253
|
-
await
|
|
254
|
-
.insert()
|
|
255
|
-
.into(db_1.CodeChunk)
|
|
256
|
-
.values(records.slice(i, i + this.config.indexer.batchSize))
|
|
257
|
-
.execute();
|
|
421
|
+
await tx.insert(db_1.codeChunk).values(records.slice(i, i + this.config.indexer.batchSize));
|
|
258
422
|
}
|
|
259
|
-
await this.writeDepEdges(serviceName, relFilePath, edges,
|
|
260
|
-
await
|
|
423
|
+
await this.writeDepEdges(serviceName, relFilePath, edges, tx);
|
|
424
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, relFilePath)));
|
|
261
425
|
if (symbols.length > 0) {
|
|
262
|
-
await
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
426
|
+
await tx.insert(db_1.codeSymbol).values(symbols);
|
|
427
|
+
}
|
|
428
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, relFilePath)));
|
|
429
|
+
if (calls.length > 0) {
|
|
430
|
+
await tx.insert(db_1.codeCall).values(calls);
|
|
267
431
|
}
|
|
268
432
|
});
|
|
269
433
|
const dbMs = Date.now() - t3;
|
|
270
434
|
if (chunks.length > 0) {
|
|
271
435
|
this.logger.debug('IndexerService: file indexed', {
|
|
272
436
|
file: relFilePath, chunks: chunks.length,
|
|
273
|
-
|
|
437
|
+
astMs, embedMs, dbMs, totalMs: Date.now() - t0,
|
|
274
438
|
});
|
|
275
439
|
}
|
|
276
440
|
return { status: 'indexed', chunks: chunks.length };
|
|
@@ -4,7 +4,11 @@ export declare class LoggerService {
|
|
|
4
4
|
private readonly logger;
|
|
5
5
|
constructor(logger: pino.Logger);
|
|
6
6
|
static silent(): LoggerService;
|
|
7
|
-
static
|
|
7
|
+
static fromConfig(cfg: {
|
|
8
|
+
level: string;
|
|
9
|
+
dir?: string;
|
|
10
|
+
}, name?: string): LoggerService;
|
|
11
|
+
private static buildFromPath;
|
|
8
12
|
debug(msg: string, data?: Record<string, unknown>): void;
|
|
9
13
|
info(msg: string, data?: Record<string, unknown>): void;
|
|
10
14
|
warn(msg: string, data?: Record<string, unknown>): void;
|