codesearch-mcp 0.0.14-alpha → 0.0.16-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -0
- package/dist/src/db/index.d.ts +2 -3
- package/dist/src/db/index.js +5 -7
- package/dist/src/db/schema.d.ts +417 -0
- package/dist/src/db/schema.js +36 -0
- package/dist/src/interfaces/IAstParser.d.ts +6 -0
- package/dist/src/services/AstParserService.d.ts +3 -0
- package/dist/src/services/AstParserService.js +92 -20
- package/dist/src/services/DatabaseService.d.ts +11 -4
- package/dist/src/services/DatabaseService.js +89 -48
- package/dist/src/services/IndexerService.d.ts +2 -0
- package/dist/src/services/IndexerService.js +88 -44
- package/dist/src/services/McpServer.js +55 -16
- package/package.json +4 -4
|
@@ -51,6 +51,55 @@ class AstParserService {
|
|
|
51
51
|
this.logger = logger;
|
|
52
52
|
}
|
|
53
53
|
// ── public interface ────────────────────────────────────────────────────────
|
|
54
|
+
async analyzeProject(filePaths, serviceName, projectRoot = process.cwd()) {
|
|
55
|
+
const t0 = Date.now();
|
|
56
|
+
const analysisMap = new Map();
|
|
57
|
+
const morphProject = this.createMorphProject(projectRoot);
|
|
58
|
+
for (const filePath of filePaths) {
|
|
59
|
+
if (!/\.(ts|tsx|js|jsx)$/.test(filePath) || /\.d\.ts$/.test(filePath)) {
|
|
60
|
+
analysisMap.set(filePath, { edges: [], symbols: [] });
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
morphProject.addSourceFileAtPath(filePath);
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
this.logger.warn('AstParser: failed to add source file', { filePath, reason: err.message });
|
|
68
|
+
analysisMap.set(filePath, { edges: [], symbols: [] });
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
for (const sourceFile of morphProject.getSourceFiles()) {
|
|
72
|
+
const filePath = sourceFile.getFilePath();
|
|
73
|
+
try {
|
|
74
|
+
const edges = this.extractEdges(sourceFile, filePath, serviceName);
|
|
75
|
+
const symbols = this.extractSymbols(sourceFile, filePath, serviceName);
|
|
76
|
+
analysisMap.set(filePath, { edges, symbols });
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
this.logger.warn('AstParser: failed to analyze file', { filePath, reason: err.message });
|
|
80
|
+
analysisMap.set(filePath, { edges: [], symbols: [] });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
// Release source file memory before returning
|
|
84
|
+
for (const sourceFile of morphProject.getSourceFiles()) {
|
|
85
|
+
morphProject.removeSourceFile(sourceFile);
|
|
86
|
+
}
|
|
87
|
+
this.logger.debug('AstParser: analyzeProject done', {
|
|
88
|
+
projectRoot, total: filePaths.length, parsed: analysisMap.size, ms: Date.now() - t0,
|
|
89
|
+
});
|
|
90
|
+
return analysisMap;
|
|
91
|
+
}
|
|
92
|
+
clearCache(projectRoot) {
|
|
93
|
+
if (projectRoot) {
|
|
94
|
+
const project = this.morphProjects.get(projectRoot);
|
|
95
|
+
if (project) {
|
|
96
|
+
this.morphProjects.delete(projectRoot);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
this.morphProjects.clear();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
54
103
|
async analyzeFile(filePath, serviceName, projectRoot = path.dirname(filePath)) {
|
|
55
104
|
if (!/\.(ts|tsx|js|jsx)$/.test(filePath) || /\.d\.ts$/.test(filePath)) {
|
|
56
105
|
return { edges: [], symbols: [] };
|
|
@@ -77,20 +126,7 @@ class AstParserService {
|
|
|
77
126
|
return edges;
|
|
78
127
|
}
|
|
79
128
|
// ── private: MorphProject cache (LRU, max MORPH_CACHE_MAX entries) ──────────
|
|
80
|
-
|
|
81
|
-
if (this.morphProjects.has(projectRoot)) {
|
|
82
|
-
// Move to end = most recently used
|
|
83
|
-
const cached = this.morphProjects.get(projectRoot);
|
|
84
|
-
this.morphProjects.delete(projectRoot);
|
|
85
|
-
this.morphProjects.set(projectRoot, cached);
|
|
86
|
-
return cached;
|
|
87
|
-
}
|
|
88
|
-
if (this.morphProjects.size >= MORPH_CACHE_MAX) {
|
|
89
|
-
// Evict least recently used (first entry in insertion-order Map)
|
|
90
|
-
const lruKey = this.morphProjects.keys().next().value;
|
|
91
|
-
this.morphProjects.delete(lruKey);
|
|
92
|
-
this.logger.debug('AstParser: evicted LRU project', { lruKey });
|
|
93
|
-
}
|
|
129
|
+
createMorphProject(projectRoot) {
|
|
94
130
|
const tsconfigPath = path.join(projectRoot, 'tsconfig.json');
|
|
95
131
|
const hasTsconfig = fs.existsSync(tsconfigPath);
|
|
96
132
|
this.logger.debug('AstParser: tsconfig lookup', { projectRoot, found: hasTsconfig });
|
|
@@ -117,6 +153,23 @@ class AstParserService {
|
|
|
117
153
|
});
|
|
118
154
|
project = new ts_morph_1.Project(FALLBACK_OPTS);
|
|
119
155
|
}
|
|
156
|
+
return project;
|
|
157
|
+
}
|
|
158
|
+
getMorphProject(projectRoot) {
|
|
159
|
+
if (this.morphProjects.has(projectRoot)) {
|
|
160
|
+
// Move to end = most recently used
|
|
161
|
+
const cached = this.morphProjects.get(projectRoot);
|
|
162
|
+
this.morphProjects.delete(projectRoot);
|
|
163
|
+
this.morphProjects.set(projectRoot, cached);
|
|
164
|
+
return cached;
|
|
165
|
+
}
|
|
166
|
+
if (this.morphProjects.size >= MORPH_CACHE_MAX) {
|
|
167
|
+
// Evict least recently used (first entry in insertion-order Map)
|
|
168
|
+
const lruKey = this.morphProjects.keys().next().value;
|
|
169
|
+
this.morphProjects.delete(lruKey);
|
|
170
|
+
this.logger.debug('AstParser: evicted LRU project', { lruKey });
|
|
171
|
+
}
|
|
172
|
+
const project = this.createMorphProject(projectRoot);
|
|
120
173
|
this.morphProjects.set(projectRoot, project);
|
|
121
174
|
return project;
|
|
122
175
|
}
|
|
@@ -223,7 +276,6 @@ class AstParserService {
|
|
|
223
276
|
// ── private: symbol extraction ─────────────────────────────────────────────
|
|
224
277
|
extractSymbols(sourceFile, filePath, serviceName) {
|
|
225
278
|
const symbols = [];
|
|
226
|
-
// Find the first line matching the keyword — skips leading decorators.
|
|
227
279
|
const sig = (text, keyword) => {
|
|
228
280
|
const found = text.split('\n').find(l => keyword.test(l));
|
|
229
281
|
return found?.trim() ?? text.split('\n')[0].trim();
|
|
@@ -239,7 +291,11 @@ class AstParserService {
|
|
|
239
291
|
continue;
|
|
240
292
|
const nameNode = cls.getNameNode();
|
|
241
293
|
const { line, col } = nameNode ? getPos(nameNode) : { line: cls.getStartLineNumber(), col: 1 };
|
|
242
|
-
symbols.push({
|
|
294
|
+
symbols.push({
|
|
295
|
+
project: serviceName, filePath, name, kind: 'class',
|
|
296
|
+
signature: sig(cls.getText(), /\bclass\s/),
|
|
297
|
+
line, endLine: cls.getEndLineNumber(), col,
|
|
298
|
+
});
|
|
243
299
|
}
|
|
244
300
|
for (const fn of sourceFile.getFunctions()) {
|
|
245
301
|
const name = fn.getName();
|
|
@@ -247,20 +303,36 @@ class AstParserService {
|
|
|
247
303
|
continue;
|
|
248
304
|
const nameNode = fn.getNameNode();
|
|
249
305
|
const { line, col } = nameNode ? getPos(nameNode) : { line: fn.getStartLineNumber(), col: 1 };
|
|
250
|
-
symbols.push({
|
|
306
|
+
symbols.push({
|
|
307
|
+
project: serviceName, filePath, name, kind: 'function',
|
|
308
|
+
signature: sig(fn.getText(), /\bfunction\s/),
|
|
309
|
+
line, endLine: fn.getEndLineNumber(), col,
|
|
310
|
+
});
|
|
251
311
|
}
|
|
252
312
|
for (const iface of sourceFile.getInterfaces()) {
|
|
253
313
|
const { line, col } = getPos(iface.getNameNode());
|
|
254
|
-
symbols.push({
|
|
314
|
+
symbols.push({
|
|
315
|
+
project: serviceName, filePath, name: iface.getName(), kind: 'interface',
|
|
316
|
+
signature: sig(iface.getText(), /\binterface\s/),
|
|
317
|
+
line, endLine: iface.getEndLineNumber(), col,
|
|
318
|
+
});
|
|
255
319
|
}
|
|
256
320
|
for (const ta of sourceFile.getTypeAliases()) {
|
|
257
321
|
const { line, col } = getPos(ta.getNameNode());
|
|
258
|
-
symbols.push({
|
|
322
|
+
symbols.push({
|
|
323
|
+
project: serviceName, filePath, name: ta.getName(), kind: 'type',
|
|
324
|
+
signature: sig(ta.getText(), /\btype\s/),
|
|
325
|
+
line, endLine: ta.getEndLineNumber(), col,
|
|
326
|
+
});
|
|
259
327
|
}
|
|
260
328
|
for (const stmt of sourceFile.getVariableStatements()) {
|
|
261
329
|
for (const decl of stmt.getDeclarations()) {
|
|
262
330
|
const { line, col } = getPos(decl.getNameNode());
|
|
263
|
-
symbols.push({
|
|
331
|
+
symbols.push({
|
|
332
|
+
project: serviceName, filePath, name: decl.getName(), kind: 'variable',
|
|
333
|
+
signature: sig(stmt.getText(), /\b(const|let|var)\s/),
|
|
334
|
+
line, endLine: stmt.getEndLineNumber(), col,
|
|
335
|
+
});
|
|
264
336
|
}
|
|
265
337
|
}
|
|
266
338
|
return symbols;
|
|
@@ -1,14 +1,21 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { NodePgDatabase, NodePgTransaction } from 'drizzle-orm/node-postgres';
|
|
2
|
+
import type { ExtractTablesWithRelations } from 'drizzle-orm/relations';
|
|
2
3
|
import type { Config } from '../config';
|
|
4
|
+
import * as schema from '../db/schema';
|
|
5
|
+
type Schema = typeof schema;
|
|
6
|
+
export type DrizzleDb = NodePgDatabase<Schema>;
|
|
7
|
+
export type DrizzleTx = NodePgTransaction<Schema, ExtractTablesWithRelations<Schema>>;
|
|
3
8
|
export declare class DatabaseService {
|
|
4
9
|
private config;
|
|
5
|
-
private
|
|
10
|
+
private pool;
|
|
11
|
+
private _db;
|
|
6
12
|
constructor(config: Config);
|
|
13
|
+
get orm(): DrizzleDb;
|
|
7
14
|
query<T = any>(sql: string, params?: any[]): Promise<T>;
|
|
8
|
-
transaction<T>(fn: (
|
|
9
|
-
getRepository<T extends ObjectLiteral>(entity: EntityTarget<T>): Repository<T>;
|
|
15
|
+
transaction<T>(fn: (tx: DrizzleTx) => Promise<T>): Promise<T>;
|
|
10
16
|
destroy(): Promise<void>;
|
|
11
17
|
initDb(): Promise<void>;
|
|
12
18
|
initDbWithSchema(): Promise<void>;
|
|
13
19
|
private ensureDatabaseExists;
|
|
14
20
|
}
|
|
21
|
+
export {};
|
|
@@ -1,105 +1,146 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.DatabaseService = void 0;
|
|
4
|
-
const
|
|
37
|
+
const node_postgres_1 = require("drizzle-orm/node-postgres");
|
|
5
38
|
const pg_1 = require("pg");
|
|
6
|
-
const
|
|
39
|
+
const schema = __importStar(require("../db/schema"));
|
|
7
40
|
class DatabaseService {
|
|
8
41
|
config;
|
|
9
|
-
|
|
42
|
+
pool;
|
|
43
|
+
_db;
|
|
10
44
|
constructor(config) {
|
|
11
45
|
this.config = config;
|
|
12
|
-
this.dataSource = new typeorm_1.DataSource({
|
|
13
|
-
type: 'postgres',
|
|
14
|
-
...config.db,
|
|
15
|
-
entities: [db_1.CodeChunk, db_1.CodeDependency, db_1.CodeSymbol],
|
|
16
|
-
synchronize: false,
|
|
17
|
-
});
|
|
18
46
|
}
|
|
19
|
-
|
|
20
|
-
|
|
47
|
+
get orm() { return this._db; }
|
|
48
|
+
async query(sql, params) {
|
|
49
|
+
const result = await this.pool.query(sql, params);
|
|
50
|
+
return result.rows;
|
|
21
51
|
}
|
|
22
52
|
transaction(fn) {
|
|
23
|
-
return this.
|
|
24
|
-
}
|
|
25
|
-
getRepository(entity) {
|
|
26
|
-
return this.dataSource.getRepository(entity);
|
|
53
|
+
return this._db.transaction(fn);
|
|
27
54
|
}
|
|
28
55
|
destroy() {
|
|
29
|
-
if (!this.
|
|
56
|
+
if (!this.pool)
|
|
30
57
|
return Promise.resolve();
|
|
31
|
-
return this.
|
|
58
|
+
return this.pool.end();
|
|
32
59
|
}
|
|
33
60
|
async initDb() {
|
|
34
|
-
|
|
61
|
+
const { host, port, username, password, database } = this.config.db;
|
|
62
|
+
this.pool = new pg_1.Pool({ host, port, user: username, password, database });
|
|
63
|
+
this._db = (0, node_postgres_1.drizzle)(this.pool, { schema });
|
|
35
64
|
}
|
|
36
65
|
async initDbWithSchema() {
|
|
37
66
|
await this.ensureDatabaseExists();
|
|
38
|
-
await this.
|
|
67
|
+
await this.initDb();
|
|
39
68
|
// code_chunk table
|
|
40
|
-
const [{ exists }] = await this.
|
|
69
|
+
const [{ exists }] = await this.query(`
|
|
41
70
|
SELECT EXISTS (
|
|
42
71
|
SELECT 1 FROM information_schema.tables
|
|
43
72
|
WHERE table_schema = 'public' AND table_name = 'code_chunk'
|
|
44
73
|
)
|
|
45
74
|
`);
|
|
46
75
|
if (!exists) {
|
|
47
|
-
await this.
|
|
48
|
-
|
|
76
|
+
await this.query(`
|
|
77
|
+
CREATE TABLE code_chunk (
|
|
78
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
79
|
+
"serviceName" TEXT NOT NULL,
|
|
80
|
+
"filePath" TEXT NOT NULL,
|
|
81
|
+
"startLine" INT,
|
|
82
|
+
content TEXT NOT NULL,
|
|
83
|
+
embedding vector(1024) NOT NULL,
|
|
84
|
+
"lastModified" DOUBLE PRECISION,
|
|
85
|
+
"contentHash" VARCHAR(64)
|
|
86
|
+
)
|
|
87
|
+
`);
|
|
88
|
+
await this.query(`
|
|
49
89
|
CREATE INDEX idx_code_chunk_embedding
|
|
50
90
|
ON code_chunk USING hnsw (embedding vector_cosine_ops)
|
|
51
91
|
`);
|
|
52
92
|
}
|
|
53
|
-
await this.
|
|
93
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
94
|
+
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "contentHash" varchar(64) NULL`);
|
|
54
95
|
// code_dependency table
|
|
55
|
-
const [{ existsDep }] = await this.
|
|
96
|
+
const [{ existsDep }] = await this.query(`
|
|
56
97
|
SELECT EXISTS (
|
|
57
98
|
SELECT 1 FROM information_schema.tables
|
|
58
99
|
WHERE table_schema = 'public' AND table_name = 'code_dependency'
|
|
59
100
|
) AS "existsDep"
|
|
60
101
|
`);
|
|
61
102
|
if (!existsDep) {
|
|
62
|
-
await this.
|
|
103
|
+
await this.query(`
|
|
63
104
|
CREATE TABLE code_dependency (
|
|
64
|
-
id
|
|
65
|
-
project
|
|
66
|
-
"fromFile"
|
|
105
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
106
|
+
project TEXT NOT NULL,
|
|
107
|
+
"fromFile" TEXT NOT NULL,
|
|
67
108
|
"fromSymbol" TEXT,
|
|
68
|
-
"toFile"
|
|
69
|
-
"toSymbol"
|
|
70
|
-
"depType"
|
|
109
|
+
"toFile" TEXT NOT NULL,
|
|
110
|
+
"toSymbol" TEXT,
|
|
111
|
+
"depType" TEXT NOT NULL
|
|
71
112
|
)
|
|
72
113
|
`);
|
|
73
114
|
}
|
|
74
|
-
await this.
|
|
75
|
-
await this.
|
|
76
|
-
await this.
|
|
77
|
-
await this.
|
|
78
|
-
await this.
|
|
115
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_from ON code_dependency (project, "fromFile")`);
|
|
116
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_to ON code_dependency (project, "toFile")`);
|
|
117
|
+
await this.query(`DROP INDEX IF EXISTS idx_dep_from`);
|
|
118
|
+
await this.query(`DROP INDEX IF EXISTS idx_dep_to`);
|
|
119
|
+
await this.query(`DROP INDEX IF EXISTS idx_dep_project`);
|
|
79
120
|
// code_symbol table
|
|
80
|
-
const [{ existsSym }] = await this.
|
|
121
|
+
const [{ existsSym }] = await this.query(`
|
|
81
122
|
SELECT EXISTS (
|
|
82
123
|
SELECT 1 FROM information_schema.tables
|
|
83
124
|
WHERE table_schema = 'public' AND table_name = 'code_symbol'
|
|
84
125
|
) AS "existsSym"
|
|
85
126
|
`);
|
|
86
127
|
if (!existsSym) {
|
|
87
|
-
await this.
|
|
128
|
+
await this.query(`
|
|
88
129
|
CREATE TABLE code_symbol (
|
|
89
|
-
id
|
|
90
|
-
project
|
|
130
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
131
|
+
project TEXT NOT NULL,
|
|
91
132
|
"filePath" TEXT NOT NULL,
|
|
92
|
-
name
|
|
93
|
-
kind
|
|
94
|
-
signature
|
|
95
|
-
line
|
|
96
|
-
col
|
|
133
|
+
name TEXT NOT NULL,
|
|
134
|
+
kind TEXT NOT NULL,
|
|
135
|
+
signature TEXT,
|
|
136
|
+
line INT,
|
|
137
|
+
col INT
|
|
97
138
|
)
|
|
98
139
|
`);
|
|
99
140
|
}
|
|
100
|
-
await this.
|
|
101
|
-
await this.
|
|
102
|
-
await this.
|
|
141
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_name ON code_symbol (name, project)`);
|
|
142
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_file ON code_symbol (project, "filePath")`);
|
|
143
|
+
await this.query(`CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique ON code_symbol (project, "filePath", name, kind)`);
|
|
103
144
|
}
|
|
104
145
|
async ensureDatabaseExists() {
|
|
105
146
|
const { host, port, username, password, database } = this.config.db;
|
|
@@ -34,6 +34,8 @@ export declare class IndexerService {
|
|
|
34
34
|
private writeDepEdges;
|
|
35
35
|
scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[]): Promise<IndexStats>;
|
|
36
36
|
private getSplitter;
|
|
37
|
+
private getEnrichedPrompt;
|
|
37
38
|
private embedText;
|
|
39
|
+
private static hashContent;
|
|
38
40
|
private ingestFile;
|
|
39
41
|
}
|
|
@@ -36,8 +36,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.IndexerService = void 0;
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
|
+
const crypto_1 = require("crypto");
|
|
39
40
|
const textsplitters_1 = require("@langchain/textsplitters");
|
|
40
41
|
const cli_progress_1 = require("cli-progress");
|
|
42
|
+
const drizzle_orm_1 = require("drizzle-orm");
|
|
41
43
|
const db_1 = require("../db");
|
|
42
44
|
const file_utils_1 = require("../file-utils");
|
|
43
45
|
const LoggerService_1 = require("./LoggerService");
|
|
@@ -80,11 +82,11 @@ class IndexerService {
|
|
|
80
82
|
}
|
|
81
83
|
try {
|
|
82
84
|
const { symbols: rawSymbols } = await this.ast.analyzeFile(absPath, serviceName, projectRoot);
|
|
83
|
-
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath })).map(s => [`${s.name}:${s.kind}`, s])).values()];
|
|
84
|
-
await this.db.transaction(async (
|
|
85
|
-
await
|
|
85
|
+
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
|
|
86
|
+
await this.db.transaction(async (tx) => {
|
|
87
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, filePath)));
|
|
86
88
|
if (symbols.length > 0) {
|
|
87
|
-
await
|
|
89
|
+
await tx.insert(db_1.codeSymbol).values(symbols);
|
|
88
90
|
}
|
|
89
91
|
});
|
|
90
92
|
processed++;
|
|
@@ -109,30 +111,29 @@ class IndexerService {
|
|
|
109
111
|
.map(d => ({ name: d.name, path: path.join(rootDir, d.name) }))
|
|
110
112
|
.filter(p => fs.existsSync(path.join(p.path, 'package.json')));
|
|
111
113
|
}
|
|
112
|
-
async writeDepEdges(serviceName, relFilePath, edges,
|
|
113
|
-
const run = async (
|
|
114
|
-
await
|
|
114
|
+
async writeDepEdges(serviceName, relFilePath, edges, tx) {
|
|
115
|
+
const run = async (t) => {
|
|
116
|
+
await t.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeDependency.fromFile, relFilePath)));
|
|
115
117
|
if (edges.length > 0) {
|
|
116
|
-
await
|
|
118
|
+
await t.insert(db_1.codeDependency).values(edges);
|
|
117
119
|
}
|
|
118
120
|
};
|
|
119
|
-
if (
|
|
120
|
-
await run(
|
|
121
|
+
if (tx)
|
|
122
|
+
await run(tx);
|
|
121
123
|
else
|
|
122
124
|
await this.db.transaction(run);
|
|
123
125
|
}
|
|
124
126
|
async scanAndIndex(dir, serviceName, force = false, signal, preCollectedFiles) {
|
|
125
127
|
const startTime = Date.now();
|
|
126
|
-
const repo = this.db.getRepository(db_1.CodeChunk);
|
|
127
128
|
if (force) {
|
|
128
|
-
await this.db.query(`UPDATE code_chunk SET "lastModified" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
129
|
+
await this.db.query(`UPDATE code_chunk SET "lastModified" = NULL, "contentHash" = NULL WHERE "serviceName" = $1`, [serviceName]);
|
|
129
130
|
}
|
|
130
131
|
const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
|
|
131
132
|
const collectedRelPaths = new Set(files.map(f => path.relative(this.config.workspaceRoot, f)));
|
|
132
133
|
const dbPaths = await this.db.query(`SELECT DISTINCT "filePath" FROM code_chunk WHERE "serviceName" = $1`, [serviceName]);
|
|
133
134
|
for (const { filePath } of dbPaths) {
|
|
134
135
|
if (!collectedRelPaths.has(filePath)) {
|
|
135
|
-
await
|
|
136
|
+
await this.db.transaction(tx => tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, filePath))));
|
|
136
137
|
await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "fromFile" = $2`, [serviceName, filePath]);
|
|
137
138
|
await this.db.query(`DELETE FROM code_dependency WHERE project = $1 AND "toFile" = $2`, [serviceName, filePath]);
|
|
138
139
|
console.log(` Removed stale: ${path.basename(filePath)}`);
|
|
@@ -171,6 +172,8 @@ class IndexerService {
|
|
|
171
172
|
const stats = {
|
|
172
173
|
serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0, warnings,
|
|
173
174
|
};
|
|
175
|
+
const mtimeRefreshBatch = [];
|
|
176
|
+
const analysisMap = await this.ast.analyzeProject(files, serviceName, dir);
|
|
174
177
|
for (const filePath of files) {
|
|
175
178
|
if (signal?.aborted) {
|
|
176
179
|
bar.stop();
|
|
@@ -180,7 +183,14 @@ class IndexerService {
|
|
|
180
183
|
}
|
|
181
184
|
bar.update({ filename: path.relative(dir, filePath) });
|
|
182
185
|
const mtimeMs = fs.statSync(filePath).mtimeMs;
|
|
183
|
-
const
|
|
186
|
+
const analysis = analysisMap.get(filePath);
|
|
187
|
+
if (analysis === undefined && /\.(ts|tsx|js|jsx)$/.test(filePath) && !/\.d\.ts$/.test(filePath)) {
|
|
188
|
+
this.logger.warn('IndexerService: no pre-computed analysis for TS file', { file: filePath });
|
|
189
|
+
}
|
|
190
|
+
const result = await this.ingestFile(filePath, serviceName, mtimeMs, dir, analysis);
|
|
191
|
+
if (result.pendingMtimeRefresh) {
|
|
192
|
+
mtimeRefreshBatch.push({ relFilePath: result.pendingMtimeRefresh, mtimeMs });
|
|
193
|
+
}
|
|
184
194
|
if (result.status === 'indexed')
|
|
185
195
|
stats.indexed++;
|
|
186
196
|
else if (result.status === 'skipped')
|
|
@@ -192,29 +202,71 @@ class IndexerService {
|
|
|
192
202
|
}
|
|
193
203
|
bar.stop();
|
|
194
204
|
process.stdout.write('\x1B[J');
|
|
205
|
+
if (mtimeRefreshBatch.length > 0) {
|
|
206
|
+
try {
|
|
207
|
+
await this.db.query(`UPDATE code_chunk SET "lastModified" = t.mtime
|
|
208
|
+
FROM unnest($1::text[], $2::float8[]) AS t(fp, mtime)
|
|
209
|
+
WHERE "serviceName" = $3 AND "filePath" = t.fp`, [
|
|
210
|
+
mtimeRefreshBatch.map(r => r.relFilePath),
|
|
211
|
+
mtimeRefreshBatch.map(r => r.mtimeMs),
|
|
212
|
+
serviceName,
|
|
213
|
+
]);
|
|
214
|
+
}
|
|
215
|
+
catch (err) {
|
|
216
|
+
this.logger.error('IndexerService: mtime refresh batch failed', { reason: err.message });
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
this.ast.clearCache(dir);
|
|
195
220
|
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
196
221
|
return stats;
|
|
197
222
|
}
|
|
198
223
|
getSplitter(filePath) {
|
|
199
224
|
return SCHEMA_EXTENSIONS.test(filePath) ? this.schemaSplitter : this.jsSplitter;
|
|
200
225
|
}
|
|
226
|
+
getEnrichedPrompt(chunk, analysis, serviceName, relPath) {
|
|
227
|
+
const { pageContent, metadata } = chunk;
|
|
228
|
+
const header = `Project: ${serviceName} | Path: ${relPath}`;
|
|
229
|
+
if (!analysis || analysis.symbols.length === 0) {
|
|
230
|
+
return `${header}\n${pageContent}`;
|
|
231
|
+
}
|
|
232
|
+
const chunkStartLine = metadata?.loc?.lines?.from ?? 1;
|
|
233
|
+
const chunkEndLine = metadata?.loc?.lines?.to ?? chunkStartLine;
|
|
234
|
+
const relevantSymbols = analysis.symbols.filter(s => s.line != null && s.endLine != null && s.line <= chunkEndLine && s.endLine >= chunkStartLine);
|
|
235
|
+
if (relevantSymbols.length === 0) {
|
|
236
|
+
return `${header}\n${pageContent}`;
|
|
237
|
+
}
|
|
238
|
+
const symbolsTag = `[Symbols: ${relevantSymbols.map(s => `${s.kind} ${s.name}`).join(', ')}]`;
|
|
239
|
+
return `${header}\n${symbolsTag}\n${pageContent}`;
|
|
240
|
+
}
|
|
201
241
|
async embedText(prompt) {
|
|
202
242
|
return this.embedder.embed(prompt);
|
|
203
243
|
}
|
|
204
|
-
|
|
244
|
+
static hashContent(content) {
|
|
245
|
+
return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
246
|
+
}
|
|
247
|
+
async ingestFile(filePath, serviceName, mtimeMs, projectRoot, analysis) {
|
|
205
248
|
try {
|
|
206
|
-
const repo = this.db.getRepository(db_1.CodeChunk);
|
|
207
249
|
const relFilePath = path.relative(this.config.workspaceRoot, filePath);
|
|
208
|
-
const existing = await
|
|
209
|
-
select:
|
|
210
|
-
|
|
211
|
-
|
|
250
|
+
const [existing] = await this.db.orm
|
|
251
|
+
.select({ id: db_1.codeChunk.id, lastModified: db_1.codeChunk.lastModified, contentHash: db_1.codeChunk.contentHash })
|
|
252
|
+
.from(db_1.codeChunk)
|
|
253
|
+
.where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)))
|
|
254
|
+
.limit(1);
|
|
255
|
+
// Fast path: mtime unchanged — skip without reading file
|
|
212
256
|
if (existing?.lastModified === mtimeMs) {
|
|
213
257
|
return { status: 'skipped', chunks: 0 };
|
|
214
258
|
}
|
|
215
259
|
const t0 = Date.now();
|
|
216
|
-
const
|
|
217
|
-
|
|
260
|
+
const code = fs.readFileSync(filePath, 'utf-8');
|
|
261
|
+
const hash = IndexerService.hashContent(code);
|
|
262
|
+
// Content unchanged (mtime drifted — DB transfer, git clone, touch)
|
|
263
|
+
if (existing !== undefined && existing.contentHash !== null && existing.contentHash === hash) {
|
|
264
|
+
return { status: 'skipped', chunks: 0, pendingMtimeRefresh: relFilePath };
|
|
265
|
+
}
|
|
266
|
+
const tAst = Date.now();
|
|
267
|
+
const { edges: rawEdges, symbols: rawSymbols } = analysis ?? { edges: [], symbols: [] };
|
|
268
|
+
// astMs is near-zero — actual AST time is spent upfront in analyzeProject()
|
|
269
|
+
// Normalize edges to workspace-relative paths
|
|
218
270
|
const edges = rawEdges
|
|
219
271
|
.filter(e => e.toFile.startsWith(this.config.workspaceRoot))
|
|
220
272
|
.map(e => ({
|
|
@@ -222,10 +274,9 @@ class IndexerService {
|
|
|
222
274
|
fromFile: path.relative(this.config.workspaceRoot, e.fromFile),
|
|
223
275
|
toFile: path.relative(this.config.workspaceRoot, e.toFile),
|
|
224
276
|
}));
|
|
225
|
-
//
|
|
226
|
-
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath: relFilePath })).map(s => [`${s.name}:${s.kind}`, s])).values()];
|
|
227
|
-
const
|
|
228
|
-
const code = fs.readFileSync(filePath, 'utf-8');
|
|
277
|
+
// Normalize symbols: replace absolute filePath with workspace-relative, strip endLine (not stored)
|
|
278
|
+
const symbols = [...new Map(rawSymbols.map(s => ({ ...s, filePath: relFilePath })).map(s => [`${s.name}:${s.kind}`, s])).values()].map(({ project, filePath: fp, name, kind, signature, line, col }) => ({ project, filePath: fp, name, kind, signature, line, col }));
|
|
279
|
+
const astMs = Date.now() - tAst;
|
|
229
280
|
const chunks = await this.getSplitter(filePath).createDocuments([code]);
|
|
230
281
|
const relPath = path.relative(projectRoot, filePath);
|
|
231
282
|
const t2 = Date.now();
|
|
@@ -234,43 +285,36 @@ class IndexerService {
|
|
|
234
285
|
const batch = chunks.slice(i, i + this.config.indexer.batchSize);
|
|
235
286
|
const embedded = await Promise.all(batch.map(async (chunk) => {
|
|
236
287
|
const startLine = (chunk.metadata?.loc?.lines?.from ?? 1);
|
|
237
|
-
const prompt =
|
|
288
|
+
const prompt = this.getEnrichedPrompt(chunk, analysis, serviceName, relPath);
|
|
238
289
|
const embedding = await this.embedText(prompt);
|
|
239
|
-
return
|
|
290
|
+
return {
|
|
240
291
|
serviceName, filePath: relFilePath, startLine,
|
|
241
292
|
content: chunk.pageContent,
|
|
242
293
|
embedding: `[${embedding}]`,
|
|
243
294
|
lastModified: mtimeMs,
|
|
244
|
-
|
|
295
|
+
contentHash: hash,
|
|
296
|
+
};
|
|
245
297
|
}));
|
|
246
298
|
records.push(...embedded);
|
|
247
299
|
}
|
|
248
300
|
const embedMs = Date.now() - t2;
|
|
249
301
|
const t3 = Date.now();
|
|
250
|
-
await this.db.transaction(async (
|
|
251
|
-
await
|
|
302
|
+
await this.db.transaction(async (tx) => {
|
|
303
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)));
|
|
252
304
|
for (let i = 0; i < records.length; i += this.config.indexer.batchSize) {
|
|
253
|
-
await
|
|
254
|
-
.insert()
|
|
255
|
-
.into(db_1.CodeChunk)
|
|
256
|
-
.values(records.slice(i, i + this.config.indexer.batchSize))
|
|
257
|
-
.execute();
|
|
305
|
+
await tx.insert(db_1.codeChunk).values(records.slice(i, i + this.config.indexer.batchSize));
|
|
258
306
|
}
|
|
259
|
-
await this.writeDepEdges(serviceName, relFilePath, edges,
|
|
260
|
-
await
|
|
307
|
+
await this.writeDepEdges(serviceName, relFilePath, edges, tx);
|
|
308
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, relFilePath)));
|
|
261
309
|
if (symbols.length > 0) {
|
|
262
|
-
await
|
|
263
|
-
.insert()
|
|
264
|
-
.into(db_1.CodeSymbol)
|
|
265
|
-
.values(symbols)
|
|
266
|
-
.execute();
|
|
310
|
+
await tx.insert(db_1.codeSymbol).values(symbols);
|
|
267
311
|
}
|
|
268
312
|
});
|
|
269
313
|
const dbMs = Date.now() - t3;
|
|
270
314
|
if (chunks.length > 0) {
|
|
271
315
|
this.logger.debug('IndexerService: file indexed', {
|
|
272
316
|
file: relFilePath, chunks: chunks.length,
|
|
273
|
-
|
|
317
|
+
astMs, embedMs, dbMs, totalMs: Date.now() - t0,
|
|
274
318
|
});
|
|
275
319
|
}
|
|
276
320
|
return { status: 'indexed', chunks: chunks.length };
|