codesearch-mcp 0.0.18-alpha → 0.2.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -1
- package/README.md +194 -23
- package/dist/app.js +38 -2
- package/dist/src/commands/IndexCommand.js +55 -1
- package/dist/src/commands/McpCommand.js +6 -2
- package/dist/src/config-error.d.ts +7 -0
- package/dist/src/config-error.js +14 -0
- package/dist/src/config-store.d.ts +73 -17
- package/dist/src/config-store.js +142 -27
- package/dist/src/config.d.ts +36 -11
- package/dist/src/config.js +89 -27
- package/dist/src/constants.d.ts +42 -2
- package/dist/src/constants.js +61 -7
- package/dist/src/container.js +5 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +2 -1
- package/dist/src/db/schema.d.ts +111 -0
- package/dist/src/db/schema.js +15 -1
- package/dist/src/help.js +6 -5
- package/dist/src/interfaces/IAstParser.d.ts +14 -0
- package/dist/src/interfaces/IChunker.d.ts +24 -0
- package/dist/src/interfaces/IChunker.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +31 -0
- package/dist/src/interfaces/IEmbedder.d.ts +14 -2
- package/dist/src/interfaces/ILogger.d.ts +7 -0
- package/dist/src/interfaces/ISearcher.d.ts +1 -0
- package/dist/src/interfaces/index.d.ts +2 -1
- package/dist/src/owner-path.d.ts +10 -0
- package/dist/src/owner-path.js +27 -0
- package/dist/src/services/AstChunker.d.ts +42 -0
- package/dist/src/services/AstChunker.js +158 -0
- package/dist/src/services/AstParserService.d.ts +6 -0
- package/dist/src/services/AstParserService.js +31 -2
- package/dist/src/services/DatabaseService.d.ts +17 -1
- package/dist/src/services/DatabaseService.js +116 -25
- package/dist/src/services/EmbedderFactory.d.ts +9 -0
- package/dist/src/services/EmbedderFactory.js +26 -0
- package/dist/src/services/IndexerService.d.ts +21 -5
- package/dist/src/services/IndexerService.js +127 -44
- package/dist/src/services/InfinityEmbedder.d.ts +38 -0
- package/dist/src/services/InfinityEmbedder.js +88 -0
- package/dist/src/services/LoggerService.d.ts +3 -1
- package/dist/src/services/LoggerService.js +17 -3
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +25 -0
- package/dist/src/services/OpenAICompatibleEmbedder.js +75 -0
- package/dist/src/services/SearcherService.d.ts +21 -1
- package/dist/src/services/SearcherService.js +129 -28
- package/dist/src/services/VoyageEmbedder.d.ts +22 -0
- package/dist/src/services/VoyageEmbedder.js +72 -0
- package/dist/src/services/WebServerService.d.ts +57 -14
- package/dist/src/services/WebServerService.js +134 -26
- package/dist/src/services/http-retry.d.ts +19 -0
- package/dist/src/services/http-retry.js +54 -0
- package/dist/src/services/infinity-provisioning.d.ts +115 -0
- package/dist/src/services/infinity-provisioning.js +516 -0
- package/dist/src/services/legacy-ollama-index.d.ts +24 -0
- package/dist/src/services/legacy-ollama-index.js +61 -0
- package/dist/src/services/reranker.d.ts +15 -0
- package/dist/src/services/reranker.js +44 -0
- package/dist/src/setup.d.ts +43 -0
- package/dist/src/setup.js +227 -55
- package/dist/src/tools/SearchCodebaseTool.js +1 -1
- package/dist/src/tools/format-helpers.js +2 -1
- package/dist/src/web/dashboard.js +5 -0
- package/docker/docker-compose.infinity-gpu.yml +19 -0
- package/docker/docker-compose.yml +20 -5
- package/docs/web-dashboard.md +36 -17
- package/package.json +18 -3
- package/dist/src/ast.d.ts +0 -2
- package/dist/src/ast.js +0 -110
- package/dist/src/commands/SetupCommand.d.ts +0 -7
- package/dist/src/commands/SetupCommand.js +0 -14
- package/dist/src/db/data-source.d.ts +0 -2
- package/dist/src/db/data-source.js +0 -13
- package/dist/src/db/entities/code-chunk.entity.d.ts +0 -9
- package/dist/src/db/entities/code-chunk.entity.js +0 -54
- package/dist/src/db/entities/code-dependency.entity.d.ts +0 -9
- package/dist/src/db/entities/code-dependency.entity.js +0 -54
- package/dist/src/db/entities/code-symbol.entity.d.ts +0 -10
- package/dist/src/db/entities/code-symbol.entity.js +0 -59
- package/dist/src/db/init.d.ts +0 -2
- package/dist/src/db/init.js +0 -62
- package/dist/src/env-loader.d.ts +0 -1
- package/dist/src/env-loader.js +0 -41
- package/dist/src/indexer.d.ts +0 -17
- package/dist/src/indexer.js +0 -242
- package/dist/src/mcp.d.ts +0 -1
- package/dist/src/mcp.js +0 -403
- package/dist/src/ollama.d.ts +0 -2
- package/dist/src/ollama.js +0 -6
- package/dist/src/searcher.d.ts +0 -74
- package/dist/src/searcher.js +0 -446
- package/dist/src/services/OllamaEmbedder.d.ts +0 -14
- package/dist/src/services/OllamaEmbedder.js +0 -63
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -9
|
@@ -7,4 +7,11 @@ export interface ILogger {
|
|
|
7
7
|
info(msg: string, data?: Record<string, unknown>): void;
|
|
8
8
|
warn(msg: string, data?: Record<string, unknown>): void;
|
|
9
9
|
error(msg: string, data?: Record<string, unknown>): void;
|
|
10
|
+
/**
|
|
11
|
+
* Structured telemetry emitted at `info` **independent of the configured log
|
|
12
|
+
* level** — so an operator can collect it in production without the full
|
|
13
|
+
* `debug` firehose. No-op when file logging is disabled (never writes to
|
|
14
|
+
* stdout, which the MCP transport owns).
|
|
15
|
+
*/
|
|
16
|
+
metric(msg: string, data?: Record<string, unknown>): void;
|
|
10
17
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
export type { IEmbedder } from './IEmbedder';
|
|
1
|
+
export type { IEmbedder, InputType } from './IEmbedder';
|
|
2
2
|
export type { IAstParser, DepEdge, SymbolEntry, CallEdge, FileAnalysis } from './IAstParser';
|
|
3
|
+
export type { IChunker, ChunkPiece, ChunkOpts } from './IChunker';
|
|
3
4
|
export type { ILogger } from './ILogger';
|
|
4
5
|
export type { IDatabase } from './IDatabase';
|
|
5
6
|
export type { ISearcher } from './ISearcher';
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { OwnerRange } from './interfaces/IAstParser';
|
|
2
|
+
/**
|
|
3
|
+
* Resolve the owning declaration for a chunk given its 1-based start line and a
|
|
4
|
+
* flat list of candidate ranges (top-level symbols + method ranges). The
|
|
5
|
+
* deepest range containing the start line wins (smallest `endLine - line` span),
|
|
6
|
+
* so a method beats its enclosing class. Returns `Container.name` for methods
|
|
7
|
+
* (via `container`) or `name` for top-level declarations, and `null` when the
|
|
8
|
+
* start line lies outside every range (imports / module-scope gaps).
|
|
9
|
+
*/
|
|
10
|
+
export declare function resolveOwnerPath(startLine: number, ranges: OwnerRange[]): string | null;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveOwnerPath = resolveOwnerPath;
|
|
4
|
+
/**
|
|
5
|
+
* Resolve the owning declaration for a chunk given its 1-based start line and a
|
|
6
|
+
* flat list of candidate ranges (top-level symbols + method ranges). The
|
|
7
|
+
* deepest range containing the start line wins (smallest `endLine - line` span),
|
|
8
|
+
* so a method beats its enclosing class. Returns `Container.name` for methods
|
|
9
|
+
* (via `container`) or `name` for top-level declarations, and `null` when the
|
|
10
|
+
* start line lies outside every range (imports / module-scope gaps).
|
|
11
|
+
*/
|
|
12
|
+
function resolveOwnerPath(startLine, ranges) {
|
|
13
|
+
let best = null;
|
|
14
|
+
let bestSpan = Infinity;
|
|
15
|
+
for (const r of ranges) {
|
|
16
|
+
if (r.line <= startLine && startLine <= r.endLine) {
|
|
17
|
+
const span = r.endLine - r.line;
|
|
18
|
+
if (span < bestSpan) {
|
|
19
|
+
bestSpan = span;
|
|
20
|
+
best = r;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
if (!best)
|
|
25
|
+
return null;
|
|
26
|
+
return best.container ? `${best.container}.${best.name}` : best.name;
|
|
27
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IChunker, ChunkPiece, ChunkOpts } from '../interfaces/IChunker';
|
|
3
|
+
/**
|
|
4
|
+
* Splits TS/JS source into embeddable pieces aligned to top-level declaration
|
|
5
|
+
* ranges (AST strategy). Falls back to the character-based recursive splitter
|
|
6
|
+
* for schema files, files without symbol analysis, or when strategy is
|
|
7
|
+
* 'character' — that path is byte-for-byte the previous behavior.
|
|
8
|
+
*/
|
|
9
|
+
export declare class AstChunker implements IChunker {
|
|
10
|
+
private config;
|
|
11
|
+
private readonly jsSplitter;
|
|
12
|
+
private readonly schemaSplitter;
|
|
13
|
+
private readonly chunkSize;
|
|
14
|
+
constructor(config: Config);
|
|
15
|
+
chunk(code: string, opts: ChunkOpts): Promise<ChunkPiece[]>;
|
|
16
|
+
private produce;
|
|
17
|
+
/**
|
|
18
|
+
* True iff every significant line (blank and comment lines stripped) belongs
|
|
19
|
+
* to an `import` / re-export statement and at least one is an import or
|
|
20
|
+
* re-export keyword — so the chunk carries no value/type declaration and is
|
|
21
|
+
* never a search answer. Any `export const|let|var|function|class|interface|
|
|
22
|
+
* enum|namespace|default|declare|async function|type X = …` disqualifies
|
|
23
|
+
* exclusion. Pure. Distinguishes `export { … } from` / `export type { … }`
|
|
24
|
+
* (re-export → drop) from `export type X = …` (declaration → keep).
|
|
25
|
+
*/
|
|
26
|
+
static isImportOrReexportOnly(content: string): boolean;
|
|
27
|
+
/** Character-splitter path for a whole file; line numbers from splitter metadata. */
|
|
28
|
+
private splitWholeFile;
|
|
29
|
+
private chunkByAst;
|
|
30
|
+
/**
|
|
31
|
+
* Build disjoint, sorted line intervals from symbol ranges. Overlapping /
|
|
32
|
+
* nested ranges merge into the outermost (a merged region covers any symbol
|
|
33
|
+
* inside it). Ranges are clamped to the file's line count.
|
|
34
|
+
*/
|
|
35
|
+
private static buildIntervals;
|
|
36
|
+
/**
|
|
37
|
+
* Emit pieces for a 1-based inclusive line region. Whitespace-only gaps are
|
|
38
|
+
* dropped; regions longer than chunkSize are sub-split with the char splitter,
|
|
39
|
+
* with sub-piece line numbers offset back to absolute file lines.
|
|
40
|
+
*/
|
|
41
|
+
private regionPieces;
|
|
42
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AstChunker = void 0;
|
|
4
|
+
const textsplitters_1 = require("@langchain/textsplitters");
|
|
5
|
+
/**
|
|
6
|
+
* Splits TS/JS source into embeddable pieces aligned to top-level declaration
|
|
7
|
+
* ranges (AST strategy). Falls back to the character-based recursive splitter
|
|
8
|
+
* for schema files, files without symbol analysis, or when strategy is
|
|
9
|
+
* 'character' — that path is byte-for-byte the previous behavior.
|
|
10
|
+
*/
|
|
11
|
+
class AstChunker {
|
|
12
|
+
config;
|
|
13
|
+
jsSplitter;
|
|
14
|
+
schemaSplitter;
|
|
15
|
+
chunkSize;
|
|
16
|
+
constructor(config) {
|
|
17
|
+
this.config = config;
|
|
18
|
+
this.chunkSize = config.indexer.chunkSize;
|
|
19
|
+
this.jsSplitter = textsplitters_1.RecursiveCharacterTextSplitter.fromLanguage('js', {
|
|
20
|
+
chunkSize: config.indexer.chunkSize,
|
|
21
|
+
chunkOverlap: config.indexer.chunkOverlap,
|
|
22
|
+
});
|
|
23
|
+
this.schemaSplitter = new textsplitters_1.RecursiveCharacterTextSplitter({
|
|
24
|
+
chunkSize: config.indexer.chunkSize,
|
|
25
|
+
chunkOverlap: config.indexer.chunkOverlap,
|
|
26
|
+
separators: ['\n\n', '\n', ' '],
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
async chunk(code, opts) {
|
|
30
|
+
const pieces = await this.produce(code, opts);
|
|
31
|
+
// Single post-chunk pass: drop pieces whose only significant content is
|
|
32
|
+
// import / re-export statements. Both chunk sources — AST "gap" regions and
|
|
33
|
+
// the whole-file character path for barrel files — can emit these, so we
|
|
34
|
+
// filter once over the final pieces. Dependency edges are untouched (they
|
|
35
|
+
// come from analyzeProject → code_dependency, not from chunk text).
|
|
36
|
+
return pieces.filter(p => !AstChunker.isImportOrReexportOnly(p.content));
|
|
37
|
+
}
|
|
38
|
+
async produce(code, opts) {
|
|
39
|
+
if (opts.isSchemaFile) {
|
|
40
|
+
return this.splitWholeFile(code, this.schemaSplitter);
|
|
41
|
+
}
|
|
42
|
+
if (opts.strategy === 'character' || opts.symbols.length === 0) {
|
|
43
|
+
return this.splitWholeFile(code, this.jsSplitter);
|
|
44
|
+
}
|
|
45
|
+
return this.chunkByAst(code, opts.symbols);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* True iff every significant line (blank and comment lines stripped) belongs
|
|
49
|
+
* to an `import` / re-export statement and at least one is an import or
|
|
50
|
+
* re-export keyword — so the chunk carries no value/type declaration and is
|
|
51
|
+
* never a search answer. Any `export const|let|var|function|class|interface|
|
|
52
|
+
* enum|namespace|default|declare|async function|type X = …` disqualifies
|
|
53
|
+
* exclusion. Pure. Distinguishes `export { … } from` / `export type { … }`
|
|
54
|
+
* (re-export → drop) from `export type X = …` (declaration → keep).
|
|
55
|
+
*/
|
|
56
|
+
static isImportOrReexportOnly(content) {
|
|
57
|
+
const lines = content
|
|
58
|
+
.replace(/\/\*[\s\S]*?\*\//g, '') // strip block comments
|
|
59
|
+
.split('\n')
|
|
60
|
+
.map(l => l.trim())
|
|
61
|
+
.filter(l => l !== '' && !l.startsWith('//') && !l.startsWith('*'));
|
|
62
|
+
if (lines.length === 0)
|
|
63
|
+
return false;
|
|
64
|
+
const IMPORT = /^import\b/;
|
|
65
|
+
const REEXPORT = /^export\s+(\*|type\s+\{|\{)/; // export * | export type { | export {
|
|
66
|
+
const DISQUALIFY = /^export\s+(default|async\b|function\b|const\b|let\b|var\b|class\b|abstract\b|interface\b|enum\b|namespace\b|declare\b|type\s+[A-Za-z_$])/;
|
|
67
|
+
const CONTINUATION = /^(\}?\s*from\s+['"][^'"]+['"];?|[{}]|(type\s+)?[A-Za-z_$][\w$]*(\s+as\s+[A-Za-z_$][\w$]*)?,?|['"][^'"]*['"];?)$/;
|
|
68
|
+
let keywords = 0;
|
|
69
|
+
for (const line of lines) {
|
|
70
|
+
if (DISQUALIFY.test(line))
|
|
71
|
+
return false;
|
|
72
|
+
if (IMPORT.test(line) || REEXPORT.test(line)) {
|
|
73
|
+
keywords++;
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (CONTINUATION.test(line))
|
|
77
|
+
continue;
|
|
78
|
+
return false; // unrecognized line → real code, keep the chunk
|
|
79
|
+
}
|
|
80
|
+
return keywords >= 1;
|
|
81
|
+
}
|
|
82
|
+
/** Character-splitter path for a whole file; line numbers from splitter metadata. */
|
|
83
|
+
async splitWholeFile(code, splitter) {
|
|
84
|
+
const docs = await splitter.createDocuments([code]);
|
|
85
|
+
return docs.map((d) => {
|
|
86
|
+
const from = (d.metadata?.loc?.lines?.from ?? 1);
|
|
87
|
+
const to = (d.metadata?.loc?.lines?.to ?? from);
|
|
88
|
+
return { content: d.pageContent, startLine: from, endLine: to };
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
async chunkByAst(code, symbols) {
|
|
92
|
+
const lines = code.split('\n');
|
|
93
|
+
const total = lines.length;
|
|
94
|
+
const intervals = AstChunker.buildIntervals(symbols, total);
|
|
95
|
+
const pieces = [];
|
|
96
|
+
let cursor = 1;
|
|
97
|
+
for (const iv of intervals) {
|
|
98
|
+
if (cursor < iv.start) {
|
|
99
|
+
pieces.push(...await this.regionPieces(lines, cursor, iv.start - 1, true));
|
|
100
|
+
}
|
|
101
|
+
pieces.push(...await this.regionPieces(lines, iv.start, iv.end, false));
|
|
102
|
+
cursor = iv.end + 1;
|
|
103
|
+
}
|
|
104
|
+
if (cursor <= total) {
|
|
105
|
+
pieces.push(...await this.regionPieces(lines, cursor, total, true));
|
|
106
|
+
}
|
|
107
|
+
return pieces;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Build disjoint, sorted line intervals from symbol ranges. Overlapping /
|
|
111
|
+
* nested ranges merge into the outermost (a merged region covers any symbol
|
|
112
|
+
* inside it). Ranges are clamped to the file's line count.
|
|
113
|
+
*/
|
|
114
|
+
static buildIntervals(symbols, total) {
|
|
115
|
+
const raw = symbols
|
|
116
|
+
.filter(s => s.line != null && s.line >= 1 && s.line <= total)
|
|
117
|
+
.map(s => ({
|
|
118
|
+
start: s.line,
|
|
119
|
+
end: Math.min(Math.max(s.endLine ?? s.line, s.line), total),
|
|
120
|
+
}))
|
|
121
|
+
.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
122
|
+
const merged = [];
|
|
123
|
+
for (const iv of raw) {
|
|
124
|
+
const last = merged[merged.length - 1];
|
|
125
|
+
if (last && iv.start <= last.end) {
|
|
126
|
+
last.end = Math.max(last.end, iv.end);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
merged.push({ ...iv });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return merged;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Emit pieces for a 1-based inclusive line region. Whitespace-only gaps are
|
|
136
|
+
* dropped; regions longer than chunkSize are sub-split with the char splitter,
|
|
137
|
+
* with sub-piece line numbers offset back to absolute file lines.
|
|
138
|
+
*/
|
|
139
|
+
async regionPieces(lines, start, end, isGap) {
|
|
140
|
+
const text = lines.slice(start - 1, end).join('\n');
|
|
141
|
+
if (isGap && text.trim() === '')
|
|
142
|
+
return [];
|
|
143
|
+
if (text.length <= this.chunkSize) {
|
|
144
|
+
return [{ content: text, startLine: start, endLine: end }];
|
|
145
|
+
}
|
|
146
|
+
const docs = await this.jsSplitter.createDocuments([text]);
|
|
147
|
+
return docs.map((d) => {
|
|
148
|
+
const from = (d.metadata?.loc?.lines?.from ?? 1);
|
|
149
|
+
const to = (d.metadata?.loc?.lines?.to ?? from);
|
|
150
|
+
return {
|
|
151
|
+
content: d.pageContent,
|
|
152
|
+
startLine: start + from - 1,
|
|
153
|
+
endLine: start + to - 1,
|
|
154
|
+
};
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
exports.AstChunker = AstChunker;
|
|
@@ -14,6 +14,12 @@ export declare class AstParserService implements IAstParser {
|
|
|
14
14
|
private resolveFromSymbol;
|
|
15
15
|
private buildUsageMap;
|
|
16
16
|
private extractSymbols;
|
|
17
|
+
/**
|
|
18
|
+
* Method / constructor / accessor line ranges with their enclosing class, for
|
|
19
|
+
* owner-path resolution only. Deliberately NOT fed into `code_symbol` (methods
|
|
20
|
+
* stay grep-based in find_symbol), so this is separate from extractSymbols.
|
|
21
|
+
*/
|
|
22
|
+
private extractOwnerRanges;
|
|
17
23
|
private extractCalls;
|
|
18
24
|
private getContainerName;
|
|
19
25
|
}
|
|
@@ -77,7 +77,8 @@ class AstParserService {
|
|
|
77
77
|
const edges = this.extractEdges(sourceFile, filePath, serviceName);
|
|
78
78
|
const symbols = this.extractSymbols(sourceFile, filePath, serviceName);
|
|
79
79
|
const calls = this.extractCalls(sourceFile, filePath, serviceName);
|
|
80
|
-
|
|
80
|
+
const ownerRanges = this.extractOwnerRanges(sourceFile);
|
|
81
|
+
analysisMap.set(filePath, { edges, symbols, calls, ownerRanges });
|
|
81
82
|
}
|
|
82
83
|
catch (err) {
|
|
83
84
|
this.logger.warn('AstParser: failed to analyze file', { filePath, reason: err.message });
|
|
@@ -126,9 +127,10 @@ class AstParserService {
|
|
|
126
127
|
const edges = this.extractEdges(sourceFile, filePath, serviceName);
|
|
127
128
|
const symbols = this.extractSymbols(sourceFile, filePath, serviceName);
|
|
128
129
|
const calls = this.extractCalls(sourceFile, filePath, serviceName);
|
|
130
|
+
const ownerRanges = this.extractOwnerRanges(sourceFile);
|
|
129
131
|
morphProject.removeSourceFile(sourceFile);
|
|
130
132
|
this.logger.debug('AstParser: analyzeFile done', { filePath, edges: edges.length, symbols: symbols.length, calls: calls.length, ms: Date.now() - t0 });
|
|
131
|
-
return { edges, symbols, calls };
|
|
133
|
+
return { edges, symbols, calls, ownerRanges };
|
|
132
134
|
}
|
|
133
135
|
// Kept for test compatibility (extract-import-edges.test.ts).
|
|
134
136
|
async extractImportEdges(filePath, serviceName, projectRoot) {
|
|
@@ -369,6 +371,33 @@ class AstParserService {
|
|
|
369
371
|
}
|
|
370
372
|
return symbols;
|
|
371
373
|
}
|
|
374
|
+
/**
|
|
375
|
+
* Method / constructor / accessor line ranges with their enclosing class, for
|
|
376
|
+
* owner-path resolution only. Deliberately NOT fed into `code_symbol` (methods
|
|
377
|
+
* stay grep-based in find_symbol), so this is separate from extractSymbols.
|
|
378
|
+
*/
|
|
379
|
+
extractOwnerRanges(sourceFile) {
|
|
380
|
+
const ranges = [];
|
|
381
|
+
for (const cls of sourceFile.getClasses()) {
|
|
382
|
+
const container = cls.getName();
|
|
383
|
+
if (!container)
|
|
384
|
+
continue;
|
|
385
|
+
const members = [
|
|
386
|
+
...cls.getMethods().map(n => ({ name: n.getName(), node: n })),
|
|
387
|
+
...cls.getGetAccessors().map(n => ({ name: n.getName(), node: n })),
|
|
388
|
+
...cls.getSetAccessors().map(n => ({ name: n.getName(), node: n })),
|
|
389
|
+
...cls.getConstructors().map(n => ({ name: 'constructor', node: n })),
|
|
390
|
+
];
|
|
391
|
+
for (const { name, node } of members) {
|
|
392
|
+
ranges.push({
|
|
393
|
+
name, container,
|
|
394
|
+
line: node.getStartLineNumber(),
|
|
395
|
+
endLine: node.getEndLineNumber(),
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
return ranges;
|
|
400
|
+
}
|
|
372
401
|
// ── private: call graph extraction ────────────────────────────────────────
|
|
373
402
|
extractCalls(sourceFile, filePath, serviceName) {
|
|
374
403
|
const calls = [];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { NodePgDatabase, NodePgTransaction } from 'drizzle-orm/node-postgres';
|
|
2
2
|
import type { ExtractTablesWithRelations } from 'drizzle-orm/relations';
|
|
3
3
|
import type { Config } from '../config';
|
|
4
|
-
import type { IDatabase } from '../interfaces/IDatabase';
|
|
4
|
+
import type { IDatabase, IndexMeta } from '../interfaces/IDatabase';
|
|
5
5
|
import * as schema from '../db/schema';
|
|
6
6
|
type Schema = typeof schema;
|
|
7
7
|
export type DrizzleDb = NodePgDatabase<Schema>;
|
|
@@ -17,6 +17,22 @@ export declare class DatabaseService implements IDatabase {
|
|
|
17
17
|
destroy(): Promise<void>;
|
|
18
18
|
initDb(): Promise<void>;
|
|
19
19
|
initDbWithSchema(): Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Create `code_chunk` at `dimensions` if it does not exist, plus its HNSW and
|
|
22
|
+
* (serviceName, filePath) indexes and the additive `contentHash`/`ownerPath`
|
|
23
|
+
* columns for pre-existing tables. Idempotent — safe to call repeatedly.
|
|
24
|
+
*/
|
|
25
|
+
ensureChunkTable(dimensions: number): Promise<void>;
|
|
26
|
+
/** Drop and recreate `code_chunk` at `dimensions` — used on a dimension change. */
|
|
27
|
+
recreateChunkTable(dimensions: number): Promise<void>;
|
|
28
|
+
readIndexMeta(): Promise<IndexMeta | null>;
|
|
29
|
+
writeIndexMeta(meta: IndexMeta): Promise<void>;
|
|
30
|
+
listKnownFiles(project: string): Promise<string[]>;
|
|
31
|
+
deleteFiles(project: string, filePaths: string[]): Promise<void>;
|
|
32
|
+
listIndexedProjects(): Promise<string[]>;
|
|
33
|
+
deleteProject(name: string): Promise<void>;
|
|
34
|
+
/** Guard against SQL injection — the dimension is interpolated into DDL. */
|
|
35
|
+
private assertDim;
|
|
20
36
|
private ensureDatabaseExists;
|
|
21
37
|
}
|
|
22
38
|
export {};
|
|
@@ -36,7 +36,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.DatabaseService = void 0;
|
|
37
37
|
const node_postgres_1 = require("drizzle-orm/node-postgres");
|
|
38
38
|
const pg_1 = require("pg");
|
|
39
|
+
const drizzle_orm_1 = require("drizzle-orm");
|
|
39
40
|
const schema = __importStar(require("../db/schema"));
|
|
41
|
+
const db_1 = require("../db");
|
|
40
42
|
class DatabaseService {
|
|
41
43
|
config;
|
|
42
44
|
pool;
|
|
@@ -66,33 +68,20 @@ class DatabaseService {
|
|
|
66
68
|
async initDbWithSchema() {
|
|
67
69
|
await this.ensureDatabaseExists();
|
|
68
70
|
await this.initDb();
|
|
69
|
-
// code_chunk table
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
71
|
+
// code_chunk table — created at the legacy default width (1024) so search
|
|
72
|
+
// tools always have a table. IndexerService recreates it at the resolved
|
|
73
|
+
// width before indexing when the active model's dimension differs.
|
|
74
|
+
await this.ensureChunkTable(1024);
|
|
75
|
+
// index_meta table — source of truth for the model that built the index.
|
|
76
|
+
await this.query(`
|
|
77
|
+
CREATE TABLE IF NOT EXISTS index_meta (
|
|
78
|
+
id BOOLEAN PRIMARY KEY DEFAULT true,
|
|
79
|
+
provider TEXT NOT NULL,
|
|
80
|
+
model TEXT NOT NULL,
|
|
81
|
+
dimensions INT NOT NULL,
|
|
82
|
+
"indexedAt" TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
74
83
|
)
|
|
75
84
|
`);
|
|
76
|
-
if (!exists) {
|
|
77
|
-
await this.query(`
|
|
78
|
-
CREATE TABLE code_chunk (
|
|
79
|
-
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
80
|
-
"serviceName" TEXT NOT NULL,
|
|
81
|
-
"filePath" TEXT NOT NULL,
|
|
82
|
-
"startLine" INT,
|
|
83
|
-
content TEXT NOT NULL,
|
|
84
|
-
embedding vector(1024) NOT NULL,
|
|
85
|
-
"lastModified" DOUBLE PRECISION,
|
|
86
|
-
"contentHash" VARCHAR(64)
|
|
87
|
-
)
|
|
88
|
-
`);
|
|
89
|
-
await this.query(`
|
|
90
|
-
CREATE INDEX idx_code_chunk_embedding
|
|
91
|
-
ON code_chunk USING hnsw (embedding vector_cosine_ops)
|
|
92
|
-
`);
|
|
93
|
-
}
|
|
94
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
95
|
-
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "contentHash" varchar(64) NULL`);
|
|
96
85
|
// code_dependency table
|
|
97
86
|
const [{ existsDep }] = await this.query(`
|
|
98
87
|
SELECT EXISTS (
|
|
@@ -160,6 +149,108 @@ class DatabaseService {
|
|
|
160
149
|
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_callee_file ON code_call (project, "calleeFile")`);
|
|
161
150
|
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_from ON code_call (project, "fromFile")`);
|
|
162
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* Create `code_chunk` at `dimensions` if it does not exist, plus its HNSW and
|
|
154
|
+
* (serviceName, filePath) indexes and the additive `contentHash`/`ownerPath`
|
|
155
|
+
* columns for pre-existing tables. Idempotent — safe to call repeatedly.
|
|
156
|
+
*/
|
|
157
|
+
async ensureChunkTable(dimensions) {
|
|
158
|
+
const dim = this.assertDim(dimensions);
|
|
159
|
+
await this.query(`
|
|
160
|
+
CREATE TABLE IF NOT EXISTS code_chunk (
|
|
161
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
162
|
+
"serviceName" TEXT NOT NULL,
|
|
163
|
+
"filePath" TEXT NOT NULL,
|
|
164
|
+
"startLine" INT,
|
|
165
|
+
content TEXT NOT NULL,
|
|
166
|
+
embedding vector(${dim}) NOT NULL,
|
|
167
|
+
"lastModified" DOUBLE PRECISION,
|
|
168
|
+
"contentHash" VARCHAR(64),
|
|
169
|
+
"ownerPath" VARCHAR
|
|
170
|
+
)
|
|
171
|
+
`);
|
|
172
|
+
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "contentHash" varchar(64) NULL`);
|
|
173
|
+
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "ownerPath" varchar NULL`);
|
|
174
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_embedding ON code_chunk USING hnsw (embedding vector_cosine_ops)`);
|
|
175
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
176
|
+
}
|
|
177
|
+
/** Drop and recreate `code_chunk` at `dimensions` — used on a dimension change. */
|
|
178
|
+
async recreateChunkTable(dimensions) {
|
|
179
|
+
const dim = this.assertDim(dimensions);
|
|
180
|
+
await this.query(`DROP TABLE IF EXISTS code_chunk`);
|
|
181
|
+
await this.ensureChunkTable(dim);
|
|
182
|
+
}
|
|
183
|
+
async readIndexMeta() {
|
|
184
|
+
try {
|
|
185
|
+
const rows = await this.query(`SELECT provider, model, dimensions FROM index_meta LIMIT 1`);
|
|
186
|
+
const row = rows[0];
|
|
187
|
+
return row ? { provider: row.provider, model: row.model, dimensions: Number(row.dimensions) } : null;
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
// Table absent (un-migrated DB) — treat as "no metadata".
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
async writeIndexMeta(meta) {
|
|
195
|
+
await this.query(`INSERT INTO index_meta (id, provider, model, dimensions, "indexedAt")
|
|
196
|
+
VALUES (true, $1, $2, $3, now())
|
|
197
|
+
ON CONFLICT (id) DO UPDATE
|
|
198
|
+
SET provider = EXCLUDED.provider,
|
|
199
|
+
model = EXCLUDED.model,
|
|
200
|
+
dimensions = EXCLUDED.dimensions,
|
|
201
|
+
"indexedAt" = now()`, [meta.provider, meta.model, meta.dimensions]);
|
|
202
|
+
}
|
|
203
|
+
async listKnownFiles(project) {
|
|
204
|
+
const rows = await this.query(`SELECT "filePath" AS fp FROM code_chunk WHERE "serviceName" = $1
|
|
205
|
+
UNION
|
|
206
|
+
SELECT "filePath" AS fp FROM code_symbol WHERE project = $1
|
|
207
|
+
UNION
|
|
208
|
+
SELECT "fromFile" AS fp FROM code_call WHERE project = $1
|
|
209
|
+
UNION
|
|
210
|
+
SELECT "fromFile" AS fp FROM code_dependency WHERE project = $1`, [project]);
|
|
211
|
+
return rows.map(r => r.fp);
|
|
212
|
+
}
|
|
213
|
+
async deleteFiles(project, filePaths) {
|
|
214
|
+
if (filePaths.length === 0)
|
|
215
|
+
return;
|
|
216
|
+
// PostgreSQL limits ~32 767 bind params per statement; OR queries bind the
|
|
217
|
+
// array twice (fromFile + calleeFile), so cap each batch at 10 000 entries.
|
|
218
|
+
const BATCH = 10_000;
|
|
219
|
+
await this.transaction(async (tx) => {
|
|
220
|
+
for (let i = 0; i < filePaths.length; i += BATCH) {
|
|
221
|
+
const batch = filePaths.slice(i, i + BATCH);
|
|
222
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, project), (0, drizzle_orm_1.inArray)(db_1.codeChunk.filePath, batch)));
|
|
223
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, project), (0, drizzle_orm_1.inArray)(db_1.codeSymbol.filePath, batch)));
|
|
224
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, project), (0, drizzle_orm_1.or)((0, drizzle_orm_1.inArray)(db_1.codeCall.fromFile, batch), (0, drizzle_orm_1.inArray)(db_1.codeCall.calleeFile, batch))));
|
|
225
|
+
await tx.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, project), (0, drizzle_orm_1.or)((0, drizzle_orm_1.inArray)(db_1.codeDependency.fromFile, batch), (0, drizzle_orm_1.inArray)(db_1.codeDependency.toFile, batch))));
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
async listIndexedProjects() {
|
|
230
|
+
const rows = await this.query(`SELECT "serviceName" AS name FROM code_chunk
|
|
231
|
+
UNION
|
|
232
|
+
SELECT project AS name FROM code_symbol
|
|
233
|
+
UNION
|
|
234
|
+
SELECT project AS name FROM code_call
|
|
235
|
+
UNION
|
|
236
|
+
SELECT project AS name FROM code_dependency`);
|
|
237
|
+
return rows.map(r => r.name);
|
|
238
|
+
}
|
|
239
|
+
async deleteProject(name) {
|
|
240
|
+
await this.transaction(async (tx) => {
|
|
241
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, name));
|
|
242
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, name));
|
|
243
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, name));
|
|
244
|
+
await tx.delete(db_1.codeDependency).where((0, drizzle_orm_1.eq)(db_1.codeDependency.project, name));
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
/** Guard against SQL injection — the dimension is interpolated into DDL. */
|
|
248
|
+
assertDim(dimensions) {
|
|
249
|
+
if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > 2000) {
|
|
250
|
+
throw new Error(`Invalid vector dimension: ${dimensions} (must be an integer 1–2000)`);
|
|
251
|
+
}
|
|
252
|
+
return dimensions;
|
|
253
|
+
}
|
|
163
254
|
async ensureDatabaseExists() {
|
|
164
255
|
const { host, port, username, password, database } = this.config.db;
|
|
165
256
|
const client = new pg_1.Client({ host, port, user: username, password, database: 'postgres' });
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
import { type EmbedderDeps } from './OpenAICompatibleEmbedder';
|
|
5
|
+
/**
|
|
6
|
+
* Construct the `IEmbedder` implementation selected by `config.embedding.provider`.
|
|
7
|
+
* The default (`infinity`) is the recommended local backend.
|
|
8
|
+
*/
|
|
9
|
+
export declare function createEmbedder(config: Config, logger?: ILogger, deps?: EmbedderDeps): IEmbedder;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createEmbedder = createEmbedder;
|
|
4
|
+
const LoggerService_1 = require("./LoggerService");
|
|
5
|
+
const OpenAICompatibleEmbedder_1 = require("./OpenAICompatibleEmbedder");
|
|
6
|
+
const InfinityEmbedder_1 = require("./InfinityEmbedder");
|
|
7
|
+
const VoyageEmbedder_1 = require("./VoyageEmbedder");
|
|
8
|
+
/**
|
|
9
|
+
* Construct the `IEmbedder` implementation selected by `config.embedding.provider`.
|
|
10
|
+
* The default (`infinity`) is the recommended local backend.
|
|
11
|
+
*/
|
|
12
|
+
function createEmbedder(config, logger = LoggerService_1.LoggerService.silent(), deps = {}) {
|
|
13
|
+
switch (config.embedding.provider) {
|
|
14
|
+
case 'openai-compatible':
|
|
15
|
+
return new OpenAICompatibleEmbedder_1.OpenAICompatibleEmbedder(config, logger, deps);
|
|
16
|
+
case 'infinity':
|
|
17
|
+
return new InfinityEmbedder_1.InfinityEmbedder(config, logger, deps);
|
|
18
|
+
case 'voyage':
|
|
19
|
+
return new VoyageEmbedder_1.VoyageEmbedder(config, logger, deps);
|
|
20
|
+
default: {
|
|
21
|
+
// The Zod enum should already reject this; guard anyway for a clear message.
|
|
22
|
+
const p = config.embedding.provider;
|
|
23
|
+
throw new Error(`Unknown embedding provider: "${p}". Expected one of: openai-compatible, voyage, infinity.`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { Config } from '../config';
|
|
2
2
|
import type { IEmbedder } from '../interfaces/IEmbedder';
|
|
3
3
|
import type { IAstParser } from '../interfaces/IAstParser';
|
|
4
|
+
import type { IChunker } from '../interfaces/IChunker';
|
|
4
5
|
import type { IDatabase } from '../interfaces/IDatabase';
|
|
5
6
|
import type { ILogger } from '../interfaces/ILogger';
|
|
7
|
+
import type { IndexMeta } from '../interfaces/IDatabase';
|
|
6
8
|
export interface IndexStats {
|
|
7
9
|
serviceName: string;
|
|
8
10
|
indexed: number;
|
|
@@ -16,11 +18,23 @@ export declare class IndexerService {
|
|
|
16
18
|
private db;
|
|
17
19
|
private embedder;
|
|
18
20
|
private ast;
|
|
21
|
+
private chunker;
|
|
19
22
|
private config;
|
|
20
23
|
private logger;
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
+
constructor(db: IDatabase, embedder: IEmbedder, ast: IAstParser, chunker: IChunker, config: Config, logger?: ILogger);
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the active embedding dimension (config override → registry → probe),
|
|
27
|
+
* reconcile it against `index_meta`, and prepare `code_chunk` accordingly:
|
|
28
|
+
* a dimension change drops+recreates the table; a same-dimension model/provider
|
|
29
|
+
* change clears its rows (vector spaces are not comparable across models); an
|
|
30
|
+
* unchanged model keeps the rows for incremental indexing. Back-compat: an
|
|
31
|
+
* existing populated table with no `index_meta` row is treated as the legacy
|
|
32
|
+
* `ollama`/`bge-m3`/1024 index. Writes `index_meta` to reflect the active model.
|
|
33
|
+
* Call once per run, before the scan loop.
|
|
34
|
+
*/
|
|
35
|
+
prepareEmbeddingSchema(): Promise<IndexMeta>;
|
|
36
|
+
private reconcileIndex;
|
|
37
|
+
private countChunks;
|
|
24
38
|
backfillSymbols(signal?: AbortSignal): Promise<{
|
|
25
39
|
total: number;
|
|
26
40
|
processed: number;
|
|
@@ -37,10 +51,12 @@ export declare class IndexerService {
|
|
|
37
51
|
scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[], opts?: {
|
|
38
52
|
rootTsConfigPath?: string;
|
|
39
53
|
}): Promise<IndexStats>;
|
|
40
|
-
|
|
54
|
+
pruneProjects(keep: string[]): Promise<{
|
|
55
|
+
removed: string[];
|
|
56
|
+
failed: string[];
|
|
57
|
+
}>;
|
|
41
58
|
private static inferRole;
|
|
42
59
|
private getEnrichedPrompt;
|
|
43
|
-
private embedText;
|
|
44
60
|
private static hashContent;
|
|
45
61
|
private ingestFile;
|
|
46
62
|
}
|