codesearch-mcp 0.3.0-alpha → 0.4.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -6
- package/LICENSE +661 -21
- package/README.md +33 -17
- package/dist/app.js +13 -20
- package/dist/src/ast/CallExtractor.d.ts +8 -0
- package/dist/src/ast/CallExtractor.js +68 -0
- package/dist/src/ast/GraphqlNameExtractor.d.ts +22 -0
- package/dist/src/ast/GraphqlNameExtractor.js +240 -0
- package/dist/src/ast/ImportEdgeExtractor.d.ts +11 -0
- package/dist/src/ast/ImportEdgeExtractor.js +129 -0
- package/dist/src/ast/IndexedNodeResolver.d.ts +14 -0
- package/dist/src/ast/IndexedNodeResolver.js +217 -0
- package/dist/src/ast/MemberExtractor.d.ts +8 -0
- package/dist/src/ast/MemberExtractor.js +129 -0
- package/dist/src/ast/TopLevelSymbolExtractor.d.ts +5 -0
- package/dist/src/ast/TopLevelSymbolExtractor.js +86 -0
- package/dist/src/ast/graphql-grammar.d.ts +24 -0
- package/dist/src/ast/graphql-grammar.js +93 -0
- package/dist/src/ast/node-helpers.d.ts +11 -0
- package/dist/src/ast/node-helpers.js +55 -0
- package/dist/src/commands/EvalCommand.d.ts +7 -3
- package/dist/src/commands/EvalCommand.js +36 -158
- package/dist/src/commands/IndexCommand.d.ts +10 -4
- package/dist/src/commands/IndexCommand.js +52 -124
- package/dist/src/commands/McpCommand.d.ts +6 -3
- package/dist/src/commands/McpCommand.js +19 -14
- package/dist/src/commands/WebCommand.d.ts +9 -3
- package/dist/src/commands/WebCommand.js +17 -8
- package/dist/src/commands/eval/baseline-comparison.d.ts +6 -0
- package/dist/src/commands/eval/baseline-comparison.js +54 -0
- package/dist/src/commands/eval/eval-args.d.ts +6 -0
- package/dist/src/commands/eval/eval-args.js +71 -0
- package/dist/src/commands/eval/eval-corpora.d.ts +6 -0
- package/dist/src/commands/eval/eval-corpora.js +76 -0
- package/dist/src/commands/eval/eval-metadata.d.ts +17 -0
- package/dist/src/commands/eval/eval-metadata.js +75 -0
- package/dist/src/commands/eval/index-inspector.d.ts +3 -0
- package/dist/src/commands/eval/index-inspector.js +21 -0
- package/dist/src/commands/eval/staged-search.d.ts +3 -0
- package/dist/src/commands/eval/staged-search.js +20 -0
- package/dist/src/commands/eval/suite-loader.d.ts +8 -0
- package/dist/src/commands/eval/suite-loader.js +52 -0
- package/dist/src/commands/index/backend-probe.d.ts +2 -0
- package/dist/src/commands/index/backend-probe.js +15 -0
- package/dist/src/commands/index/embedding-setup.d.ts +3 -0
- package/dist/src/commands/index/embedding-setup.js +16 -0
- package/dist/src/commands/index/index-summary.d.ts +2 -0
- package/dist/src/commands/index/index-summary.js +22 -0
- package/dist/src/commands/index/legacy-ollama-guard.d.ts +3 -0
- package/dist/src/commands/index/legacy-ollama-guard.js +23 -0
- package/dist/src/commands/index/prune-step.d.ts +3 -0
- package/dist/src/commands/index/prune-step.js +22 -0
- package/dist/src/commands/index/scan-targets.d.ts +8 -0
- package/dist/src/commands/index/scan-targets.js +107 -0
- package/dist/src/config-store.d.ts +0 -1
- package/dist/src/config-store.js +3 -4
- package/dist/src/config.d.ts +1 -1
- package/dist/src/config.js +1 -1
- package/dist/src/constants.d.ts +15 -0
- package/dist/src/constants.js +34 -6
- package/dist/src/container.d.ts +4 -2
- package/dist/src/container.js +7 -5
- package/dist/src/db/drizzle-types.d.ts +7 -0
- package/dist/src/db/index.d.ts +1 -0
- package/dist/src/db/schema-steps.d.ts +8 -0
- package/dist/src/db/schema-steps.js +105 -0
- package/dist/src/declarations/DeclarationTree.d.ts +1 -1
- package/dist/src/eval-results.js +16 -2
- package/dist/src/eval-vector-index.d.ts +5 -0
- package/dist/src/eval-vector-index.js +60 -0
- package/dist/src/help.js +9 -1
- package/dist/src/index-args.d.ts +13 -0
- package/dist/src/index-args.js +48 -0
- package/dist/src/indexing/AnalysisBatcher.d.ts +12 -0
- package/dist/src/indexing/AnalysisBatcher.js +28 -0
- package/dist/src/indexing/ChunkEmbedder.d.ts +24 -0
- package/dist/src/indexing/ChunkEmbedder.js +37 -0
- package/dist/src/indexing/ConsoleScanReporter.d.ts +10 -0
- package/dist/src/indexing/ConsoleScanReporter.js +73 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.d.ts +14 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.js +66 -0
- package/dist/src/indexing/FileIngestor.d.ts +22 -0
- package/dist/src/indexing/FileIngestor.js +116 -0
- package/dist/src/indexing/IndexStore.d.ts +24 -0
- package/dist/src/indexing/IndexStore.js +69 -0
- package/dist/src/indexing/StaleFilePruner.d.ts +8 -0
- package/dist/src/indexing/StaleFilePruner.js +55 -0
- package/dist/src/indexing/analysis-rows.d.ts +4 -0
- package/dist/src/indexing/analysis-rows.js +65 -0
- package/dist/src/indexing/project-notices.d.ts +2 -0
- package/dist/src/indexing/project-notices.js +63 -0
- package/dist/src/interfaces/IAstParser.d.ts +2 -1
- package/dist/src/interfaces/ICallGraph.d.ts +28 -0
- package/dist/src/interfaces/ICallGraph.js +2 -0
- package/dist/src/interfaces/IChunkSearch.d.ts +42 -0
- package/dist/src/interfaces/IChunkSearch.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +6 -2
- package/dist/src/interfaces/IDependencyGraph.d.ts +24 -0
- package/dist/src/interfaces/IDependencyGraph.js +2 -0
- package/dist/src/interfaces/IIndexIdentityGuard.d.ts +3 -0
- package/dist/src/interfaces/IIndexIdentityGuard.js +2 -0
- package/dist/src/interfaces/IIndexedNodeResolver.d.ts +10 -0
- package/dist/src/interfaces/IIndexedNodeResolver.js +2 -0
- package/dist/src/interfaces/IMcpReadiness.d.ts +12 -0
- package/dist/src/interfaces/IMcpReadiness.js +2 -0
- package/dist/src/interfaces/IProjectCatalog.d.ts +13 -0
- package/dist/src/interfaces/IProjectCatalog.js +2 -0
- package/dist/src/interfaces/IProjectRoots.d.ts +3 -0
- package/dist/src/interfaces/IProjectRoots.js +2 -0
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +21 -2
- package/dist/src/interfaces/IScanReporter.d.ts +13 -0
- package/dist/src/interfaces/IScanReporter.js +2 -0
- package/dist/src/interfaces/ISymbolLookup.d.ts +49 -0
- package/dist/src/interfaces/ISymbolLookup.js +2 -0
- package/dist/src/interfaces/IToolPaths.d.ts +62 -0
- package/dist/src/interfaces/IToolPaths.js +2 -0
- package/dist/src/interfaces/IWorkspaceFiles.d.ts +28 -0
- package/dist/src/interfaces/IWorkspaceFiles.js +2 -0
- package/dist/src/interfaces/SymbolKind.d.ts +1 -0
- package/dist/src/interfaces/SymbolKind.js +2 -0
- package/dist/src/interfaces/index.d.ts +12 -1
- package/dist/src/project-scope.d.ts +1 -0
- package/dist/src/project-scope.js +5 -0
- package/dist/src/retrieval-eval.d.ts +2 -1
- package/dist/src/retrieval-eval.js +45 -2
- package/dist/src/search/CallGraph.d.ts +9 -0
- package/dist/src/search/CallGraph.js +84 -0
- package/dist/src/search/ChunkSearch.d.ts +23 -0
- package/dist/src/search/ChunkSearch.js +115 -0
- package/dist/src/search/DependencyGraph.d.ts +19 -0
- package/dist/src/search/DependencyGraph.js +180 -0
- package/dist/src/search/IndexIdentityGuard.d.ts +11 -0
- package/dist/src/search/IndexIdentityGuard.js +37 -0
- package/dist/src/search/ProjectCatalog.d.ts +11 -0
- package/dist/src/search/ProjectCatalog.js +38 -0
- package/dist/src/search/SymbolLookup.d.ts +24 -0
- package/dist/src/search/SymbolLookup.js +97 -0
- package/dist/src/search/chunks/CandidateFetcher.d.ts +19 -0
- package/dist/src/search/chunks/CandidateFetcher.js +40 -0
- package/dist/src/search/chunks/IterativeScanGate.d.ts +10 -0
- package/dist/src/search/chunks/IterativeScanGate.js +42 -0
- package/dist/src/search/chunks/RerankStage.d.ts +15 -0
- package/dist/src/search/chunks/RerankStage.js +42 -0
- package/dist/src/search/chunks/search-metrics.d.ts +18 -0
- package/dist/src/search/chunks/search-metrics.js +32 -0
- package/dist/src/search/chunks/search-plan.d.ts +18 -0
- package/dist/src/search/chunks/search-plan.js +19 -0
- package/dist/src/search/create-search-services.d.ts +26 -0
- package/dist/src/search/create-search-services.js +29 -0
- package/dist/src/search/index-paths.d.ts +1 -0
- package/dist/src/search/index-paths.js +5 -0
- package/dist/src/search/symbol-sql.d.ts +13 -0
- package/dist/src/search/symbol-sql.js +66 -0
- package/dist/src/search/symbols/SymbolGrepFallback.d.ts +19 -0
- package/dist/src/search/symbols/SymbolGrepFallback.js +106 -0
- package/dist/src/search/symbols/SymbolIndexQuery.d.ts +14 -0
- package/dist/src/search/symbols/SymbolIndexQuery.js +52 -0
- package/dist/src/search/symbols/SymbolTextSearch.d.ts +13 -0
- package/dist/src/search/symbols/SymbolTextSearch.js +84 -0
- package/dist/src/search/symbols/TargetColumnGate.d.ts +7 -0
- package/dist/src/search/symbols/TargetColumnGate.js +23 -0
- package/dist/src/search/symbols/find-symbol-constants.d.ts +4 -0
- package/dist/src/search/symbols/find-symbol-constants.js +8 -0
- package/dist/src/search/symbols/grep-args.d.ts +3 -0
- package/dist/src/search/symbols/grep-args.js +7 -0
- package/dist/src/search/symbols/grep-patterns.d.ts +4 -0
- package/dist/src/search/symbols/grep-patterns.js +60 -0
- package/dist/src/services/AstParserService.d.ts +8 -44
- package/dist/src/services/AstParserService.js +19 -938
- package/dist/src/services/DatabaseService.d.ts +8 -10
- package/dist/src/services/DatabaseService.js +29 -86
- package/dist/src/services/IndexerService.d.ts +7 -10
- package/dist/src/services/IndexerService.js +36 -255
- package/dist/src/services/McpReadiness.d.ts +18 -0
- package/dist/src/services/McpReadiness.js +73 -0
- package/dist/src/services/McpServer.d.ts +32 -9
- package/dist/src/services/McpServer.js +78 -41
- package/dist/src/services/WebServerService.d.ts +4 -3
- package/dist/src/services/WebServerService.js +16 -8
- package/dist/src/setup.d.ts +16 -4
- package/dist/src/setup.js +115 -40
- package/dist/src/symbol-name.d.ts +1 -0
- package/dist/src/symbol-name.js +16 -12
- package/dist/src/tools/FindSymbolTool.d.ts +17 -7
- package/dist/src/tools/FindSymbolTool.js +54 -35
- package/dist/src/tools/GetCallersTool.d.ts +16 -8
- package/dist/src/tools/GetCallersTool.js +65 -35
- package/dist/src/tools/GetDependenciesTool.d.ts +16 -8
- package/dist/src/tools/GetDependenciesTool.js +35 -28
- package/dist/src/tools/GetDependentsTool.d.ts +20 -8
- package/dist/src/tools/GetDependentsTool.js +47 -33
- package/dist/src/tools/ListFilesTool.d.ts +16 -8
- package/dist/src/tools/ListFilesTool.js +20 -26
- package/dist/src/tools/ListProjectsTool.d.ts +11 -7
- package/dist/src/tools/ListProjectsTool.js +13 -11
- package/dist/src/tools/McpTool.d.ts +17 -0
- package/dist/src/tools/McpTool.js +28 -0
- package/dist/src/tools/ReadFileRangeTool.d.ts +16 -8
- package/dist/src/tools/ReadFileRangeTool.js +23 -25
- package/dist/src/tools/ReadFileTool.d.ts +15 -8
- package/dist/src/tools/ReadFileTool.js +19 -24
- package/dist/src/tools/SearchCodebaseTool.d.ts +22 -10
- package/dist/src/tools/SearchCodebaseTool.js +69 -51
- package/dist/src/tools/format-helpers.d.ts +9 -3
- package/dist/src/tools/format-helpers.js +59 -14
- package/dist/src/tools/tool-args.d.ts +38 -0
- package/dist/src/tools/tool-args.js +95 -0
- package/dist/src/workspace/GrepRunner.d.ts +15 -0
- package/dist/src/workspace/GrepRunner.js +104 -0
- package/dist/src/workspace/ProjectRoots.d.ts +22 -0
- package/dist/src/workspace/ProjectRoots.js +102 -0
- package/dist/src/workspace/ToolPaths.d.ts +31 -0
- package/dist/src/workspace/ToolPaths.js +312 -0
- package/dist/src/workspace/WorkspaceFiles.d.ts +19 -0
- package/dist/src/workspace/WorkspaceFiles.js +285 -0
- package/dist/src/workspace/fs-access.d.ts +9 -0
- package/dist/src/workspace/fs-access.js +47 -0
- package/dist/src/workspace/path-helpers.d.ts +3 -0
- package/dist/src/workspace/path-helpers.js +43 -0
- package/package.json +4 -3
- package/dist/src/interfaces/ISearcher.d.ts +0 -46
- package/dist/src/services/SearcherService.d.ts +0 -235
- package/dist/src/services/SearcherService.js +0 -1118
- /package/dist/src/{interfaces/ISearcher.js → db/drizzle-types.js} +0 -0
|
@@ -1,23 +1,22 @@
|
|
|
1
|
-
import type { NodePgDatabase, NodePgTransaction } from 'drizzle-orm/node-postgres';
|
|
2
|
-
import type { ExtractTablesWithRelations } from 'drizzle-orm/relations';
|
|
3
1
|
import type { Config } from '../config';
|
|
4
|
-
import type { IDatabase, IndexMeta } from '../interfaces/IDatabase';
|
|
5
|
-
import
|
|
6
|
-
type
|
|
7
|
-
export type DrizzleDb = NodePgDatabase<Schema>;
|
|
8
|
-
export type DrizzleTx = NodePgTransaction<Schema, ExtractTablesWithRelations<Schema>>;
|
|
2
|
+
import type { IDatabase, IndexMeta, InitDbOptions } from '../interfaces/IDatabase';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
import type { DrizzleDb, DrizzleTx } from '../db';
|
|
9
5
|
export declare class DatabaseService implements IDatabase {
|
|
10
6
|
private config;
|
|
7
|
+
private logger;
|
|
11
8
|
private pool;
|
|
12
9
|
private _db;
|
|
13
|
-
|
|
10
|
+
private probeTimeoutMs;
|
|
11
|
+
constructor(config: Config, logger?: ILogger);
|
|
14
12
|
get orm(): DrizzleDb;
|
|
15
13
|
query<T = any>(sql: string, params?: any[]): Promise<T>;
|
|
16
14
|
queryWithLocalSettings<T = any>(settings: Record<string, string | number>, sql: string, params?: any[]): Promise<T>;
|
|
17
15
|
pgvectorVersion(): Promise<string | null>;
|
|
18
16
|
transaction<T>(fn: (tx: DrizzleTx) => Promise<T>): Promise<T>;
|
|
19
17
|
destroy(): Promise<void>;
|
|
20
|
-
initDb(): Promise<void>;
|
|
18
|
+
initDb(options?: InitDbOptions): Promise<void>;
|
|
19
|
+
probe(): Promise<void>;
|
|
21
20
|
initDbWithSchema(): Promise<void>;
|
|
22
21
|
ensureChunkTable(dimensions: number): Promise<void>;
|
|
23
22
|
recreateChunkTable(dimensions: number): Promise<void>;
|
|
@@ -31,4 +30,3 @@ export declare class DatabaseService implements IDatabase {
|
|
|
31
30
|
private assertMinPostgresVersion;
|
|
32
31
|
private ensureDatabaseExists;
|
|
33
32
|
}
|
|
34
|
-
export {};
|
|
@@ -34,20 +34,26 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.DatabaseService = void 0;
|
|
37
|
+
const async_hooks_1 = require("async_hooks");
|
|
37
38
|
const node_postgres_1 = require("drizzle-orm/node-postgres");
|
|
38
39
|
const pg_1 = require("pg");
|
|
39
40
|
const drizzle_orm_1 = require("drizzle-orm");
|
|
41
|
+
const LoggerService_1 = require("./LoggerService");
|
|
40
42
|
const schema = __importStar(require("../db/schema"));
|
|
41
43
|
const db_1 = require("../db");
|
|
44
|
+
const schema_steps_1 = require("../db/schema-steps");
|
|
42
45
|
const MIN_PG_VERSION_NUM = 150000;
|
|
43
46
|
const SETTING_NAME = /^[a-z_]+(\.[a-z_]+)*$/;
|
|
44
47
|
const SETTING_VALUE = /^[A-Za-z0-9_]+$/;
|
|
45
48
|
class DatabaseService {
|
|
46
49
|
config;
|
|
50
|
+
logger;
|
|
47
51
|
pool;
|
|
48
52
|
_db;
|
|
49
|
-
|
|
53
|
+
probeTimeoutMs;
|
|
54
|
+
constructor(config, logger = LoggerService_1.LoggerService.silent()) {
|
|
50
55
|
this.config = config;
|
|
56
|
+
this.logger = logger;
|
|
51
57
|
}
|
|
52
58
|
get orm() { return this._db; }
|
|
53
59
|
async query(sql, params) {
|
|
@@ -97,96 +103,31 @@ class DatabaseService {
|
|
|
97
103
|
return Promise.resolve();
|
|
98
104
|
return this.pool.end();
|
|
99
105
|
}
|
|
100
|
-
async initDb() {
|
|
106
|
+
async initDb(options = {}) {
|
|
101
107
|
const { host, port, username, password, database } = this.config.db;
|
|
102
|
-
|
|
108
|
+
const { connectionTimeoutMs } = options;
|
|
109
|
+
this.probeTimeoutMs = connectionTimeoutMs;
|
|
110
|
+
this.pool = new pg_1.Pool({
|
|
111
|
+
host, port, user: username, password, database,
|
|
112
|
+
...(connectionTimeoutMs !== undefined ? { connectionTimeoutMillis: connectionTimeoutMs } : {}),
|
|
113
|
+
});
|
|
114
|
+
this.pool.on('error', async_hooks_1.AsyncResource.bind((err) => {
|
|
115
|
+
this.logger.warn('DatabaseService: idle client error', { err });
|
|
116
|
+
}));
|
|
103
117
|
this._db = (0, node_postgres_1.drizzle)(this.pool, { schema });
|
|
104
118
|
await this.pool.query('SELECT 1');
|
|
105
119
|
}
|
|
120
|
+
async probe() {
|
|
121
|
+
if (!this.pool)
|
|
122
|
+
throw new Error('DatabaseService: probe() called before initDb()');
|
|
123
|
+
const query = { text: 'SELECT 1', query_timeout: this.probeTimeoutMs };
|
|
124
|
+
await this.pool.query(query);
|
|
125
|
+
}
|
|
106
126
|
async initDbWithSchema() {
|
|
107
127
|
await this.ensureDatabaseExists();
|
|
108
128
|
await this.initDb();
|
|
109
129
|
await this.assertMinPostgresVersion();
|
|
110
|
-
await
|
|
111
|
-
await this.query(`
|
|
112
|
-
CREATE TABLE IF NOT EXISTS index_meta (
|
|
113
|
-
id BOOLEAN PRIMARY KEY DEFAULT true,
|
|
114
|
-
provider TEXT NOT NULL,
|
|
115
|
-
model TEXT NOT NULL,
|
|
116
|
-
dimensions INT NOT NULL,
|
|
117
|
-
"indexedAt" TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
118
|
-
)
|
|
119
|
-
`);
|
|
120
|
-
const [{ existsDep }] = await this.query(`
|
|
121
|
-
SELECT EXISTS (
|
|
122
|
-
SELECT 1 FROM information_schema.tables
|
|
123
|
-
WHERE table_schema = 'public' AND table_name = 'code_dependency'
|
|
124
|
-
) AS "existsDep"
|
|
125
|
-
`);
|
|
126
|
-
if (!existsDep) {
|
|
127
|
-
await this.query(`
|
|
128
|
-
CREATE TABLE code_dependency (
|
|
129
|
-
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
130
|
-
project TEXT NOT NULL,
|
|
131
|
-
"fromFile" TEXT NOT NULL,
|
|
132
|
-
"fromSymbol" TEXT,
|
|
133
|
-
"toFile" TEXT NOT NULL,
|
|
134
|
-
"toSymbol" TEXT,
|
|
135
|
-
"depType" TEXT NOT NULL
|
|
136
|
-
)
|
|
137
|
-
`);
|
|
138
|
-
}
|
|
139
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_from ON code_dependency (project, "fromFile")`);
|
|
140
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_dep_project_to ON code_dependency (project, "toFile")`);
|
|
141
|
-
await this.query(`DROP INDEX IF EXISTS idx_dep_from`);
|
|
142
|
-
await this.query(`DROP INDEX IF EXISTS idx_dep_to`);
|
|
143
|
-
await this.query(`DROP INDEX IF EXISTS idx_dep_project`);
|
|
144
|
-
const [{ existsSym }] = await this.query(`
|
|
145
|
-
SELECT EXISTS (
|
|
146
|
-
SELECT 1 FROM information_schema.tables
|
|
147
|
-
WHERE table_schema = 'public' AND table_name = 'code_symbol'
|
|
148
|
-
) AS "existsSym"
|
|
149
|
-
`);
|
|
150
|
-
if (!existsSym) {
|
|
151
|
-
await this.query(`
|
|
152
|
-
CREATE TABLE code_symbol (
|
|
153
|
-
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
154
|
-
project TEXT NOT NULL,
|
|
155
|
-
"filePath" TEXT NOT NULL,
|
|
156
|
-
name TEXT NOT NULL,
|
|
157
|
-
kind TEXT NOT NULL,
|
|
158
|
-
signature TEXT,
|
|
159
|
-
line INT,
|
|
160
|
-
col INT
|
|
161
|
-
)
|
|
162
|
-
`);
|
|
163
|
-
}
|
|
164
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_name ON code_symbol (name, project)`);
|
|
165
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_symbol_file ON code_symbol (project, "filePath")`);
|
|
166
|
-
await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS container TEXT`);
|
|
167
|
-
await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS "isStatic" BOOLEAN NOT NULL DEFAULT false`);
|
|
168
|
-
await this.query(`ALTER TABLE code_symbol ADD COLUMN IF NOT EXISTS "target" TEXT`);
|
|
169
|
-
await this.transaction(async (tx) => {
|
|
170
|
-
await tx.execute('DROP INDEX IF EXISTS idx_symbol_unique');
|
|
171
|
-
await tx.execute('CREATE UNIQUE INDEX IF NOT EXISTS idx_symbol_unique_v2 ' +
|
|
172
|
-
'ON code_symbol (project, "filePath", container, name, kind, line) NULLS NOT DISTINCT');
|
|
173
|
-
});
|
|
174
|
-
await this.query(`
|
|
175
|
-
CREATE TABLE IF NOT EXISTS code_call (
|
|
176
|
-
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
177
|
-
project TEXT NOT NULL,
|
|
178
|
-
"fromFile" TEXT NOT NULL,
|
|
179
|
-
"fromSymbol" TEXT,
|
|
180
|
-
"calleeName" TEXT NOT NULL,
|
|
181
|
-
"calleeFile" TEXT NOT NULL,
|
|
182
|
-
"calleeSymbol" TEXT,
|
|
183
|
-
line INT,
|
|
184
|
-
col INT
|
|
185
|
-
)
|
|
186
|
-
`);
|
|
187
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_callee ON code_call (project, "calleeName")`);
|
|
188
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_callee_file ON code_call (project, "calleeFile")`);
|
|
189
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_call_from ON code_call (project, "fromFile")`);
|
|
130
|
+
await (0, schema_steps_1.runSchemaSteps)(this);
|
|
190
131
|
}
|
|
191
132
|
async ensureChunkTable(dimensions) {
|
|
192
133
|
const dim = this.assertDim(dimensions);
|
|
@@ -207,7 +148,7 @@ class DatabaseService {
|
|
|
207
148
|
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "contentHash" varchar(64) NULL`);
|
|
208
149
|
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "ownerPath" varchar NULL`);
|
|
209
150
|
await this.query(`ALTER TABLE code_chunk ADD COLUMN IF NOT EXISTS "endLine" integer NULL`);
|
|
210
|
-
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_embedding ON code_chunk USING hnsw (embedding vector_cosine_ops)`);
|
|
151
|
+
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_embedding ON code_chunk USING hnsw (embedding vector_cosine_ops) WITH (m = 32, ef_construction = 200)`);
|
|
211
152
|
await this.query(`CREATE INDEX IF NOT EXISTS idx_code_chunk_service_file ON code_chunk ("serviceName", "filePath")`);
|
|
212
153
|
}
|
|
213
154
|
async recreateChunkTable(dimensions) {
|
|
@@ -221,8 +162,10 @@ class DatabaseService {
|
|
|
221
162
|
const row = rows[0];
|
|
222
163
|
return row ? { provider: row.provider, model: row.model, dimensions: Number(row.dimensions) } : null;
|
|
223
164
|
}
|
|
224
|
-
catch {
|
|
225
|
-
|
|
165
|
+
catch (err) {
|
|
166
|
+
if (err?.code === '42P01')
|
|
167
|
+
return null;
|
|
168
|
+
throw err;
|
|
226
169
|
}
|
|
227
170
|
}
|
|
228
171
|
async writeIndexMeta(meta) {
|
|
@@ -2,10 +2,9 @@ import type { Config } from '../config';
|
|
|
2
2
|
import type { IEmbedder } from '../interfaces/IEmbedder';
|
|
3
3
|
import type { IAstParser } from '../interfaces/IAstParser';
|
|
4
4
|
import type { IChunker } from '../interfaces/IChunker';
|
|
5
|
-
import type { IDatabase } from '../interfaces/IDatabase';
|
|
5
|
+
import type { IDatabase, IndexMeta } from '../interfaces/IDatabase';
|
|
6
6
|
import type { DiscoveredProject } from '../interfaces/IProjectDiscovery';
|
|
7
7
|
import type { ILogger } from '../interfaces/ILogger';
|
|
8
|
-
import type { IndexMeta } from '../interfaces/IDatabase';
|
|
9
8
|
export interface IndexStats {
|
|
10
9
|
serviceName: string;
|
|
11
10
|
indexed: number;
|
|
@@ -17,16 +16,16 @@ export interface IndexStats {
|
|
|
17
16
|
}
|
|
18
17
|
export declare class IndexerService {
|
|
19
18
|
private db;
|
|
20
|
-
private embedder;
|
|
21
19
|
private ast;
|
|
22
|
-
private chunker;
|
|
23
|
-
private config;
|
|
24
20
|
private logger;
|
|
21
|
+
private readonly store;
|
|
22
|
+
private readonly stalePruner;
|
|
23
|
+
private readonly batcher;
|
|
24
|
+
private readonly chunkEmbedder;
|
|
25
|
+
private readonly ingestor;
|
|
26
|
+
private readonly schema;
|
|
25
27
|
constructor(db: IDatabase, embedder: IEmbedder, ast: IAstParser, chunker: IChunker, config: Config, logger?: ILogger);
|
|
26
28
|
prepareEmbeddingSchema(): Promise<IndexMeta>;
|
|
27
|
-
private reconcileIndex;
|
|
28
|
-
private countChunks;
|
|
29
|
-
private writeDepEdges;
|
|
30
29
|
scanAndIndex(dir: string, serviceName: string, force?: boolean, signal?: AbortSignal, preCollectedFiles?: string[], opts?: {
|
|
31
30
|
rootTsConfigPath?: string;
|
|
32
31
|
}): Promise<IndexStats>;
|
|
@@ -35,6 +34,4 @@ export declare class IndexerService {
|
|
|
35
34
|
removed: string[];
|
|
36
35
|
failed: string[];
|
|
37
36
|
}>;
|
|
38
|
-
private static hashContent;
|
|
39
|
-
private ingestFile;
|
|
40
37
|
}
|
|
@@ -36,180 +36,74 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.IndexerService = void 0;
|
|
37
37
|
const fs = __importStar(require("fs"));
|
|
38
38
|
const path = __importStar(require("path"));
|
|
39
|
-
const crypto_1 = require("crypto");
|
|
40
|
-
const cli_progress_1 = require("cli-progress");
|
|
41
|
-
const drizzle_orm_1 = require("drizzle-orm");
|
|
42
|
-
const owner_path_1 = require("../owner-path");
|
|
43
|
-
const DeclarationTree_1 = require("../declarations/DeclarationTree");
|
|
44
|
-
const SourceLines_1 = require("../declarations/SourceLines");
|
|
45
|
-
const enriched_prompt_1 = require("../enriched-prompt");
|
|
46
|
-
const symbol_rows_1 = require("../symbol-rows");
|
|
47
|
-
const db_1 = require("../db");
|
|
48
39
|
const file_utils_1 = require("../file-utils");
|
|
49
40
|
const LoggerService_1 = require("./LoggerService");
|
|
50
|
-
const
|
|
51
|
-
const
|
|
52
|
-
const
|
|
53
|
-
const
|
|
41
|
+
const IndexStore_1 = require("../indexing/IndexStore");
|
|
42
|
+
const StaleFilePruner_1 = require("../indexing/StaleFilePruner");
|
|
43
|
+
const AnalysisBatcher_1 = require("../indexing/AnalysisBatcher");
|
|
44
|
+
const ChunkEmbedder_1 = require("../indexing/ChunkEmbedder");
|
|
45
|
+
const FileIngestor_1 = require("../indexing/FileIngestor");
|
|
46
|
+
const project_notices_1 = require("../indexing/project-notices");
|
|
47
|
+
const ConsoleScanReporter_1 = require("../indexing/ConsoleScanReporter");
|
|
48
|
+
const EmbeddingSchemaPreparer_1 = require("../indexing/EmbeddingSchemaPreparer");
|
|
54
49
|
class IndexerService {
|
|
55
50
|
db;
|
|
56
|
-
embedder;
|
|
57
51
|
ast;
|
|
58
|
-
chunker;
|
|
59
|
-
config;
|
|
60
52
|
logger;
|
|
53
|
+
store;
|
|
54
|
+
stalePruner;
|
|
55
|
+
batcher;
|
|
56
|
+
chunkEmbedder;
|
|
57
|
+
ingestor;
|
|
58
|
+
schema;
|
|
61
59
|
constructor(db, embedder, ast, chunker, config, logger = LoggerService_1.LoggerService.silent()) {
|
|
62
60
|
this.db = db;
|
|
63
|
-
this.embedder = embedder;
|
|
64
61
|
this.ast = ast;
|
|
65
|
-
this.chunker = chunker;
|
|
66
|
-
this.config = config;
|
|
67
62
|
this.logger = logger;
|
|
63
|
+
this.store = new IndexStore_1.IndexStore(db, config);
|
|
64
|
+
this.stalePruner = new StaleFilePruner_1.StaleFilePruner(db, config);
|
|
65
|
+
this.batcher = new AnalysisBatcher_1.AnalysisBatcher(ast, config);
|
|
66
|
+
this.chunkEmbedder = new ChunkEmbedder_1.ChunkEmbedder(embedder, config);
|
|
67
|
+
this.ingestor = new FileIngestor_1.FileIngestor(this.store, this.chunkEmbedder, chunker, config, logger);
|
|
68
|
+
this.schema = new EmbeddingSchemaPreparer_1.EmbeddingSchemaPreparer(db, embedder, config, logger);
|
|
68
69
|
}
|
|
69
70
|
async prepareEmbeddingSchema() {
|
|
70
|
-
|
|
71
|
-
const next = {
|
|
72
|
-
provider,
|
|
73
|
-
model,
|
|
74
|
-
dimensions: await (0, constants_1.resolveDimension)(override, model, async () => {
|
|
75
|
-
const probe = await this.embedder.embed('probe', 'document');
|
|
76
|
-
return probe.length;
|
|
77
|
-
}),
|
|
78
|
-
};
|
|
79
|
-
const meta = await this.db.readIndexMeta();
|
|
80
|
-
if (!meta) {
|
|
81
|
-
const chunkCount = await this.countChunks();
|
|
82
|
-
if (chunkCount === 0) {
|
|
83
|
-
await this.db.recreateChunkTable(next.dimensions);
|
|
84
|
-
await this.db.writeIndexMeta(next);
|
|
85
|
-
this.logger.debug('IndexerService: prepared fresh index', { ...next });
|
|
86
|
-
return next;
|
|
87
|
-
}
|
|
88
|
-
await this.reconcileIndex({ provider: 'ollama', model: 'bge-m3', dimensions: 1024 }, next);
|
|
89
|
-
return next;
|
|
90
|
-
}
|
|
91
|
-
await this.reconcileIndex(meta, next);
|
|
92
|
-
return next;
|
|
93
|
-
}
|
|
94
|
-
async reconcileIndex(prev, next) {
|
|
95
|
-
if (prev.dimensions !== next.dimensions) {
|
|
96
|
-
console.log(`Embedding dimension changed (${prev.dimensions} → ${next.dimensions}) — recreating code_chunk.`);
|
|
97
|
-
this.logger.info('IndexerService: dimension change, recreating code_chunk', { prev, next });
|
|
98
|
-
await this.db.recreateChunkTable(next.dimensions);
|
|
99
|
-
}
|
|
100
|
-
else if (prev.provider !== next.provider || prev.model !== next.model) {
|
|
101
|
-
console.log(`Embedding model changed (${prev.provider}/${prev.model} → ${next.provider}/${next.model}) — clearing index.`);
|
|
102
|
-
this.logger.info('IndexerService: model change, clearing code_chunk', { prev, next });
|
|
103
|
-
await this.db.query(`DELETE FROM code_chunk`);
|
|
104
|
-
}
|
|
105
|
-
await this.db.writeIndexMeta(next);
|
|
106
|
-
}
|
|
107
|
-
async countChunks() {
|
|
108
|
-
try {
|
|
109
|
-
const rows = await this.db.query(`SELECT COUNT(*)::int AS n FROM code_chunk`);
|
|
110
|
-
return Number(rows[0]?.n ?? 0);
|
|
111
|
-
}
|
|
112
|
-
catch (err) {
|
|
113
|
-
if (/does not exist/i.test(err.message))
|
|
114
|
-
return 0;
|
|
115
|
-
throw err;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
async writeDepEdges(serviceName, relFilePath, edges, tx) {
|
|
119
|
-
const run = async (t) => {
|
|
120
|
-
await t.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeDependency.fromFile, relFilePath)));
|
|
121
|
-
if (edges.length > 0) {
|
|
122
|
-
await t.insert(db_1.codeDependency).values(edges);
|
|
123
|
-
}
|
|
124
|
-
};
|
|
125
|
-
if (tx)
|
|
126
|
-
await run(tx);
|
|
127
|
-
else
|
|
128
|
-
await this.db.transaction(run);
|
|
71
|
+
return this.schema.prepare();
|
|
129
72
|
}
|
|
130
73
|
async scanAndIndex(dir, serviceName, force = false, signal, preCollectedFiles, opts) {
|
|
131
74
|
const startTime = Date.now();
|
|
75
|
+
const reporter = new ConsoleScanReporter_1.ConsoleScanReporter();
|
|
132
76
|
this.logger.debug('IndexerService: scanAndIndex start', { serviceName, dir, force });
|
|
133
77
|
if (force) {
|
|
134
|
-
await this.
|
|
135
|
-
await tx.update(db_1.codeChunk)
|
|
136
|
-
.set({ lastModified: null, contentHash: null })
|
|
137
|
-
.where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName));
|
|
138
|
-
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName));
|
|
139
|
-
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName));
|
|
140
|
-
});
|
|
78
|
+
await this.store.resetProject(serviceName);
|
|
141
79
|
}
|
|
142
80
|
const files = preCollectedFiles ?? (0, file_utils_1.collectFiles)(dir, (0, file_utils_1.loadIgnore)(dir), dir);
|
|
143
81
|
this.logger.debug('IndexerService: scanAndIndex files collected', { serviceName, count: files.length, force });
|
|
144
|
-
const
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
if (staleFiles.length > 0) {
|
|
148
|
-
await this.db.deleteFiles(serviceName, staleFiles);
|
|
149
|
-
for (const f of staleFiles) {
|
|
150
|
-
console.log(` Removed stale: ${path.basename(f)}`);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
const bar = new cli_progress_1.SingleBar({
|
|
154
|
-
format: `{serviceName} {bar} {percentage}% | {value}/{total} | {duration_formatted} elapsed | ~{eta_formatted} left | {filename}`,
|
|
155
|
-
barCompleteChar: '█',
|
|
156
|
-
barIncompleteChar: '░',
|
|
157
|
-
hideCursor: true,
|
|
158
|
-
clearOnComplete: false,
|
|
159
|
-
}, cli_progress_1.Presets.shades_grey);
|
|
160
|
-
const warnings = [];
|
|
161
|
-
console.log(`\nIndexing ${serviceName}: ${files.length} files`);
|
|
162
|
-
const meta = (0, file_utils_1.readRagMeta)(dir);
|
|
163
|
-
if (meta === 'invalid') {
|
|
164
|
-
const msg = `.ragmeta.json found but invalid JSON`;
|
|
165
|
-
console.log(` ⚠ ${msg}`);
|
|
166
|
-
warnings.push(msg);
|
|
167
|
-
}
|
|
168
|
-
else if (meta !== 'missing') {
|
|
169
|
-
const parts = [meta.description, meta.stack && `[${meta.stack}]`].filter(Boolean).join(' ');
|
|
170
|
-
console.log(` ✓ .ragmeta.json: ${parts || '(no fields)'}`);
|
|
171
|
-
}
|
|
172
|
-
else {
|
|
173
|
-
const msg = `No .ragmeta.json — add {"description":"...","stack":"...","notes":"..."} so list_projects can describe this project`;
|
|
174
|
-
console.log(` ⚠ ${msg}`);
|
|
175
|
-
warnings.push(msg);
|
|
82
|
+
const staleFiles = await this.stalePruner.prune(serviceName, files);
|
|
83
|
+
for (const f of staleFiles) {
|
|
84
|
+
reporter.staleRemoved(f);
|
|
176
85
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
warnings.push(msg);
|
|
181
|
-
}
|
|
182
|
-
if (files.length > this.config.indexer.astBatchSize) {
|
|
183
|
-
console.log(` ⚠ Large project (${files.length} files) — AST analysis uses batches of ${this.config.indexer.astBatchSize}; cross-batch dependency edges may be incomplete`);
|
|
184
|
-
}
|
|
185
|
-
bar.start(files.length, 0, { serviceName, filename: '' });
|
|
86
|
+
const notices = (0, project_notices_1.collectProjectNotices)(dir, files.length, this.batcher.batchSize);
|
|
87
|
+
reporter.begin(serviceName, files.length, notices);
|
|
88
|
+
const warnings = notices.filter(n => n.warning).map(n => n.message);
|
|
186
89
|
const stats = {
|
|
187
90
|
serviceName, indexed: 0, skipped: 0, failed: 0, chunks: 0, elapsedSec: 0, warnings,
|
|
188
91
|
};
|
|
189
92
|
const mtimeRefreshBatch = [];
|
|
190
|
-
const
|
|
191
|
-
for (let batchStart = 0; batchStart < files.length; batchStart += this.config.indexer.astBatchSize) {
|
|
192
|
-
if (signal?.aborted)
|
|
193
|
-
break;
|
|
194
|
-
const batch = files.slice(batchStart, batchStart + this.config.indexer.astBatchSize);
|
|
195
|
-
const analysisMap = await this.ast.analyzeProject(batch, serviceName, dir, {
|
|
196
|
-
skipDependencyResolution: isLargeProject,
|
|
197
|
-
rootTsConfigPath: opts?.rootTsConfigPath,
|
|
198
|
-
});
|
|
93
|
+
for await (const { batch, analysisMap } of this.batcher.batches(files, serviceName, dir, signal, opts?.rootTsConfigPath)) {
|
|
199
94
|
for (const filePath of batch) {
|
|
200
95
|
if (signal?.aborted) {
|
|
201
|
-
|
|
202
|
-
console.log(`\n Interrupted — ${serviceName}`);
|
|
96
|
+
reporter.interrupted(serviceName);
|
|
203
97
|
stats.elapsedSec = Math.round((Date.now() - startTime) / 100) / 10;
|
|
204
98
|
return stats;
|
|
205
99
|
}
|
|
206
|
-
|
|
100
|
+
reporter.file(path.relative(dir, filePath));
|
|
207
101
|
const mtimeMs = fs.statSync(filePath).mtimeMs;
|
|
208
102
|
const analysis = analysisMap.get(filePath);
|
|
209
103
|
if (analysis === undefined && /\.(ts|tsx|js|jsx)$/.test(filePath) && !/\.d\.ts$/.test(filePath)) {
|
|
210
104
|
this.logger.warn('IndexerService: no pre-computed analysis for TS file', { file: filePath });
|
|
211
105
|
}
|
|
212
|
-
const result = await this.ingestFile(filePath, serviceName, mtimeMs, dir, analysis
|
|
106
|
+
const result = await this.ingestor.ingestFile(filePath, serviceName, mtimeMs, dir, analysis);
|
|
213
107
|
if (result.pendingMtimeRefresh) {
|
|
214
108
|
mtimeRefreshBatch.push({ relFilePath: result.pendingMtimeRefresh, mtimeMs });
|
|
215
109
|
}
|
|
@@ -220,20 +114,13 @@ class IndexerService {
|
|
|
220
114
|
else
|
|
221
115
|
stats.failed++;
|
|
222
116
|
stats.chunks += result.chunks;
|
|
223
|
-
|
|
117
|
+
reporter.advance();
|
|
224
118
|
}
|
|
225
119
|
}
|
|
226
|
-
|
|
227
|
-
process.stdout.write('\x1B[J');
|
|
120
|
+
reporter.end();
|
|
228
121
|
if (mtimeRefreshBatch.length > 0) {
|
|
229
122
|
try {
|
|
230
|
-
await this.
|
|
231
|
-
FROM unnest($1::text[], $2::float8[]) AS t(fp, mtime)
|
|
232
|
-
WHERE "serviceName" = $3 AND "filePath" = t.fp`, [
|
|
233
|
-
mtimeRefreshBatch.map(r => r.relFilePath),
|
|
234
|
-
mtimeRefreshBatch.map(r => r.mtimeMs),
|
|
235
|
-
serviceName,
|
|
236
|
-
]);
|
|
123
|
+
await this.store.refreshMtimes(serviceName, mtimeRefreshBatch);
|
|
237
124
|
}
|
|
238
125
|
catch (err) {
|
|
239
126
|
this.logger.error('IndexerService: mtime refresh batch failed', { err });
|
|
@@ -282,111 +169,5 @@ class IndexerService {
|
|
|
282
169
|
}
|
|
283
170
|
return { removed, failed };
|
|
284
171
|
}
|
|
285
|
-
static hashContent(content) {
|
|
286
|
-
return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
287
|
-
}
|
|
288
|
-
async ingestFile(filePath, serviceName, mtimeMs, projectRoot, analysis, meta) {
|
|
289
|
-
try {
|
|
290
|
-
const relFilePath = path.relative(this.config.workspaceRoot, filePath);
|
|
291
|
-
const [existing] = await this.db.orm
|
|
292
|
-
.select({ id: db_1.codeChunk.id, lastModified: db_1.codeChunk.lastModified, contentHash: db_1.codeChunk.contentHash })
|
|
293
|
-
.from(db_1.codeChunk)
|
|
294
|
-
.where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)))
|
|
295
|
-
.limit(1);
|
|
296
|
-
if (existing?.lastModified === mtimeMs) {
|
|
297
|
-
return { status: 'skipped', chunks: 0 };
|
|
298
|
-
}
|
|
299
|
-
const t0 = Date.now();
|
|
300
|
-
const code = fs.readFileSync(filePath, 'utf-8');
|
|
301
|
-
const hash = IndexerService.hashContent(code);
|
|
302
|
-
if (existing !== undefined && existing.contentHash !== null && existing.contentHash === hash) {
|
|
303
|
-
return { status: 'skipped', chunks: 0, pendingMtimeRefresh: relFilePath };
|
|
304
|
-
}
|
|
305
|
-
const tAst = Date.now();
|
|
306
|
-
const { edges: rawEdges, symbols: rawSymbols, members: rawMembers = [], calls: rawCalls, graphqlNames: rawGraphqlNames = [], } = analysis ?? { edges: [], symbols: [], calls: [] };
|
|
307
|
-
const edges = rawEdges
|
|
308
|
-
.filter(e => e.toFile.startsWith(this.config.workspaceRoot))
|
|
309
|
-
.map(e => ({
|
|
310
|
-
...e,
|
|
311
|
-
fromFile: path.relative(this.config.workspaceRoot, e.fromFile),
|
|
312
|
-
toFile: path.relative(this.config.workspaceRoot, e.toFile),
|
|
313
|
-
}));
|
|
314
|
-
const calls = rawCalls
|
|
315
|
-
.filter(c => c.calleeFile.startsWith(this.config.workspaceRoot))
|
|
316
|
-
.map(c => ({
|
|
317
|
-
project: serviceName,
|
|
318
|
-
fromFile: relFilePath,
|
|
319
|
-
fromSymbol: c.fromSymbol,
|
|
320
|
-
calleeName: c.calleeName,
|
|
321
|
-
calleeFile: path.relative(this.config.workspaceRoot, c.calleeFile),
|
|
322
|
-
calleeSymbol: c.calleeSymbol,
|
|
323
|
-
line: c.line,
|
|
324
|
-
col: c.col,
|
|
325
|
-
}));
|
|
326
|
-
const symbolRows = (0, symbol_rows_1.buildSymbolRows)(rawSymbols, rawMembers, rawGraphqlNames).map(s => ({ ...s, filePath: relFilePath }));
|
|
327
|
-
const promptEdges = edges.filter(e => e.depType !== 'import-side-effect');
|
|
328
|
-
const astMs = Date.now() - tAst;
|
|
329
|
-
const jsx = JSX_EXTENSIONS.test(filePath);
|
|
330
|
-
const chunks = await this.chunker.chunk(code, {
|
|
331
|
-
strategy: this.config.indexer.chunkStrategy,
|
|
332
|
-
isSchemaFile: SCHEMA_EXTENSIONS.test(filePath),
|
|
333
|
-
declarations: analysis?.declarations,
|
|
334
|
-
jsx,
|
|
335
|
-
});
|
|
336
|
-
const relPath = path.relative(projectRoot, filePath);
|
|
337
|
-
const sourceLines = new SourceLines_1.SourceLines(code, { jsx });
|
|
338
|
-
const ownerTree = analysis?.declarations ?? DeclarationTree_1.DeclarationTree.EMPTY;
|
|
339
|
-
const t2 = Date.now();
|
|
340
|
-
const records = [];
|
|
341
|
-
const embedBatchSize = this.config.embedding.batchSize;
|
|
342
|
-
for (let i = 0; i < chunks.length; i += embedBatchSize) {
|
|
343
|
-
const batch = chunks.slice(i, i + embedBatchSize);
|
|
344
|
-
const prompts = batch.map(piece => (0, enriched_prompt_1.getEnrichedPrompt)(piece, relPath, serviceName, promptEdges, rawSymbols));
|
|
345
|
-
const vectors = await this.embedder.embedBatch(prompts, 'document');
|
|
346
|
-
for (let j = 0; j < batch.length; j++) {
|
|
347
|
-
const piece = batch[j];
|
|
348
|
-
records.push({
|
|
349
|
-
serviceName, filePath: relFilePath, startLine: piece.startLine,
|
|
350
|
-
content: piece.content,
|
|
351
|
-
embedding: `[${vectors[j]}]`,
|
|
352
|
-
lastModified: mtimeMs,
|
|
353
|
-
contentHash: hash,
|
|
354
|
-
ownerPath: (0, owner_path_1.ownerOf)(piece, ownerTree, sourceLines),
|
|
355
|
-
endLine: piece.startLine + (piece.content.match(/\n/g)?.length ?? 0),
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
const embedMs = Date.now() - t2;
|
|
360
|
-
const t3 = Date.now();
|
|
361
|
-
await this.db.transaction(async (tx) => {
|
|
362
|
-
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)));
|
|
363
|
-
for (let i = 0; i < records.length; i += this.config.indexer.batchSize) {
|
|
364
|
-
await tx.insert(db_1.codeChunk).values(records.slice(i, i + this.config.indexer.batchSize));
|
|
365
|
-
}
|
|
366
|
-
await this.writeDepEdges(serviceName, relFilePath, edges, tx);
|
|
367
|
-
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, relFilePath)));
|
|
368
|
-
if (symbolRows.length > 0) {
|
|
369
|
-
await tx.insert(db_1.codeSymbol).values(symbolRows);
|
|
370
|
-
}
|
|
371
|
-
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, relFilePath)));
|
|
372
|
-
if (calls.length > 0) {
|
|
373
|
-
await tx.insert(db_1.codeCall).values(calls);
|
|
374
|
-
}
|
|
375
|
-
});
|
|
376
|
-
const dbMs = Date.now() - t3;
|
|
377
|
-
if (chunks.length > 0) {
|
|
378
|
-
this.logger.debug('IndexerService: file indexed', {
|
|
379
|
-
file: relFilePath, chunks: chunks.length,
|
|
380
|
-
astMs, embedMs, dbMs, totalMs: Date.now() - t0,
|
|
381
|
-
});
|
|
382
|
-
}
|
|
383
|
-
return { status: 'indexed', chunks: chunks.length };
|
|
384
|
-
}
|
|
385
|
-
catch (err) {
|
|
386
|
-
console.error(`\n[indexer] Failed: ${filePath} — ${(0, error_record_1.rootCauseMessage)(err)}`);
|
|
387
|
-
this.logger.error('IndexerService: failed to index file', { file: filePath, err });
|
|
388
|
-
return { status: 'failed', chunks: 0 };
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
172
|
}
|
|
392
173
|
exports.IndexerService = IndexerService;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
2
|
+
import type { IIndexIdentityGuard } from '../interfaces/IIndexIdentityGuard';
|
|
3
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
4
|
+
import type { IMcpReadiness, ReadinessCheck } from '../interfaces/IMcpReadiness';
|
|
5
|
+
export declare const CHECK_DEADLINE_MS: number;
|
|
6
|
+
export declare class McpReadiness implements IMcpReadiness {
|
|
7
|
+
private db;
|
|
8
|
+
private identity;
|
|
9
|
+
private logger;
|
|
10
|
+
private isReady;
|
|
11
|
+
private lastCause;
|
|
12
|
+
private inFlight;
|
|
13
|
+
constructor(db: Pick<IDatabase, 'initDb' | 'probe'>, identity: IIndexIdentityGuard, logger: ILogger);
|
|
14
|
+
get ready(): boolean;
|
|
15
|
+
start(): Promise<ReadinessCheck>;
|
|
16
|
+
ensureReady(): Promise<ReadinessCheck>;
|
|
17
|
+
private recheck;
|
|
18
|
+
}
|