codesearch-mcp 0.3.0-alpha → 0.4.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -6
- package/LICENSE +661 -21
- package/README.md +33 -17
- package/dist/app.js +13 -20
- package/dist/src/ast/CallExtractor.d.ts +8 -0
- package/dist/src/ast/CallExtractor.js +68 -0
- package/dist/src/ast/GraphqlNameExtractor.d.ts +22 -0
- package/dist/src/ast/GraphqlNameExtractor.js +240 -0
- package/dist/src/ast/ImportEdgeExtractor.d.ts +11 -0
- package/dist/src/ast/ImportEdgeExtractor.js +129 -0
- package/dist/src/ast/IndexedNodeResolver.d.ts +14 -0
- package/dist/src/ast/IndexedNodeResolver.js +217 -0
- package/dist/src/ast/MemberExtractor.d.ts +8 -0
- package/dist/src/ast/MemberExtractor.js +129 -0
- package/dist/src/ast/TopLevelSymbolExtractor.d.ts +5 -0
- package/dist/src/ast/TopLevelSymbolExtractor.js +86 -0
- package/dist/src/ast/graphql-grammar.d.ts +24 -0
- package/dist/src/ast/graphql-grammar.js +93 -0
- package/dist/src/ast/node-helpers.d.ts +11 -0
- package/dist/src/ast/node-helpers.js +55 -0
- package/dist/src/commands/EvalCommand.d.ts +7 -3
- package/dist/src/commands/EvalCommand.js +36 -158
- package/dist/src/commands/IndexCommand.d.ts +10 -4
- package/dist/src/commands/IndexCommand.js +52 -124
- package/dist/src/commands/McpCommand.d.ts +6 -3
- package/dist/src/commands/McpCommand.js +19 -14
- package/dist/src/commands/WebCommand.d.ts +9 -3
- package/dist/src/commands/WebCommand.js +17 -8
- package/dist/src/commands/eval/baseline-comparison.d.ts +6 -0
- package/dist/src/commands/eval/baseline-comparison.js +54 -0
- package/dist/src/commands/eval/eval-args.d.ts +6 -0
- package/dist/src/commands/eval/eval-args.js +71 -0
- package/dist/src/commands/eval/eval-corpora.d.ts +6 -0
- package/dist/src/commands/eval/eval-corpora.js +76 -0
- package/dist/src/commands/eval/eval-metadata.d.ts +17 -0
- package/dist/src/commands/eval/eval-metadata.js +75 -0
- package/dist/src/commands/eval/index-inspector.d.ts +3 -0
- package/dist/src/commands/eval/index-inspector.js +21 -0
- package/dist/src/commands/eval/staged-search.d.ts +3 -0
- package/dist/src/commands/eval/staged-search.js +20 -0
- package/dist/src/commands/eval/suite-loader.d.ts +8 -0
- package/dist/src/commands/eval/suite-loader.js +52 -0
- package/dist/src/commands/index/backend-probe.d.ts +2 -0
- package/dist/src/commands/index/backend-probe.js +15 -0
- package/dist/src/commands/index/embedding-setup.d.ts +3 -0
- package/dist/src/commands/index/embedding-setup.js +16 -0
- package/dist/src/commands/index/index-summary.d.ts +2 -0
- package/dist/src/commands/index/index-summary.js +22 -0
- package/dist/src/commands/index/legacy-ollama-guard.d.ts +3 -0
- package/dist/src/commands/index/legacy-ollama-guard.js +23 -0
- package/dist/src/commands/index/prune-step.d.ts +3 -0
- package/dist/src/commands/index/prune-step.js +22 -0
- package/dist/src/commands/index/scan-targets.d.ts +8 -0
- package/dist/src/commands/index/scan-targets.js +107 -0
- package/dist/src/config-store.d.ts +0 -1
- package/dist/src/config-store.js +3 -4
- package/dist/src/config.d.ts +1 -1
- package/dist/src/config.js +1 -1
- package/dist/src/constants.d.ts +15 -0
- package/dist/src/constants.js +34 -6
- package/dist/src/container.d.ts +4 -2
- package/dist/src/container.js +7 -5
- package/dist/src/db/drizzle-types.d.ts +7 -0
- package/dist/src/db/index.d.ts +1 -0
- package/dist/src/db/schema-steps.d.ts +8 -0
- package/dist/src/db/schema-steps.js +105 -0
- package/dist/src/declarations/DeclarationTree.d.ts +1 -1
- package/dist/src/eval-results.js +16 -2
- package/dist/src/eval-vector-index.d.ts +5 -0
- package/dist/src/eval-vector-index.js +60 -0
- package/dist/src/help.js +9 -1
- package/dist/src/index-args.d.ts +13 -0
- package/dist/src/index-args.js +48 -0
- package/dist/src/indexing/AnalysisBatcher.d.ts +12 -0
- package/dist/src/indexing/AnalysisBatcher.js +28 -0
- package/dist/src/indexing/ChunkEmbedder.d.ts +24 -0
- package/dist/src/indexing/ChunkEmbedder.js +37 -0
- package/dist/src/indexing/ConsoleScanReporter.d.ts +10 -0
- package/dist/src/indexing/ConsoleScanReporter.js +73 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.d.ts +14 -0
- package/dist/src/indexing/EmbeddingSchemaPreparer.js +66 -0
- package/dist/src/indexing/FileIngestor.d.ts +22 -0
- package/dist/src/indexing/FileIngestor.js +116 -0
- package/dist/src/indexing/IndexStore.d.ts +24 -0
- package/dist/src/indexing/IndexStore.js +69 -0
- package/dist/src/indexing/StaleFilePruner.d.ts +8 -0
- package/dist/src/indexing/StaleFilePruner.js +55 -0
- package/dist/src/indexing/analysis-rows.d.ts +4 -0
- package/dist/src/indexing/analysis-rows.js +65 -0
- package/dist/src/indexing/project-notices.d.ts +2 -0
- package/dist/src/indexing/project-notices.js +63 -0
- package/dist/src/interfaces/IAstParser.d.ts +2 -1
- package/dist/src/interfaces/ICallGraph.d.ts +28 -0
- package/dist/src/interfaces/ICallGraph.js +2 -0
- package/dist/src/interfaces/IChunkSearch.d.ts +42 -0
- package/dist/src/interfaces/IChunkSearch.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +6 -2
- package/dist/src/interfaces/IDependencyGraph.d.ts +24 -0
- package/dist/src/interfaces/IDependencyGraph.js +2 -0
- package/dist/src/interfaces/IIndexIdentityGuard.d.ts +3 -0
- package/dist/src/interfaces/IIndexIdentityGuard.js +2 -0
- package/dist/src/interfaces/IIndexedNodeResolver.d.ts +10 -0
- package/dist/src/interfaces/IIndexedNodeResolver.js +2 -0
- package/dist/src/interfaces/IMcpReadiness.d.ts +12 -0
- package/dist/src/interfaces/IMcpReadiness.js +2 -0
- package/dist/src/interfaces/IProjectCatalog.d.ts +13 -0
- package/dist/src/interfaces/IProjectCatalog.js +2 -0
- package/dist/src/interfaces/IProjectRoots.d.ts +3 -0
- package/dist/src/interfaces/IProjectRoots.js +2 -0
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +21 -2
- package/dist/src/interfaces/IScanReporter.d.ts +13 -0
- package/dist/src/interfaces/IScanReporter.js +2 -0
- package/dist/src/interfaces/ISymbolLookup.d.ts +49 -0
- package/dist/src/interfaces/ISymbolLookup.js +2 -0
- package/dist/src/interfaces/IToolPaths.d.ts +62 -0
- package/dist/src/interfaces/IToolPaths.js +2 -0
- package/dist/src/interfaces/IWorkspaceFiles.d.ts +28 -0
- package/dist/src/interfaces/IWorkspaceFiles.js +2 -0
- package/dist/src/interfaces/SymbolKind.d.ts +1 -0
- package/dist/src/interfaces/SymbolKind.js +2 -0
- package/dist/src/interfaces/index.d.ts +12 -1
- package/dist/src/project-scope.d.ts +1 -0
- package/dist/src/project-scope.js +5 -0
- package/dist/src/retrieval-eval.d.ts +2 -1
- package/dist/src/retrieval-eval.js +45 -2
- package/dist/src/search/CallGraph.d.ts +9 -0
- package/dist/src/search/CallGraph.js +84 -0
- package/dist/src/search/ChunkSearch.d.ts +23 -0
- package/dist/src/search/ChunkSearch.js +115 -0
- package/dist/src/search/DependencyGraph.d.ts +19 -0
- package/dist/src/search/DependencyGraph.js +180 -0
- package/dist/src/search/IndexIdentityGuard.d.ts +11 -0
- package/dist/src/search/IndexIdentityGuard.js +37 -0
- package/dist/src/search/ProjectCatalog.d.ts +11 -0
- package/dist/src/search/ProjectCatalog.js +38 -0
- package/dist/src/search/SymbolLookup.d.ts +24 -0
- package/dist/src/search/SymbolLookup.js +97 -0
- package/dist/src/search/chunks/CandidateFetcher.d.ts +19 -0
- package/dist/src/search/chunks/CandidateFetcher.js +40 -0
- package/dist/src/search/chunks/IterativeScanGate.d.ts +10 -0
- package/dist/src/search/chunks/IterativeScanGate.js +42 -0
- package/dist/src/search/chunks/RerankStage.d.ts +15 -0
- package/dist/src/search/chunks/RerankStage.js +42 -0
- package/dist/src/search/chunks/search-metrics.d.ts +18 -0
- package/dist/src/search/chunks/search-metrics.js +32 -0
- package/dist/src/search/chunks/search-plan.d.ts +18 -0
- package/dist/src/search/chunks/search-plan.js +19 -0
- package/dist/src/search/create-search-services.d.ts +26 -0
- package/dist/src/search/create-search-services.js +29 -0
- package/dist/src/search/index-paths.d.ts +1 -0
- package/dist/src/search/index-paths.js +5 -0
- package/dist/src/search/symbol-sql.d.ts +13 -0
- package/dist/src/search/symbol-sql.js +66 -0
- package/dist/src/search/symbols/SymbolGrepFallback.d.ts +19 -0
- package/dist/src/search/symbols/SymbolGrepFallback.js +106 -0
- package/dist/src/search/symbols/SymbolIndexQuery.d.ts +14 -0
- package/dist/src/search/symbols/SymbolIndexQuery.js +52 -0
- package/dist/src/search/symbols/SymbolTextSearch.d.ts +13 -0
- package/dist/src/search/symbols/SymbolTextSearch.js +84 -0
- package/dist/src/search/symbols/TargetColumnGate.d.ts +7 -0
- package/dist/src/search/symbols/TargetColumnGate.js +23 -0
- package/dist/src/search/symbols/find-symbol-constants.d.ts +4 -0
- package/dist/src/search/symbols/find-symbol-constants.js +8 -0
- package/dist/src/search/symbols/grep-args.d.ts +3 -0
- package/dist/src/search/symbols/grep-args.js +7 -0
- package/dist/src/search/symbols/grep-patterns.d.ts +4 -0
- package/dist/src/search/symbols/grep-patterns.js +60 -0
- package/dist/src/services/AstParserService.d.ts +8 -44
- package/dist/src/services/AstParserService.js +19 -938
- package/dist/src/services/DatabaseService.d.ts +8 -10
- package/dist/src/services/DatabaseService.js +29 -86
- package/dist/src/services/IndexerService.d.ts +7 -10
- package/dist/src/services/IndexerService.js +36 -255
- package/dist/src/services/McpReadiness.d.ts +18 -0
- package/dist/src/services/McpReadiness.js +73 -0
- package/dist/src/services/McpServer.d.ts +32 -9
- package/dist/src/services/McpServer.js +78 -41
- package/dist/src/services/WebServerService.d.ts +4 -3
- package/dist/src/services/WebServerService.js +16 -8
- package/dist/src/setup.d.ts +16 -4
- package/dist/src/setup.js +115 -40
- package/dist/src/symbol-name.d.ts +1 -0
- package/dist/src/symbol-name.js +16 -12
- package/dist/src/tools/FindSymbolTool.d.ts +17 -7
- package/dist/src/tools/FindSymbolTool.js +54 -35
- package/dist/src/tools/GetCallersTool.d.ts +16 -8
- package/dist/src/tools/GetCallersTool.js +65 -35
- package/dist/src/tools/GetDependenciesTool.d.ts +16 -8
- package/dist/src/tools/GetDependenciesTool.js +35 -28
- package/dist/src/tools/GetDependentsTool.d.ts +20 -8
- package/dist/src/tools/GetDependentsTool.js +47 -33
- package/dist/src/tools/ListFilesTool.d.ts +16 -8
- package/dist/src/tools/ListFilesTool.js +20 -26
- package/dist/src/tools/ListProjectsTool.d.ts +11 -7
- package/dist/src/tools/ListProjectsTool.js +13 -11
- package/dist/src/tools/McpTool.d.ts +17 -0
- package/dist/src/tools/McpTool.js +28 -0
- package/dist/src/tools/ReadFileRangeTool.d.ts +16 -8
- package/dist/src/tools/ReadFileRangeTool.js +23 -25
- package/dist/src/tools/ReadFileTool.d.ts +15 -8
- package/dist/src/tools/ReadFileTool.js +19 -24
- package/dist/src/tools/SearchCodebaseTool.d.ts +22 -10
- package/dist/src/tools/SearchCodebaseTool.js +69 -51
- package/dist/src/tools/format-helpers.d.ts +9 -3
- package/dist/src/tools/format-helpers.js +59 -14
- package/dist/src/tools/tool-args.d.ts +38 -0
- package/dist/src/tools/tool-args.js +95 -0
- package/dist/src/workspace/GrepRunner.d.ts +15 -0
- package/dist/src/workspace/GrepRunner.js +104 -0
- package/dist/src/workspace/ProjectRoots.d.ts +22 -0
- package/dist/src/workspace/ProjectRoots.js +102 -0
- package/dist/src/workspace/ToolPaths.d.ts +31 -0
- package/dist/src/workspace/ToolPaths.js +312 -0
- package/dist/src/workspace/WorkspaceFiles.d.ts +19 -0
- package/dist/src/workspace/WorkspaceFiles.js +285 -0
- package/dist/src/workspace/fs-access.d.ts +9 -0
- package/dist/src/workspace/fs-access.js +47 -0
- package/dist/src/workspace/path-helpers.d.ts +3 -0
- package/dist/src/workspace/path-helpers.js +43 -0
- package/package.json +4 -3
- package/dist/src/interfaces/ISearcher.d.ts +0 -46
- package/dist/src/services/SearcherService.d.ts +0 -235
- package/dist/src/services/SearcherService.js +0 -1118
- /package/dist/src/{interfaces/ISearcher.js → db/drizzle-types.js} +0 -0
package/dist/src/help.js
CHANGED
|
@@ -16,7 +16,15 @@ function printHelp(command) {
|
|
|
16
16
|
console.log(' --project [name] Scope this run to one discovered project or monorepo unit');
|
|
17
17
|
console.log(' by exact name, bypassing selection and pruning; omit the');
|
|
18
18
|
console.log(' name for an interactive single-choice prompt');
|
|
19
|
+
console.log(' --require-cwd Needs --project <name>; the run is refused unless the current');
|
|
20
|
+
console.log(' directory is that project\'s directory');
|
|
19
21
|
console.log(' -h, --help Show this help message');
|
|
22
|
+
console.log('');
|
|
23
|
+
console.log('Exit codes:');
|
|
24
|
+
console.log(' 0 No file failed and nothing else failed; includes an interrupted run,');
|
|
25
|
+
console.log(' which may have left files unindexed');
|
|
26
|
+
console.log(' 2 The run finished with failed files and the rest of the index is usable');
|
|
27
|
+
console.log(' 1 Any other failure');
|
|
20
28
|
break;
|
|
21
29
|
case 'rag-mcp':
|
|
22
30
|
case 'mcp':
|
|
@@ -75,7 +83,7 @@ function printHelp(command) {
|
|
|
75
83
|
console.log(' -h, --help Show this help message');
|
|
76
84
|
break;
|
|
77
85
|
default:
|
|
78
|
-
console.log('Usage: rag-index [--reindex] [--select] [--project [name]]');
|
|
86
|
+
console.log('Usage: rag-index [--reindex] [--select] [--project [name]] [--require-cwd]');
|
|
79
87
|
console.log(' rag-mcp');
|
|
80
88
|
console.log(' rag-setup');
|
|
81
89
|
console.log(' rag-config <show|get|set|reset|edit>');
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare function findUnexpectedIndexArg(args: string[]): string | undefined;
|
|
2
|
+
export declare function unreadableCwdCause(args: string[]): string | undefined;
|
|
3
|
+
export interface ProjectFlag {
|
|
4
|
+
present: boolean;
|
|
5
|
+
name?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface IndexArgs {
|
|
8
|
+
force: boolean;
|
|
9
|
+
forceSelect: boolean;
|
|
10
|
+
projectFlag: ProjectFlag;
|
|
11
|
+
requireCwd: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare function parseIndexArgs(args: string[]): IndexArgs;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.findUnexpectedIndexArg = findUnexpectedIndexArg;
|
|
4
|
+
exports.unreadableCwdCause = unreadableCwdCause;
|
|
5
|
+
exports.parseIndexArgs = parseIndexArgs;
|
|
6
|
+
const BOOL_FLAGS = new Set(['--reindex', '--select', '--require-cwd']);
|
|
7
|
+
function findUnexpectedIndexArg(args) {
|
|
8
|
+
let projectSeen = false;
|
|
9
|
+
for (let i = 0; i < args.length; i++) {
|
|
10
|
+
const a = args[i];
|
|
11
|
+
if (BOOL_FLAGS.has(a))
|
|
12
|
+
continue;
|
|
13
|
+
if (a === '--project' && !projectSeen) {
|
|
14
|
+
projectSeen = true;
|
|
15
|
+
const next = args[i + 1];
|
|
16
|
+
if (next !== undefined && !next.startsWith('-'))
|
|
17
|
+
i++;
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
return a;
|
|
21
|
+
}
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
function unreadableCwdCause(args) {
|
|
25
|
+
if (!args.includes('--require-cwd'))
|
|
26
|
+
return undefined;
|
|
27
|
+
try {
|
|
28
|
+
process.cwd();
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
return err.message;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function parseProjectFlag(args) {
|
|
36
|
+
const i = args.indexOf('--project');
|
|
37
|
+
if (i === -1)
|
|
38
|
+
return { present: false };
|
|
39
|
+
const next = args[i + 1];
|
|
40
|
+
return next !== undefined && !next.startsWith('-') ? { present: true, name: next } : { present: true };
|
|
41
|
+
}
|
|
42
|
+
function parseIndexArgs(args) {
|
|
43
|
+
const force = args.includes('--reindex');
|
|
44
|
+
const forceSelect = args.includes('--select');
|
|
45
|
+
const projectFlag = parseProjectFlag(args);
|
|
46
|
+
const requireCwd = args.includes('--require-cwd');
|
|
47
|
+
return { force, forceSelect, projectFlag, requireCwd };
|
|
48
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IAstParser, FileAnalysis } from '../interfaces/IAstParser';
|
|
3
|
+
export declare class AnalysisBatcher {
|
|
4
|
+
private ast;
|
|
5
|
+
private config;
|
|
6
|
+
constructor(ast: IAstParser, config: Config);
|
|
7
|
+
get batchSize(): number;
|
|
8
|
+
batches(files: string[], serviceName: string, dir: string, signal?: AbortSignal, rootTsConfigPath?: string): AsyncGenerator<{
|
|
9
|
+
batch: string[];
|
|
10
|
+
analysisMap: Map<string, FileAnalysis>;
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AnalysisBatcher = void 0;
|
|
4
|
+
class AnalysisBatcher {
|
|
5
|
+
ast;
|
|
6
|
+
config;
|
|
7
|
+
constructor(ast, config) {
|
|
8
|
+
this.ast = ast;
|
|
9
|
+
this.config = config;
|
|
10
|
+
}
|
|
11
|
+
get batchSize() {
|
|
12
|
+
return this.config.indexer.astBatchSize;
|
|
13
|
+
}
|
|
14
|
+
async *batches(files, serviceName, dir, signal, rootTsConfigPath) {
|
|
15
|
+
const isLargeProject = files.length > this.config.indexer.astBatchSize;
|
|
16
|
+
for (let batchStart = 0; batchStart < files.length; batchStart += this.config.indexer.astBatchSize) {
|
|
17
|
+
if (signal?.aborted)
|
|
18
|
+
return;
|
|
19
|
+
const batch = files.slice(batchStart, batchStart + this.config.indexer.astBatchSize);
|
|
20
|
+
const analysisMap = await this.ast.analyzeProject(batch, serviceName, dir, {
|
|
21
|
+
skipDependencyResolution: isLargeProject,
|
|
22
|
+
rootTsConfigPath,
|
|
23
|
+
});
|
|
24
|
+
yield { batch, analysisMap };
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
exports.AnalysisBatcher = AnalysisBatcher;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { DepEdge, SymbolEntry } from '../interfaces/IAstParser';
|
|
4
|
+
import type { ChunkPiece } from '../interfaces/IChunker';
|
|
5
|
+
import type { DeclarationTree } from '../declarations/DeclarationTree';
|
|
6
|
+
import type { SourceLines } from '../declarations/SourceLines';
|
|
7
|
+
import type { CodeChunkInsert } from '../db';
|
|
8
|
+
export interface ChunkRecordContext {
|
|
9
|
+
serviceName: string;
|
|
10
|
+
relFilePath: string;
|
|
11
|
+
relPath: string;
|
|
12
|
+
promptEdges: DepEdge[];
|
|
13
|
+
rawSymbols: SymbolEntry[];
|
|
14
|
+
mtimeMs: number;
|
|
15
|
+
hash: string;
|
|
16
|
+
ownerTree: DeclarationTree;
|
|
17
|
+
sourceLines: SourceLines;
|
|
18
|
+
}
|
|
19
|
+
export declare class ChunkEmbedder {
|
|
20
|
+
private embedder;
|
|
21
|
+
private config;
|
|
22
|
+
constructor(embedder: IEmbedder, config: Config);
|
|
23
|
+
records(chunks: ChunkPiece[], ctx: ChunkRecordContext): Promise<CodeChunkInsert[]>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ChunkEmbedder = void 0;
|
|
4
|
+
const owner_path_1 = require("../owner-path");
|
|
5
|
+
const enriched_prompt_1 = require("../enriched-prompt");
|
|
6
|
+
class ChunkEmbedder {
|
|
7
|
+
embedder;
|
|
8
|
+
config;
|
|
9
|
+
constructor(embedder, config) {
|
|
10
|
+
this.embedder = embedder;
|
|
11
|
+
this.config = config;
|
|
12
|
+
}
|
|
13
|
+
async records(chunks, ctx) {
|
|
14
|
+
const { serviceName, relFilePath, relPath, promptEdges, rawSymbols, mtimeMs, hash, ownerTree, sourceLines } = ctx;
|
|
15
|
+
const records = [];
|
|
16
|
+
const embedBatchSize = this.config.embedding.batchSize;
|
|
17
|
+
for (let i = 0; i < chunks.length; i += embedBatchSize) {
|
|
18
|
+
const batch = chunks.slice(i, i + embedBatchSize);
|
|
19
|
+
const prompts = batch.map(piece => (0, enriched_prompt_1.getEnrichedPrompt)(piece, relPath, serviceName, promptEdges, rawSymbols));
|
|
20
|
+
const vectors = await this.embedder.embedBatch(prompts, 'document');
|
|
21
|
+
for (let j = 0; j < batch.length; j++) {
|
|
22
|
+
const piece = batch[j];
|
|
23
|
+
records.push({
|
|
24
|
+
serviceName, filePath: relFilePath, startLine: piece.startLine,
|
|
25
|
+
content: piece.content,
|
|
26
|
+
embedding: `[${vectors[j]}]`,
|
|
27
|
+
lastModified: mtimeMs,
|
|
28
|
+
contentHash: hash,
|
|
29
|
+
ownerPath: (0, owner_path_1.ownerOf)(piece, ownerTree, sourceLines),
|
|
30
|
+
endLine: piece.startLine + (piece.content.match(/\n/g)?.length ?? 0),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return records;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
exports.ChunkEmbedder = ChunkEmbedder;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { IScanReporter, ProjectNotice } from '../interfaces';
|
|
2
|
+
export declare class ConsoleScanReporter implements IScanReporter {
|
|
3
|
+
private bar;
|
|
4
|
+
staleRemoved(relFile: string): void;
|
|
5
|
+
begin(serviceName: string, fileCount: number, notices: ProjectNotice[]): void;
|
|
6
|
+
file(relToDir: string): void;
|
|
7
|
+
advance(): void;
|
|
8
|
+
interrupted(serviceName: string): void;
|
|
9
|
+
end(): void;
|
|
10
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.ConsoleScanReporter = void 0;
|
|
37
|
+
const path = __importStar(require("path"));
|
|
38
|
+
const cli_progress_1 = require("cli-progress");
|
|
39
|
+
class ConsoleScanReporter {
|
|
40
|
+
bar;
|
|
41
|
+
staleRemoved(relFile) {
|
|
42
|
+
console.log(` Removed stale: ${path.basename(relFile)}`);
|
|
43
|
+
}
|
|
44
|
+
begin(serviceName, fileCount, notices) {
|
|
45
|
+
this.bar = new cli_progress_1.SingleBar({
|
|
46
|
+
format: `{serviceName} {bar} {percentage}% | {value}/{total} | {duration_formatted} elapsed | ~{eta_formatted} left | {filename}`,
|
|
47
|
+
barCompleteChar: '█',
|
|
48
|
+
barIncompleteChar: '░',
|
|
49
|
+
hideCursor: true,
|
|
50
|
+
clearOnComplete: false,
|
|
51
|
+
}, cli_progress_1.Presets.shades_grey);
|
|
52
|
+
console.log(`\nIndexing ${serviceName}: ${fileCount} files`);
|
|
53
|
+
for (const { mark, message } of notices) {
|
|
54
|
+
console.log(` ${mark} ${message}`);
|
|
55
|
+
}
|
|
56
|
+
this.bar.start(fileCount, 0, { serviceName, filename: '' });
|
|
57
|
+
}
|
|
58
|
+
file(relToDir) {
|
|
59
|
+
this.bar?.update({ filename: relToDir });
|
|
60
|
+
}
|
|
61
|
+
advance() {
|
|
62
|
+
this.bar?.increment();
|
|
63
|
+
}
|
|
64
|
+
interrupted(serviceName) {
|
|
65
|
+
this.bar?.stop();
|
|
66
|
+
console.log(`\n Interrupted — ${serviceName}`);
|
|
67
|
+
}
|
|
68
|
+
end() {
|
|
69
|
+
this.bar?.stop();
|
|
70
|
+
process.stdout.write('\x1B[J');
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
exports.ConsoleScanReporter = ConsoleScanReporter;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IEmbedder } from '../interfaces/IEmbedder';
|
|
3
|
+
import type { IDatabase, IndexMeta } from '../interfaces/IDatabase';
|
|
4
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
5
|
+
export declare class EmbeddingSchemaPreparer {
|
|
6
|
+
private db;
|
|
7
|
+
private embedder;
|
|
8
|
+
private config;
|
|
9
|
+
private logger;
|
|
10
|
+
constructor(db: IDatabase, embedder: IEmbedder, config: Config, logger: ILogger);
|
|
11
|
+
prepare(): Promise<IndexMeta>;
|
|
12
|
+
private reconcileIndex;
|
|
13
|
+
private countChunks;
|
|
14
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EmbeddingSchemaPreparer = void 0;
|
|
4
|
+
const constants_1 = require("../constants");
|
|
5
|
+
class EmbeddingSchemaPreparer {
|
|
6
|
+
db;
|
|
7
|
+
embedder;
|
|
8
|
+
config;
|
|
9
|
+
logger;
|
|
10
|
+
constructor(db, embedder, config, logger) {
|
|
11
|
+
this.db = db;
|
|
12
|
+
this.embedder = embedder;
|
|
13
|
+
this.config = config;
|
|
14
|
+
this.logger = logger;
|
|
15
|
+
}
|
|
16
|
+
async prepare() {
|
|
17
|
+
const { provider, model, dimensions: override } = this.config.embedding;
|
|
18
|
+
const next = {
|
|
19
|
+
provider,
|
|
20
|
+
model,
|
|
21
|
+
dimensions: await (0, constants_1.resolveDimension)(override, model, async () => {
|
|
22
|
+
const probe = await this.embedder.embed('probe', 'document');
|
|
23
|
+
return probe.length;
|
|
24
|
+
}),
|
|
25
|
+
};
|
|
26
|
+
const meta = await this.db.readIndexMeta();
|
|
27
|
+
if (!meta) {
|
|
28
|
+
const chunkCount = await this.countChunks();
|
|
29
|
+
if (chunkCount === 0) {
|
|
30
|
+
await this.db.recreateChunkTable(next.dimensions);
|
|
31
|
+
await this.db.writeIndexMeta(next);
|
|
32
|
+
this.logger.debug('IndexerService: prepared fresh index', { ...next });
|
|
33
|
+
return next;
|
|
34
|
+
}
|
|
35
|
+
await this.reconcileIndex({ provider: 'ollama', model: 'bge-m3', dimensions: 1024 }, next);
|
|
36
|
+
return next;
|
|
37
|
+
}
|
|
38
|
+
await this.reconcileIndex(meta, next);
|
|
39
|
+
return next;
|
|
40
|
+
}
|
|
41
|
+
async reconcileIndex(prev, next) {
|
|
42
|
+
if (prev.dimensions !== next.dimensions) {
|
|
43
|
+
console.log(`Embedding dimension changed (${prev.dimensions} → ${next.dimensions}) — recreating code_chunk.`);
|
|
44
|
+
this.logger.info('IndexerService: dimension change, recreating code_chunk', { prev, next });
|
|
45
|
+
await this.db.recreateChunkTable(next.dimensions);
|
|
46
|
+
}
|
|
47
|
+
else if (prev.provider !== next.provider || prev.model !== next.model) {
|
|
48
|
+
console.log(`Embedding model changed (${prev.provider}/${prev.model} → ${next.provider}/${next.model}) — clearing index.`);
|
|
49
|
+
this.logger.info('IndexerService: model change, clearing code_chunk', { prev, next });
|
|
50
|
+
await this.db.query(`DELETE FROM code_chunk`);
|
|
51
|
+
}
|
|
52
|
+
await this.db.writeIndexMeta(next);
|
|
53
|
+
}
|
|
54
|
+
async countChunks() {
|
|
55
|
+
try {
|
|
56
|
+
const rows = await this.db.query(`SELECT COUNT(*)::int AS n FROM code_chunk`);
|
|
57
|
+
return Number(rows[0]?.n ?? 0);
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
if (/does not exist/i.test(err.message))
|
|
61
|
+
return 0;
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
exports.EmbeddingSchemaPreparer = EmbeddingSchemaPreparer;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { FileAnalysis } from '../interfaces/IAstParser';
|
|
3
|
+
import type { IChunker } from '../interfaces/IChunker';
|
|
4
|
+
import type { ILogger } from '../interfaces/ILogger';
|
|
5
|
+
import type { IndexStore } from './IndexStore';
|
|
6
|
+
import type { ChunkEmbedder } from './ChunkEmbedder';
|
|
7
|
+
export type IngestStatus = 'indexed' | 'skipped' | 'failed';
|
|
8
|
+
export type IngestResult = {
|
|
9
|
+
status: IngestStatus;
|
|
10
|
+
chunks: number;
|
|
11
|
+
pendingMtimeRefresh?: string;
|
|
12
|
+
};
|
|
13
|
+
export declare class FileIngestor {
|
|
14
|
+
private store;
|
|
15
|
+
private chunkEmbedder;
|
|
16
|
+
private chunker;
|
|
17
|
+
private config;
|
|
18
|
+
private logger;
|
|
19
|
+
constructor(store: IndexStore, chunkEmbedder: ChunkEmbedder, chunker: IChunker, config: Config, logger: ILogger);
|
|
20
|
+
private static hashContent;
|
|
21
|
+
ingestFile(filePath: string, serviceName: string, mtimeMs: number, projectRoot: string, analysis?: FileAnalysis): Promise<IngestResult>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.FileIngestor = void 0;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const crypto_1 = require("crypto");
|
|
40
|
+
const DeclarationTree_1 = require("../declarations/DeclarationTree");
|
|
41
|
+
const SourceLines_1 = require("../declarations/SourceLines");
|
|
42
|
+
const symbol_rows_1 = require("../symbol-rows");
|
|
43
|
+
const error_record_1 = require("../error-record");
|
|
44
|
+
const analysis_rows_1 = require("./analysis-rows");
|
|
45
|
+
const SCHEMA_EXTENSIONS = /\.(prisma|graphql|gql)$/;
|
|
46
|
+
const JSX_EXTENSIONS = /\.(tsx|jsx|js)$/;
|
|
47
|
+
class FileIngestor {
|
|
48
|
+
store;
|
|
49
|
+
chunkEmbedder;
|
|
50
|
+
chunker;
|
|
51
|
+
config;
|
|
52
|
+
logger;
|
|
53
|
+
constructor(store, chunkEmbedder, chunker, config, logger) {
|
|
54
|
+
this.store = store;
|
|
55
|
+
this.chunkEmbedder = chunkEmbedder;
|
|
56
|
+
this.chunker = chunker;
|
|
57
|
+
this.config = config;
|
|
58
|
+
this.logger = logger;
|
|
59
|
+
}
|
|
60
|
+
static hashContent(content) {
|
|
61
|
+
return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
62
|
+
}
|
|
63
|
+
async ingestFile(filePath, serviceName, mtimeMs, projectRoot, analysis) {
|
|
64
|
+
try {
|
|
65
|
+
const relFilePath = path.relative(this.config.workspaceRoot, filePath);
|
|
66
|
+
const existing = await this.store.fileState(serviceName, relFilePath);
|
|
67
|
+
if (existing?.lastModified === mtimeMs) {
|
|
68
|
+
return { status: 'skipped', chunks: 0 };
|
|
69
|
+
}
|
|
70
|
+
const t0 = Date.now();
|
|
71
|
+
const code = fs.readFileSync(filePath, 'utf-8');
|
|
72
|
+
const hash = FileIngestor.hashContent(code);
|
|
73
|
+
if (existing !== undefined && existing.contentHash !== null && existing.contentHash === hash) {
|
|
74
|
+
return { status: 'skipped', chunks: 0, pendingMtimeRefresh: relFilePath };
|
|
75
|
+
}
|
|
76
|
+
const tAst = Date.now();
|
|
77
|
+
const { edges: rawEdges, symbols: rawSymbols, members: rawMembers = [], calls: rawCalls, graphqlNames: rawGraphqlNames = [], } = analysis ?? { edges: [], symbols: [], calls: [] };
|
|
78
|
+
const edges = (0, analysis_rows_1.workspaceEdges)(rawEdges, this.config.workspaceRoot);
|
|
79
|
+
const calls = (0, analysis_rows_1.workspaceCalls)(rawCalls, serviceName, relFilePath, this.config.workspaceRoot);
|
|
80
|
+
const symbolRows = (0, symbol_rows_1.buildSymbolRows)(rawSymbols, rawMembers, rawGraphqlNames).map(s => ({ ...s, filePath: relFilePath }));
|
|
81
|
+
const promptEdges = (0, analysis_rows_1.toPromptEdges)(edges);
|
|
82
|
+
const astMs = Date.now() - tAst;
|
|
83
|
+
const jsx = JSX_EXTENSIONS.test(filePath);
|
|
84
|
+
const chunks = await this.chunker.chunk(code, {
|
|
85
|
+
strategy: this.config.indexer.chunkStrategy,
|
|
86
|
+
isSchemaFile: SCHEMA_EXTENSIONS.test(filePath),
|
|
87
|
+
declarations: analysis?.declarations,
|
|
88
|
+
jsx,
|
|
89
|
+
});
|
|
90
|
+
const relPath = path.relative(projectRoot, filePath);
|
|
91
|
+
const sourceLines = new SourceLines_1.SourceLines(code, { jsx });
|
|
92
|
+
const ownerTree = analysis?.declarations ?? DeclarationTree_1.DeclarationTree.EMPTY;
|
|
93
|
+
const t2 = Date.now();
|
|
94
|
+
const records = await this.chunkEmbedder.records(chunks, {
|
|
95
|
+
serviceName, relFilePath, relPath, promptEdges, rawSymbols, mtimeMs, hash, ownerTree, sourceLines,
|
|
96
|
+
});
|
|
97
|
+
const embedMs = Date.now() - t2;
|
|
98
|
+
const t3 = Date.now();
|
|
99
|
+
await this.store.writeFile(serviceName, relFilePath, records, edges, symbolRows, calls);
|
|
100
|
+
const dbMs = Date.now() - t3;
|
|
101
|
+
if (chunks.length > 0) {
|
|
102
|
+
this.logger.debug('IndexerService: file indexed', {
|
|
103
|
+
file: relFilePath, chunks: chunks.length,
|
|
104
|
+
astMs, embedMs, dbMs, totalMs: Date.now() - t0,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return { status: 'indexed', chunks: chunks.length };
|
|
108
|
+
}
|
|
109
|
+
catch (err) {
|
|
110
|
+
console.error(`\n[indexer] Failed: ${filePath} — ${(0, error_record_1.rootCauseMessage)(err)}`);
|
|
111
|
+
this.logger.error('IndexerService: failed to index file', { file: filePath, err });
|
|
112
|
+
return { status: 'failed', chunks: 0 };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
exports.FileIngestor = FileIngestor;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { DepEdge } from '../interfaces/IAstParser';
|
|
3
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
4
|
+
import { codeSymbol, codeCall } from '../db';
|
|
5
|
+
import type { CodeChunkInsert } from '../db';
|
|
6
|
+
export type CodeSymbolInsert = typeof codeSymbol.$inferInsert;
|
|
7
|
+
export type CodeCallInsert = typeof codeCall.$inferInsert;
|
|
8
|
+
export declare class IndexStore {
|
|
9
|
+
private db;
|
|
10
|
+
private config;
|
|
11
|
+
constructor(db: IDatabase, config: Config);
|
|
12
|
+
resetProject(serviceName: string): Promise<void>;
|
|
13
|
+
fileState(serviceName: string, relFilePath: string): Promise<{
|
|
14
|
+
id: string;
|
|
15
|
+
lastModified: number | null;
|
|
16
|
+
contentHash: string | null;
|
|
17
|
+
}>;
|
|
18
|
+
writeFile(serviceName: string, relFilePath: string, records: CodeChunkInsert[], edges: DepEdge[], symbolRows: CodeSymbolInsert[], calls: CodeCallInsert[]): Promise<void>;
|
|
19
|
+
refreshMtimes(serviceName: string, batch: Array<{
|
|
20
|
+
relFilePath: string;
|
|
21
|
+
mtimeMs: number;
|
|
22
|
+
}>): Promise<void>;
|
|
23
|
+
private writeDepEdges;
|
|
24
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.IndexStore = void 0;
|
|
4
|
+
const drizzle_orm_1 = require("drizzle-orm");
|
|
5
|
+
const db_1 = require("../db");
|
|
6
|
+
class IndexStore {
|
|
7
|
+
db;
|
|
8
|
+
config;
|
|
9
|
+
constructor(db, config) {
|
|
10
|
+
this.db = db;
|
|
11
|
+
this.config = config;
|
|
12
|
+
}
|
|
13
|
+
async resetProject(serviceName) {
|
|
14
|
+
await this.db.transaction(async (tx) => {
|
|
15
|
+
await tx.update(db_1.codeChunk)
|
|
16
|
+
.set({ lastModified: null, contentHash: null })
|
|
17
|
+
.where((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName));
|
|
18
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName));
|
|
19
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName));
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
async fileState(serviceName, relFilePath) {
|
|
23
|
+
const [existing] = await this.db.orm
|
|
24
|
+
.select({ id: db_1.codeChunk.id, lastModified: db_1.codeChunk.lastModified, contentHash: db_1.codeChunk.contentHash })
|
|
25
|
+
.from(db_1.codeChunk)
|
|
26
|
+
.where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)))
|
|
27
|
+
.limit(1);
|
|
28
|
+
return existing;
|
|
29
|
+
}
|
|
30
|
+
async writeFile(serviceName, relFilePath, records, edges, symbolRows, calls) {
|
|
31
|
+
await this.db.transaction(async (tx) => {
|
|
32
|
+
await tx.delete(db_1.codeChunk).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeChunk.serviceName, serviceName), (0, drizzle_orm_1.eq)(db_1.codeChunk.filePath, relFilePath)));
|
|
33
|
+
for (let i = 0; i < records.length; i += this.config.indexer.batchSize) {
|
|
34
|
+
await tx.insert(db_1.codeChunk).values(records.slice(i, i + this.config.indexer.batchSize));
|
|
35
|
+
}
|
|
36
|
+
await this.writeDepEdges(serviceName, relFilePath, edges, tx);
|
|
37
|
+
await tx.delete(db_1.codeSymbol).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeSymbol.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeSymbol.filePath, relFilePath)));
|
|
38
|
+
if (symbolRows.length > 0) {
|
|
39
|
+
await tx.insert(db_1.codeSymbol).values(symbolRows);
|
|
40
|
+
}
|
|
41
|
+
await tx.delete(db_1.codeCall).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeCall.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeCall.fromFile, relFilePath)));
|
|
42
|
+
if (calls.length > 0) {
|
|
43
|
+
await tx.insert(db_1.codeCall).values(calls);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
async refreshMtimes(serviceName, batch) {
|
|
48
|
+
await this.db.query(`UPDATE code_chunk SET "lastModified" = t.mtime
|
|
49
|
+
FROM unnest($1::text[], $2::float8[]) AS t(fp, mtime)
|
|
50
|
+
WHERE "serviceName" = $3 AND "filePath" = t.fp`, [
|
|
51
|
+
batch.map(r => r.relFilePath),
|
|
52
|
+
batch.map(r => r.mtimeMs),
|
|
53
|
+
serviceName,
|
|
54
|
+
]);
|
|
55
|
+
}
|
|
56
|
+
async writeDepEdges(serviceName, relFilePath, edges, tx) {
|
|
57
|
+
const run = async (t) => {
|
|
58
|
+
await t.delete(db_1.codeDependency).where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(db_1.codeDependency.project, serviceName), (0, drizzle_orm_1.eq)(db_1.codeDependency.fromFile, relFilePath)));
|
|
59
|
+
if (edges.length > 0) {
|
|
60
|
+
await t.insert(db_1.codeDependency).values(edges);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
if (tx)
|
|
64
|
+
await run(tx);
|
|
65
|
+
else
|
|
66
|
+
await this.db.transaction(run);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
exports.IndexStore = IndexStore;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IDatabase } from '../interfaces/IDatabase';
|
|
3
|
+
export declare class StaleFilePruner {
|
|
4
|
+
private db;
|
|
5
|
+
private config;
|
|
6
|
+
constructor(db: IDatabase, config: Config);
|
|
7
|
+
prune(serviceName: string, files: string[]): Promise<string[]>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.StaleFilePruner = void 0;
|
|
37
|
+
const path = __importStar(require("path"));
|
|
38
|
+
class StaleFilePruner {
|
|
39
|
+
db;
|
|
40
|
+
config;
|
|
41
|
+
constructor(db, config) {
|
|
42
|
+
this.db = db;
|
|
43
|
+
this.config = config;
|
|
44
|
+
}
|
|
45
|
+
async prune(serviceName, files) {
|
|
46
|
+
const collectedRelPaths = new Set(files.map(f => path.relative(this.config.workspaceRoot, f)));
|
|
47
|
+
const knownFiles = await this.db.listKnownFiles(serviceName);
|
|
48
|
+
const staleFiles = knownFiles.filter(f => !collectedRelPaths.has(f));
|
|
49
|
+
if (staleFiles.length > 0) {
|
|
50
|
+
await this.db.deleteFiles(serviceName, staleFiles);
|
|
51
|
+
}
|
|
52
|
+
return staleFiles;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
exports.StaleFilePruner = StaleFilePruner;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { CallEdge, DepEdge } from '../interfaces/IAstParser';
|
|
2
|
+
export declare function workspaceEdges(rawEdges: DepEdge[], workspaceRoot: string): DepEdge[];
|
|
3
|
+
export declare function workspaceCalls(rawCalls: CallEdge[], serviceName: string, relFilePath: string, workspaceRoot: string): CallEdge[];
|
|
4
|
+
export declare function toPromptEdges(edges: DepEdge[]): DepEdge[];
|