codevault 1.7.5 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/semantic-chunker.d.ts.map +1 -1
- package/dist/chunking/semantic-chunker.js +16 -13
- package/dist/chunking/semantic-chunker.js.map +1 -1
- package/dist/chunking/token-counter.d.ts.map +1 -1
- package/dist/chunking/token-counter.js +12 -11
- package/dist/chunking/token-counter.js.map +1 -1
- package/dist/cli/commands/index-cmd.js +2 -2
- package/dist/cli/commands/index-cmd.js.map +1 -1
- package/dist/cli.js +0 -0
- package/dist/codemap/io.d.ts +1 -0
- package/dist/codemap/io.d.ts.map +1 -1
- package/dist/codemap/io.js +8 -0
- package/dist/codemap/io.js.map +1 -1
- package/dist/config/constants.d.ts +8 -6
- package/dist/config/constants.d.ts.map +1 -1
- package/dist/config/constants.js +6 -4
- package/dist/config/constants.js.map +1 -1
- package/dist/config/loader.js +6 -6
- package/dist/config/loader.js.map +1 -1
- package/dist/config/resolver.d.ts +3 -0
- package/dist/config/resolver.d.ts.map +1 -1
- package/dist/config/resolver.js +4 -2
- package/dist/config/resolver.js.map +1 -1
- package/dist/config/types.d.ts +20 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/core/IndexerEngine.d.ts +1 -0
- package/dist/core/IndexerEngine.d.ts.map +1 -1
- package/dist/core/IndexerEngine.js +21 -7
- package/dist/core/IndexerEngine.js.map +1 -1
- package/dist/core/SearchService.js +3 -3
- package/dist/core/SearchService.js.map +1 -1
- package/dist/core/batch-indexer.d.ts +0 -4
- package/dist/core/batch-indexer.d.ts.map +1 -1
- package/dist/core/batch-indexer.js +37 -30
- package/dist/core/batch-indexer.js.map +1 -1
- package/dist/core/indexing/FileProcessor.d.ts +3 -1
- package/dist/core/indexing/FileProcessor.d.ts.map +1 -1
- package/dist/core/indexing/FileProcessor.js +17 -3
- package/dist/core/indexing/FileProcessor.js.map +1 -1
- package/dist/core/indexing/IndexFinalizationStage.d.ts +7 -1
- package/dist/core/indexing/IndexFinalizationStage.d.ts.map +1 -1
- package/dist/core/indexing/IndexFinalizationStage.js +55 -8
- package/dist/core/indexing/IndexFinalizationStage.js.map +1 -1
- package/dist/core/indexing/PersistManager.d.ts +18 -0
- package/dist/core/indexing/PersistManager.d.ts.map +1 -0
- package/dist/core/indexing/PersistManager.js +67 -0
- package/dist/core/indexing/PersistManager.js.map +1 -0
- package/dist/core/indexing/chunk-pipeline.d.ts.map +1 -1
- package/dist/core/indexing/chunk-pipeline.js +12 -8
- package/dist/core/indexing/chunk-pipeline.js.map +1 -1
- package/dist/core/metadata.d.ts +3 -2
- package/dist/core/metadata.d.ts.map +1 -1
- package/dist/core/metadata.js +2 -2
- package/dist/core/metadata.js.map +1 -1
- package/dist/core/search/HybridFusion.d.ts +4 -4
- package/dist/core/search/HybridFusion.d.ts.map +1 -1
- package/dist/core/search/HybridFusion.js +8 -8
- package/dist/core/search/HybridFusion.js.map +1 -1
- package/dist/core/search/ResultMapper.d.ts.map +1 -1
- package/dist/core/search/ResultMapper.js +4 -4
- package/dist/core/search/ResultMapper.js.map +1 -1
- package/dist/core/unified-metadata.d.ts +23 -0
- package/dist/core/unified-metadata.d.ts.map +1 -0
- package/dist/core/unified-metadata.js +58 -0
- package/dist/core/unified-metadata.js.map +1 -0
- package/dist/database/db.d.ts +4 -0
- package/dist/database/db.d.ts.map +1 -1
- package/dist/database/db.js +22 -0
- package/dist/database/db.js.map +1 -1
- package/dist/indexer/merkle.d.ts +1 -0
- package/dist/indexer/merkle.d.ts.map +1 -1
- package/dist/indexer/merkle.js +8 -0
- package/dist/indexer/merkle.js.map +1 -1
- package/dist/mcp-server.js +0 -0
- package/dist/providers/base.d.ts +1 -1
- package/dist/providers/base.d.ts.map +1 -1
- package/dist/providers/base.js +9 -4
- package/dist/providers/base.js.map +1 -1
- package/dist/providers/chat-llm.d.ts +2 -0
- package/dist/providers/chat-llm.d.ts.map +1 -1
- package/dist/providers/chat-llm.js +21 -6
- package/dist/providers/chat-llm.js.map +1 -1
- package/dist/providers/openai.d.ts +2 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +55 -10
- package/dist/providers/openai.js.map +1 -1
- package/dist/storage/encrypted-chunks.d.ts +3 -3
- package/dist/storage/encrypted-chunks.d.ts.map +1 -1
- package/dist/storage/encrypted-chunks.js +28 -22
- package/dist/storage/encrypted-chunks.js.map +1 -1
- package/dist/symbols/extract.d.ts +7 -0
- package/dist/symbols/extract.d.ts.map +1 -1
- package/dist/symbols/extract.js +3 -3
- package/dist/symbols/extract.js.map +1 -1
- package/dist/synthesis/conversational-synthesizer.js +1 -1
- package/dist/synthesis/conversational-synthesizer.js.map +1 -1
- package/dist/synthesis/synthesizer.js +1 -1
- package/dist/synthesis/synthesizer.js.map +1 -1
- package/dist/tests/semantic-chunker.test.js +18 -0
- package/dist/tests/semantic-chunker.test.js.map +1 -1
- package/dist/utils/cli-ui.d.ts +1 -1
- package/dist/utils/cli-ui.d.ts.map +1 -1
- package/dist/utils/cli-ui.js +22 -1
- package/dist/utils/cli-ui.js.map +1 -1
- package/dist/utils/indexer-with-progress.d.ts +1 -1
- package/dist/utils/indexer-with-progress.d.ts.map +1 -1
- package/dist/utils/indexer-with-progress.js +6 -1
- package/dist/utils/indexer-with-progress.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexerEngine.d.ts","sourceRoot":"","sources":["../../src/core/IndexerEngine.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"IndexerEngine.d.ts","sourceRoot":"","sources":["../../src/core/IndexerEngine.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAS1E;;;;;;GAMG;AACH,qBAAa,aAAa;IACZ,OAAO,CAAC,OAAO;gBAAP,OAAO,GAAE,mBAAwB;IAExC,KAAK,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAsEjD,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,qBAAqB;YAMf,kBAAkB;CAuBjC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
|
+
import os from 'node:os';
|
|
2
3
|
import { normalizeToProjectPath } from '../indexer/merkle.js';
|
|
3
4
|
import { FileScanner } from './indexing/file-scanner.js';
|
|
4
5
|
import { IndexContext } from './indexing/IndexContext.js';
|
|
@@ -6,6 +7,7 @@ import { IndexState } from './indexing/IndexState.js';
|
|
|
6
7
|
import { FileProcessor } from './indexing/FileProcessor.js';
|
|
7
8
|
import { IndexFinalizationStage } from './indexing/IndexFinalizationStage.js';
|
|
8
9
|
import { INDEXING_CONSTANTS } from '../config/constants.js';
|
|
10
|
+
import { PersistManager } from './indexing/PersistManager.js';
|
|
9
11
|
/**
|
|
10
12
|
* IndexerEngine orchestrates the code indexing process using a stage-based architecture:
|
|
11
13
|
* - IndexContext: Setup and initialization
|
|
@@ -41,7 +43,8 @@ export class IndexerEngine {
|
|
|
41
43
|
// Stage 1: Setup and initialization
|
|
42
44
|
const context = await IndexContext.prepare(this.options);
|
|
43
45
|
const state = new IndexState(context.codemap, context.updatedMerkle);
|
|
44
|
-
const
|
|
46
|
+
const persistManager = new PersistManager(context, state, 1500);
|
|
47
|
+
const fileProcessor = new FileProcessor(context, state, onProgress, persistManager);
|
|
45
48
|
// Stage 2: Process files
|
|
46
49
|
const concurrency = this.resolveConcurrency();
|
|
47
50
|
await this.runWithConcurrency(files, concurrency, async (rel) => {
|
|
@@ -62,17 +65,28 @@ export class IndexerEngine {
|
|
|
62
65
|
}
|
|
63
66
|
}
|
|
64
67
|
// Stage 4: Finalization
|
|
65
|
-
const finalizer = new IndexFinalizationStage(context, state, onProgress);
|
|
68
|
+
const finalizer = new IndexFinalizationStage(context, state, onProgress, persistManager);
|
|
66
69
|
return await finalizer.finalize();
|
|
67
70
|
}
|
|
68
71
|
resolveConcurrency() {
|
|
72
|
+
const { DEFAULT_CONCURRENCY, MAX_CONCURRENCY } = INDEXING_CONSTANTS;
|
|
69
73
|
const requested = this.options.concurrency;
|
|
70
|
-
const
|
|
71
|
-
const
|
|
74
|
+
const envValue = process.env.CODEVAULT_INDEXING_CONCURRENCY;
|
|
75
|
+
const parsedEnv = envValue ? Number.parseInt(envValue, 10) : null;
|
|
76
|
+
const optionValue = typeof requested === 'number' && Number.isFinite(requested)
|
|
72
77
|
? requested
|
|
73
|
-
:
|
|
74
|
-
const
|
|
75
|
-
|
|
78
|
+
: null;
|
|
79
|
+
const dynamicDefault = this.computeDynamicDefault(MAX_CONCURRENCY);
|
|
80
|
+
const fallbackDefault = Number.isFinite(DEFAULT_CONCURRENCY) ? DEFAULT_CONCURRENCY : dynamicDefault;
|
|
81
|
+
const rawValue = optionValue ?? parsedEnv ?? dynamicDefault ?? fallbackDefault;
|
|
82
|
+
const safeValue = Number.isFinite(rawValue) ? rawValue : 1;
|
|
83
|
+
const bounded = Math.min(MAX_CONCURRENCY, Math.max(1, Math.floor(safeValue)));
|
|
84
|
+
return bounded;
|
|
85
|
+
}
|
|
86
|
+
computeDynamicDefault(maxConcurrency) {
|
|
87
|
+
const cpuCount = typeof os.cpus === 'function' ? os.cpus().length : 1;
|
|
88
|
+
const calculated = Math.max(1, cpuCount * 2);
|
|
89
|
+
return Math.min(calculated, maxConcurrency);
|
|
76
90
|
}
|
|
77
91
|
async runWithConcurrency(items, concurrency, worker) {
|
|
78
92
|
if (!items.length)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexerEngine.js","sourceRoot":"","sources":["../../src/core/IndexerEngine.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"IndexerEngine.js","sourceRoot":"","sources":["../../src/core/IndexerEngine.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IACJ;IAApB,YAAoB,UAA+B,EAAE;QAAjC,YAAO,GAAP,OAAO,CAA0B;IAAG,CAAC;IAElD,KAAK,CAAC,KAAK;QAChB,MAAM,EACJ,QAAQ,GAAG,GAAG,EACd,UAAU,GAAG,IAAI,EACjB,YAAY,GAAG,IAAI,EACnB,YAAY,GAAG,EAAE,EAClB,GAAG,IAAI,CAAC,OAAO,CAAC;QAEjB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEpC,uBAAuB;QACvB,MAAM,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YACnD,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAChB,YAAY;iBACT,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,sBAAsB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;iBAC/C,MAAM,CAAC,OAAO,CAAa,CAC/B,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAC1C,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,sBAAsB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;aAC/C,MAAM,CAAC,OAAO,CAAa,CAC/B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE9C,4BAA4B;QAC5B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAExE,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,eAAe,GAAG,iBAAiB,KAAK,IAAI,CAAC;QAEnD,oCAAoC;QACpC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAChE,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;QAEpF,yBAAyB;QACzB,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9C,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAC,GAAG,EAAC,EAAE;YAC5D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,MAAM,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,gCAAgC;QAChC,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM,aAAa,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,4CAA4C;QAC5C,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YACxC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBACnC,MAAM,aAAa,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,MAAM,SAAS,GAAG,IAAI,sBAAsB,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;QACzF,OAAO,MAAM,SAAS,CAAC,QAAQ,EAAE,CAAC;IACpC,CAAC;IAEO,kBAAkB;QACxB,MAAM,EAAE,mBAAmB,EAAE,eAAe,EAAE,GAAG,kBAAkB,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAC3C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;QAC5D,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElE,MAAM,WAAW,GAAG,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC7E,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;QACnE,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,cAAc,CAAC;QAEpG,MAAM,QAAQ,GAAG,WAAW,IAAI,SAAS,IAAI,cAAc,IAAI,eAAe,CAAC;QAC/E,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9E,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,qBAAqB,CAAC,cAAsB;QAClD,MAAM,QAAQ,GAAG,OAAO,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9C,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,KAAU,EACV,WAAmB,EACnB,MAAkC;QAElC,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,OAAO;QAE1B,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAC/C,KAAK,IAAI,EAAE;YACT,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,OAAO,IAAI,KAAK,WAAW,EAAE,CAAC;oBAChC,MAAM;gBACR,CAAC;gBACD,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC,CACF,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -88,7 +88,7 @@ export class SearchService {
|
|
|
88
88
|
let bm25PrefilterResults;
|
|
89
89
|
let vectorCandidates = scopedChunks;
|
|
90
90
|
if (hybridEnabled && bm25Enabled) {
|
|
91
|
-
const prefilter = this.fusion.prefilterCandidates({
|
|
91
|
+
const prefilter = await this.fusion.prefilterCandidates({
|
|
92
92
|
hybridEnabled,
|
|
93
93
|
bm25Enabled,
|
|
94
94
|
query: normalizedQuery,
|
|
@@ -124,7 +124,7 @@ export class SearchService {
|
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
126
|
// Hybrid fusion
|
|
127
|
-
const { fusedResults, bm25Fused, bm25CandidateCount: fusedBm25CandidateCount } = this.fusion.fuseResults({
|
|
127
|
+
const { fusedResults, bm25Fused, bm25CandidateCount: fusedBm25CandidateCount } = await this.fusion.fuseResults({
|
|
128
128
|
hybridEnabled,
|
|
129
129
|
bm25Enabled,
|
|
130
130
|
selectionBudget,
|
|
@@ -228,7 +228,7 @@ export class SearchService {
|
|
|
228
228
|
const basePath = path.resolve(workingPath);
|
|
229
229
|
const chunkDir = path.join(basePath, '.codevault/chunks');
|
|
230
230
|
try {
|
|
231
|
-
const result = readChunkFromDisk({ chunkDir, sha });
|
|
231
|
+
const result = await readChunkFromDisk({ chunkDir, sha });
|
|
232
232
|
if (!result) {
|
|
233
233
|
return { success: false, error: 'Chunk not found' };
|
|
234
234
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SearchService.js","sourceRoot":"","sources":["../../src/core/SearchService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAChB,cAAc,GAAgC,IAAI,CAAC;IACnD,SAAS,CAAqB;IAC9B,MAAM,CAAe;IACrB,MAAM,CAAe;IACrB,eAAe,GAAkB,IAAI,CAAC;IAE9C;QACE,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,cAAsB,GAAG,EACzB,WAAmB,MAAM;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAEM,WAAW;QAChB,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,KAAa,EACb,QAAgB,gBAAgB,CAAC,oBAAoB,EACrD,WAAmB,MAAM,EACzB,cAAsB,GAAG,EACzB,eAA6B,EAAE;QAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC9C,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;QAClC,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC3C,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAErC,MAAM,eAAe,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,iBAAiB,GAAG,eAAe,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAC/D,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,KAAK,KAAK,CAAC;QACvD,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,KAAK,KAAK,CAAC;QACnD,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,KAAK,KAAK,CAAC;QAElE,IAAI,CAAC;YACH,mCAAmC;YACnC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAE/D,6BAA6B;YAC7B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,IAAI,CAAC,iBAAiB,CAC3B,iBAAiB,EACjB,yBAAyB,EACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAC1B,eAAe,EACf,aAAa,EACb,WAAW,EACX,kBAAkB,CACnB,CAAC;YACJ,CAAC;YAED,wBAAwB;YACxB,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,EAAE,eAAe,CAAU,CAAC;YAClE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CACjC,eAAe,EACf,gBAAgB,CAAC,oBAAoB,CACtC,CAAC;YAEF,kEAAkE;YAClE,IAAI,oBAAsE,CAAC;YAC3E,IAAI,gBAAgB,GAAG,YAAY,CAAC;YACpC,IAAI,aAAa,IAAI,WAAW,EAAE,CAAC;gBACjC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"SearchService.js","sourceRoot":"","sources":["../../src/core/SearchService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAGjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAChB,cAAc,GAAgC,IAAI,CAAC;IACnD,SAAS,CAAqB;IAC9B,MAAM,CAAe;IACrB,MAAM,CAAe;IACrB,eAAe,GAAkB,IAAI,CAAC;IAE9C;QACE,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,cAAsB,GAAG,EACzB,WAAmB,MAAM;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAEM,WAAW;QAChB,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,KAAa,EACb,QAAgB,gBAAgB,CAAC,oBAAoB,EACrD,WAAmB,MAAM,EACzB,cAAsB,GAAG,EACzB,eAA6B,EAAE;QAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC9C,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,cAAc,GAAG,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;QAClC,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC3C,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAErC,MAAM,eAAe,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,iBAAiB,GAAG,eAAe,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAC/D,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,KAAK,KAAK,CAAC;QACvD,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,KAAK,KAAK,CAAC;QACnD,MAAM,kBAAkB,GAAG,eAAe,CAAC,YAAY,KAAK,KAAK,CAAC;QAElE,IAAI,CAAC;YACH,mCAAmC;YACnC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAE/D,6BAA6B;YAC7B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAEvD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,IAAI,CAAC,iBAAiB,CAC3B,iBAAiB,EACjB,yBAAyB,EACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAC1B,eAAe,EACf,aAAa,EACb,WAAW,EACX,kBAAkB,CACnB,CAAC;YACJ,CAAC;YAED,wBAAwB;YACxB,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,EAAE,eAAe,CAAU,CAAC;YAClE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CACjC,eAAe,EACf,gBAAgB,CAAC,oBAAoB,CACtC,CAAC;YAEF,kEAAkE;YAClE,IAAI,oBAAsE,CAAC;YAC3E,IAAI,gBAAgB,GAAG,YAAY,CAAC;YACpC,IAAI,aAAa,IAAI,WAAW,EAAE,CAAC;gBACjC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC;oBACtD,aAAa;oBACb,WAAW;oBACX,KAAK,EAAE,eAAe;oBACtB,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE;oBACxC,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE;oBACpD,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,QAAQ;oBACR,YAAY;oBACZ,KAAK,EAAE,kBAAkB;iBAC1B,CAAC,CAAC;gBAEH,oBAAoB,GAAG,SAAS,CAAC,WAAW,CAAC;gBAC7C,IAAI,SAAS,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC3C,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CACrC,CAAC;oBACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACxB,gBAAgB,GAAG,QAAQ,CAAC;oBAC9B,CAAC;gBACH,CAAC;YACH,CAAC;YAED,oBAAoB;YACpB,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CACxE,gBAAgB,EAChB,OAAO,CAAC,QAAQ,EAChB,eAAe,CAChB,CAAC;YAEF,qBAAqB;YACrB,IAAI,kBAAkB,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,gBAAgB,CAAC,UAAU,EAAE;wBAC3B,KAAK,EAAE,eAAe;wBACtB,OAAO,EAAE,OAAO,CAAC,OAAO;qBACzB,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE;wBAC3D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;qBAC9D,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,MAAM,EACJ,YAAY,EACZ,SAAS,EACT,kBAAkB,EAAE,uBAAuB,EAC5C,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBAChC,aAAa;gBACb,WAAW;gBACX,eAAe;gBACf,KAAK,EAAE,eAAe;gBACtB,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE;gBACxC,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE;gBACpD,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ;gBACR,YAAY,EAAE,gBAAgB;gBAC9B,aAAa;gBACb,UAAU;gBACV,mBAAmB,EAAE,oBAAoB;aAC1C,CAAC,CAAC;YACH,MAAM,kBAAkB,GACtB,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC;gBACrD,CAAC,CAAC,oBAAoB,CAAC,MAAM;gBAC7B,CAAC,CAAC,uBAAuB,CAAC;YAE9B,0BAA0B;YAC1B,IAAI,OAAO,GACT,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;YAChF,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;YACvE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAElC,sBAAsB;YACtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;gBAC7D,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CACvC,eAAe,EACf,OAAO,EACP,OAAO,CAAC,QAAQ,EAChB,QAAQ,EACR,OAAO,CAAC,eAAe,CACxB,CAAC;YACJ,CAAC;YAED,uBAAuB;YACvB,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAElD,wBAAwB;YACxB,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAC1C,OAAO,EACP,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAChC,CAAC;YAEF,gBAAgB;YAChB,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;YAE7D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,IAAI,CAAC,iBAAiB,CAC3B,qBAAqB,EACrB,kCAAkC,KAAK,GAAG,EAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAC1B,eAAe,EACf,aAAa,EACb,WAAW,EACX,kBAAkB,CACnB,CAAC;YACJ,CAAC;YAED,qCAAqC;YACrC,IACE,kBAAkB;gBAClB,aAAa,CAAC,MAAM,GAAG,CAAC;gBACxB,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,EACjC,CAAC;gBACD,MAAM,OAAO,CAAC,EAAE,CAAC,eAAe,CAC9B,eAAe,EACf,KAAK,EACL,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,EACpB,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAC5B,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,eAAe;iBAC5B,OAAO,CAAC,qBAAqB,EAAE,WAAW,CAAC;iBAC3C,OAAO,CAAC,cAAc,EAAE,oBAAoB,CAAC;iBAC7C,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC;iBACxC,OAAO,CAAC,qBAAqB,EAAE,cAAc,CAAC;iBAC9C,IAAI,EAAE,CAAC;YAEV,MAAM,OAAO,CAAC,EAAE,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAE7C,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,KAAK;gBACL,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;gBAC3C,aAAa,EAAE,OAAO,CAAC,MAAM;gBAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE;gBACpC,KAAK,EAAE,eAAe;gBACtB,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,MAAM,EAAE;oBACN,OAAO,EAAE,aAAa;oBACtB,WAAW;oBACX,KAAK,EAAE,SAAS;oBAChB,cAAc,EAAE,kBAAkB;iBACnC;gBACD,WAAW,EAAE;oBACX,OAAO,EAAE,kBAAkB;oBAC3B,OAAO,EACL,kBAAkB;wBAClB,OAAO,CAAC,IAAI,CACV,CAAC,MAAW,EAAE,EAAE,CACd,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,CACnE;iBACJ;gBACD,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE;gBAC3D,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;gBACrC,OAAO,EAAE,aAAa;aACvB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAC3C,OAAO,IAAI,CAAC,iBAAiB,CAC3B,cAAc,EACb,KAAe,CAAC,OAAO,EACxB,QAAQ,EACR,eAAe,EACf,aAAa,EACb,WAAW,EACX,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,EAAE,cAAsB,GAAG;QACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;YAC3H,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACjD,EAAE,CAAC,KAAK,EAAE,CAAC;YAEX,MAAM,OAAO,GAAmB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACnD,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,SAAS;gBACrB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,IAAI;gBACV,IAAI,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE;aACzD,CAAC,CAAC,CAAC;YAEJ,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAG,KAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC3H,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,WAAW,GAAG,GAAG;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;QAE1D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;YACtD,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,yBAAyB,EAAE,CAAC;gBACtD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,yDAAyD,EAAE,CAAC;YAC9F,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QAClD,CAAC;IACH,CAAC;IAED,UAAU;IAEF,iBAAiB,CAAC,KAAa,EAAE,OAAe,EAAE,QAAgB,EAAE,KAAU,EAAE,MAAe,EAAE,IAAa,EAAE,WAAoB;QAC1I,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE;YAC9C,WAAW,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE;YACrD,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,kBAAkB;IAEV,cAAc,CAAC,KAAa;QAClC,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9E,CAAC;CACF"}
|
|
@@ -36,10 +36,6 @@ export declare class BatchEmbeddingProcessor {
|
|
|
36
36
|
* Process any remaining chunks in the batch
|
|
37
37
|
*/
|
|
38
38
|
flush(): Promise<void>;
|
|
39
|
-
/**
|
|
40
|
-
* Process the current batch (with mutex protection)
|
|
41
|
-
*/
|
|
42
|
-
private processBatch;
|
|
43
39
|
/**
|
|
44
40
|
* Process a batch with smart error handling and retry logic
|
|
45
41
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-indexer.d.ts","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AA8BlD,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,iBAAiB,EAAE,GAAG,CAAC;QACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;QAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,WAAW,EAAE,GAAG,CAAC;KAClB,CAAC;CACH;AAED;;;;GAIG;AACH,qBAAa,uBAAuB;IAMhC,OAAO,CAAC,iBAAiB;IACzB,OAAO,CAAC,EAAE;IANZ,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAe;gBAGlB,iBAAiB,EAAE,iBAAiB,EACpC,EAAE,EAAE,QAAQ,EACpB,SAAS,GAAE,MAAmB;IAKhC;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"batch-indexer.d.ts","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AA8BlD,UAAU,YAAY;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,iBAAiB,EAAE,GAAG,CAAC;QACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;QAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,WAAW,EAAE,GAAG,CAAC;KAClB,CAAC;CACH;AAED;;;;GAIG;AACH,qBAAa,uBAAuB;IAMhC,OAAO,CAAC,iBAAiB;IACzB,OAAO,CAAC,EAAE;IANZ,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAe;gBAGlB,iBAAiB,EAAE,iBAAiB,EACpC,EAAE,EAAE,QAAQ,EACpB,SAAS,GAAE,MAAmB;IAKhC;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAYlD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B;;OAEG;YACW,qBAAqB;IAkCnC;;OAEG;YACW,oBAAoB;IAsDlC;;OAEG;YACW,8BAA8B;IA4C5C;;OAEG;IACH,aAAa,IAAI,MAAM;CAGxB"}
|
|
@@ -40,27 +40,19 @@ export class BatchEmbeddingProcessor {
|
|
|
40
40
|
* Add a chunk to the batch queue
|
|
41
41
|
*/
|
|
42
42
|
async addChunk(chunk) {
|
|
43
|
-
this.batch.push(chunk);
|
|
44
|
-
// Process batch when it reaches the threshold
|
|
45
|
-
if (this.batch.length >= this.batchSize) {
|
|
46
|
-
await this.processBatch();
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Process any remaining chunks in the batch
|
|
51
|
-
*/
|
|
52
|
-
async flush() {
|
|
53
43
|
await this.mutex.runExclusive(async () => {
|
|
54
|
-
|
|
44
|
+
this.batch.push(chunk);
|
|
45
|
+
// Process batch when it reaches the threshold
|
|
46
|
+
if (this.batch.length >= this.batchSize) {
|
|
55
47
|
await this.processBatchWithRetry(this.batch, 0);
|
|
56
48
|
this.batch = [];
|
|
57
49
|
}
|
|
58
50
|
});
|
|
59
51
|
}
|
|
60
52
|
/**
|
|
61
|
-
* Process
|
|
53
|
+
* Process any remaining chunks in the batch
|
|
62
54
|
*/
|
|
63
|
-
async
|
|
55
|
+
async flush() {
|
|
64
56
|
await this.mutex.runExclusive(async () => {
|
|
65
57
|
if (this.batch.length > 0) {
|
|
66
58
|
await this.processBatchWithRetry(this.batch, 0);
|
|
@@ -118,23 +110,38 @@ export class BatchEmbeddingProcessor {
|
|
|
118
110
|
const embeddings = await this.embeddingProvider.generateEmbeddings(texts);
|
|
119
111
|
log.debug(`Batch complete (${texts.length} embeddings generated)`);
|
|
120
112
|
// Store all embeddings in database within a transaction
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
113
|
+
const dbParams = batch.map((chunk, i) => {
|
|
114
|
+
const embedding = embeddings[i];
|
|
115
|
+
// Validate each embedding before insertion
|
|
116
|
+
if (!embedding || !Array.isArray(embedding) || embedding.length === 0) {
|
|
117
|
+
log.error(`Invalid embedding at index ${i} for chunk ${chunk.chunkId}`, {
|
|
118
|
+
embeddingType: typeof embedding,
|
|
119
|
+
isArray: Array.isArray(embedding),
|
|
120
|
+
length: embedding?.length,
|
|
121
|
+
embeddingsArrayLength: embeddings.length,
|
|
122
|
+
batchLength: batch.length
|
|
123
|
+
});
|
|
124
|
+
throw new Error(`Invalid embedding at index ${i}: type=${typeof embedding}, isArray=${Array.isArray(embedding)}, length=${embedding?.length}`);
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
id: chunk.chunkId,
|
|
128
|
+
file_path: chunk.params.rel,
|
|
129
|
+
symbol: chunk.params.symbol,
|
|
130
|
+
sha: chunk.params.sha,
|
|
131
|
+
lang: chunk.params.lang,
|
|
132
|
+
chunk_type: chunk.params.chunkType,
|
|
133
|
+
embedding: embedding,
|
|
134
|
+
embedding_provider: this.embeddingProvider.getName(),
|
|
135
|
+
embedding_dimensions: this.embeddingProvider.getDimensions(),
|
|
136
|
+
codevault_tags: chunk.params.codevaultMetadata.tags,
|
|
137
|
+
codevault_intent: chunk.params.codevaultMetadata.intent,
|
|
138
|
+
codevault_description: chunk.params.codevaultMetadata.description,
|
|
139
|
+
doc_comments: chunk.params.docComments,
|
|
140
|
+
variables_used: chunk.params.importantVariables,
|
|
141
|
+
context_info: chunk.params.contextInfo
|
|
142
|
+
};
|
|
143
|
+
});
|
|
144
|
+
this.db.insertChunks(dbParams);
|
|
138
145
|
}
|
|
139
146
|
/**
|
|
140
147
|
* Fall back to processing chunks individually
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAEzC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD;;;;GAIG;AACH,MAAM,OAAO,uBAAuB;IAMxB;IACA;IANF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAClB,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE5B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,
|
|
1
|
+
{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAEzC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD;;;;GAIG;AACH,MAAM,OAAO,uBAAuB;IAMxB;IACA;IANF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAClB,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE5B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEvB,8CAA8C;YAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,YAA4B,EAAE,UAAkB;QAClF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,4CAA4C;gBAC5C,GAAG,CAAC,IAAI,CAAC,yBAAyB,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACzF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,kCAAkC;gBAClC,MAAM,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;iBAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,iBAAiB,EAAE,CAAC;gBACrE,yCAAyC;gBACzC,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,GAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,eAAe,UAAU,GAAG,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;gBAC/D,OAAO;YACT,CAAC;YAED,2EAA2E;YAC3E,GAAG,CAAC,KAAK,CAAC,+BAA+B,YAAY,CAAC,MAAM,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YAExE,MAAM,IAAI,CAAC,8BAA8B,CAAC,YAAY,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,KAAqB;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,8BAA8B;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE9D,gEAAgE;QAChE,GAAG,CAAC,KAAK,CAAC,uBAAuB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAExD,yDAAyD;QACzD,4EAA4E;QAC5E,4CAA4C;QAC5C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE1E,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAEnE,wDAAwD;QACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAEhC,2CAA2C;YAC3C,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtE,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,cAAc,KAAK,CAAC,OAAO,EAAE,EAAE;oBACtE,aAAa,EAAE,OAAO,SAAS;oBAC/B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;oBACjC,MAAM,EAAE,SAAS,EAAE,MAAM;oBACzB,qBAAqB,EAAE,UAAU,CAAC,MAAM;oBACxC,WAAW,EAAE,KAAK,CAAC,MAAM;iBAC1B,CAAC,CAAC;gBACH,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,UAAU,OAAO,SAAS,aAAa,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,YAAY,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YACjJ,CAAC;YAED,OAAO;gBACL,EAAE,EAAE,KAAK,CAAC,OAAO;gBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;gBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;gBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;gBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;gBAClC,SAAS,EAAE,SAAS;gBACpB,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;gBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;gBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;gBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;gBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;gBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;gBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;gBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;aACvC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAAC,KAAqB;QAChE,uEAAuE;QACvE,MAAM,MAAM,GAA6C,EAAE,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAE9F,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,GAAG,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;gBACvE,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,eAAwB;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,uCAAuC,CAAC,CAAC;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,kCAAkC;gBAClC,MAAM,IAAI,KAAK,CAAC,OAAO,MAAM,CAAC,MAAM,8BAA8B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { IndexContextData } from './IndexContext.js';
|
|
2
2
|
import type { IndexState } from './IndexState.js';
|
|
3
|
+
import { PersistManager } from './PersistManager.js';
|
|
3
4
|
/**
|
|
4
5
|
* FileProcessor handles the processing of individual files:
|
|
5
6
|
* - Parsing and chunking
|
|
@@ -11,13 +12,14 @@ export declare class FileProcessor {
|
|
|
11
12
|
private context;
|
|
12
13
|
private state;
|
|
13
14
|
private onProgress;
|
|
15
|
+
private persistManager;
|
|
14
16
|
private chunkPipeline;
|
|
15
17
|
constructor(context: IndexContextData, state: IndexState, onProgress: ((event: {
|
|
16
18
|
type: string;
|
|
17
19
|
file?: string;
|
|
18
20
|
symbol?: string;
|
|
19
21
|
chunkId?: string;
|
|
20
|
-
}) => void) | null);
|
|
22
|
+
}) => void) | null, persistManager: PersistManager);
|
|
21
23
|
/**
|
|
22
24
|
* Process a single file
|
|
23
25
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileProcessor.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/FileProcessor.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"FileProcessor.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/FileProcessor.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAIlD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD;;;;;;GAMG;AACH,qBAAa,aAAa;IAItB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,cAAc;IANxB,OAAO,CAAC,aAAa,CAAuB;gBAGlC,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,EACxG,cAAc,EAAE,cAAc;IAGxC;;OAEG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6E7C;;OAEG;YACW,qBAAqB;IAuFnC;;OAEG;YACW,aAAa;IA8E3B;;OAEG;YACW,YAAY;IAsB1B;;OAEG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAkB1D"}
|
|
@@ -17,11 +17,13 @@ export class FileProcessor {
|
|
|
17
17
|
context;
|
|
18
18
|
state;
|
|
19
19
|
onProgress;
|
|
20
|
+
persistManager;
|
|
20
21
|
chunkPipeline = new ChunkPipeline();
|
|
21
|
-
constructor(context, state, onProgress) {
|
|
22
|
+
constructor(context, state, onProgress, persistManager) {
|
|
22
23
|
this.context = context;
|
|
23
24
|
this.state = state;
|
|
24
25
|
this.onProgress = onProgress;
|
|
26
|
+
this.persistManager = persistManager;
|
|
25
27
|
}
|
|
26
28
|
/**
|
|
27
29
|
* Process a single file
|
|
@@ -54,6 +56,7 @@ export class FileProcessor {
|
|
|
54
56
|
if (staleChunkIds.size > 0) {
|
|
55
57
|
await this.deleteChunks(Array.from(staleChunkIds), existingChunks);
|
|
56
58
|
this.state.markIndexMutated();
|
|
59
|
+
this.persistManager.scheduleCodemapSave();
|
|
57
60
|
}
|
|
58
61
|
// Update merkle tree
|
|
59
62
|
if (fileHash) {
|
|
@@ -62,6 +65,7 @@ export class FileProcessor {
|
|
|
62
65
|
chunkShas: chunkMerkleHashes
|
|
63
66
|
};
|
|
64
67
|
this.state.markMerkleDirty();
|
|
68
|
+
this.persistManager.scheduleMerkleSave();
|
|
65
69
|
}
|
|
66
70
|
}
|
|
67
71
|
catch (error) {
|
|
@@ -121,6 +125,7 @@ export class FileProcessor {
|
|
|
121
125
|
});
|
|
122
126
|
this.state.incrementProcessedChunks();
|
|
123
127
|
this.state.markIndexMutated();
|
|
128
|
+
this.persistManager.scheduleCodemapSave();
|
|
124
129
|
if (this.onProgress) {
|
|
125
130
|
this.onProgress({ type: 'chunk_processed', file: rel, symbol: fallbackSymbol, chunkId });
|
|
126
131
|
}
|
|
@@ -128,6 +133,7 @@ export class FileProcessor {
|
|
|
128
133
|
if (staleChunkIds.size > 0) {
|
|
129
134
|
await this.deleteChunks(Array.from(staleChunkIds), existingChunks);
|
|
130
135
|
this.state.markIndexMutated();
|
|
136
|
+
this.persistManager.scheduleCodemapSave();
|
|
131
137
|
}
|
|
132
138
|
chunkMerkleHashes.length = 0;
|
|
133
139
|
chunkMerkleHashes.push(chunkMerkleHash);
|
|
@@ -136,6 +142,7 @@ export class FileProcessor {
|
|
|
136
142
|
chunkShas: [...chunkMerkleHashes]
|
|
137
143
|
};
|
|
138
144
|
this.state.markMerkleDirty();
|
|
145
|
+
this.persistManager.scheduleMerkleSave();
|
|
139
146
|
}
|
|
140
147
|
catch (fallbackError) {
|
|
141
148
|
this.state.addError({
|
|
@@ -170,8 +177,9 @@ export class FileProcessor {
|
|
|
170
177
|
}
|
|
171
178
|
});
|
|
172
179
|
this.state.markIndexMutated();
|
|
180
|
+
this.persistManager.scheduleCodemapSave();
|
|
173
181
|
await fs.promises.mkdir(this.context.chunkDir, { recursive: true });
|
|
174
|
-
const writeResult = writeChunkToDisk({
|
|
182
|
+
const writeResult = await writeChunkToDisk({
|
|
175
183
|
chunkDir: this.context.chunkDir,
|
|
176
184
|
sha: params.sha,
|
|
177
185
|
code: params.code,
|
|
@@ -196,6 +204,7 @@ export class FileProcessor {
|
|
|
196
204
|
symbol_return: params.symbolData && params.symbolData.returnType ? params.symbolData.returnType : undefined,
|
|
197
205
|
symbol_calls: params.symbolData && Array.isArray(params.symbolData.calls) ? params.symbolData.calls : undefined
|
|
198
206
|
}, previousMetadata);
|
|
207
|
+
this.persistManager.scheduleCodemapSave();
|
|
199
208
|
}
|
|
200
209
|
catch (error) {
|
|
201
210
|
this.state.addError({
|
|
@@ -219,10 +228,13 @@ export class FileProcessor {
|
|
|
219
228
|
for (const chunkId of chunkIds) {
|
|
220
229
|
const metadata = metadataLookup.get(chunkId) || this.state.codemap[chunkId];
|
|
221
230
|
if (metadata && metadata.sha) {
|
|
222
|
-
removeChunkArtifacts(this.context.chunkDir, metadata.sha);
|
|
231
|
+
await removeChunkArtifacts(this.context.chunkDir, metadata.sha);
|
|
223
232
|
}
|
|
224
233
|
delete this.state.codemap[chunkId];
|
|
225
234
|
}
|
|
235
|
+
if (chunkIds.length > 0) {
|
|
236
|
+
this.persistManager.scheduleCodemapSave();
|
|
237
|
+
}
|
|
226
238
|
}
|
|
227
239
|
/**
|
|
228
240
|
* Remove all artifacts for a deleted file
|
|
@@ -234,11 +246,13 @@ export class FileProcessor {
|
|
|
234
246
|
const metadataLookup = new Map(entries);
|
|
235
247
|
await this.deleteChunks(entries.map(([chunkId]) => chunkId), metadataLookup);
|
|
236
248
|
this.state.markIndexMutated();
|
|
249
|
+
this.persistManager.scheduleCodemapSave();
|
|
237
250
|
}
|
|
238
251
|
// Remove from merkle tree
|
|
239
252
|
const { removeMerkleEntry } = await import('../../indexer/merkle.js');
|
|
240
253
|
if (removeMerkleEntry(this.state.updatedMerkle, fileRel)) {
|
|
241
254
|
this.state.markMerkleDirty();
|
|
255
|
+
this.persistManager.scheduleMerkleSave();
|
|
242
256
|
}
|
|
243
257
|
}
|
|
244
258
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileProcessor.js","sourceRoot":"","sources":["../../../src/core/indexing/FileProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,OAAO,EAAC,sBAAsB,EAAC,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,gBAAgB,EAAE,oBAAoB,EAAC,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"FileProcessor.js","sourceRoot":"","sources":["../../../src/core/indexing/FileProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,OAAO,EAAC,sBAAsB,EAAC,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,gBAAgB,EAAE,oBAAoB,EAAC,MAAM,mCAAmC,CAAC;AAGzF;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAId;IACA;IACA;IACA;IANF,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;IAE5C,YACU,OAAyB,EACzB,KAAiB,EACjB,UAAwG,EACxG,cAA8B;QAH9B,YAAO,GAAP,OAAO,CAAkB;QACzB,UAAK,GAAL,KAAK,CAAY;QACjB,eAAU,GAAV,UAAU,CAA8F;QACxG,mBAAc,GAAd,cAAc,CAAgB;IACrC,CAAC;IAEJ;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;aAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,IAAK,QAAgB,CAAC,IAAI,KAAK,GAAG,CAAoB,CAC3F,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,MAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,IAAI,QAAQ,GAAkB,IAAI,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACvD,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEzC,8BAA8B;YAC9B,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,cAAc,IAAI,cAAc,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC1D,OAAO;YACT,CAAC;YAED,0BAA0B;YAC1B,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAClF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CACpD,cAAc,EACd,MAAM,EACN,IAAI,CAAC,OAAO,CAAC,YAAY,EACzB,IAAI,CACL,CAAC;YAEF,iBAAiB;YACjB,MAAM,IAAI,CAAC,aAAa,CAAC,aAAa,CACpC,UAAU,EACV,MAAM,EACN,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,MAAM,EACnB,IAAI,CAAC,OAAO,CAAC,YAAY,EACzB,GAAG,EACH,EAAE,aAAa,EAAE,cAAc,EAAE,EACjC,iBAAiB,EACjB,IAAI,CAAC,UAAU,EACf,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EACtC,IAAI,CAAC,KAAK,CAAC,aAAa,CACzB,CAAC;YAEF,sBAAsB;YACtB,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC,CAAC;gBACnE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAC5C,CAAC;YAED,qBAAqB;YACrB,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG;oBAC9B,OAAO,EAAE,QAAQ;oBACjB,SAAS,EAAE,iBAAiB;iBAC7B,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC7B,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,kBAAkB;gBACxB,IAAI,EAAE,GAAG;gBACT,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;YAEH,0BAA0B;YAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,GAAW,EACX,GAAW,EACX,IAAyC,EACzC,cAAgC,EAChC,aAA0B,EAC1B,iBAA2B;QAE3B,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;YAED,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;YACjD,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,GAAG,GAAG,aAAa,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEtD,MAAM,gBAAgB,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACvE,MAAM,WAAW,GAAG;gBAClB,QAAQ,EAAE,MAAM;gBAChB,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAClC,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,gBAAgB,EAAE,KAAK;gBACvB,aAAa,EAAE,CAAC;aACjB,CAAC;YAEF,MAAM,IAAI,CAAC,aAAa,CAAC;gBACvB,IAAI,EAAE,MAAM;gBACZ,qBAAqB,EAAE,MAAM;gBAC7B,OAAO;gBACP,GAAG;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,GAAG;gBACH,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,MAAM;gBACjB,iBAAiB,EAAE,gBAAgB;gBACnC,kBAAkB,EAAE,EAAE;gBACtB,WAAW,EAAE,IAAI;gBACjB,WAAW;gBACX,UAAU,EAAE;oBACV,SAAS,EAAE,GAAG,cAAc,IAAI;oBAChC,UAAU,EAAE,EAAE;oBACd,UAAU,EAAE,IAAI;oBAChB,KAAK,EAAE,EAAE;iBACV;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;YACtC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAE1C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;YAED,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC9B,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC,CAAC;gBACnE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;gBAC9B,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAC5C,CAAC;YAED,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7B,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAExC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG;gBAC9B,OAAO,EAAE,eAAe;gBACxB,SAAS,EAAE,CAAC,GAAG,iBAAiB,CAAC;aAClC,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;YAC7B,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,CAAC;QAC3C,CAAC;QAAC,OAAO,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,GAAG;gBACT,KAAK,EAAG,aAAuB,CAAC,OAAO;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,MAc3B;QACC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;gBACzC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;gBACnD,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;oBAC3C,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;oBAC7C,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAE1C,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC;gBACzC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAC/B,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;aAC9C,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,sBAAsB,CAAC;gBAC1D,IAAI,EAAE,MAAM,CAAC,GAAG;gBAChB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACjD,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,aAAa,EAAE;gBACzD,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBAC1G,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;gBAC5C,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,WAAW;gBACtC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC9F,SAAS,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC,SAAS,CAAC;gBACnD,gBAAgB,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;gBAC5G,iBAAiB,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC9H,aAAa,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC3G,YAAY,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aAChH,EAAE,gBAAgB,CAAC,CAAC;YACrB,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,QAAkB,EAAE,iBAAiB,IAAI,GAAG,EAAe;QACpF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC/C,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC5E,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QAC5C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,OAAe;QACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;aAC/C,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,IAAK,QAAgB,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;QAE5E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,OAA0B,CAAC,CAAC;YAC3D,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YAC7E,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE,CAAC;QAC5C,CAAC;QAED,0BAA0B;QAC1B,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;QACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzD,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;YAC7B,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,CAAC;QAC3C,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { IndexContextData } from './IndexContext.js';
|
|
2
2
|
import type { IndexState } from './IndexState.js';
|
|
3
3
|
import type { IndexProjectResult } from '../types.js';
|
|
4
|
+
import { PersistManager } from './PersistManager.js';
|
|
4
5
|
/**
|
|
5
6
|
* IndexFinalizationStage handles the finalization of the indexing process:
|
|
6
7
|
* - Flushing batch processor
|
|
@@ -14,7 +15,8 @@ export declare class IndexFinalizationStage {
|
|
|
14
15
|
private context;
|
|
15
16
|
private state;
|
|
16
17
|
private onProgress;
|
|
17
|
-
|
|
18
|
+
private persistManager;
|
|
19
|
+
constructor(context: IndexContextData, state: IndexState, onProgress: ((event: any) => void) | null, persistManager: PersistManager);
|
|
18
20
|
/**
|
|
19
21
|
* Finalize the indexing process
|
|
20
22
|
*/
|
|
@@ -31,6 +33,10 @@ export declare class IndexFinalizationStage {
|
|
|
31
33
|
* Build the final result object
|
|
32
34
|
*/
|
|
33
35
|
private buildResult;
|
|
36
|
+
/**
|
|
37
|
+
* Remove orphaned chunks whose source files no longer exist
|
|
38
|
+
*/
|
|
39
|
+
private cleanupOrphanedChunks;
|
|
34
40
|
/**
|
|
35
41
|
* Clean up resources
|
|
36
42
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexFinalizationStage.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/IndexFinalizationStage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"IndexFinalizationStage.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/IndexFinalizationStage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAIrD;;;;;;;;GAQG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,cAAc;gBAHd,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,GAAG,IAAI,EACzC,cAAc,EAAE,cAAc;IAGxC;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,kBAAkB,CAAC;IA2C7C;;OAEG;YACW,mBAAmB;IAajC;;OAEG;IACH,OAAO,CAAC,aAAa;IAUrB;;OAEG;IACH,OAAO,CAAC,WAAW;IAYnB;;OAEG;YACW,qBAAqB;IAsCnC;;OAEG;YACW,OAAO;CAatB"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { saveMerkleAsync } from '../../indexer/merkle.js';
|
|
2
|
+
import { writeCodemapAsync } from '../../codemap/io.js';
|
|
3
3
|
import { attachSymbolGraphToCodemap } from '../../symbols/graph.js';
|
|
4
4
|
import { getTokenCountStats } from '../../chunking/token-counter.js';
|
|
5
5
|
import { logger } from '../../utils/logger.js';
|
|
6
|
+
import fs from 'fs';
|
|
7
|
+
import path from 'path';
|
|
6
8
|
/**
|
|
7
9
|
* IndexFinalizationStage handles the finalization of the indexing process:
|
|
8
10
|
* - Flushing batch processor
|
|
@@ -16,10 +18,12 @@ export class IndexFinalizationStage {
|
|
|
16
18
|
context;
|
|
17
19
|
state;
|
|
18
20
|
onProgress;
|
|
19
|
-
|
|
21
|
+
persistManager;
|
|
22
|
+
constructor(context, state, onProgress, persistManager) {
|
|
20
23
|
this.context = context;
|
|
21
24
|
this.state = state;
|
|
22
25
|
this.onProgress = onProgress;
|
|
26
|
+
this.persistManager = persistManager;
|
|
23
27
|
}
|
|
24
28
|
/**
|
|
25
29
|
* Finalize the indexing process
|
|
@@ -32,13 +36,21 @@ export class IndexFinalizationStage {
|
|
|
32
36
|
}
|
|
33
37
|
// Flush any remaining embeddings
|
|
34
38
|
await this.flushBatchProcessor();
|
|
35
|
-
// Save merkle tree if modified
|
|
36
|
-
if (this.state.merkleDirty) {
|
|
37
|
-
saveMerkle(this.context.repo, this.state.updatedMerkle);
|
|
38
|
-
}
|
|
39
39
|
// Build symbol graph and write codemap
|
|
40
40
|
attachSymbolGraphToCodemap(this.state.codemap);
|
|
41
|
-
this.state.
|
|
41
|
+
this.state.markIndexMutated();
|
|
42
|
+
await this.cleanupOrphanedChunks();
|
|
43
|
+
// Persist any pending data (debounced during processing)
|
|
44
|
+
await this.persistManager.flush();
|
|
45
|
+
// Final guard: ensure codemap and merkle are written
|
|
46
|
+
if (this.state.indexMutated) {
|
|
47
|
+
this.state.codemap = await writeCodemapAsync(this.context.codemapPath, this.state.codemap);
|
|
48
|
+
this.state.indexMutated = false;
|
|
49
|
+
}
|
|
50
|
+
if (this.state.merkleDirty) {
|
|
51
|
+
await saveMerkleAsync(this.context.repo, this.state.updatedMerkle);
|
|
52
|
+
this.state.merkleDirty = false;
|
|
53
|
+
}
|
|
42
54
|
// Get token statistics
|
|
43
55
|
const tokenStats = getTokenCountStats();
|
|
44
56
|
// Log statistics
|
|
@@ -93,6 +105,41 @@ export class IndexFinalizationStage {
|
|
|
93
105
|
tokenStats: this.context.modelProfile.useTokens ? tokenStats : undefined
|
|
94
106
|
};
|
|
95
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Remove orphaned chunks whose source files no longer exist
|
|
110
|
+
*/
|
|
111
|
+
async cleanupOrphanedChunks() {
|
|
112
|
+
if (!this.context.db)
|
|
113
|
+
return;
|
|
114
|
+
const paths = this.context.db.getAllFilePaths();
|
|
115
|
+
if (!Array.isArray(paths) || paths.length === 0)
|
|
116
|
+
return;
|
|
117
|
+
const base = this.context.repo;
|
|
118
|
+
const orphaned = [];
|
|
119
|
+
for (const rel of paths) {
|
|
120
|
+
const full = path.join(base, rel);
|
|
121
|
+
if (!fs.existsSync(full)) {
|
|
122
|
+
orphaned.push(rel);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (orphaned.length === 0) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
logger.info(`Removing ${orphaned.length} orphaned files from index`);
|
|
129
|
+
for (const rel of orphaned) {
|
|
130
|
+
this.context.db.deleteChunksByFilePath(rel);
|
|
131
|
+
for (const [chunkId, meta] of Object.entries(this.state.codemap)) {
|
|
132
|
+
if (meta?.file === rel) {
|
|
133
|
+
delete this.state.codemap[chunkId];
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
delete this.state.updatedMerkle[rel];
|
|
137
|
+
}
|
|
138
|
+
this.state.markIndexMutated();
|
|
139
|
+
this.state.markMerkleDirty();
|
|
140
|
+
this.persistManager.scheduleCodemapSave();
|
|
141
|
+
this.persistManager.scheduleMerkleSave();
|
|
142
|
+
}
|
|
96
143
|
/**
|
|
97
144
|
* Clean up resources
|
|
98
145
|
*/
|