codevault 1.7.3 → 1.7.4

package/dist/core/indexing/IndexContext.js

@@ -0,0 +1,133 @@
+import path from 'path';
+import fs from 'fs';
+import { createEmbeddingProvider, getModelProfile, getSizeLimits } from '../../providers/index.js';
+import { BATCH_SIZE } from '../../providers/base.js';
+import { readCodemap } from '../../codemap/io.js';
+import { loadMerkle, cloneMerkle } from '../../indexer/merkle.js';
+import { resolveEncryptionPreference } from '../../storage/encrypted-chunks.js';
+import { Database, initDatabase } from '../../database/db.js';
+import { BatchEmbeddingProcessor } from '../batch-indexer.js';
+import { logger } from '../../utils/logger.js';
+import { resolveProviderContext } from '../../config/resolver.js';
+/**
+ * IndexContext prepares the indexing environment by:
+ * - Validating repository
+ * - Initializing embedding provider
+ * - Setting up database
+ * - Loading codemap and merkle tree
+ * - Creating batch processor
+ */
+export class IndexContext {
+    static async prepare(options) {
+        const { repoPath = '.', provider = 'auto', changedFiles = null, embeddingProviderOverride = null, encryptMode = undefined } = options;
+        const repo = path.resolve(repoPath);
+        // Validate repository exists
+        try {
+            await fs.promises.access(repo);
+        }
+        catch {
+            throw new Error(`Directory ${repo} does not exist`);
+        }
+        // Setup provider context and embedding provider
+        const providerContext = resolveProviderContext(repo);
+        const providerInstance = embeddingProviderOverride ||
+            createEmbeddingProvider(provider, providerContext.embedding);
+        if (!embeddingProviderOverride && providerInstance.init) {
+            await providerInstance.init();
+        }
+        // Get provider and model information
+        const providerName = providerInstance.getName();
+        const modelName = providerInstance.getModelName ? providerInstance.getModelName() : null;
+        const modelProfile = await getModelProfile(providerName, modelName || providerName);
+        const limits = getSizeLimits(modelProfile);
+        // Log configuration
+        if (!process.env.CODEVAULT_QUIET) {
+            logger.info(`Chunking Configuration`, {
+                provider: providerName,
+                model: modelName,
+                dimensions: providerInstance.getDimensions(),
+                mode: limits.unit
+            });
+        }
+        // Initialize database
+        await initDatabase(providerInstance.getDimensions(), repo);
+        // Setup paths
+        const codemapPath = path.join(repo, 'codevault.codemap.json');
+        const chunkDir = path.join(repo, '.codevault/chunks');
+        const dbPath = path.join(repo, '.codevault/codevault.db');
+        // Check for dimension mismatches
+        await IndexContext.checkDimensionMismatch(dbPath, providerInstance);
+        // Setup encryption
+        const encryptionPreference = resolveEncryptionPreference({
+            mode: encryptMode,
+            logger: console
+        });
+        // Load existing state
+        const codemap = readCodemap(codemapPath);
+        const merkle = loadMerkle(repo);
+        const updatedMerkle = cloneMerkle(merkle);
+        // Create database connection
+        const db = new Database(dbPath);
+        // Create batch processor
+        const batchProcessor = new BatchEmbeddingProcessor(providerInstance, db, BATCH_SIZE);
+        const isPartialUpdate = changedFiles !== null;
+        return {
+            repo,
+            repoPath,
+            provider,
+            providerInstance,
+            providerName,
+            modelName,
+            modelProfile,
+            limits,
+            codemapPath,
+            chunkDir,
+            dbPath,
+            encryptionPreference,
+            codemap,
+            merkle,
+            updatedMerkle,
+            db,
+            batchProcessor,
+            isPartialUpdate
+        };
+    }
+    /**
+     * Check if there's a dimension or provider mismatch and warn user
+     */
+    static async checkDimensionMismatch(dbPath, embeddingProvider) {
+        try {
+            await fs.promises.access(dbPath);
+        }
+        catch {
+            return; // DB doesn't exist yet
+        }
+        const db = new Database(dbPath);
+        try {
+            const existingDimensions = await db.getExistingDimensions();
+            if (existingDimensions.length > 0) {
+                const currentProvider = embeddingProvider.getName();
+                const currentDimensions = embeddingProvider.getDimensions();
+                const hasMismatch = existingDimensions.some(row => row.embedding_provider !== currentProvider ||
+                    row.embedding_dimensions !== currentDimensions);
+                if (hasMismatch) {
+                    logger.warn('Dimension/Provider Mismatch Detected!', {
+                        existing: existingDimensions,
+                        current: { provider: currentProvider, dimensions: currentDimensions },
+                        recommendation: 'Full re-index recommended'
+                    });
+                    await new Promise(resolve => setTimeout(resolve, 2000));
+                }
+            }
+        }
+        catch (error) {
+            logger.debug('Migration check encountered an error (continuing)', {
+                error: error instanceof Error ? error.message : String(error)
+            });
+        }
+        finally {
+            db.close();
+        }
+    }
+}
+//# sourceMappingURL=IndexContext.js.map

package/dist/core/indexing/IndexContext.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexContext.js","sourceRoot":"","sources":["../../../src/core/indexing/IndexContext.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAAE,aAAa,EAA0B,MAAM,0BAA0B,CAAC;AAC3H,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAgB,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAmB,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,2BAA2B,EAAE,MAAM,mCAAmC,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAwBlE;;;;;;;GAOG;AACH,MAAM,OAAO,YAAY;IACvB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAA4B;QAC/C,MAAM,EACJ,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,MAAM,EACjB,YAAY,GAAG,IAAI,EACnB,yBAAyB,GAAG,IAAI,EAChC,WAAW,GAAG,SAAS,EACxB,GAAG,OAAO,CAAC;QAEZ,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEpC,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,aAAa,IAAI,iBAAiB,CAAC,CAAC;QACtD,CAAC;QAED,gDAAgD;QAChD,MAAM,eAAe,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,gBAAgB,GAAG,yBAAyB;YAChD,uBAAuB,CAAC,QAAQ,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;QAE/D,IAAI,CAAC,yBAAyB,IAAI,gBAAgB,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC;QAED,qCAAqC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,SAAS,IAAI,YAAY,CAAC,CAAC;QACpF,MAAM,MAAM,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;QAE3C,oBAAoB;QACpB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACpC,QAAQ,EAAE,YAAY;gBACtB,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,gBAAgB,CAAC,aAAa,EAAE;gBAC5C,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,MAAM,YAAY,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,CAAC;QAE3D,cAAc;QACd,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAE1D,iCAAiC;QACjC,MAAM,YAAY,CAAC,sBAAsB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAEpE,mBAAmB;QACnB,MAAM,oBAAoB,GAAG,2BAA2B,CAAC;YACvD,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,OAAO;SAChB,CAAC,CAAC;QAEH,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAE1C,6BAA6B;QAC7B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEhC,yBAAyB;QACzB,MAAM,cAAc,GAAG,IAAI,uBAAuB,CAAC,gBAAgB,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;QAErF,MAAM,eAAe,GAAG,YAAY,KAAK,IAAI,CAAC;QAE9C,OAAO;YACL,IAAI;YACJ,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,MAAM;YACN,WAAW;YACX,QAAQ;YACR,MAAM;YACN,oBAAoB;YACpB,OAAO;YACP,MAAM;YACN,aAAa;YACb,EAAE;YACF,cAAc;YACd,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACzC,MAAc,EACd,iBAAoC;QAEpC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,uBAAuB;QACjC,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,kBAAkB,GAAG,MAAM,EAAE,CAAC,qBAAqB,EAAE,CAAC;YAE5D,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,eAAe,GAAG,iBAAiB,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,aAAa,EAAE,CAAC;gBAE5D,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CACzC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,eAAe;oBAC1C,GAAG,CAAC,oBAAoB,KAAK,iBAAiB,CACtD,CAAC;gBAEF,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE;wBACnD,QAAQ,EAAE,kBAAkB;wBAC5B,OAAO,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,iBAAiB,EAAE;wBACrE,cAAc,EAAE,2BAA2B;qBAC5C,CAAC,CAAC;oBAEH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,mDAAmD,EAAE;gBAChE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}

package/dist/core/indexing/IndexFinalizationStage.d.ts

@@ -0,0 +1,39 @@
+import type { IndexContextData } from './IndexContext.js';
+import type { IndexState } from './IndexState.js';
+import type { IndexProjectResult } from '../types.js';
+/**
+ * IndexFinalizationStage handles the finalization of the indexing process:
+ * - Flushing batch processor
+ * - Saving merkle tree
+ * - Building symbol graph
+ * - Writing codemap
+ * - Building result object
+ * - Cleaning up resources
+ */
+export declare class IndexFinalizationStage {
+    private context;
+    private state;
+    private onProgress;
+    constructor(context: IndexContextData, state: IndexState, onProgress: ((event: any) => void) | null);
+    /**
+     * Finalize the indexing process
+     */
+    finalize(): Promise<IndexProjectResult>;
+    /**
+     * Flush the batch processor
+     */
+    private flushBatchProcessor;
+    /**
+     * Log chunking and processing statistics
+     */
+    private logStatistics;
+    /**
+     * Build the final result object
+     */
+    private buildResult;
+    /**
+     * Clean up resources
+     */
+    private cleanup;
+}
+//# sourceMappingURL=IndexFinalizationStage.d.ts.map

package/dist/core/indexing/IndexFinalizationStage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexFinalizationStage.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/IndexFinalizationStage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD;;;;;;;;GAQG;AACH,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,UAAU;gBAFV,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,KAAK,IAAI,CAAC,GAAG,IAAI;IAGnD;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAiC7C;;OAEG;YACW,mBAAmB;IAajC;;OAEG;IACH,OAAO,CAAC,aAAa;IAUrB;;OAEG;IACH,OAAO,CAAC,WAAW;IAYnB;;OAEG;YACW,OAAO;CAatB"}

package/dist/core/indexing/IndexFinalizationStage.js

@@ -0,0 +1,114 @@
+import { saveMerkle } from '../../indexer/merkle.js';
+import { writeCodemap } from '../../codemap/io.js';
+import { attachSymbolGraphToCodemap } from '../../symbols/graph.js';
+import { getTokenCountStats } from '../../chunking/token-counter.js';
+import { logger } from '../../utils/logger.js';
+/**
+ * IndexFinalizationStage handles the finalization of the indexing process:
+ * - Flushing batch processor
+ * - Saving merkle tree
+ * - Building symbol graph
+ * - Writing codemap
+ * - Building result object
+ * - Cleaning up resources
+ */
+export class IndexFinalizationStage {
+    context;
+    state;
+    onProgress;
+    constructor(context, state, onProgress) {
+        this.context = context;
+        this.state = state;
+        this.onProgress = onProgress;
+    }
+    /**
+     * Finalize the indexing process
+     */
+    async finalize() {
+        try {
+            // Notify progress
+            if (this.onProgress) {
+                this.onProgress({ type: 'finalizing' });
+            }
+            // Flush any remaining embeddings
+            await this.flushBatchProcessor();
+            // Save merkle tree if modified
+            if (this.state.merkleDirty) {
+                saveMerkle(this.context.repo, this.state.updatedMerkle);
+            }
+            // Build symbol graph and write codemap
+            attachSymbolGraphToCodemap(this.state.codemap);
+            this.state.codemap = writeCodemap(this.context.codemapPath, this.state.codemap);
+            // Get token statistics
+            const tokenStats = getTokenCountStats();
+            // Log statistics
+            this.logStatistics();
+            // Build result
+            return this.buildResult(tokenStats);
+        }
+        finally {
+            // Clean up resources
+            await this.cleanup();
+        }
+    }
+    /**
+     * Flush the batch processor
+     */
+    async flushBatchProcessor() {
+        try {
+            if (this.context.batchProcessor) {
+                await this.context.batchProcessor.flush();
+            }
+        }
+        catch (error) {
+            this.state.addError({
+                type: 'finalize_error',
+                error: error.message
+            });
+        }
+    }
+    /**
+     * Log chunking and processing statistics
+     */
+    logStatistics() {
+        if (!process.env.CODEVAULT_QUIET) {
+            logger.info('Chunking Statistics', {
+                stats: { ...this.state.chunkingStats },
+                processedChunks: this.state.processedChunks,
+                totalChunks: Object.keys(this.state.codemap).length
+            });
+        }
+    }
+    /**
+     * Build the final result object
+     */
+    buildResult(tokenStats) {
+        return {
+            success: true,
+            processedChunks: this.state.processedChunks,
+            totalChunks: Object.keys(this.state.codemap).length,
+            provider: this.context.providerInstance.getName(),
+            errors: this.state.errors,
+            chunkingStats: this.state.chunkingStats,
+            tokenStats: this.context.modelProfile.useTokens ? tokenStats : undefined
+        };
+    }
+    /**
+     * Clean up resources
+     */
+    async cleanup() {
+        // Close database connection
+        try {
+            if (this.context.db) {
+                this.context.db.close();
+            }
+        }
+        catch (error) {
+            this.state.addError({
+                type: 'db_close_error',
+                error: error.message
+            });
+        }
+    }
+}
+//# sourceMappingURL=IndexFinalizationStage.js.map

package/dist/core/indexing/IndexFinalizationStage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexFinalizationStage.js","sourceRoot":"","sources":["../../../src/core/indexing/IndexFinalizationStage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAK/C;;;;;;;;GAQG;AACH,MAAM,OAAO,sBAAsB;IAEvB;IACA;IACA;IAHV,YACU,OAAyB,EACzB,KAAiB,EACjB,UAAyC;QAFzC,YAAO,GAAP,OAAO,CAAkB;QACzB,UAAK,GAAL,KAAK,CAAY;QACjB,eAAU,GAAV,UAAU,CAA+B;IAChD,CAAC;IAEJ;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC;YACH,kBAAkB;YAClB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;YAC1C,CAAC;YAED,iCAAiC;YACjC,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,+BAA+B;YAC/B,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC1D,CAAC;YAED,uCAAuC;YACvC,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEhF,uBAAuB;YACvB,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;YAExC,iBAAiB;YACjB,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,eAAe;YACf,OAAO,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;gBAAS,CAAC;YACT,qBAAqB;YACrB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB;QAC/B,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;gBACjC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE;gBACtC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe;gBAC3C,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM;aACpD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,UAAe;QACjC,OAAO;YACL,OAAO,EAAE,IAAI;YACb,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe;YAC3C,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM;YACnD,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE;YACjD,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACzB,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;YACvC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,OAAO;QACnB,4BAA4B;QAC5B,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;gBACpB,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/core/indexing/IndexState.d.ts

@@ -0,0 +1,50 @@
+import type { Codemap } from '../../codemap/io.js';
+import type { MerkleTree } from '../../indexer/merkle.js';
+import type { ChunkingStats } from '../types.js';
+/**
+ * IndexState tracks mutable state during the indexing process
+ */
+export declare class IndexState {
+    codemap: Codemap;
+    updatedMerkle: MerkleTree;
+    merkleDirty: boolean;
+    indexMutated: boolean;
+    processedChunks: number;
+    errors: any[];
+    chunkingStats: ChunkingStats;
+    constructor(codemap: Codemap, updatedMerkle: MerkleTree);
+    /**
+     * Add an error to the error list
+     */
+    addError(error: {
+        type: string;
+        file?: string;
+        chunkId?: string;
+        error: string;
+    }): void;
+    /**
+     * Mark merkle tree as modified
+     */
+    markMerkleDirty(): void;
+    /**
+     * Mark index as modified
+     */
+    markIndexMutated(): void;
+    /**
+     * Increment processed chunk counter
+     */
+    incrementProcessedChunks(): void;
+    /**
+     * Update chunking stats
+     */
+    updateChunkingStats(stats: Partial<ChunkingStats>): void;
+    /**
+     * Get current error count
+     */
+    getErrorCount(): number;
+    /**
+     * Check if index was modified
+     */
+    wasModified(): boolean;
+}
+//# sourceMappingURL=IndexState.d.ts.map

package/dist/core/indexing/IndexState.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexState.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/IndexState.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,UAAU,CAAC;IAC1B,WAAW,UAAS;IACpB,YAAY,UAAS;IACrB,eAAe,SAAK;IACpB,MAAM,EAAE,GAAG,EAAE,CAAM;IACnB,aAAa,EAAE,aAAa,CAO1B;gBAEU,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU;IAKvD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAIvF;;OAEG;IACH,eAAe,IAAI,IAAI;IAIvB;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAIxB;;OAEG;IACH,wBAAwB,IAAI,IAAI;IAIhC;;OAEG;IACH,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI;IAIxD;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;OAEG;IACH,WAAW,IAAI,OAAO;CAGvB"}

package/dist/core/indexing/IndexState.js

@@ -0,0 +1,66 @@
+/**
+ * IndexState tracks mutable state during the indexing process
+ */
+export class IndexState {
+    codemap;
+    updatedMerkle;
+    merkleDirty = false;
+    indexMutated = false;
+    processedChunks = 0;
+    errors = [];
+    chunkingStats = {
+        totalNodes: 0,
+        skippedSmall: 0,
+        subdivided: 0,
+        statementFallback: 0,
+        normalChunks: 0,
+        mergedSmall: 0
+    };
+    constructor(codemap, updatedMerkle) {
+        this.codemap = codemap;
+        this.updatedMerkle = updatedMerkle;
+    }
+    /**
+     * Add an error to the error list
+     */
+    addError(error) {
+        this.errors.push(error);
+    }
+    /**
+     * Mark merkle tree as modified
+     */
+    markMerkleDirty() {
+        this.merkleDirty = true;
+    }
+    /**
+     * Mark index as modified
+     */
+    markIndexMutated() {
+        this.indexMutated = true;
+    }
+    /**
+     * Increment processed chunk counter
+     */
+    incrementProcessedChunks() {
+        this.processedChunks++;
+    }
+    /**
+     * Update chunking stats
+     */
+    updateChunkingStats(stats) {
+        this.chunkingStats = { ...this.chunkingStats, ...stats };
+    }
+    /**
+     * Get current error count
+     */
+    getErrorCount() {
+        return this.errors.length;
+    }
+    /**
+     * Check if index was modified
+     */
+    wasModified() {
+        return this.indexMutated || this.merkleDirty;
+    }
+}
+//# sourceMappingURL=IndexState.js.map

package/dist/core/indexing/IndexState.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexState.js","sourceRoot":"","sources":["../../../src/core/indexing/IndexState.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,MAAM,OAAO,UAAU;IACrB,OAAO,CAAU;IACjB,aAAa,CAAa;IAC1B,WAAW,GAAG,KAAK,CAAC;IACpB,YAAY,GAAG,KAAK,CAAC;IACrB,eAAe,GAAG,CAAC,CAAC;IACpB,MAAM,GAAU,EAAE,CAAC;IACnB,aAAa,GAAkB;QAC7B,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;QACf,UAAU,EAAE,CAAC;QACb,iBAAiB,EAAE,CAAC;QACpB,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACf,CAAC;IAEF,YAAY,OAAgB,EAAE,aAAyB;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,KAAuE;QAC9E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,eAAe;QACb,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,wBAAwB;QACtB,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,KAA6B;QAC/C,IAAI,CAAC,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,GAAG,KAAK,EAAE,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;IAC/C,CAAC;CACF"}

package/dist/core/indexing/chunk-pipeline.d.ts

@@ -1,3 +1,6 @@
+import Parser from 'tree-sitter';
+import { type NodeGroup } from '../../chunking/file-grouper.js';
+import type { TreeSitterNode } from '../../types/ast.js';
 import type { LanguageRule } from '../../languages/rules.js';
 import type { ModelProfile } from '../../providers/base.js';
 type SizeLimits = {
@@ -7,6 +10,10 @@ type SizeLimits = {
     overlap: number;
     unit: string;
 };
+export interface OversizedChunk {
+    code: string;
+    part: number;
+}
 interface ExistingChunks {
     staleChunkIds: Set<string>;
     existingChunks: Map<string, any>;
@@ -26,12 +33,41 @@ interface EmbedStoreParams {
     contextInfo: any;
     symbolData: any;
 }
-export declare class ChunkPipeline {
+/**
+ * Collects candidate AST nodes for chunking using a reusable parser instance.
+ */
+export declare class ASTTraverser {
     private parser;
+    constructor(parser?: Parser);
+    collectNodesForFile(source: string, rule: LanguageRule): TreeSitterNode[];
+    private buildTree;
+}
+export interface OverlapStrategy {
+    split(node: TreeSitterNode, source: string, limits: SizeLimits, profile: ModelProfile): Promise<OversizedChunk[]>;
+}
+/**
+ * Default overlap strategy that falls back to statement-level chunking with 20% overlap.
+ */
+export declare class StatementOverlapStrategy implements OverlapStrategy {
+    split(node: TreeSitterNode, source: string, limits: SizeLimits, profile: ModelProfile): Promise<OversizedChunk[]>;
+}
+export declare class ChunkGrouper {
+    groupNodes(nodes: TreeSitterNode[], source: string, profile: ModelProfile, rule: LanguageRule): Promise<NodeGroup[]>;
+}
+export interface ChunkPipelineDependencies {
+    traverser?: ASTTraverser;
+    chunkGrouper?: ChunkGrouper;
+    overlapStrategy?: OverlapStrategy;
+}
+export declare class ChunkPipeline {
     private processedNodes;
-    constructor();
+    private traverser;
+    private chunkGrouper;
+    private overlapStrategy;
+    constructor(deps?: ChunkPipelineDependencies);
     collectNodesForFile(source: string, rule: LanguageRule): Promise<import("tree-sitter").SyntaxNode[]>;
-    processGroups(nodeGroups: any[], source: string, rule: LanguageRule, limits: SizeLimits, modelProfile: ModelProfile, rel: string, existing: ExistingChunks, chunkMerkleHashes: string[], onProgress: any, embedAndStore: (params: EmbedStoreParams) => Promise<void>, chunkingStats: any): Promise<void>;
+    groupNodes(nodes: TreeSitterNode[], source: string, profile: ModelProfile, rule: LanguageRule): Promise<NodeGroup[]>;
+    processGroups(nodeGroups: NodeGroup[], source: string, rule: LanguageRule, limits: SizeLimits, modelProfile: ModelProfile, rel: string, existing: ExistingChunks, chunkMerkleHashes: string[], onProgress: any, embedAndStore: (params: EmbedStoreParams) => Promise<void>, chunkingStats: any): Promise<void>;
     private yieldChunk;
     private processChunk;
 }

package/dist/core/indexing/chunk-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunk-pipeline.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/chunk-pipeline.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,KAAK,UAAU,GAAG;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,UAAU,cAAc;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,GAAG,CAAC;IACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,GAAG,CAAC;IACjB,UAAU,EAAE,GAAG,CAAC;CACjB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAqB;;IAMrC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY;IA6DtD,aAAa,CACjB,UAAU,EAAE,GAAG,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,cAAc,EACxB,iBAAiB,EAAE,MAAM,EAAE,EAC3B,UAAU,EAAE,GAAG,EACf,aAAa,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,EAC1D,aAAa,EAAE,GAAG,GACjB,OAAO,CAAC,IAAI,CAAC;YAgCF,UAAU;YAqGV,YAAY;CAmF3B"}
+{"version":3,"file":"chunk-pipeline.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/chunk-pipeline.ts"],"names":[],"mappings":"AAGA,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAA8C,KAAK,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAY5G,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,KAAK,UAAU,GAAG;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,UAAU,cAAc;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,GAAG,CAAC;IACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,GAAG,CAAC;IACjB,UAAU,EAAE,GAAG,CAAC;CACjB;AAED;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM;IAI3B,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,EAAE;IAkDzE,OAAO,CAAC,SAAS;CAWlB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;CACnH;AAED;;GAEG;AACH,qBAAa,wBAAyB,YAAW,eAAe;IACxD,KAAK,CACT,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,cAAc,EAAE,CAAC;CAO7B;AAED,qBAAa,YAAY;IACjB,UAAU,CACd,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,SAAS,EAAE,CAAC;CAGxB;AAED,MAAM,WAAW,yBAAyB;IACxC,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,SAAS,CAAe;IAChC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,eAAe,CAAkB;gBAE7B,IAAI,GAAE,yBAA8B;IAM1C,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY;IAItD,UAAU,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY;IAI7F,aAAa,CACjB,UAAU,EAAE,SAAS,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,cAAc,EACxB,iBAAiB,EAAE,MAAM,EAAE,EAC3B,UAAU,EAAE,GAAG,EACf,aAAa,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,EAC1D,aAAa,EAAE,GAAG,GACjB,OAAO,CAAC,IAAI,CAAC;YAgCF,UAAU;YAgHV,YAAY;CAmF3B"}

package/dist/core/indexing/chunk-pipeline.js

@@ -1,33 +1,24 @@
-import Parser from 'tree-sitter';
 import crypto from 'crypto';
+import Parser from 'tree-sitter';
 import { analyzeNodeForChunking, batchAnalyzeNodes, yieldStatementChunks } from '../../chunking/semantic-chunker.js';
-import { createCombinedChunk } from '../../chunking/file-grouper.js';
+import { groupNodesForChunking, createCombinedChunk } from '../../chunking/file-grouper.js';
 import { extractSymbolMetadata } from '../../symbols/extract.js';
 import { extractSymbolName } from '../symbol-extractor.js';
 import { extractCodevaultMetadata, extractSemanticTags, extractImportantVariables, extractDocComments, generateEnhancedEmbeddingText } from '../metadata.js';
 import { computeFastHash } from '../../indexer/merkle.js';
 import { SIZE_THRESHOLD, CHUNK_SIZE } from '../../config/constants.js';
-export class ChunkPipeline {
+/**
+ * Collects candidate AST nodes for chunking using a reusable parser instance.
+ */
+export class ASTTraverser {
     parser;
-    processedNodes = new Set();
-    constructor() {
-        this.parser = new Parser();
+    constructor(parser) {
+        this.parser = parser ?? new Parser();
     }
-    async collectNodesForFile(source, rule) {
+    collectNodesForFile(source, rule) {
         this.parser.setLanguage(rule.ts);
-        let tree;
-        if (source.length > SIZE_THRESHOLD) {
-            tree = this.parser.parse((index) => {
-                if (index < source.length) {
-                    return source.slice(index, Math.min(index + CHUNK_SIZE, source.length));
-                }
-                return null;
-            });
-        }
-        else {
-            tree = this.parser.parse(source);
-        }
-        if (!tree || !tree.rootNode) {
+        const tree = this.buildTree(source);
+        if (!tree?.rootNode) {
             throw new Error('Failed to create syntax tree');
         }
         const collectedNodes = [];
@@ -68,6 +59,51 @@ export class ChunkPipeline {
         collectNodes(tree.rootNode);
         return collectedNodes;
     }
+    buildTree(source) {
+        if (source.length > SIZE_THRESHOLD) {
+            return this.parser.parse((index) => {
+                if (index < source.length) {
+                    return source.slice(index, Math.min(index + CHUNK_SIZE, source.length));
+                }
+                return null;
+            });
+        }
+        return this.parser.parse(source);
+    }
+}
+/**
+ * Default overlap strategy that falls back to statement-level chunking with 20% overlap.
+ */
+export class StatementOverlapStrategy {
+    async split(node, source, limits, profile) {
+        const statementChunks = await yieldStatementChunks(node, source, limits.max, limits.overlap, profile);
+        return statementChunks.map((chunk, index) => ({
+            code: chunk.code,
+            part: index + 1
+        }));
+    }
+}
+export class ChunkGrouper {
+    async groupNodes(nodes, source, profile, rule) {
+        return groupNodesForChunking(nodes, source, profile, rule);
+    }
+}
+export class ChunkPipeline {
+    processedNodes = new Set();
+    traverser;
+    chunkGrouper;
+    overlapStrategy;
+    constructor(deps = {}) {
+        this.traverser = deps.traverser ?? new ASTTraverser();
+        this.chunkGrouper = deps.chunkGrouper ?? new ChunkGrouper();
+        this.overlapStrategy = deps.overlapStrategy ?? new StatementOverlapStrategy();
+    }
+    async collectNodesForFile(source, rule) {
+        return this.traverser.collectNodesForFile(source, rule);
+    }
+    async groupNodes(nodes, source, profile, rule) {
+        return this.chunkGrouper.groupNodes(nodes, source, profile, rule);
+    }
     async processGroups(nodeGroups, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats) {
         this.processedNodes = new Set();
         for (const nodeGroup of nodeGroups) {
@@ -139,10 +175,9 @@ export class ChunkPipeline {
         }
         else if (analysis.size > limits.max) {
             chunkingStats.statementFallback++;
-            const statementChunks = await yieldStatementChunks(node, source, limits.max, limits.overlap, modelProfile);
-            for (let i = 0; i < statementChunks.length; i++) {
-                const stmtChunk = statementChunks[i];
-                await this.processChunk(node, stmtChunk.code, `${i + 1}`, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+            const oversizedChunks = await this.overlapStrategy.split(node, source, limits, modelProfile);
+            for (const stmtChunk of oversizedChunks) {
+                await this.processChunk(node, stmtChunk.code, `${stmtChunk.part}`, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
             }
             return;
         }