codevault 1.6.0 → 1.6.1

package/dist/core/batch-indexer.js

@@ -0,0 +1,189 @@
+import { BATCH_SIZE } from '../providers/base.js';
+import { Mutex } from '../utils/mutex.js';
+import { log } from '../utils/logger.js';
+const MAX_BATCH_RETRIES = 3;
+const INITIAL_RETRY_DELAY_MS = 1000;
+function isRateLimitError(error) {
+    const message = error?.message || String(error);
+    return (error?.status === 429 ||
+        error?.statusCode === 429 ||
+        message.includes('rate limit') ||
+        message.includes('Rate limit') ||
+        message.includes('too many requests') ||
+        message.includes('429'));
+}
+function isBatchSizeError(error) {
+    const message = error?.message || String(error);
+    return (error?.status === 413 ||
+        message.includes('too large') ||
+        message.includes('payload') ||
+        message.includes('request size') ||
+        message.includes('token limit'));
+}
+export class BatchEmbeddingProcessor {
+    embeddingProvider;
+    db;
+    batch = [];
+    batchSize;
+    mutex = new Mutex();
+    constructor(embeddingProvider, db, batchSize = BATCH_SIZE) {
+        this.embeddingProvider = embeddingProvider;
+        this.db = db;
+        this.batchSize = batchSize;
+    }
+    /**
+     * Add a chunk to the batch queue
+     */
+    async addChunk(chunk) {
+        this.batch.push(chunk);
+        // Process batch when it reaches the threshold
+        if (this.batch.length >= this.batchSize) {
+            await this.processBatch();
+        }
+    }
+    /**
+     * Process any remaining chunks in the batch
+     */
+    async flush() {
+        await this.mutex.runExclusive(async () => {
+            if (this.batch.length > 0) {
+                await this.processBatchWithRetry(this.batch, 0);
+                this.batch = [];
+            }
+        });
+    }
+    /**
+     * Process the current batch (with mutex protection)
+     */
+    async processBatch() {
+        await this.mutex.runExclusive(async () => {
+            if (this.batch.length > 0) {
+                await this.processBatchWithRetry(this.batch, 0);
+                this.batch = [];
+            }
+        });
+    }
+    /**
+     * Process a batch with smart error handling and retry logic
+     */
+    async processBatchWithRetry(currentBatch, retryCount) {
+        try {
+            await this.processBatchInternal(currentBatch);
+        }
+        catch (error) {
+            // Smart error handling based on error type
+            if (isBatchSizeError(error) && currentBatch.length > 1) {
+                // Batch too large - split in half and retry
+                log.warn(`Batch size too large (${currentBatch.length} chunks), splitting and retrying`);
+                const mid = Math.floor(currentBatch.length / 2);
+                const firstHalf = currentBatch.slice(0, mid);
+                const secondHalf = currentBatch.slice(mid);
+                // Process both halves recursively
+                await this.processBatchWithRetry(firstHalf, 0);
+                await this.processBatchWithRetry(secondHalf, 0);
+                return;
+            }
+            else if (isRateLimitError(error) && retryCount < MAX_BATCH_RETRIES) {
+                // Rate limit error - exponential backoff
+                const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, retryCount);
+                log.warn(`Rate limit hit, retrying batch in ${delay}ms (attempt ${retryCount + 1}/${MAX_BATCH_RETRIES})`);
+                await new Promise(resolve => setTimeout(resolve, delay));
+                await this.processBatchWithRetry(currentBatch, retryCount + 1);
+                return;
+            }
+            // Other errors or max retries reached - fall back to individual processing
+            log.error(`Batch processing failed for ${currentBatch.length} chunks`, error);
+            log.warn('Falling back to individual processing (this will be slower)');
+            await this.fallbackToIndividualProcessing(currentBatch);
+        }
+    }
+    /**
+     * Internal batch processing implementation with database transactions
+     */
+    async processBatchInternal(batch) {
+        if (batch.length === 0)
+            return;
+        // Extract texts for embedding
+        const texts = batch.map(chunk => chunk.enhancedEmbeddingText);
+        // Log batching activity
+        log.info(`Processing batch of ${texts.length} chunks`);
+        // Generate embeddings in batch (single API call for all)
+        const embeddings = await this.embeddingProvider.generateEmbeddings(texts);
+        log.info(`Batch complete (${texts.length} embeddings generated)`);
+        // Store all embeddings in database within a transaction
+        await this.db.transaction(() => {
+            for (let i = 0; i < batch.length; i++) {
+                const chunk = batch[i];
+                const embedding = embeddings[i];
+                this.db.insertChunk({
+                    id: chunk.chunkId,
+                    file_path: chunk.params.rel,
+                    symbol: chunk.params.symbol,
+                    sha: chunk.params.sha,
+                    lang: chunk.params.lang,
+                    chunk_type: chunk.params.chunkType,
+                    embedding,
+                    embedding_provider: this.embeddingProvider.getName(),
+                    embedding_dimensions: this.embeddingProvider.getDimensions(),
+                    codevault_tags: chunk.params.codevaultMetadata.tags,
+                    codevault_intent: chunk.params.codevaultMetadata.intent,
+                    codevault_description: chunk.params.codevaultMetadata.description,
+                    doc_comments: chunk.params.docComments,
+                    variables_used: chunk.params.importantVariables,
+                    context_info: chunk.params.contextInfo
+                });
+            }
+        });
+    }
+    /**
+     * Fall back to processing chunks individually
+     */
+    async fallbackToIndividualProcessing(batch) {
+        // Collect errors but continue processing all chunks to avoid data loss
+        const errors = [];
+        for (const chunk of batch) {
+            try {
+                const embedding = await this.embeddingProvider.generateEmbedding(chunk.enhancedEmbeddingText);
+                this.db.insertChunk({
+                    id: chunk.chunkId,
+                    file_path: chunk.params.rel,
+                    symbol: chunk.params.symbol,
+                    sha: chunk.params.sha,
+                    lang: chunk.params.lang,
+                    chunk_type: chunk.params.chunkType,
+                    embedding,
+                    embedding_provider: this.embeddingProvider.getName(),
+                    embedding_dimensions: this.embeddingProvider.getDimensions(),
+                    codevault_tags: chunk.params.codevaultMetadata.tags,
+                    codevault_intent: chunk.params.codevaultMetadata.intent,
+                    codevault_description: chunk.params.codevaultMetadata.description,
+                    doc_comments: chunk.params.docComments,
+                    variables_used: chunk.params.importantVariables,
+                    context_info: chunk.params.contextInfo
+                });
+            }
+            catch (individualError) {
+                log.error(`Failed to process chunk ${chunk.chunkId}`, individualError);
+                errors.push({
+                    chunkId: chunk.chunkId,
+                    error: individualError
+                });
+            }
+        }
+        // Report errors but don't throw to allow indexing to continue
+        if (errors.length > 0) {
+            log.warn(`${errors.length}/${batch.length} chunks failed in fallback processing`);
+            if (errors.length === batch.length) {
+                // Only throw if ALL chunks failed
+                throw new Error(`All ${errors.length} chunks failed to process: ${errors[0].error.message}`);
+            }
+        }
+    }
+    /**
+     * Get current batch size for monitoring
+     */
+    getBatchCount() {
+        return this.batch.length;
+    }
+}
+//# sourceMappingURL=batch-indexer.js.map

package/dist/core/batch-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAEzC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD,MAAM,OAAO,uBAAuB;IAMxB;IACA;IANF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAClB,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE5B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,8CAA8C;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,YAA4B,EAAE,UAAkB;QAClF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,4CAA4C;gBAC5C,GAAG,CAAC,IAAI,CAAC,yBAAyB,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACzF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,kCAAkC;gBAClC,MAAM,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;iBAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,iBAAiB,EAAE,CAAC;gBACrE,yCAAyC;gBACzC,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,GAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,eAAe,UAAU,GAAG,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;gBAC/D,OAAO;YACT,CAAC;YAED,2EAA2E;YAC3E,GAAG,CAAC,KAAK,CAAC,+BAA+B,YAAY,CAAC,MAAM,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YAExE,MAAM,IAAI,CAAC,8BAA8B,CAAC,YAAY,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,KAAqB;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,8BAA8B;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE9D,wBAAwB;QACxB,GAAG,CAAC,IAAI,CAAC,uBAAuB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAEvD,yDAAyD;QACzD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE1E,GAAG,CAAC,IAAI,CAAC,mBAAmB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAElE,wDAAwD;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAEhC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAAC,KAAqB;QAChE,uEAAuE;QACvE,MAAM,MAAM,GAA6C,EAAE,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAE9F,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,GAAG,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;gBACvE,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,eAAwB;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,uCAAuC,CAAC,CAAC;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,kCAAkC;gBAClC,MAAM,IAAI,KAAK,CAAC,OAAO,MAAM,CAAC,MAAM,8BAA8B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}

package/dist/core/indexer.d.ts

@@ -0,0 +1,3 @@
+import type { IndexProjectOptions, IndexProjectResult } from './types.js';
+export declare function indexProject({ repoPath, provider, onProgress, changedFiles, deletedFiles, embeddingProviderOverride, encryptMode }?: IndexProjectOptions): Promise<IndexProjectResult>;
+//# sourceMappingURL=indexer.d.ts.map

package/dist/core/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AAuCA,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAiB,MAAM,YAAY,CAAC;AAMzF,wBAAsB,YAAY,CAAC,EACjC,QAAc,EACd,QAAiB,EACjB,UAAiB,EACjB,YAAmB,EACnB,YAAiB,EACjB,yBAAgC,EAChC,WAAuB,EACxB,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA8sBxD"}