codevault 1.5.0 → 1.6.0

package/dist/core/batch-indexer.js

@@ -1,161 +0,0 @@
-import { BATCH_SIZE } from '../providers/base.js';
-const MAX_BATCH_RETRIES = 3;
-const INITIAL_RETRY_DELAY_MS = 1000;
-function isRateLimitError(error) {
-    const message = error?.message || String(error);
-    return (error?.status === 429 ||
-        error?.statusCode === 429 ||
-        message.includes('rate limit') ||
-        message.includes('Rate limit') ||
-        message.includes('too many requests') ||
-        message.includes('429'));
-}
-function isBatchSizeError(error) {
-    const message = error?.message || String(error);
-    return (error?.status === 413 ||
-        message.includes('too large') ||
-        message.includes('payload') ||
-        message.includes('request size') ||
-        message.includes('token limit'));
-}
-export class BatchEmbeddingProcessor {
-    embeddingProvider;
-    db;
-    batch = [];
-    batchSize;
-    // FIX: Add mutex to prevent concurrent batch processing
-    processing = false;
-    constructor(embeddingProvider, db, batchSize = BATCH_SIZE) {
-        this.embeddingProvider = embeddingProvider;
-        this.db = db;
-        this.batchSize = batchSize;
-    }
-    /**
-     * Add a chunk to the batch queue
-     */
-    async addChunk(chunk) {
-        this.batch.push(chunk);
-        // FIX: Wait for any ongoing processing to complete before checking batch size
-        while (this.processing) {
-            await new Promise(resolve => setTimeout(resolve, 10));
-        }
-        // Process batch when it reaches the threshold
-        if (this.batch.length >= this.batchSize) {
-            await this.processBatch();
-        }
-    }
-    /**
-     * Process any remaining chunks in the batch
-     */
-    async flush() {
-        // FIX: Wait for any ongoing processing to complete
-        while (this.processing) {
-            await new Promise(resolve => setTimeout(resolve, 10));
-        }
-        if (this.batch.length > 0) {
-            await this.processBatch();
-        }
-    }
-    /**
-     * Process the current batch of chunks
-     */
-    async processBatch() {
-        // FIX: Prevent concurrent batch processing
-        if (this.batch.length === 0 || this.processing)
-            return;
-        this.processing = true;
-        const currentBatch = [...this.batch];
-        this.batch = [];
-        try {
-            // Extract texts for embedding
-            const texts = currentBatch.map(chunk => chunk.enhancedEmbeddingText);
-            // Log batching activity
-            if (!process.env.CODEVAULT_QUIET) {
-                console.log(`🚀 Processing batch of ${texts.length} chunks...`);
-            }
-            // Generate embeddings in batch (single API call for all)
-            const embeddings = await this.embeddingProvider.generateEmbeddings(texts);
-            if (!process.env.CODEVAULT_QUIET) {
-                console.log(`✓ Batch complete (${texts.length} embeddings generated)`);
-            }
-            // Store all embeddings in database
-            for (let i = 0; i < currentBatch.length; i++) {
-                const chunk = currentBatch[i];
-                const embedding = embeddings[i];
-                await this.db.insertChunk({
-                    id: chunk.chunkId,
-                    file_path: chunk.params.rel,
-                    symbol: chunk.params.symbol,
-                    sha: chunk.params.sha,
-                    lang: chunk.params.lang,
-                    chunk_type: chunk.params.chunkType,
-                    embedding,
-                    embedding_provider: this.embeddingProvider.getName(),
-                    embedding_dimensions: this.embeddingProvider.getDimensions(),
-                    codevault_tags: chunk.params.codevaultMetadata.tags,
-                    codevault_intent: chunk.params.codevaultMetadata.intent,
-                    codevault_description: chunk.params.codevaultMetadata.description,
-                    doc_comments: chunk.params.docComments,
-                    variables_used: chunk.params.importantVariables,
-                    context_info: chunk.params.contextInfo
-                });
-            }
-        }
-        catch (error) {
-            // On batch failure, fall back to individual processing
-            console.error(`\n❌ Batch processing failed for ${currentBatch.length} chunks`);
-            console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
-            console.log(`⚠️  Falling back to individual processing (this will be slower)...\n`);
-            // FIX: Collect errors but continue processing all chunks to avoid data loss
-            const errors = [];
-            for (const chunk of currentBatch) {
-                try {
-                    const embedding = await this.embeddingProvider.generateEmbedding(chunk.enhancedEmbeddingText);
-                    await this.db.insertChunk({
-                        id: chunk.chunkId,
-                        file_path: chunk.params.rel,
-                        symbol: chunk.params.symbol,
-                        sha: chunk.params.sha,
-                        lang: chunk.params.lang,
-                        chunk_type: chunk.params.chunkType,
-                        embedding,
-                        embedding_provider: this.embeddingProvider.getName(),
-                        embedding_dimensions: this.embeddingProvider.getDimensions(),
-                        codevault_tags: chunk.params.codevaultMetadata.tags,
-                        codevault_intent: chunk.params.codevaultMetadata.intent,
-                        codevault_description: chunk.params.codevaultMetadata.description,
-                        doc_comments: chunk.params.docComments,
-                        variables_used: chunk.params.importantVariables,
-                        context_info: chunk.params.contextInfo
-                    });
-                }
-                catch (individualError) {
-                    console.error(`Failed to process chunk ${chunk.chunkId}:`, individualError);
-                    errors.push({
-                        chunkId: chunk.chunkId,
-                        error: individualError
-                    });
-                }
-            }
-            // Report errors but don't throw to allow indexing to continue
-            if (errors.length > 0) {
-                console.warn(`⚠️  ${errors.length}/${currentBatch.length} chunks failed in fallback processing`);
-                if (errors.length === currentBatch.length) {
-                    // Only throw if ALL chunks failed
-                    throw new Error(`All ${errors.length} chunks failed to process: ${errors[0].error.message}`);
-                }
-            }
-        }
-        finally {
-            // FIX: Always release the lock, even on error
-            this.processing = false;
-        }
-    }
-    /**
-     * Get current batch size for monitoring
-     */
-    getBatchCount() {
-        return this.batch.length;
-    }
-}
-//# sourceMappingURL=batch-indexer.js.map

package/dist/core/batch-indexer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAIlD,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD,MAAM,OAAO,uBAAuB;IAOxB;IACA;IAPF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAC1B,wDAAwD;IAChD,UAAU,GAAG,KAAK,CAAC;IAE3B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,8EAA8E;QAC9E,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;YACvB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,8CAA8C;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,mDAAmD;QACnD,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;YACvB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,2CAA2C;QAC3C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAEvD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAEhB,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAErE,wBAAwB;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,CAAC,MAAM,YAAY,CAAC,CAAC;YAClE,CAAC;YAED,yDAAyD;YACzD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAE1E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;YACzE,CAAC;YAED,mCAAmC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAEhC,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBACxB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,uDAAuD;YACvD,OAAO,CAAC,KAAK,CAAC,mCAAmC,YAAY,CAAC,MAAM,SAAS,CAAC,CAAC;YAC/E,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,OAAO,CAAC,GAAG,CAAC,sEAAsE,CAAC,CAAC;YAEpF,4EAA4E;YAC5E,MAAM,MAAM,GAA6C,EAAE,CAAC;YAE5D,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;oBAE9F,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;wBACxB,EAAE,EAAE,KAAK,CAAC,OAAO;wBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;wBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;wBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;wBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;wBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;wBAClC,SAAS;wBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;wBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;wBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;wBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;wBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;wBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;wBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;wBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;qBACvC,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,eAAe,EAAE,CAAC;oBACzB,OAAO,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,GAAG,EAAE,eAAe,CAAC,CAAC;oBAC5E,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,KAAK,EAAE,eAAwB;qBAChC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,8DAA8D;YAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC,OAAO,MAAM,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,uCAAuC,CAAC,CAAC;gBACjG,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;oBAC1C,kCAAkC;oBAClC,MAAM,IAAI,KAAK,CAAC,OAAO,MAAM,CAAC,MAAM,8BAA8B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/F,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,8CAA8C;YAC9C,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}

package/dist/core/indexer.d.ts

@@ -1,3 +0,0 @@
-import type { IndexProjectOptions, IndexProjectResult } from './types.js';
-export declare function indexProject({ repoPath, provider, onProgress, changedFiles, deletedFiles, embeddingProviderOverride, encryptMode }?: IndexProjectOptions): Promise<IndexProjectResult>;
-//# sourceMappingURL=indexer.d.ts.map

package/dist/core/indexer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AAuCA,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAiB,MAAM,YAAY,CAAC;AAIzF,wBAAsB,YAAY,CAAC,EACjC,QAAc,EACd,QAAiB,EACjB,UAAiB,EACjB,YAAmB,EACnB,YAAiB,EACjB,yBAAgC,EAChC,WAAuB,EACxB,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAyuBxD"}