codevault 1.6.0 → 1.7.0

package/dist/core/batch-indexer.js

@@ -0,0 +1,192 @@
+import { BATCH_SIZE } from '../providers/base.js';
+import { Mutex } from '../utils/mutex.js';
+import { log } from '../utils/logger.js';
+const MAX_BATCH_RETRIES = 3;
+const INITIAL_RETRY_DELAY_MS = 1000;
+function isRateLimitError(error) {
+    const message = error?.message || String(error);
+    return (error?.status === 429 ||
+        error?.statusCode === 429 ||
+        message.includes('rate limit') ||
+        message.includes('Rate limit') ||
+        message.includes('too many requests') ||
+        message.includes('429'));
+}
+function isBatchSizeError(error) {
+    const message = error?.message || String(error);
+    return (error?.status === 413 ||
+        message.includes('too large') ||
+        message.includes('payload') ||
+        message.includes('request size') ||
+        message.includes('token limit'));
+}
+export class BatchEmbeddingProcessor {
+    embeddingProvider;
+    db;
+    batch = [];
+    batchSize;
+    mutex = new Mutex();
+    constructor(embeddingProvider, db, batchSize = BATCH_SIZE) {
+        this.embeddingProvider = embeddingProvider;
+        this.db = db;
+        this.batchSize = batchSize;
+    }
+    /**
+     * Add a chunk to the batch queue
+     */
+    async addChunk(chunk) {
+        this.batch.push(chunk);
+        // Process batch when it reaches the threshold
+        if (this.batch.length >= this.batchSize) {
+            await this.processBatch();
+        }
+    }
+    /**
+     * Process any remaining chunks in the batch
+     */
+    async flush() {
+        await this.mutex.runExclusive(async () => {
+            if (this.batch.length > 0) {
+                await this.processBatchWithRetry(this.batch, 0);
+                this.batch = [];
+            }
+        });
+    }
+    /**
+     * Process the current batch (with mutex protection)
+     */
+    async processBatch() {
+        await this.mutex.runExclusive(async () => {
+            if (this.batch.length > 0) {
+                await this.processBatchWithRetry(this.batch, 0);
+                this.batch = [];
+            }
+        });
+    }
+    /**
+     * Process a batch with smart error handling and retry logic
+     */
+    async processBatchWithRetry(currentBatch, retryCount) {
+        try {
+            await this.processBatchInternal(currentBatch);
+        }
+        catch (error) {
+            // Smart error handling based on error type
+            if (isBatchSizeError(error) && currentBatch.length > 1) {
+                // Batch too large - split in half and retry
+                log.warn(`Batch size too large (${currentBatch.length} chunks), splitting and retrying`);
+                const mid = Math.floor(currentBatch.length / 2);
+                const firstHalf = currentBatch.slice(0, mid);
+                const secondHalf = currentBatch.slice(mid);
+                // Process both halves recursively
+                await this.processBatchWithRetry(firstHalf, 0);
+                await this.processBatchWithRetry(secondHalf, 0);
+                return;
+            }
+            else if (isRateLimitError(error) && retryCount < MAX_BATCH_RETRIES) {
+                // Rate limit error - exponential backoff
+                const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, retryCount);
+                log.warn(`Rate limit hit, retrying batch in ${delay}ms (attempt ${retryCount + 1}/${MAX_BATCH_RETRIES})`);
+                await new Promise(resolve => setTimeout(resolve, delay));
+                await this.processBatchWithRetry(currentBatch, retryCount + 1);
+                return;
+            }
+            // Other errors or max retries reached - fall back to individual processing
+            log.error(`Batch processing failed for ${currentBatch.length} chunks`, error);
+            log.warn('Falling back to individual processing (this will be slower)');
+            await this.fallbackToIndividualProcessing(currentBatch);
+        }
+    }
+    /**
+     * Internal batch processing implementation with database transactions
+     */
+    async processBatchInternal(batch) {
+        if (batch.length === 0)
+            return;
+        // Extract texts for embedding
+        const texts = batch.map(chunk => chunk.enhancedEmbeddingText);
+        // Log batching activity at debug to reduce noise in normal runs
+        log.debug(`Processing batch of ${texts.length} chunks`);
+        // Generate embeddings in batch (single API call for all)
+        const generate = async () => this.embeddingProvider.generateEmbeddings(texts);
+        const embeddings = this.embeddingProvider.rateLimiter
+            ? await this.embeddingProvider.rateLimiter.execute(generate)
+            : await generate();
+        log.debug(`Batch complete (${texts.length} embeddings generated)`);
+        // Store all embeddings in database within a transaction
+        await this.db.transaction(() => {
+            for (let i = 0; i < batch.length; i++) {
+                const chunk = batch[i];
+                const embedding = embeddings[i];
+                this.db.insertChunk({
+                    id: chunk.chunkId,
+                    file_path: chunk.params.rel,
+                    symbol: chunk.params.symbol,
+                    sha: chunk.params.sha,
+                    lang: chunk.params.lang,
+                    chunk_type: chunk.params.chunkType,
+                    embedding,
+                    embedding_provider: this.embeddingProvider.getName(),
+                    embedding_dimensions: this.embeddingProvider.getDimensions(),
+                    codevault_tags: chunk.params.codevaultMetadata.tags,
+                    codevault_intent: chunk.params.codevaultMetadata.intent,
+                    codevault_description: chunk.params.codevaultMetadata.description,
+                    doc_comments: chunk.params.docComments,
+                    variables_used: chunk.params.importantVariables,
+                    context_info: chunk.params.contextInfo
+                });
+            }
+        });
+    }
+    /**
+     * Fall back to processing chunks individually
+     */
+    async fallbackToIndividualProcessing(batch) {
+        // Collect errors but continue processing all chunks to avoid data loss
+        const errors = [];
+        for (const chunk of batch) {
+            try {
+                const embedding = await this.embeddingProvider.generateEmbedding(chunk.enhancedEmbeddingText);
+                this.db.insertChunk({
+                    id: chunk.chunkId,
+                    file_path: chunk.params.rel,
+                    symbol: chunk.params.symbol,
+                    sha: chunk.params.sha,
+                    lang: chunk.params.lang,
+                    chunk_type: chunk.params.chunkType,
+                    embedding,
+                    embedding_provider: this.embeddingProvider.getName(),
+                    embedding_dimensions: this.embeddingProvider.getDimensions(),
+                    codevault_tags: chunk.params.codevaultMetadata.tags,
+                    codevault_intent: chunk.params.codevaultMetadata.intent,
+                    codevault_description: chunk.params.codevaultMetadata.description,
+                    doc_comments: chunk.params.docComments,
+                    variables_used: chunk.params.importantVariables,
+                    context_info: chunk.params.contextInfo
+                });
+            }
+            catch (individualError) {
+                log.error(`Failed to process chunk ${chunk.chunkId}`, individualError);
+                errors.push({
+                    chunkId: chunk.chunkId,
+                    error: individualError
+                });
+            }
+        }
+        // Report errors but don't throw to allow indexing to continue
+        if (errors.length > 0) {
+            log.warn(`${errors.length}/${batch.length} chunks failed in fallback processing`);
+            if (errors.length === batch.length) {
+                // Only throw if ALL chunks failed
+                throw new Error(`All ${errors.length} chunks failed to process: ${errors[0].error.message}`);
+            }
+        }
+    }
+    /**
+     * Get current batch size for monitoring
+     */
+    getBatchCount() {
+        return this.batch.length;
+    }
+}
+//# sourceMappingURL=batch-indexer.js.map

package/dist/core/batch-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAEzC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD,MAAM,OAAO,uBAAuB;IAMxB;IACA;IANF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAClB,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE5B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,8CAA8C;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,YAA4B,EAAE,UAAkB;QAClF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,4CAA4C;gBAC5C,GAAG,CAAC,IAAI,CAAC,yBAAyB,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACzF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,kCAAkC;gBAClC,MAAM,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;iBAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,iBAAiB,EAAE,CAAC;gBACrE,yCAAyC;gBACzC,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,GAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,eAAe,UAAU,GAAG,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;gBAC/D,OAAO;YACT,CAAC;YAED,2EAA2E;YAC3E,GAAG,CAAC,KAAK,CAAC,+BAA+B,YAAY,CAAC,MAAM,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YAExE,MAAM,IAAI,CAAC,8BAA8B,CAAC,YAAY,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,KAAqB;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,8BAA8B;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE9D,gEAAgE;QAChE,GAAG,CAAC,KAAK,CAAC,uBAAuB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAExD,yDAAyD;QACzD,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC9E,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW;YACnD,CAAC,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC5D,CAAC,CAAC,MAAM,QAAQ,EAAE,CAAC;QAErB,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAEnE,wDAAwD;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAEhC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAAC,KAAqB;QAChE,uEAAuE;QACvE,MAAM,MAAM,GAA6C,EAAE,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAE9F,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,GAAG,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;gBACvE,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,eAAwB;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,uCAAuC,CAAC,CAAC;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,kCAAkC;gBAClC,MAAM,IAAI,KAAK,CAAC,OAAO,MAAM,CAAC,MAAM,8BAA8B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}

package/dist/core/indexer.d.ts

@@ -0,0 +1,3 @@
+import type { IndexProjectOptions, IndexProjectResult } from './types.js';
+export declare function indexProject(options?: IndexProjectOptions): Promise<IndexProjectResult>;
+//# sourceMappingURL=indexer.d.ts.map

package/dist/core/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE1E,wBAAsB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAGjG"}

package/dist/core/indexer.js

@@ -0,0 +1,6 @@
+import { IndexerEngine } from './IndexerEngine.js';
+export async function indexProject(options = {}) {
+    const engine = new IndexerEngine(options);
+    return await engine.index();
+}
+//# sourceMappingURL=indexer.js.map

package/dist/core/indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAA+B,EAAE;IAChE,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;AAChC,CAAC"}

package/dist/core/indexing/chunk-pipeline.d.ts

@@ -0,0 +1,39 @@
+import type { LanguageRule } from '../../languages/rules.js';
+import type { ModelProfile } from '../../providers/base.js';
+type SizeLimits = {
+    optimal: number;
+    min: number;
+    max: number;
+    overlap: number;
+    unit: string;
+};
+interface ExistingChunks {
+    staleChunkIds: Set<string>;
+    existingChunks: Map<string, any>;
+}
+interface EmbedStoreParams {
+    code: string;
+    enhancedEmbeddingText: string;
+    chunkId: string;
+    sha: string;
+    lang: string;
+    rel: string;
+    symbol: string;
+    chunkType: string;
+    codevaultMetadata: any;
+    importantVariables: any[];
+    docComments: string | null;
+    contextInfo: any;
+    symbolData: any;
+}
+export declare class ChunkPipeline {
+    private parser;
+    private processedNodes;
+    constructor();
+    collectNodesForFile(source: string, rule: LanguageRule): Promise<import("tree-sitter").SyntaxNode[]>;
+    processGroups(nodeGroups: any[], source: string, rule: LanguageRule, limits: SizeLimits, modelProfile: ModelProfile, rel: string, existing: ExistingChunks, chunkMerkleHashes: string[], onProgress: any, embedAndStore: (params: EmbedStoreParams) => Promise<void>, chunkingStats: any): Promise<void>;
+    private yieldChunk;
+    private processChunk;
+}
+export {};
+//# sourceMappingURL=chunk-pipeline.d.ts.map

package/dist/core/indexing/chunk-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-pipeline.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/chunk-pipeline.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,KAAK,UAAU,GAAG;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,UAAU,cAAc;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,GAAG,CAAC;IACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,GAAG,CAAC;IACjB,UAAU,EAAE,GAAG,CAAC;CACjB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAqB;;IAMrC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY;IA6DtD,aAAa,CACjB,UAAU,EAAE,GAAG,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,cAAc,EACxB,iBAAiB,EAAE,MAAM,EAAE,EAC3B,UAAU,EAAE,GAAG,EACf,aAAa,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,EAC1D,aAAa,EAAE,GAAG,GACjB,OAAO,CAAC,IAAI,CAAC;YAgCF,UAAU;YAqGV,YAAY;CAmF3B"}

package/dist/core/indexing/chunk-pipeline.js

@@ -0,0 +1,210 @@
+import Parser from 'tree-sitter';
+import crypto from 'crypto';
+import { analyzeNodeForChunking, batchAnalyzeNodes, yieldStatementChunks } from '../../chunking/semantic-chunker.js';
+import { createCombinedChunk } from '../../chunking/file-grouper.js';
+import { extractSymbolMetadata } from '../../symbols/extract.js';
+import { extractSymbolName } from '../symbol-extractor.js';
+import { extractCodevaultMetadata, extractSemanticTags, extractImportantVariables, extractDocComments, generateEnhancedEmbeddingText } from '../metadata.js';
+import { computeFastHash } from '../../indexer/merkle.js';
+import { SIZE_THRESHOLD, CHUNK_SIZE } from '../../config/constants.js';
+export class ChunkPipeline {
+    parser;
+    processedNodes = new Set();
+    constructor() {
+        this.parser = new Parser();
+    }
+    async collectNodesForFile(source, rule) {
+        this.parser.setLanguage(rule.ts);
+        let tree;
+        if (source.length > SIZE_THRESHOLD) {
+            tree = this.parser.parse((index) => {
+                if (index < source.length) {
+                    return source.slice(index, Math.min(index + CHUNK_SIZE, source.length));
+                }
+                return null;
+            });
+        }
+        else {
+            tree = this.parser.parse(source);
+        }
+        if (!tree || !tree.rootNode) {
+            throw new Error('Failed to create syntax tree');
+        }
+        const collectedNodes = [];
+        const collectNodes = (node) => {
+            if (node.type === 'export_statement') {
+                let hasDeclaration = false;
+                for (let i = 0; i < node.childCount; i++) {
+                    const child = node.child(i);
+                    if (child && ['function_declaration', 'class_declaration', 'method_definition'].includes(child.type)) {
+                        hasDeclaration = true;
+                        break;
+                    }
+                }
+                if (!hasDeclaration && rule.nodeTypes.includes(node.type)) {
+                    collectedNodes.push(node);
+                    return;
+                }
+                if (hasDeclaration) {
+                    for (let i = 0; i < node.childCount; i++) {
+                        const child = node.child(i);
+                        if (child) {
+                            collectNodes(child);
+                        }
+                    }
+                    return;
+                }
+            }
+            if (rule.nodeTypes.includes(node.type)) {
+                collectedNodes.push(node);
+            }
+            for (let i = 0; i < node.childCount; i++) {
+                const child = node.child(i);
+                if (child) {
+                    collectNodes(child);
+                }
+            }
+        };
+        collectNodes(tree.rootNode);
+        return collectedNodes;
+    }
+    async processGroups(nodeGroups, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats) {
+        this.processedNodes = new Set();
+        for (const nodeGroup of nodeGroups) {
+            if (nodeGroup.nodes.length === 1) {
+                await this.yieldChunk(nodeGroup.nodes[0], source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+            }
+            else {
+                const combinedChunk = createCombinedChunk(nodeGroup, source);
+                if (combinedChunk) {
+                    chunkingStats.totalNodes += nodeGroup.nodes.length;
+                    chunkingStats.fileGrouped = (chunkingStats.fileGrouped || 0) + 1;
+                    chunkingStats.functionsGrouped = (chunkingStats.functionsGrouped || 0) + nodeGroup.nodes.length;
+                    await this.processChunk(combinedChunk.node, combinedChunk.code, `group_${nodeGroup.nodes.length}funcs`, null, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+                }
+            }
+        }
+    }
+    async yieldChunk(node, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats, parentNode = null) {
+        chunkingStats.totalNodes++;
+        const analysis = await analyzeNodeForChunking(node, source, rule, modelProfile);
+        if (analysis.size < limits.min && parentNode !== null) {
+            chunkingStats.skippedSmall++;
+            return;
+        }
+        if (analysis.needsSubdivision && analysis.subdivisionCandidates.length > 0) {
+            chunkingStats.subdivided++;
+            const subAnalyses = await batchAnalyzeNodes(analysis.subdivisionCandidates, source, rule, modelProfile, true);
+            const smallChunks = [];
+            for (let i = 0; i < subAnalyses.length; i++) {
+                const subAnalysis = subAnalyses[i];
+                const subNode = subAnalysis.node;
+                if (subAnalysis.size < limits.min) {
+                    const subCode = source.slice(subNode.startIndex, subNode.endIndex);
+                    smallChunks.push({
+                        node: subNode,
+                        code: subCode,
+                        size: subAnalysis.size
+                    });
+                    if (subNode.id !== undefined) {
+                        this.processedNodes.add(subNode.id);
+                    }
+                }
+                else {
+                    if (subNode.id !== undefined) {
+                        this.processedNodes.add(subNode.id);
+                    }
+                    await this.yieldChunk(subNode, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats, node);
+                }
+            }
+            if (smallChunks.length > 0) {
+                const totalSmallSize = smallChunks.reduce((sum, c) => sum + c.size, 0);
+                if (totalSmallSize >= limits.min || smallChunks.length >= 3) {
+                    const mergedCode = smallChunks.map((c) => c.code).join('\n\n');
+                    const mergedNode = {
+                        ...node,
+                        type: `${node.type}_merged`,
+                        startIndex: smallChunks[0].node.startIndex,
+                        endIndex: smallChunks[smallChunks.length - 1].node.endIndex
+                    };
+                    const suffix = `small_methods_${smallChunks.length}`;
+                    chunkingStats.mergedSmall++;
+                    await this.processChunk(mergedNode, mergedCode, suffix, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+                }
+                else {
+                    chunkingStats.skippedSmall += smallChunks.length;
+                }
+            }
+            return;
+        }
+        else if (analysis.size > limits.max) {
+            chunkingStats.statementFallback++;
+            const statementChunks = await yieldStatementChunks(node, source, limits.max, limits.overlap, modelProfile);
+            for (let i = 0; i < statementChunks.length; i++) {
+                const stmtChunk = statementChunks[i];
+                await this.processChunk(node, stmtChunk.code, `${i + 1}`, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+            }
+            return;
+        }
+        chunkingStats.normalChunks++;
+        const code = source.slice(node.startIndex, node.endIndex);
+        await this.processChunk(node, code, null, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+    }
+    async processChunk(node, code, suffix, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats) {
+        let symbol = extractSymbolName(node, source);
+        if (!symbol)
+            return;
+        if (suffix) {
+            symbol = `${symbol}_part${suffix}`;
+        }
+        const docComments = extractDocComments(source, node, rule);
+        const codevaultMetadata = extractCodevaultMetadata(docComments);
+        const automaticTags = extractSemanticTags(rel, symbol, code);
+        const allTags = [...new Set([...codevaultMetadata.tags, ...automaticTags])];
+        codevaultMetadata.tags = allTags;
+        const importantVariables = extractImportantVariables(node, source, rule);
+        const symbolData = extractSymbolMetadata({ node, source, symbol });
+        const enhancedEmbeddingText = generateEnhancedEmbeddingText(code, codevaultMetadata, importantVariables, docComments);
+        const chunkType = node.type.includes('class') ? 'class' :
+            node.type.includes('method') ? 'method' : 'function';
+        const contextInfo = {
+            nodeType: node.type,
+            startLine: source.slice(0, node.startIndex).split('\n').length,
+            endLine: source.slice(0, node.endIndex).split('\n').length,
+            codeLength: code.length,
+            hasDocumentation: !!docComments,
+            variableCount: importantVariables.length,
+            isSubdivision: !!suffix,
+            hasParentContext: !!parentNode
+        };
+        const sha = crypto.createHash('sha1').update(code).digest('hex');
+        const chunkId = `${rel}:${symbol}:${sha.substring(0, 8)}`;
+        const chunkMerkleHash = await computeFastHash(code);
+        if (existing.existingChunks.has(chunkId)) {
+            existing.staleChunkIds.delete(chunkId);
+            chunkMerkleHashes.push(chunkMerkleHash);
+            return;
+        }
+        await embedAndStore({
+            code,
+            enhancedEmbeddingText,
+            chunkId,
+            sha,
+            lang: rule.lang,
+            rel,
+            symbol,
+            chunkType,
+            codevaultMetadata,
+            importantVariables,
+            docComments,
+            contextInfo,
+            symbolData
+        });
+        existing.staleChunkIds.delete(chunkId);
+        chunkMerkleHashes.push(chunkMerkleHash);
+        if (onProgress) {
+            onProgress({ type: 'chunk_processed', file: rel, symbol, chunkId });
+        }
+    }
+}
+//# sourceMappingURL=chunk-pipeline.js.map

package/dist/core/indexing/chunk-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-pipeline.js","sourceRoot":"","sources":["../../../src/core/indexing/chunk-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AACrH,OAAO,EAAyB,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC5F,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EACL,wBAAwB,EACxB,mBAAmB,EACnB,yBAAyB,EACzB,kBAAkB,EAClB,6BAA6B,EAC9B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAkCvE,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAE3C;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,MAAc,EAAE,IAAkB;QAC1D,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC;QACT,IAAI,MAAM,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAa,EAAE,EAAE;gBACzC,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC1E,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,CAAC,IAAoB,EAAE,EAAE;YAC3C,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACtC,IAAI,cAAc,GAAG,KAAK,CAAC;gBAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC5B,IAAI,KAAK,IAAI,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;wBACrG,cAAc,GAAG,IAAI,CAAC;wBACtB,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1D,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBAED,IAAI,cAAc,EAAE,CAAC;oBACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;wBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC5B,IAAI,KAAK,EAAE,CAAC;4BACV,YAAY,CAAC,KAAK,CAAC,CAAC;wBACtB,CAAC;oBACH,CAAC;oBACD,OAAO;gBACT,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,IAAI,KAAK,EAAE,CAAC;oBACV,YAAY,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,UAAiB,EACjB,MAAc,EACd,IAAkB,EAClB,MAAkB,EAClB,YAA0B,EAC1B,GAAW,EACX,QAAwB,EACxB,iBAA2B,EAC3B,UAAe,EACf,aAA0D,EAC1D,aAAkB;QAElB,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAExC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YAC5J,CAAC;iBAAM,CAAC;gBACN,MAAM,aAAa,GAAG,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC7D,IAAI,aAAa,EAAE,CAAC;oBAClB,aAAa,CAAC,UAAU,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;oBACnD,aAAa,CAAC,WAAW,GAAG,CAAC,aAAa,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBACjE,aAAa,CAAC,gBAAgB,GAAG,CAAC,aAAa,CAAC,gBAAgB,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;oBAEhG,MAAM,IAAI,CAAC,YAAY,CACrB,aAAa,CAAC,IAAI,EAClB,aAAa,CAAC,IAAI,EAClB,SAAS,SAAS,CAAC,KAAK,CAAC,MAAM,OAAO,EACtC,IAAI,EACJ,MAAM,EACN,GAAG,EACH,IAAI,EACJ,QAAQ,EACR,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,aAAa,CACd,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,IAAoB,EACpB,MAAc,EACd,IAAkB,EAClB,MAAkB,EAClB,YAA0B,EAC1B,GAAW,EACX,QAAwB,EACxB,iBAA2B,EAC3B,UAAe,EACf,aAA0D,EAC1D,aAAkB,EAClB,aAAoC,IAAI;QAE1C,aAAa,CAAC,UAAU,EAAE,CAAC;QAE3B,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QAEhF,IAAI,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACtD,aAAa,CAAC,YAAY,EAAE,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3E,aAAa,CAAC,UAAU,EAAE,CAAC;YAE3B,MAAM,WAAW,GAAG,MAAM,iBAAiB,CACzC,QAAQ,CAAC,qBAAqB,EAC9B,MAAM,EACN,IAAI,EACJ,YAAY,EACZ,IAAI,CACL,CAAC;YAEF,MAAM,WAAW,GAAU,EAAE,CAAC;YAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,MAAM,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAEjC,IAAI,WAAW,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;oBACnE,WAAW,CAAC,IAAI,CAAC;wBACf,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,WAAW,CAAC,IAAI;qBACvB,CAAC,CAAC;oBACH,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBACtC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBACtC,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;gBACvJ,CAAC;YACH,CAAC;YAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,CAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAEpF,IAAI,cAAc,IAAI,MAAM,CAAC,GAAG,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC5D,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,UAAU,GAAmB;wBACjC,GAAG,IAAI;wBACP,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,SAAS;wBAC3B,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU;wBAC1C,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ;qBAC5D,CAAC;oBACF,MAAM,MAAM,GAAG,iBAAiB,WAAW,CAAC,MAAM,EAAE,CAAC;oBAErD,aAAa,CAAC,WAAW,EAAE,CAAC;oBAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;gBAChK,CAAC;qBAAM,CAAC;oBACN,aAAa,CAAC,YAAY,IAAI,WAAW,CAAC,MAAM,CAAC;gBACnD,CAAC;YACH,CAAC;YAED,OAAO;QACT,CAAC;aAAM,IAAI,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YACtC,aAAa,CAAC,iBAAiB,EAAE,CAAC;YAClC,MAAM,eAAe,GAAG,MAAM,oBAAoB,CAChD,IAAI,EACJ,MAAM,EACN,MAAM,CAAC,GAAG,EACV,MAAM,CAAC,OAAO,EACd,YAAY,CACb,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACrC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YAClK,CAAC;YACD,OAAO;QACT,CAAC;QAED,aAAa,CAAC,YAAY,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IAClJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACtB,IAAoB,EACpB,IAAY,EACZ,MAAqB,EACrB,UAAiC,EACjC,MAAc,EACd,GAAW,EACX,IAAkB,EAClB,QAAwB,EACxB,iBAA2B,EAC3B,UAAe,EACf,aAA0D,EAC1D,aAAkB;QAEpB,IAAI,MAAM,GAAG,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,MAAM,QAAQ,MAAM,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAChE,MAAM,aAAa,GAAG,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,iBAAiB,CAAC,IAAI,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC5E,iBAAiB,CAAC,IAAI,GAAG,OAAO,CAAC;QAEjC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnE,MAAM,qBAAqB,GAAG,6BAA6B,CACzD,IAAI,EACJ,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,CACZ,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC;QAEvD,MAAM,WAAW,GAAG;YAClB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;YAC9D,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;YAC1D,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,gBAAgB,EAAE,CAAC,CAAC,WAAW;YAC/B,aAAa,EAAE,kBAAkB,CAAC,MAAM;YACxC,aAAa,EAAE,CAAC,CAAC,MAAM;YACvB,gBAAgB,EAAE,CAAC,CAAC,UAAU;SAC/B,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1D,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;QAED,MAAM,aAAa,CAAC;YAClB,IAAI;YACJ,qBAAqB;YACrB,OAAO;YACP,GAAG;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,GAAG;YACH,MAAM;YACN,SAAS;YACT,iBAAiB;YACjB,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,UAAU;SACX,CAAC,CAAC;QAEH,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;CACF"}

package/dist/core/indexing/file-scanner.d.ts

@@ -0,0 +1,11 @@
+export interface ScanResult {
+    files: string[];
+    toDelete: string[];
+}
+/**
+ * Lightweight helper responsible only for discovering files eligible for indexing.
+ */
+export declare class FileScanner {
+    scan(repo: string, normalizedChanged: string[] | null): Promise<ScanResult>;
+}
+//# sourceMappingURL=file-scanner.d.ts.map

package/dist/core/indexing/file-scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-scanner.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/file-scanner.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;GAEG;AACH,qBAAa,WAAW;IAChB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC;CAyClF"}

package/dist/core/indexing/file-scanner.js

@@ -0,0 +1,49 @@
+import fg from 'fast-glob';
+import path from 'path';
+import fs from 'fs';
+import { LANG_RULES, getSupportedLanguageExtensions } from '../../languages/rules.js';
+import { DEFAULT_SCAN_IGNORES } from '../../utils/scan-patterns.js';
+/**
+ * Lightweight helper responsible only for discovering files eligible for indexing.
+ */
+export class FileScanner {
+    async scan(repo, normalizedChanged) {
+        const languagePatterns = getSupportedLanguageExtensions().map(ext => `**/*${ext}`);
+        let files = [];
+        if (normalizedChanged === null) {
+            files = await fg(languagePatterns, {
+                cwd: repo,
+                absolute: false,
+                followSymbolicLinks: false,
+                ignore: DEFAULT_SCAN_IGNORES,
+                onlyFiles: true,
+                dot: false
+            });
+        }
+        else {
+            files = normalizedChanged.filter(rel => {
+                const ext = path.extname(rel).toLowerCase();
+                return !!LANG_RULES[ext];
+            });
+        }
+        const uniqueFiles = [];
+        const toDelete = [];
+        const seenFiles = new Set();
+        for (const rel of files) {
+            if (!rel || seenFiles.has(rel)) {
+                continue;
+            }
+            const absPath = path.join(repo, rel);
+            try {
+                await fs.promises.access(absPath);
+                seenFiles.add(rel);
+                uniqueFiles.push(rel);
+            }
+            catch {
+                toDelete.push(rel);
+            }
+        }
+        return { files: uniqueFiles, toDelete };
+    }
+}
+//# sourceMappingURL=file-scanner.js.map

package/dist/core/indexing/file-scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-scanner.js","sourceRoot":"","sources":["../../../src/core/indexing/file-scanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,UAAU,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAOpE;;GAEG;AACH,MAAM,OAAO,WAAW;IACtB,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,iBAAkC;QACzD,MAAM,gBAAgB,GAAG,8BAA8B,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACnF,IAAI,KAAK,GAAa,EAAE,CAAC;QAEzB,IAAI,iBAAiB,KAAK,IAAI,EAAE,CAAC;YAC/B,KAAK,GAAG,MAAM,EAAE,CAAC,gBAAgB,EAAE;gBACjC,GAAG,EAAE,IAAI;gBACT,QAAQ,EAAE,KAAK;gBACf,mBAAmB,EAAE,KAAK;gBAC1B,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,IAAI;gBACf,GAAG,EAAE,KAAK;aACX,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;gBACrC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC5C,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QAEpC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAClC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACnB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IAC1C,CAAC;CACF"}

package/dist/core/metadata.d.ts

@@ -0,0 +1,19 @@
+import type { LanguageRule } from '../languages/rules.js';
+import type { TreeSitterNode } from '../types/ast.js';
+export interface CodevaultMetadata {
+    tags: string[];
+    intent: string | null;
+    description: string | null;
+}
+export declare function extractCodevaultMetadata(commentText: string | null): CodevaultMetadata;
+export declare function extractSemanticTags(filePath: string, symbolName: string | null, code: string): string[];
+interface ImportantVariable {
+    type: string;
+    name: string;
+    value: string;
+}
+export declare function extractImportantVariables(node: TreeSitterNode, source: string, rule: LanguageRule): ImportantVariable[];
+export declare function extractDocComments(source: string, node: TreeSitterNode, rule: LanguageRule): string | null;
+export declare function generateEnhancedEmbeddingText(code: string, metadata: CodevaultMetadata, variables: ImportantVariable[], docComments: string | null): string;
+export {};
+//# sourceMappingURL=metadata.d.ts.map

package/dist/core/metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../src/core/metadata.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,iBAAiB,CAyBtF;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAgEvG;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,iBAAiB,EAAE,CA0BvH;AAkCD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,GAAG,MAAM,GAAG,IAAI,CAY1G;AAED,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,SAAS,EAAE,iBAAiB,EAAE,EAC9B,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,MAAM,CAyBR"}