codevault 1.7.3 → 1.7.5

package/dist/core/batch-indexer.js

@@ -20,6 +20,11 @@ function isBatchSizeError(error) {
         message.includes('request size') ||
         message.includes('token limit'));
 }
+/**
+ * Batches chunk embeddings for efficiency and handles retry/backoff with a SQLite sink.
+ *
+ * Use `addChunk` to enqueue work and `flush` to force processing of remaining items.
+ */
 export class BatchEmbeddingProcessor {
     embeddingProvider;
     db;
@@ -113,29 +118,23 @@ export class BatchEmbeddingProcessor {
         const embeddings = await this.embeddingProvider.generateEmbeddings(texts);
         log.debug(`Batch complete (${texts.length} embeddings generated)`);
         // Store all embeddings in database within a transaction
-        await this.db.transaction(() => {
-            for (let i = 0; i < batch.length; i++) {
-                const chunk = batch[i];
-                const embedding = embeddings[i];
-                this.db.insertChunk({
-                    id: chunk.chunkId,
-                    file_path: chunk.params.rel,
-                    symbol: chunk.params.symbol,
-                    sha: chunk.params.sha,
-                    lang: chunk.params.lang,
-                    chunk_type: chunk.params.chunkType,
-                    embedding,
-                    embedding_provider: this.embeddingProvider.getName(),
-                    embedding_dimensions: this.embeddingProvider.getDimensions(),
-                    codevault_tags: chunk.params.codevaultMetadata.tags,
-                    codevault_intent: chunk.params.codevaultMetadata.intent,
-                    codevault_description: chunk.params.codevaultMetadata.description,
-                    doc_comments: chunk.params.docComments,
-                    variables_used: chunk.params.importantVariables,
-                    context_info: chunk.params.contextInfo
-                });
-            }
-        });
+        this.db.insertChunks(batch.map((chunk, i) => ({
+            id: chunk.chunkId,
+            file_path: chunk.params.rel,
+            symbol: chunk.params.symbol,
+            sha: chunk.params.sha,
+            lang: chunk.params.lang,
+            chunk_type: chunk.params.chunkType,
+            embedding: embeddings[i],
+            embedding_provider: this.embeddingProvider.getName(),
+            embedding_dimensions: this.embeddingProvider.getDimensions(),
+            codevault_tags: chunk.params.codevaultMetadata.tags,
+            codevault_intent: chunk.params.codevaultMetadata.intent,
+            codevault_description: chunk.params.codevaultMetadata.description,
+            doc_comments: chunk.params.docComments,
+            variables_used: chunk.params.importantVariables,
+            context_info: chunk.params.contextInfo
+        })));
     }
     /**
      * Fall back to processing chunks individually

package/dist/core/batch-indexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAEzC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD,MAAM,OAAO,uBAAuB;IAMxB;IACA;IANF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAClB,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE5B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,8CAA8C;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,YAA4B,EAAE,UAAkB;QAClF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,4CAA4C;gBAC5C,GAAG,CAAC,IAAI,CAAC,yBAAyB,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACzF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,kCAAkC;gBAClC,MAAM,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;iBAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,iBAAiB,EAAE,CAAC;gBACrE,yCAAyC;gBACzC,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,GAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,eAAe,UAAU,GAAG,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;gBAC/D,OAAO;YACT,CAAC;YAED,2EAA2E;YAC3E,GAAG,CAAC,KAAK,CAAC,+BAA+B,YAAY,CAAC,MAAM,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YAExE,MAAM,IAAI,CAAC,8BAA8B,CAAC,YAAY,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,KAAqB;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,8BAA8B;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE9D,gEAAgE;QAChE,GAAG,CAAC,KAAK,CAAC,uBAAuB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAExD,yDAAyD;QACzD,4EAA4E;QAC5E,4CAA4C;QAC5C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE1E,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAEnE,wDAAwD;QACxD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAEhC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAAC,KAAqB;QAChE,uEAAuE;QACvE,MAAM,MAAM,GAA6C,EAAE,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAE9F,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,GAAG,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;gBACvE,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,eAAwB;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,uCAAuC,CAAC,CAAC;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,kCAAkC;gBAClC,MAAM,IAAI,KAAK,CAAC,OAAO,MAAM,CAAC,MAAM,8BAA8B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}
+{"version":3,"file":"batch-indexer.js","sourceRoot":"","sources":["../../src/core/batch-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAEzC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,KAAK,EAAE,UAAU,KAAK,GAAG;QACzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAU;IAClC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,CACL,KAAK,EAAE,MAAM,KAAK,GAAG;QACrB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC7B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAChC,CAAC;AACJ,CAAC;AAmBD;;;;GAIG;AACH,MAAM,OAAO,uBAAuB;IAMxB;IACA;IANF,KAAK,GAAmB,EAAE,CAAC;IAC3B,SAAS,CAAS;IAClB,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE5B,YACU,iBAAoC,EACpC,EAAY,EACpB,YAAoB,UAAU;QAFtB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,OAAE,GAAF,EAAE,CAAU;QAGpB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAmB;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEvB,8CAA8C;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACvC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAChD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAC,YAA4B,EAAE,UAAkB;QAClF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,4CAA4C;gBAC5C,GAAG,CAAC,IAAI,CAAC,yBAAyB,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACzF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAChD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,kCAAkC;gBAClC,MAAM,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;iBAAM,IAAI,gBAAgB,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,iBAAiB,EAAE,CAAC;gBACrE,yCAAyC;gBACzC,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,GAAG,CAAC,IAAI,CAAC,qCAAqC,KAAK,eAAe,UAAU,GAAG,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBAC1G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;gBAC/D,OAAO;YACT,CAAC;YAED,2EAA2E;YAC3E,GAAG,CAAC,KAAK,CAAC,+BAA+B,YAAY,CAAC,MAAM,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YAExE,MAAM,IAAI,CAAC,8BAA8B,CAAC,YAAY,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,KAAqB;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,8BAA8B;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE9D,gEAAgE;QAChE,GAAG,CAAC,KAAK,CAAC,uBAAuB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAExD,yDAAyD;QACzD,4EAA4E;QAC5E,4CAA4C;QAC5C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE1E,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAEnE,wDAAwD;QACxD,IAAI,CAAC,EAAE,CAAC,YAAY,CAClB,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,EAAE,EAAE,KAAK,CAAC,OAAO;YACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;YAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;YAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;YACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;YACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;YAClC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;YACxB,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;YACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;YAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;YACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;YACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;YACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;YACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;YAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;SACvC,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAAC,KAAqB;QAChE,uEAAuE;QACvE,MAAM,MAAM,GAA6C,EAAE,CAAC;QAE5D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAE9F,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC;oBAClB,EAAE,EAAE,KAAK,CAAC,OAAO;oBACjB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG;oBACrB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI;oBACvB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBAClC,SAAS;oBACT,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE;oBACpD,oBAAoB,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE;oBAC5D,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI;oBACnD,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;oBACvD,qBAAqB,EAAE,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW;oBACjE,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;oBACtC,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;oBAC/C,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW;iBACvC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,GAAG,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;gBACvE,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,eAAwB;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,uCAAuC,CAAC,CAAC;YAClF,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,kCAAkC;gBAClC,MAAM,IAAI,KAAK,CAAC,OAAO,MAAM,CAAC,MAAM,8BAA8B,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;CACF"}

package/dist/core/indexer.d.ts

@@ -1,3 +1,16 @@
 import type { IndexProjectOptions, IndexProjectResult } from './types.js';
+/**
+ * Indexes a project into CodeVault's SQLite + chunk store for hybrid search.
+ *
+ * The indexer performs Tree-sitter based chunking, embedding generation, codemap
+ * construction, Merkle tracking, and optional encryption. It supports both full
+ * and incremental runs by passing `changedFiles` / `deletedFiles`.
+ *
+ * @param options - Indexing configuration (repo path, provider, hooks, encryption)
+ * @returns Summary of processed chunks, provider used, and any encountered errors
+ * @throws {Error} When the target repository path does not exist or cannot be read
+ * @example
+ * await indexProject({ repoPath: '/path/to/repo', provider: 'openai' });
+ */
 export declare function indexProject(options?: IndexProjectOptions): Promise<IndexProjectResult>;
 //# sourceMappingURL=indexer.d.ts.map

package/dist/core/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE1E,wBAAsB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAGjG"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE1E;;;;;;;;;;;;GAYG;AACH,wBAAsB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAGjG"}

package/dist/core/indexer.js

@@ -1,4 +1,17 @@
 import { IndexerEngine } from './IndexerEngine.js';
+/**
+ * Indexes a project into CodeVault's SQLite + chunk store for hybrid search.
+ *
+ * The indexer performs Tree-sitter based chunking, embedding generation, codemap
+ * construction, Merkle tracking, and optional encryption. It supports both full
+ * and incremental runs by passing `changedFiles` / `deletedFiles`.
+ *
+ * @param options - Indexing configuration (repo path, provider, hooks, encryption)
+ * @returns Summary of processed chunks, provider used, and any encountered errors
+ * @throws {Error} When the target repository path does not exist or cannot be read
+ * @example
+ * await indexProject({ repoPath: '/path/to/repo', provider: 'openai' });
+ */
 export async function indexProject(options = {}) {
     const engine = new IndexerEngine(options);
     return await engine.index();

package/dist/core/indexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAA+B,EAAE;IAChE,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;AAChC,CAAC"}
+{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/core/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAA+B,EAAE;IAClE,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;AAC9B,CAAC"}

package/dist/core/indexing/FileProcessor.d.ts

@@ -0,0 +1,42 @@
+import type { IndexContextData } from './IndexContext.js';
+import type { IndexState } from './IndexState.js';
+/**
+ * FileProcessor handles the processing of individual files:
+ * - Parsing and chunking
+ * - Change detection via merkle tree
+ * - Embedding and storage
+ * - Fallback handling on errors
+ */
+export declare class FileProcessor {
+    private context;
+    private state;
+    private onProgress;
+    private chunkPipeline;
+    constructor(context: IndexContextData, state: IndexState, onProgress: ((event: {
+        type: string;
+        file?: string;
+        symbol?: string;
+        chunkId?: string;
+    }) => void) | null);
+    /**
+     * Process a single file
+     */
+    processFile(rel: string): Promise<void>;
+    /**
+     * Fallback processing when normal processing fails
+     */
+    private tryFallbackProcessing;
+    /**
+     * Embed and store a chunk
+     */
+    private embedAndStore;
+    /**
+     * Delete chunks from database and file system
+     */
+    private deleteChunks;
+    /**
+     * Remove all artifacts for a deleted file
+     */
+    removeFileArtifacts(fileRel: string): Promise<void>;
+}
+//# sourceMappingURL=FileProcessor.d.ts.map

package/dist/core/indexing/FileProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileProcessor.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/FileProcessor.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAKlD;;;;;;GAMG;AACH,qBAAa,aAAa;IAItB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,UAAU;IALpB,OAAO,CAAC,aAAa,CAAuB;gBAGlC,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI;IAGlH;;OAEG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2E7C;;OAEG;YACW,qBAAqB;IAoFnC;;OAEG;YACW,aAAa;IA4E3B;;OAEG;YACW,YAAY;IAkB1B;;OAEG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAgB1D"}

package/dist/core/indexing/FileProcessor.js

@@ -0,0 +1,245 @@
+import crypto from 'crypto';
+import fs from 'fs';
+import path from 'path';
+import { LANG_RULES } from '../../languages/rules.js';
+import { computeFastHash } from '../../indexer/merkle.js';
+import { ChunkPipeline } from './chunk-pipeline.js';
+import { normalizeChunkMetadata } from '../../types/codemap.js';
+import { writeChunkToDisk, removeChunkArtifacts } from '../../storage/encrypted-chunks.js';
+/**
+ * FileProcessor handles the processing of individual files:
+ * - Parsing and chunking
+ * - Change detection via merkle tree
+ * - Embedding and storage
+ * - Fallback handling on errors
+ */
+export class FileProcessor {
+    context;
+    state;
+    onProgress;
+    chunkPipeline = new ChunkPipeline();
+    constructor(context, state, onProgress) {
+        this.context = context;
+        this.state = state;
+        this.onProgress = onProgress;
+    }
+    /**
+     * Process a single file
+     */
+    async processFile(rel) {
+        const abs = path.join(this.context.repo, rel);
+        const ext = path.extname(rel).toLowerCase();
+        const rule = LANG_RULES[ext];
+        if (!rule)
+            return;
+        const existingChunks = new Map(Object.entries(this.state.codemap)
+            .filter(([, metadata]) => metadata && metadata.file === rel));
+        const staleChunkIds = new Set(existingChunks.keys());
+        const chunkMerkleHashes = [];
+        let fileHash = null;
+        try {
+            const source = await fs.promises.readFile(abs, 'utf8');
+            fileHash = await computeFastHash(source);
+            // Skip if file hasn't changed
+            const previousMerkle = this.context.merkle[rel];
+            if (previousMerkle && previousMerkle.shaFile === fileHash) {
+                return;
+            }
+            // Collect and group nodes
+            const collectedNodes = await this.chunkPipeline.collectNodesForFile(source, rule);
+            const nodeGroups = await this.chunkPipeline.groupNodes(collectedNodes, source, this.context.modelProfile, rule);
+            // Process groups
+            await this.chunkPipeline.processGroups(nodeGroups, source, rule, this.context.limits, this.context.modelProfile, rel, { staleChunkIds, existingChunks }, chunkMerkleHashes, this.onProgress, (params) => this.embedAndStore(params), this.state.chunkingStats);
+            // Delete stale chunks
+            if (staleChunkIds.size > 0) {
+                await this.deleteChunks(Array.from(staleChunkIds), existingChunks);
+                this.state.markIndexMutated();
+            }
+            // Update merkle tree
+            if (fileHash) {
+                this.state.updatedMerkle[rel] = {
+                    shaFile: fileHash,
+                    chunkShas: chunkMerkleHashes
+                };
+                this.state.markMerkleDirty();
+            }
+        }
+        catch (error) {
+            this.state.addError({
+                type: 'processing_error',
+                file: rel,
+                error: error.message
+            });
+            // Try fallback processing
+            await this.tryFallbackProcessing(rel, abs, rule, existingChunks, staleChunkIds, chunkMerkleHashes);
+        }
+    }
+    /**
+     * Fallback processing when normal processing fails
+     */
+    async tryFallbackProcessing(rel, abs, rule, existingChunks, staleChunkIds, chunkMerkleHashes) {
+        try {
+            if (!fs.existsSync(abs)) {
+                return;
+            }
+            if (!rule) {
+                return;
+            }
+            const source = await fs.promises.readFile(abs, 'utf8');
+            const fallbackSymbol = path.basename(rel) || rel;
+            const sha = crypto.createHash('sha1').update(source).digest('hex');
+            const chunkId = `${rel}:fallback:${sha.substring(0, 8)}`;
+            const chunkMerkleHash = await computeFastHash(source);
+            const fallbackMetadata = { tags: [], intent: null, description: null };
+            const contextInfo = {
+                nodeType: 'file',
+                startLine: 1,
+                endLine: source.split('\n').length,
+                codeLength: source.length,
+                hasDocumentation: false,
+                variableCount: 0
+            };
+            await this.embedAndStore({
+                code: source,
+                enhancedEmbeddingText: source,
+                chunkId,
+                sha,
+                lang: rule.lang,
+                rel,
+                symbol: fallbackSymbol,
+                chunkType: 'file',
+                codevaultMetadata: fallbackMetadata,
+                importantVariables: [],
+                docComments: null,
+                contextInfo,
+                symbolData: {
+                    signature: `${fallbackSymbol}()`,
+                    parameters: [],
+                    returnType: null,
+                    calls: []
+                }
+            });
+            this.state.incrementProcessedChunks();
+            this.state.markIndexMutated();
+            if (this.onProgress) {
+                this.onProgress({ type: 'chunk_processed', file: rel, symbol: fallbackSymbol, chunkId });
+            }
+            staleChunkIds.delete(chunkId);
+            if (staleChunkIds.size > 0) {
+                await this.deleteChunks(Array.from(staleChunkIds), existingChunks);
+                this.state.markIndexMutated();
+            }
+            chunkMerkleHashes.length = 0;
+            chunkMerkleHashes.push(chunkMerkleHash);
+            this.state.updatedMerkle[rel] = {
+                shaFile: chunkMerkleHash,
+                chunkShas: [...chunkMerkleHashes]
+            };
+            this.state.markMerkleDirty();
+        }
+        catch (fallbackError) {
+            this.state.addError({
+                type: 'fallback_error',
+                file: rel,
+                error: fallbackError.message
+            });
+        }
+    }
+    /**
+     * Embed and store a chunk
+     */
+    async embedAndStore(params) {
+        try {
+            if (!this.context.batchProcessor) {
+                throw new Error('Batch processor not initialized');
+            }
+            await this.context.batchProcessor.addChunk({
+                chunkId: params.chunkId,
+                enhancedEmbeddingText: params.enhancedEmbeddingText,
+                params: {
+                    code: params.code,
+                    sha: params.sha,
+                    lang: params.lang,
+                    rel: params.rel,
+                    symbol: params.symbol,
+                    chunkType: params.chunkType,
+                    codevaultMetadata: params.codevaultMetadata,
+                    importantVariables: params.importantVariables,
+                    docComments: params.docComments,
+                    contextInfo: params.contextInfo
+                }
+            });
+            this.state.markIndexMutated();
+            await fs.promises.mkdir(this.context.chunkDir, { recursive: true });
+            const writeResult = writeChunkToDisk({
+                chunkDir: this.context.chunkDir,
+                sha: params.sha,
+                code: params.code,
+                encryption: this.context.encryptionPreference
+            });
+            const previousMetadata = this.state.codemap[params.chunkId];
+            this.state.codemap[params.chunkId] = normalizeChunkMetadata({
+                file: params.rel,
+                symbol: params.symbol,
+                sha: params.sha,
+                lang: params.lang,
+                chunkType: params.chunkType,
+                provider: this.context.providerInstance.getName(),
+                dimensions: this.context.providerInstance.getDimensions(),
+                hasCodevaultTags: Array.isArray(params.codevaultMetadata.tags) && params.codevaultMetadata.tags.length > 0,
+                hasIntent: !!params.codevaultMetadata.intent,
+                hasDocumentation: !!params.docComments,
+                variableCount: Array.isArray(params.importantVariables) ? params.importantVariables.length : 0,
+                encrypted: !!(writeResult && writeResult.encrypted),
+                symbol_signature: params.symbolData && params.symbolData.signature ? params.symbolData.signature : undefined,
+                symbol_parameters: params.symbolData && Array.isArray(params.symbolData.parameters) ? params.symbolData.parameters : undefined,
+                symbol_return: params.symbolData && params.symbolData.returnType ? params.symbolData.returnType : undefined,
+                symbol_calls: params.symbolData && Array.isArray(params.symbolData.calls) ? params.symbolData.calls : undefined
+            }, previousMetadata);
+        }
+        catch (error) {
+            this.state.addError({
+                type: 'indexing_error',
+                chunkId: params.chunkId,
+                error: error.message
+            });
+            throw error;
+        }
+    }
+    /**
+     * Delete chunks from database and file system
+     */
+    async deleteChunks(chunkIds, metadataLookup = new Map()) {
+        if (!Array.isArray(chunkIds) || chunkIds.length === 0) {
+            return;
+        }
+        if (this.context.db) {
+            await this.context.db.deleteChunks(chunkIds);
+        }
+        for (const chunkId of chunkIds) {
+            const metadata = metadataLookup.get(chunkId) || this.state.codemap[chunkId];
+            if (metadata && metadata.sha) {
+                removeChunkArtifacts(this.context.chunkDir, metadata.sha);
+            }
+            delete this.state.codemap[chunkId];
+        }
+    }
+    /**
+     * Remove all artifacts for a deleted file
+     */
+    async removeFileArtifacts(fileRel) {
+        const entries = Object.entries(this.state.codemap)
+            .filter(([, metadata]) => metadata && metadata.file === fileRel);
+        if (entries.length > 0) {
+            const metadataLookup = new Map(entries);
+            await this.deleteChunks(entries.map(([chunkId]) => chunkId), metadataLookup);
+            this.state.markIndexMutated();
+        }
+        // Remove from merkle tree
+        const { removeMerkleEntry } = await import('../../indexer/merkle.js');
+        if (removeMerkleEntry(this.state.updatedMerkle, fileRel)) {
+            this.state.markMerkleDirty();
+        }
+    }
+}
+//# sourceMappingURL=FileProcessor.js.map

package/dist/core/indexing/FileProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileProcessor.js","sourceRoot":"","sources":["../../../src/core/indexing/FileProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,OAAO,EAAC,sBAAsB,EAAC,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,gBAAgB,EAAE,oBAAoB,EAAC,MAAM,mCAAmC,CAAC;AAEzF;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAId;IACA;IACA;IALF,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;IAE5C,YACU,OAAyB,EACzB,KAAiB,EACjB,UAAwG;QAFxG,YAAO,GAAP,OAAO,CAAkB;QACzB,UAAK,GAAL,KAAK,CAAY;QACjB,eAAU,GAAV,UAAU,CAA8F;IAC/G,CAAC;IAEJ;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;aAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,IAAK,QAAgB,CAAC,IAAI,KAAK,GAAG,CAAoB,CAC3F,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,MAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,IAAI,QAAQ,GAAkB,IAAI,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACvD,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEzC,8BAA8B;YAC9B,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,cAAc,IAAI,cAAc,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC1D,OAAO;YACT,CAAC;YAED,0BAA0B;YAC1B,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAClF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CACpD,cAAc,EACd,MAAM,EACN,IAAI,CAAC,OAAO,CAAC,YAAY,EACzB,IAAI,CACL,CAAC;YAEF,iBAAiB;YACjB,MAAM,IAAI,CAAC,aAAa,CAAC,aAAa,CACpC,UAAU,EACV,MAAM,EACN,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,MAAM,EACnB,IAAI,CAAC,OAAO,CAAC,YAAY,EACzB,GAAG,EACH,EAAE,aAAa,EAAE,cAAc,EAAE,EACjC,iBAAiB,EACjB,IAAI,CAAC,UAAU,EACf,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EACtC,IAAI,CAAC,KAAK,CAAC,aAAa,CACzB,CAAC;YAEF,sBAAsB;YACtB,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC,CAAC;gBACnE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAChC,CAAC;YAED,qBAAqB;YACrB,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG;oBAC9B,OAAO,EAAE,QAAQ;oBACjB,SAAS,EAAE,iBAAiB;iBAC7B,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,kBAAkB;gBACxB,IAAI,EAAE,GAAG;gBACT,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;YAEH,0BAA0B;YAC1B,MAAM,IAAI,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,GAAW,EACX,GAAW,EACX,IAAyC,EACzC,cAAgC,EAChC,aAA0B,EAC1B,iBAA2B;QAE3B,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;YAED,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;YACjD,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,OAAO,GAAG,GAAG,GAAG,aAAa,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;YAEtD,MAAM,gBAAgB,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACvE,MAAM,WAAW,GAAG;gBAClB,QAAQ,EAAE,MAAM;gBAChB,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAClC,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,gBAAgB,EAAE,KAAK;gBACvB,aAAa,EAAE,CAAC;aACjB,CAAC;YAEF,MAAM,IAAI,CAAC,aAAa,CAAC;gBACvB,IAAI,EAAE,MAAM;gBACZ,qBAAqB,EAAE,MAAM;gBAC7B,OAAO;gBACP,GAAG;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,GAAG;gBACH,MAAM,EAAE,cAAc;gBACtB,SAAS,EAAE,MAAM;gBACjB,iBAAiB,EAAE,gBAAgB;gBACnC,kBAAkB,EAAE,EAAE;gBACtB,WAAW,EAAE,IAAI;gBACjB,WAAW;gBACX,UAAU,EAAE;oBACV,SAAS,EAAE,GAAG,cAAc,IAAI;oBAChC,UAAU,EAAE,EAAE;oBACd,UAAU,EAAE,IAAI;oBAChB,KAAK,EAAE,EAAE;iBACV;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;YACtC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAE9B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3F,CAAC;YAED,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC9B,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC,CAAC;gBACnE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAChC,CAAC;YAED,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7B,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAExC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG;gBAC9B,OAAO,EAAE,eAAe;gBACxB,SAAS,EAAE,CAAC,GAAG,iBAAiB,CAAC;aAClC,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,GAAG;gBACT,KAAK,EAAG,aAAuB,CAAC,OAAO;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,MAc3B;QACC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;gBACzC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;gBACnD,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;oBAC3C,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;oBAC7C,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAE9B,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,MAAM,WAAW,GAAG,gBAAgB,CAAC;gBACnC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAC/B,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;aAC9C,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,sBAAsB,CAAC;gBAC1D,IAAI,EAAE,MAAM,CAAC,GAAG;gBAChB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACjD,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,aAAa,EAAE;gBACzD,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBAC1G,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;gBAC5C,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,WAAW;gBACtC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC9F,SAAS,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC,SAAS,CAAC;gBACnD,gBAAgB,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;gBAC5G,iBAAiB,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC9H,aAAa,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC3G,YAAY,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aAChH,EAAE,gBAAgB,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClB,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,QAAkB,EAAE,iBAAiB,IAAI,GAAG,EAAe;QACpF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC/C,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC5E,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAC7B,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,OAAe;QACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;aAC/C,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,IAAK,QAAgB,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;QAE5E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,OAA0B,CAAC,CAAC;YAC3D,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YAC7E,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAChC,CAAC;QAED,0BAA0B;QAC1B,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;QACtE,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YACzD,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/core/indexing/IndexContext.d.ts

@@ -0,0 +1,42 @@
+import { type EmbeddingProvider } from '../../providers/index.js';
+import { type Codemap } from '../../codemap/io.js';
+import { type MerkleTree } from '../../indexer/merkle.js';
+import { Database } from '../../database/db.js';
+import { BatchEmbeddingProcessor } from '../batch-indexer.js';
+import type { IndexProjectOptions } from '../types.js';
+export interface IndexContextData {
+    repo: string;
+    repoPath: string;
+    provider: string;
+    providerInstance: EmbeddingProvider;
+    providerName: string;
+    modelName: string | null;
+    modelProfile: any;
+    limits: any;
+    codemapPath: string;
+    chunkDir: string;
+    dbPath: string;
+    encryptionPreference: any;
+    codemap: Codemap;
+    merkle: MerkleTree;
+    updatedMerkle: MerkleTree;
+    db: Database;
+    batchProcessor: BatchEmbeddingProcessor;
+    isPartialUpdate: boolean;
+}
+/**
+ * IndexContext prepares the indexing environment by:
+ * - Validating repository
+ * - Initializing embedding provider
+ * - Setting up database
+ * - Loading codemap and merkle tree
+ * - Creating batch processor
+ */
+export declare class IndexContext {
+    static prepare(options: IndexProjectOptions): Promise<IndexContextData>;
+    /**
+     * Check if there's a dimension or provider mismatch and warn user
+     */
+    private static checkDimensionMismatch;
+}
+//# sourceMappingURL=IndexContext.d.ts.map

package/dist/core/indexing/IndexContext.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexContext.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/IndexContext.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2D,KAAK,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE3H,OAAO,EAAe,KAAK,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAA2B,KAAK,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAEnF,OAAO,EAAE,QAAQ,EAAgB,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAG9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,iBAAiB,CAAC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,GAAG,CAAC;IAClB,MAAM,EAAE,GAAG,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,GAAG,CAAC;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,aAAa,EAAE,UAAU,CAAC;IAC1B,EAAE,EAAE,QAAQ,CAAC;IACb,cAAc,EAAE,uBAAuB,CAAC;IACxC,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;;;GAOG;AACH,qBAAa,YAAY;WACV,OAAO,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA+F7E;;OAEG;mBACkB,sBAAsB;CAyC5C"}

package/dist/core/indexing/IndexContext.js

@@ -0,0 +1,133 @@
+import path from 'path';
+import fs from 'fs';
+import { createEmbeddingProvider, getModelProfile, getSizeLimits } from '../../providers/index.js';
+import { BATCH_SIZE } from '../../providers/base.js';
+import { readCodemap } from '../../codemap/io.js';
+import { loadMerkle, cloneMerkle } from '../../indexer/merkle.js';
+import { resolveEncryptionPreference } from '../../storage/encrypted-chunks.js';
+import { Database, initDatabase } from '../../database/db.js';
+import { BatchEmbeddingProcessor } from '../batch-indexer.js';
+import { logger } from '../../utils/logger.js';
+import { resolveProviderContext } from '../../config/resolver.js';
+/**
+ * IndexContext prepares the indexing environment by:
+ * - Validating repository
+ * - Initializing embedding provider
+ * - Setting up database
+ * - Loading codemap and merkle tree
+ * - Creating batch processor
+ */
+export class IndexContext {
+    static async prepare(options) {
+        const { repoPath = '.', provider = 'auto', changedFiles = null, embeddingProviderOverride = null, encryptMode = undefined } = options;
+        const repo = path.resolve(repoPath);
+        // Validate repository exists
+        try {
+            await fs.promises.access(repo);
+        }
+        catch {
+            throw new Error(`Directory ${repo} does not exist`);
+        }
+        // Setup provider context and embedding provider
+        const providerContext = resolveProviderContext(repo);
+        const providerInstance = embeddingProviderOverride ||
+            createEmbeddingProvider(provider, providerContext.embedding);
+        if (!embeddingProviderOverride && providerInstance.init) {
+            await providerInstance.init();
+        }
+        // Get provider and model information
+        const providerName = providerInstance.getName();
+        const modelName = providerInstance.getModelName ? providerInstance.getModelName() : null;
+        const modelProfile = await getModelProfile(providerName, modelName || providerName);
+        const limits = getSizeLimits(modelProfile);
+        // Log configuration
+        if (!process.env.CODEVAULT_QUIET) {
+            logger.info(`Chunking Configuration`, {
+                provider: providerName,
+                model: modelName,
+                dimensions: providerInstance.getDimensions(),
+                mode: limits.unit
+            });
+        }
+        // Initialize database
+        await initDatabase(providerInstance.getDimensions(), repo);
+        // Setup paths
+        const codemapPath = path.join(repo, 'codevault.codemap.json');
+        const chunkDir = path.join(repo, '.codevault/chunks');
+        const dbPath = path.join(repo, '.codevault/codevault.db');
+        // Check for dimension mismatches
+        await IndexContext.checkDimensionMismatch(dbPath, providerInstance);
+        // Setup encryption
+        const encryptionPreference = resolveEncryptionPreference({
+            mode: encryptMode,
+            logger: console
+        });
+        // Load existing state
+        const codemap = readCodemap(codemapPath);
+        const merkle = loadMerkle(repo);
+        const updatedMerkle = cloneMerkle(merkle);
+        // Create database connection
+        const db = new Database(dbPath);
+        // Create batch processor
+        const batchProcessor = new BatchEmbeddingProcessor(providerInstance, db, BATCH_SIZE);
+        const isPartialUpdate = changedFiles !== null;
+        return {
+            repo,
+            repoPath,
+            provider,
+            providerInstance,
+            providerName,
+            modelName,
+            modelProfile,
+            limits,
+            codemapPath,
+            chunkDir,
+            dbPath,
+            encryptionPreference,
+            codemap,
+            merkle,
+            updatedMerkle,
+            db,
+            batchProcessor,
+            isPartialUpdate
+        };
+    }
+    /**
+     * Check if there's a dimension or provider mismatch and warn user
+     */
+    static async checkDimensionMismatch(dbPath, embeddingProvider) {
+        try {
+            await fs.promises.access(dbPath);
+        }
+        catch {
+            return; // DB doesn't exist yet
+        }
+        const db = new Database(dbPath);
+        try {
+            const existingDimensions = await db.getExistingDimensions();
+            if (existingDimensions.length > 0) {
+                const currentProvider = embeddingProvider.getName();
+                const currentDimensions = embeddingProvider.getDimensions();
+                const hasMismatch = existingDimensions.some(row => row.embedding_provider !== currentProvider ||
+                    row.embedding_dimensions !== currentDimensions);
+                if (hasMismatch) {
+                    logger.warn('Dimension/Provider Mismatch Detected!', {
+                        existing: existingDimensions,
+                        current: { provider: currentProvider, dimensions: currentDimensions },
+                        recommendation: 'Full re-index recommended'
+                    });
+                    await new Promise(resolve => setTimeout(resolve, 2000));
+                }
+            }
+        }
+        catch (error) {
+            logger.debug('Migration check encountered an error (continuing)', {
+                error: error instanceof Error ? error.message : String(error)
+            });
+        }
+        finally {
+            db.close();
+        }
+    }
+}
+//# sourceMappingURL=IndexContext.js.map

package/dist/core/indexing/IndexContext.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexContext.js","sourceRoot":"","sources":["../../../src/core/indexing/IndexContext.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAAE,aAAa,EAA0B,MAAM,0BAA0B,CAAC;AAC3H,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAgB,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAmB,MAAM,yBAAyB,CAAC;AACnF,OAAO,EAAE,2BAA2B,EAAE,MAAM,mCAAmC,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAwBlE;;;;;;;GAOG;AACH,MAAM,OAAO,YAAY;IACvB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAA4B;QAC/C,MAAM,EACJ,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,MAAM,EACjB,YAAY,GAAG,IAAI,EACnB,yBAAyB,GAAG,IAAI,EAChC,WAAW,GAAG,SAAS,EACxB,GAAG,OAAO,CAAC;QAEZ,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEpC,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,aAAa,IAAI,iBAAiB,CAAC,CAAC;QACtD,CAAC;QAED,gDAAgD;QAChD,MAAM,eAAe,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,gBAAgB,GAAG,yBAAyB;YAChD,uBAAuB,CAAC,QAAQ,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;QAE/D,IAAI,CAAC,yBAAyB,IAAI,gBAAgB,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC;QAED,qCAAqC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzF,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,SAAS,IAAI,YAAY,CAAC,CAAC;QACpF,MAAM,MAAM,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;QAE3C,oBAAoB;QACpB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACpC,QAAQ,EAAE,YAAY;gBACtB,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,gBAAgB,CAAC,aAAa,EAAE;gBAC5C,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,MAAM,YAAY,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,CAAC;QAE3D,cAAc;QACd,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAE1D,iCAAiC;QACjC,MAAM,YAAY,CAAC,sBAAsB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAEpE,mBAAmB;QACnB,MAAM,oBAAoB,GAAG,2BAA2B,CAAC;YACvD,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,OAAO;SAChB,CAAC,CAAC;QAEH,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAE1C,6BAA6B;QAC7B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEhC,yBAAyB;QACzB,MAAM,cAAc,GAAG,IAAI,uBAAuB,CAAC,gBAAgB,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;QAErF,MAAM,eAAe,GAAG,YAAY,KAAK,IAAI,CAAC;QAE9C,OAAO;YACL,IAAI;YACJ,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,MAAM;YACN,WAAW;YACX,QAAQ;YACR,MAAM;YACN,oBAAoB;YACpB,OAAO;YACP,MAAM;YACN,aAAa;YACb,EAAE;YACF,cAAc;YACd,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACzC,MAAc,EACd,iBAAoC;QAEpC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,uBAAuB;QACjC,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,kBAAkB,GAAG,MAAM,EAAE,CAAC,qBAAqB,EAAE,CAAC;YAE5D,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,eAAe,GAAG,iBAAiB,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,aAAa,EAAE,CAAC;gBAE5D,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CACzC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,eAAe;oBAC1C,GAAG,CAAC,oBAAoB,KAAK,iBAAiB,CACtD,CAAC;gBAEF,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE;wBACnD,QAAQ,EAAE,kBAAkB;wBAC5B,OAAO,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,iBAAiB,EAAE;wBACrE,cAAc,EAAE,2BAA2B;qBAC5C,CAAC,CAAC;oBAEH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,mDAAmD,EAAE;gBAChE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}