@vivantel/rag-core 1.1.2 → 2.0.0

package/dist/core/chunk-processor.js

@@ -8,22 +8,31 @@ export class ChunkProcessor {
         this.chunkers = new Map(chunkers.map((c) => [c.name, c]));
     }
     async processFile(filePath, commitHash, chunker) {
-        try {
-            const chunks = await chunker.chunk(filePath, commitHash);
-            for (const chunk of chunks) {
-                chunk.contentHash = computeContentHash(chunk.content);
-                chunk.sourceFile = filePath;
-                chunk.commitHash = commitHash;
-            }
-            return chunks;
-        }
-        catch (error) {
-            console.error(`  ❌ Error processing ${filePath}: ${error}`);
-            return [];
+        const chunks = await chunker.chunk(filePath, commitHash);
+        for (const chunk of chunks) {
+            chunk.contentHash = computeContentHash(chunk.content);
+            chunk.sourceFile = filePath;
+            chunk.commitHash = commitHash;
         }
+        return chunks;
     }
-    async processFiles(files, fileState) {
+    async processFiles(files, fileState, existingChunks = []) {
         const allChunks = [];
+        let errorCount = 0;
+        // Build resume cache: sourceFile → { commitHash, chunks }
+        const resumeCache = new Map();
+        for (const chunk of existingChunks) {
+            const entry = resumeCache.get(chunk.sourceFile);
+            if (!entry) {
+                resumeCache.set(chunk.sourceFile, {
+                    commitHash: chunk.commitHash,
+                    chunks: [chunk],
+                });
+            }
+            else {
+                entry.chunks.push(chunk);
+            }
+        }
         for (let i = 0; i < files.length; i++) {
             const filePath = files[i];
             const info = fileState.get(filePath);
@@ -31,16 +40,33 @@ export class ChunkProcessor {
                 console.log(`  ⚠️ No chunker for: ${filePath}`);
                 continue;
             }
+            // Resume: reuse cached chunks when commitHash matches
+            const cached = resumeCache.get(filePath);
+            if (cached && cached.commitHash === info.commitHash) {
+                console.log(`  [${i + 1}/${files.length}] ${filePath}`);
+                console.log(`    ⏭️  Cached (${cached.chunks.length} chunk(s))`);
+                allChunks.push(...cached.chunks);
+                continue;
+            }
             console.log(`  [${i + 1}/${files.length}] ${filePath}`);
-            const chunks = await this.processFile(filePath, info.commitHash, info.chunker);
-            if (chunks.length > 0) {
-                allChunks.push(...chunks);
-                console.log(`    ✅ Generated ${chunks.length} chunk(s)`);
+            try {
+                const chunks = await this.processFile(filePath, info.commitHash, info.chunker);
+                if (chunks.length > 0) {
+                    allChunks.push(...chunks);
+                    console.log(`    ✅ Generated ${chunks.length} chunk(s)`);
+                }
+                else {
+                    console.log(`    ⚠️ No chunks generated (skipped)`);
+                }
             }
-            else {
-                console.log(`    ⚠️ No chunks generated (skipped)`);
+            catch (error) {
+                errorCount++;
+                console.error(`    ❌ Error processing ${filePath}: ${error}`);
             }
         }
+        if (errorCount > 0) {
+            console.warn(`\n⚠️ ${errorCount} file(s) failed during chunking.`);
+        }
         return allChunks;
     }
     async saveChunksLocal(chunks, outputFile) {
@@ -50,10 +76,13 @@ export class ChunkProcessor {
         let existing = [];
         try {
             const content = await readFile(outputFile, "utf-8");
-            existing = JSON.parse(content);
+            const parsed = JSON.parse(content);
+            if (Array.isArray(parsed)) {
+                existing = parsed;
+            }
         }
         catch {
-            // File doesn't exist
+            // File doesn't exist or is not valid JSON — start fresh
         }
         const processedFiles = new Set(chunks.map((c) => c.sourceFile));
         const filtered = existing.filter((c) => !processedFiles.has(c.sourceFile));

package/dist/core/chunk-processor.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunk-processor.js","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,kBAAkB,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,QAAQ,CAA2B;IAE3C,YAAY,QAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,UAAkB,EAClB,OAAoB;QAEpB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACtD,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC;gBAC5B,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;YAChC,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wBAAwB,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;YAC5D,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAe,EACf,SAAoE;QAEpE,MAAM,SAAS,GAAY,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAErC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;YAExD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,QAAQ,EACR,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAAe,EAAE,UAAkB;QACvD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAEnE,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtD,IAAI,QAAQ,GAAY,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;QAE3C,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,cAAc,UAAU,EAAE,CAAC,CAAC;IACxE,CAAC;CACF"}
+{"version":3,"file":"chunk-processor.js","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,kBAAkB,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,QAAQ,CAA2B;IAE3C,YAAY,QAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,UAAkB,EAClB,OAAoB;QAEpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAe,EACf,SAAoE,EACpE,iBAA0B,EAAE;QAE5B,MAAM,SAAS,GAAY,EAAE,CAAC;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,0DAA0D;QAC1D,MAAM,WAAW,GAAG,IAAI,GAAG,EAGxB,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE;oBAChC,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,CAAC,KAAK,CAAC;iBAChB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAErC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YAED,sDAAsD;YACtD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAI,MAAM,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;gBACxD,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,CAAC,CAAC;gBACjE,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;YAExD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,QAAQ,EACR,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,CACb,CAAC;gBAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC;gBAC3D,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,0BAA0B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,UAAU,kCAAkC,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAAe,EAAE,UAAkB;QACvD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAEnE,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtD,IAAI,QAAQ,GAAY,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,QAAQ,GAAG,MAAiB,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;QAE3C,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,cAAc,UAAU,EAAE,CAAC,CAAC;IACxE,CAAC;CACF"}

package/dist/core/embedder.d.ts

@@ -1,13 +1,17 @@
 import { EmbeddingProvider, EmbeddedChunk, Chunk } from "../interfaces/index.js";
+import { RetryOptions } from "./utils.js";
 export declare class EmbedderProcessor {
     private provider;
     private rateLimitMs;
     private batchSize;
+    private retryOptions;
+    private concurrency;
     constructor(provider: EmbeddingProvider, options?: {
         rateLimitMs?: number;
         batchSize?: number;
+        retry?: RetryOptions;
+        concurrency?: number;
     });
-    private sleep;
     embedChunk(chunk: Chunk): Promise<EmbeddedChunk>;
     embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
     getChunksToEmbed(chunksFile: string, force?: boolean): Promise<{

package/dist/core/embedder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,KAAK,EACN,MAAM,wBAAwB,CAAC;AAUhC,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;gBAGxB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO;YAO9C,KAAK;IAIb,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;IAUhD,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAqCrD,gBAAgB,CACpB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,aAAa,EAAE,KAAK,EAAE,CAAC;KACxB,CAAC;IA4CI,cAAc,CAClB,aAAa,EAAE,aAAa,EAAE,EAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,IAAI,CAAC;IAoCV,GAAG,CACP,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,aAAa,EAAE,CAAC;CAiB5B"}
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,KAAK,EACN,MAAM,wBAAwB,CAAC;AAKhC,OAAO,EAAqC,YAAY,EAAE,MAAM,YAAY,CAAC;AAO7E,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAS;gBAG1B,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;KACjB;IASF,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;IAahD,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA+CrD,gBAAgB,CACpB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,aAAa,EAAE,KAAK,EAAE,CAAC;KACxB,CAAC;IAuDI,cAAc,CAClB,aAAa,EAAE,aAAa,EAAE,EAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,IAAI,CAAC;IAoCV,GAAG,CACP,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,aAAa,EAAE,CAAC;CAiB5B"}

package/dist/core/embedder.js

@@ -1,6 +1,8 @@
 import { readFile, writeFile, mkdir } from "fs/promises";
 import { dirname } from "path";
 import { createHash } from "crypto";
+import { EmbedError } from "./errors.js";
+import { sleep, withRetry, withConcurrency } from "./utils.js";
 function chunkContentHash(chunk) {
     if (chunk.contentHash)
         return chunk.contentHash;
@@ -10,16 +12,17 @@ export class EmbedderProcessor {
     provider;
     rateLimitMs;
     batchSize;
+    retryOptions;
+    concurrency;
     constructor(provider, options = {}) {
         this.provider = provider;
         this.rateLimitMs = options.rateLimitMs ?? 500;
         this.batchSize = options.batchSize ?? 10;
-    }
-    async sleep(ms) {
-        return new Promise((resolve) => setTimeout(resolve, ms));
+        this.retryOptions = options.retry ?? {};
+        this.concurrency = options.concurrency ?? 1;
     }
     async embedChunk(chunk) {
-        const embedding = await this.provider.embed(chunk.content);
+        const embedding = await withRetry(() => this.provider.embed(chunk.content), this.retryOptions);
         return {
             ...chunk,
             embedding,
@@ -27,43 +30,51 @@ export class EmbedderProcessor {
         };
     }
     async embedBatch(chunks) {
-        const results = [];
         if (this.provider.embedBatch && chunks.length >= this.batchSize) {
             const texts = chunks.map((c) => c.content);
-            const embeddings = await this.provider.embedBatch(texts);
-            for (let i = 0; i < chunks.length; i++) {
-                results.push({
-                    ...chunks[i],
-                    embedding: embeddings[i],
-                    embeddedAt: Date.now() / 1000,
+            const embeddings = await withRetry(() => this.provider.embedBatch(texts), this.retryOptions);
+            if (embeddings.length !== chunks.length) {
+                throw new EmbedError(`embedBatch returned ${embeddings.length} embeddings for ${chunks.length} chunks`, {
+                    suggestion: "Check that your EmbeddingProvider.embedBatch() returns one vector per input text.",
                 });
             }
+            return chunks.map((chunk, i) => ({
+                ...chunk,
+                embedding: embeddings[i],
+                embeddedAt: Date.now() / 1000,
+            }));
         }
-        else {
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                const eventType = chunk.metadata.event_type ||
-                    chunk.metadata.title ||
-                    chunk.sourceFile.split("/").pop() ||
-                    "unknown";
-                console.log(`  [${i + 1}/${chunks.length}] ${eventType}`);
-                const embedded = await this.embedChunk(chunk);
-                results.push(embedded);
-                if (this.rateLimitMs > 0 && i < chunks.length - 1) {
-                    await this.sleep(this.rateLimitMs);
-                }
+        let completed = 0;
+        const tasks = chunks.map((chunk) => async () => {
+            const label = chunk.metadata.event_type ||
+                chunk.metadata.title ||
+                chunk.sourceFile.split("/").pop() ||
+                "unknown";
+            const embedded = await this.embedChunk(chunk);
+            completed++;
+            console.log(`  [${completed}/${chunks.length}] ${label}`);
+            if (this.rateLimitMs > 0) {
+                await sleep(this.rateLimitMs);
             }
-        }
-        return results;
+            return embedded;
+        });
+        return withConcurrency(tasks, this.concurrency);
     }
     async getChunksToEmbed(chunksFile, force = false) {
         let chunks;
         try {
             const content = await readFile(chunksFile, "utf-8");
-            chunks = JSON.parse(content);
+            const parsed = JSON.parse(content);
+            if (!Array.isArray(parsed)) {
+                throw new Error("chunks file does not contain a JSON array");
+            }
+            chunks = parsed;
         }
-        catch {
-            throw new Error(`Chunks file not found: ${chunksFile}`);
+        catch (err) {
+            throw new EmbedError(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, {
+                suggestion: "Run the pipeline without --skip-upload to regenerate chunks first.",
+                cause: err,
+            });
         }
         console.log(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
         if (force) {

package/dist/core/embedder.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAE1B,YACE,QAA2B,EAC3B,UAAwD,EAAE;QAE1D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE3D,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,MAAM,OAAO,GAAoB,EAAE,CAAC;QAEpC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,MAAM,CAAC,CAAC,CAAC;oBACZ,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;oBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;iBAC9B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GACZ,KAAK,CAAC,QAAQ,CAAC,UAAqB;oBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;oBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;oBACjC,SAAS,CAAC;gBAEZ,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC;gBAE1D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC9C,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEvB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAItB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,0BAA0B,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QAEpE,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,IAAI,kBAAkB,GAAoB,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YACxD,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,IAAI,SAAS,CAAC,CAAC;QAEpE,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,QAAiB,KAAK;QAEtB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;gBACxD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,KAAK;YACjB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QAEP,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QAE7B,MAAM,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,WAAW,aAAa,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,CACpF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAE/D,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEjE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAE5D,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,eAAe,EAAgB,MAAM,YAAY,CAAC;AAE7E,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,YAAY,CAAe;IAC3B,WAAW,CAAS;IAE5B,YACE,QAA2B,EAC3B,UAKI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,IAAI,CAAC,YAAY,CAClB,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,IAAI,CAAC,YAAY,CAClB,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,MAAM,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAE1D,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAItB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QAEpE,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,IAAI,kBAAkB,GAAoB,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YACxD,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,IAAI,SAAS,CAAC,CAAC;QAEpE,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,QAAiB,KAAK;QAEtB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;gBACxD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,KAAK;YACjB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QAEP,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QAE7B,MAAM,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,WAAW,aAAa,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,CACpF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAE/D,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEjE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAE5D,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}

package/dist/core/errors.d.ts

@@ -0,0 +1,16 @@
+export declare class RagError extends Error {
+    suggestion?: string;
+    constructor(message: string, options?: {
+        suggestion?: string;
+        cause?: unknown;
+    });
+}
+export declare class ConfigError extends RagError {
+}
+export declare class ChunkError extends RagError {
+}
+export declare class EmbedError extends RagError {
+}
+export declare class UploadError extends RagError {
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/core/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,qBAAa,QAAS,SAAQ,KAAK;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;gBAGlB,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CASrD;AAED,qBAAa,WAAY,SAAQ,QAAQ;CAAG;AAC5C,qBAAa,UAAW,SAAQ,QAAQ;CAAG;AAC3C,qBAAa,UAAW,SAAQ,QAAQ;CAAG;AAC3C,qBAAa,WAAY,SAAQ,QAAQ;CAAG"}

package/dist/core/errors.js

@@ -0,0 +1,17 @@
+export class RagError extends Error {
+    suggestion;
+    constructor(message, options) {
+        super(message, options?.cause !== undefined ? { cause: options.cause } : undefined);
+        this.name = this.constructor.name;
+        this.suggestion = options?.suggestion;
+    }
+}
+export class ConfigError extends RagError {
+}
+export class ChunkError extends RagError {
+}
+export class EmbedError extends RagError {
+}
+export class UploadError extends RagError {
+}
+//# sourceMappingURL=errors.js.map

package/dist/core/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,QAAS,SAAQ,KAAK;IACjC,UAAU,CAAU;IAEpB,YACE,OAAe,EACf,OAAkD;QAElD,KAAK,CACH,OAAO,EACP,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CACpE,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,CAAC;IACxC,CAAC;CACF;AAED,MAAM,OAAO,WAAY,SAAQ,QAAQ;CAAG;AAC5C,MAAM,OAAO,UAAW,SAAQ,QAAQ;CAAG;AAC3C,MAAM,OAAO,UAAW,SAAQ,QAAQ;CAAG;AAC3C,MAAM,OAAO,WAAY,SAAQ,QAAQ;CAAG"}

package/dist/core/git-tracker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"git-tracker.d.ts","sourceRoot":"","sources":["../../src/core/git-tracker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAsCrD,qBAAa,UAAU;IACrB,OAAO,CAAC,GAAG,CAAY;IACvB,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,gBAAgB,CAAwB;gBAEpC,QAAQ,EAAE,WAAW,EAAE;YAMrB,cAAc;YAWd,qBAAqB;IAYnC,OAAO,CAAC,iBAAiB;IAWzB,OAAO,CAAC,cAAc;IAOhB,kBAAkB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAKvC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAuC9D,eAAe,IAAI,OAAO,CAC9B,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,CAC1D;IA0BK,eAAe,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC;QACjE,SAAS,EAAE,MAAM,EAAE,CAAC;QACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;CA+BH"}
+{"version":3,"file":"git-tracker.d.ts","sourceRoot":"","sources":["../../src/core/git-tracker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAIrD,qBAAa,UAAU;IACrB,OAAO,CAAC,GAAG,CAAY;IACvB,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,gBAAgB,CAAwB;gBAEpC,QAAQ,EAAE,WAAW,EAAE;YAMrB,cAAc;YAOd,qBAAqB;IAQnC,OAAO,CAAC,iBAAiB;IAWzB,OAAO,CAAC,cAAc;IAOhB,kBAAkB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAKvC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAwB9D,eAAe,IAAI,OAAO,CAC9B,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,CAC1D;IA0BK,eAAe,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC;QACjE,SAAS,EAAE,MAAM,EAAE,CAAC;QACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;CA+BH"}

package/dist/core/git-tracker.js

@@ -2,31 +2,6 @@ import { simpleGit } from "simple-git";
 import { glob } from "glob";
 import { minimatch } from "minimatch";
 import path from "path";
-const MAX_FILES_PER_BATCH = 100;
-const MAX_CMD_LEN = 32000;
-function batchFiles(files) {
-    const batches = [];
-    let currentBatch = [];
-    let currentLen = 0;
-    const baseCmdLen = "git log -1 --format=%H --all -- ".length;
-    for (const file of files) {
-        const fileLen = file.length + 1;
-        if (currentBatch.length >= MAX_FILES_PER_BATCH ||
-            currentLen + fileLen > MAX_CMD_LEN) {
-            if (currentBatch.length > 0) {
-                batches.push(currentBatch);
-                currentBatch = [];
-                currentLen = baseCmdLen;
-            }
-        }
-        currentBatch.push(file);
-        currentLen += fileLen;
-    }
-    if (currentBatch.length > 0) {
-        batches.push(currentBatch);
-    }
-    return batches;
-}
 export class GitTracker {
     git;
     chunkers;
@@ -40,24 +15,14 @@ export class GitTracker {
     }
     async getCurrentHead() {
         if (!this.currentHeadCache) {
-            try {
-                this.currentHeadCache = await this.git.revparse(["HEAD"]);
-            }
-            catch {
-                this.currentHeadCache = "dev_0000000000000000000000000000000000000000";
-            }
+            this.currentHeadCache = await this.git.revparse(["HEAD"]);
         }
         return this.currentHeadCache;
     }
     async hasUncommittedChanges() {
         if (this.uncommittedCache === null) {
-            try {
-                const status = await this.git.status();
-                this.uncommittedCache = status.files.length > 0;
-            }
-            catch {
-                this.uncommittedCache = false;
-            }
+            const status = await this.git.status();
+            this.uncommittedCache = status.files.length > 0;
         }
         return this.uncommittedCache;
     }
@@ -82,37 +47,22 @@ export class GitTracker {
     }
     async getCommitHashes(files) {
         const commitMap = new Map();
-        const batches = batchFiles(files);
-        const currentHead = await this.getCurrentHead();
-        for (const batch of batches) {
+        await Promise.all(files.map(async (file) => {
             try {
                 const output = await this.git.raw([
                     "log",
                     "-1",
                     "--format=%H",
-                    "--all",
                     "--",
-                    ...batch,
+                    file,
                 ]);
-                const lines = output.trim().split("\n");
-                for (let i = 0; i < lines.length && i < batch.length; i++) {
-                    const hash = lines[i].trim();
-                    if (hash) {
-                        commitMap.set(batch[i], hash);
-                    }
-                }
-                for (const file of batch) {
-                    if (!commitMap.has(file)) {
-                        commitMap.set(file, currentHead);
-                    }
-                }
+                const hash = output.trim();
+                commitMap.set(file, hash || (await this.getCurrentHead()));
             }
             catch {
-                for (const file of batch) {
-                    commitMap.set(file, currentHead);
-                }
+                commitMap.set(file, await this.getCurrentHead());
             }
-        }
+        }));
         return commitMap;
     }
     async getCurrentState() {

package/dist/core/git-tracker.js.map

@@ -1 +1 @@
-{"version":3,"file":"git-tracker.js","sourceRoot":"","sources":["../../src/core/git-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,WAAW,GAAG,KAAK,CAAC;AAE1B,SAAS,UAAU,CAAC,KAAe;IACjC,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,UAAU,GAAG,kCAAkC,CAAC,MAAM,CAAC;IAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAEhC,IACE,YAAY,CAAC,MAAM,IAAI,mBAAmB;YAC1C,UAAU,GAAG,OAAO,GAAG,WAAW,EAClC,CAAC;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC3B,YAAY,GAAG,EAAE,CAAC;gBAClB,UAAU,GAAG,UAAU,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,UAAU,IAAI,OAAO,CAAC;IACxB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,UAAU;IACb,GAAG,CAAY;IACf,QAAQ,CAAgB;IACxB,WAAW,CAAW;IACtB,gBAAgB,GAAkB,IAAI,CAAC;IACvC,gBAAgB,GAAmB,IAAI,CAAC;IAEhD,YAAY,QAAuB;QACjC,IAAI,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,gBAAgB,GAAG,8CAA8C,CAAC;YACzE,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;gBACvC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChC,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC3C,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,QAAgB,EAAE,OAAe;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1D,MAAM,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO,SAAS,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAe;QACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAEhD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAChC,KAAK;oBACL,IAAI;oBACJ,aAAa;oBACb,OAAO;oBACP,IAAI;oBACJ,GAAG,KAAK;iBACT,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC7B,IAAI,IAAI,EAAE,CAAC;wBACT,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;oBAChC,CAAC;gBACH,CAAC;gBAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACzB,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;oBACnC,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe;QAGnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACpD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAEhD,MAAM,KAAK,GAAG,IAAI,GAAG,EAGlB,CAAC;QAEJ,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC;YACpD,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,GAAG,GAAG,UAAU,QAAQ,CAAC;YACrC,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,aAAkC;QAKtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAE7C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;gBACrC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,QAAQ,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CACT,iBAAiB,QAAQ,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CACvF,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;gBAC1C,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;CACF"}
+{"version":3,"file":"git-tracker.js","sourceRoot":"","sources":["../../src/core/git-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,MAAM,OAAO,UAAU;IACb,GAAG,CAAY;IACf,QAAQ,CAAgB;IACxB,WAAW,CAAW;IACtB,gBAAgB,GAAkB,IAAI,CAAC;IACvC,gBAAgB,GAAmB,IAAI,CAAC;IAEhD,YAAY,QAAuB;QACjC,IAAI,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACvC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC3C,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,QAAgB,EAAE,OAAe;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1D,MAAM,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO,SAAS,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAe;QACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAChC,KAAK;oBACL,IAAI;oBACJ,aAAa;oBACb,IAAI;oBACJ,IAAI;iBACL,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3B,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe;QAGnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACpD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAEhD,MAAM,KAAK,GAAG,IAAI,GAAG,EAGlB,CAAC;QAEJ,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC;YACpD,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,GAAG,GAAG,UAAU,QAAQ,CAAC;YACrC,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,aAAkC;QAKtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAE7C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;gBACrC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,QAAQ,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CACT,iBAAiB,QAAQ,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CACvF,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;gBAC1C,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;CACF"}

package/dist/core/orchestrator.d.ts

@@ -1,4 +1,5 @@
 import { FileChunker, EmbeddingProvider, VectorStore } from "../interfaces/index.js";
+import { RetryOptions } from "./utils.js";
 export interface RAGPipelineConfig {
     chunkers: FileChunker[];
     embedder: EmbeddingProvider;
@@ -8,8 +9,15 @@ export interface RAGPipelineConfig {
         embeddingsFile?: string;
         force?: boolean;
         skipUpload?: boolean;
+        dryRun?: boolean;
         rateLimitMs?: number;
         batchSize?: number;
+        retry?: RetryOptions;
+        concurrency?: number;
+        telemetry?: boolean;
+        notifications?: {
+            webhookUrl?: string;
+        };
     };
 }
 export declare class Orchestrator {

package/dist/core/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EACZ,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA4D3B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACzC,CAAC;CACH;AA+CD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA+J3B"}