@vivantel/rag-core 1.1.3 → 2.1.0

package/dist/config-loader.js

@@ -1,11 +1,79 @@
 import { pathToFileURL } from "url";
+import { ConfigError } from "./core/errors.js";
+function validateConfig(config) {
+    if (!config || typeof config !== "object") {
+        throw new ConfigError("Config must be a non-null object", {
+            suggestion: "Ensure your config file has a default export: `export default { chunkers, embedder, vectorStore }`.",
+        });
+    }
+    const c = config;
+    if (!Array.isArray(c.chunkers) || c.chunkers.length === 0) {
+        throw new ConfigError("config.chunkers must be a non-empty array", {
+            suggestion: "Add at least one chunker using createChunker() or implementing FileChunker.",
+        });
+    }
+    for (let i = 0; i < c.chunkers.length; i++) {
+        const chunker = c.chunkers[i];
+        if (typeof chunker.name !== "string" || !chunker.name) {
+            throw new ConfigError(`config.chunkers[${i}].name must be a non-empty string`);
+        }
+        if (!Array.isArray(chunker.patterns) || chunker.patterns.length === 0) {
+            throw new ConfigError(`config.chunkers[${i}] ("${chunker.name}") must have a non-empty patterns array`);
+        }
+        if (typeof chunker.chunk !== "function") {
+            throw new ConfigError(`config.chunkers[${i}] ("${chunker.name}") must implement chunk(filePath, commitHash)`);
+        }
+    }
+    if (!c.embedder || typeof c.embedder !== "object") {
+        throw new ConfigError("config.embedder is required", {
+            suggestion: "Provide an object implementing EmbeddingProvider with at least name, dimensions, and embed().",
+        });
+    }
+    const emb = c.embedder;
+    if (typeof emb.name !== "string") {
+        throw new ConfigError("config.embedder.name must be a string");
+    }
+    if (typeof emb.dimensions !== "number") {
+        throw new ConfigError("config.embedder.dimensions must be a number (the vector output size)");
+    }
+    if (typeof emb.embed !== "function") {
+        throw new ConfigError("config.embedder must implement embed(text: string): Promise<number[]>");
+    }
+    if (!c.vectorStore || typeof c.vectorStore !== "object") {
+        throw new ConfigError("config.vectorStore is required", {
+            suggestion: "Provide an object implementing VectorStore with initialize, upsert, deleteBySourceFile, getCurrentState, and search.",
+        });
+    }
+    const vs = c.vectorStore;
+    const required = [
+        "initialize",
+        "upsert",
+        "deleteBySourceFile",
+        "getCurrentState",
+        "search",
+    ];
+    for (const method of required) {
+        if (typeof vs[method] !== "function") {
+            throw new ConfigError(`config.vectorStore.${method} must be a function`, {
+                suggestion: `Implement ${method}() on your VectorStore object.`,
+            });
+        }
+    }
+}
 export async function loadConfig(configPath) {
-    const configUrl = pathToFileURL(configPath).href;
-    const configModule = await import(configUrl);
-    const config = configModule.default;
-    if (!config.chunkers || !config.embedder || !config.vectorStore) {
-        throw new Error("Invalid config: missing chunkers, embedder, or vectorStore");
+    let configModule;
+    try {
+        const configUrl = pathToFileURL(configPath).href;
+        configModule = await import(configUrl);
+    }
+    catch (err) {
+        throw new ConfigError(`Cannot load config file: ${configPath}`, {
+            suggestion: "Run `rag-update init` to generate a starter config, or check that the path is correct.",
+            cause: err,
+        });
     }
+    const config = configModule.default;
+    validateConfig(config);
     return config;
 }
 //# sourceMappingURL=config-loader.js.map

package/dist/config-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../src/config-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAGpC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAkB;IAElB,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;IACjD,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC;IAEpC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../src/config-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,SAAS,cAAc,CAAC,MAAe;IACrC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,IAAI,WAAW,CAAC,kCAAkC,EAAE;YACxD,UAAU,EACR,qGAAqG;SACxG,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,GAAG,MAAiC,CAAC;IAE5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,WAAW,CAAC,2CAA2C,EAAE;YACjE,UAAU,EACR,6EAA6E;SAChF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAA4B,CAAC;QACzD,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,IAAI,WAAW,CACnB,mBAAmB,CAAC,mCAAmC,CACxD,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,WAAW,CACnB,mBAAmB,CAAC,OAAO,OAAO,CAAC,IAAI,yCAAyC,CACjF,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACxC,MAAM,IAAI,WAAW,CACnB,mBAAmB,CAAC,OAAO,OAAO,CAAC,IAAI,+CAA+C,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,WAAW,CAAC,6BAA6B,EAAE;YACnD,UAAU,EACR,+FAA+F;SAClG,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,CAAC,QAAmC,CAAC;IAClD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,WAAW,CAAC,uCAAuC,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,IAAI,WAAW,CACnB,sEAAsE,CACvE,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QACpC,MAAM,IAAI,WAAW,CACnB,uEAAuE,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxD,MAAM,IAAI,WAAW,CAAC,gCAAgC,EAAE;YACtD,UAAU,EACR,sHAAsH;SACzH,CAAC,CAAC;IACL,CAAC;IAED,MAAM,EAAE,GAAG,CAAC,CAAC,WAAsC,CAAC;IACpD,MAAM,QAAQ,GAAG;QACf,YAAY;QACZ,QAAQ;QACR,oBAAoB;QACpB,iBAAiB;QACjB,QAAQ;KACT,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,OAAO,EAAE,CAAC,MAAM,CAAC,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,IAAI,WAAW,CAAC,sBAAsB,MAAM,qBAAqB,EAAE;gBACvE,UAAU,EAAE,aAAa,MAAM,gCAAgC;aAChE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,UAAkB;IAElB,IAAI,YAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;QACjD,YAAY,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,WAAW,CAAC,4BAA4B,UAAU,EAAE,EAAE;YAC9D,UAAU,EACR,wFAAwF;YAC1F,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC;IACpC,cAAc,CAAC,MAAM,CAAC,CAAC;IACvB,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/core/chunk-processor.d.ts

@@ -6,7 +6,7 @@ export declare class ChunkProcessor {
     processFiles(files: string[], fileState: Map<string, {
         commitHash: string;
         chunker: FileChunker;
-    }>): Promise<Chunk[]>;
+    }>, existingChunks?: Chunk[]): Promise<Chunk[]>;
     saveChunksLocal(chunks: Chunk[], outputFile: string): Promise<void>;
 }
 //# sourceMappingURL=chunk-processor.d.ts.map

package/dist/core/chunk-processor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunk-processor.d.ts","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAO5D,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAA2B;gBAE/B,QAAQ,EAAE,WAAW,EAAE;IAI7B,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAYb,YAAY,CAChB,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,GACnE,OAAO,CAAC,KAAK,EAAE,CAAC;IAyCb,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAyB1E"}
+{"version":3,"file":"chunk-processor.d.ts","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAO5D,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAA2B;gBAE/B,QAAQ,EAAE,WAAW,EAAE;IAI7B,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAYb,YAAY,CAChB,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,EACpE,cAAc,GAAE,KAAK,EAAO,GAC3B,OAAO,CAAC,KAAK,EAAE,CAAC;IAmEb,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAyB1E"}

package/dist/core/chunk-processor.js

@@ -16,9 +16,23 @@ export class ChunkProcessor {
         }
         return chunks;
     }
-    async processFiles(files, fileState) {
+    async processFiles(files, fileState, existingChunks = []) {
         const allChunks = [];
         let errorCount = 0;
+        // Build resume cache: sourceFile → { commitHash, chunks }
+        const resumeCache = new Map();
+        for (const chunk of existingChunks) {
+            const entry = resumeCache.get(chunk.sourceFile);
+            if (!entry) {
+                resumeCache.set(chunk.sourceFile, {
+                    commitHash: chunk.commitHash,
+                    chunks: [chunk],
+                });
+            }
+            else {
+                entry.chunks.push(chunk);
+            }
+        }
         for (let i = 0; i < files.length; i++) {
             const filePath = files[i];
             const info = fileState.get(filePath);
@@ -26,6 +40,14 @@ export class ChunkProcessor {
                 console.log(`  ⚠️ No chunker for: ${filePath}`);
                 continue;
             }
+            // Resume: reuse cached chunks when commitHash matches
+            const cached = resumeCache.get(filePath);
+            if (cached && cached.commitHash === info.commitHash) {
+                console.log(`  [${i + 1}/${files.length}] ${filePath}`);
+                console.log(`    ⏭️  Cached (${cached.chunks.length} chunk(s))`);
+                allChunks.push(...cached.chunks);
+                continue;
+            }
             console.log(`  [${i + 1}/${files.length}] ${filePath}`);
             try {
                 const chunks = await this.processFile(filePath, info.commitHash, info.chunker);

package/dist/core/chunk-processor.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunk-processor.js","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,kBAAkB,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,QAAQ,CAA2B;IAE3C,YAAY,QAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,UAAkB,EAClB,OAAoB;QAEpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAe,EACf,SAAoE;QAEpE,MAAM,SAAS,GAAY,EAAE,CAAC;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAErC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;YAExD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,QAAQ,EACR,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,CACb,CAAC;gBAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC;gBAC3D,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,0BAA0B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,UAAU,kCAAkC,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAAe,EAAE,UAAkB;QACvD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAEnE,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtD,IAAI,QAAQ,GAAY,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,QAAQ,GAAG,MAAiB,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;QAE3C,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,cAAc,UAAU,EAAE,CAAC,CAAC;IACxE,CAAC;CACF"}
+{"version":3,"file":"chunk-processor.js","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,kBAAkB,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,QAAQ,CAA2B;IAE3C,YAAY,QAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,UAAkB,EAClB,OAAoB;QAEpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAe,EACf,SAAoE,EACpE,iBAA0B,EAAE;QAE5B,MAAM,SAAS,GAAY,EAAE,CAAC;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,0DAA0D;QAC1D,MAAM,WAAW,GAAG,IAAI,GAAG,EAGxB,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE;oBAChC,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,CAAC,KAAK,CAAC;iBAChB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAErC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YAED,sDAAsD;YACtD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAI,MAAM,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;gBACxD,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,CAAC,CAAC;gBACjE,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;YAExD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,QAAQ,EACR,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,CACb,CAAC;gBAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC;gBAC3D,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,0BAA0B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,UAAU,kCAAkC,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAAe,EAAE,UAAkB;QACvD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAEnE,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtD,IAAI,QAAQ,GAAY,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,QAAQ,GAAG,MAAiB,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;QAE3C,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,cAAc,UAAU,EAAE,CAAC,CAAC;IACxE,CAAC;CACF"}

package/dist/core/embedder.d.ts

@@ -1,13 +1,17 @@
 import { EmbeddingProvider, EmbeddedChunk, Chunk } from "../interfaces/index.js";
+import { RetryOptions } from "./utils.js";
 export declare class EmbedderProcessor {
     private provider;
     private rateLimitMs;
     private batchSize;
+    private retryOptions;
+    private concurrency;
     constructor(provider: EmbeddingProvider, options?: {
         rateLimitMs?: number;
         batchSize?: number;
+        retry?: RetryOptions;
+        concurrency?: number;
     });
-    private sleep;
     embedChunk(chunk: Chunk): Promise<EmbeddedChunk>;
     embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
     getChunksToEmbed(chunksFile: string, force?: boolean): Promise<{

package/dist/core/embedder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,KAAK,EACN,MAAM,wBAAwB,CAAC;AAUhC,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;gBAGxB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO;YAO9C,KAAK;IAIb,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;IAUhD,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA2CrD,gBAAgB,CACpB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,aAAa,EAAE,KAAK,EAAE,CAAC;KACxB,CAAC;IAmDI,cAAc,CAClB,aAAa,EAAE,aAAa,EAAE,EAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,IAAI,CAAC;IAoCV,GAAG,CACP,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,aAAa,EAAE,CAAC;CAiB5B"}
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,KAAK,EACN,MAAM,wBAAwB,CAAC;AAKhC,OAAO,EAAqC,YAAY,EAAE,MAAM,YAAY,CAAC;AAO7E,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAS;gBAG1B,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;KACjB;IASF,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;IAahD,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA+CrD,gBAAgB,CACpB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,aAAa,EAAE,KAAK,EAAE,CAAC;KACxB,CAAC;IAuDI,cAAc,CAClB,aAAa,EAAE,aAAa,EAAE,EAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,IAAI,CAAC;IAoCV,GAAG,CACP,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,aAAa,EAAE,CAAC;CAiB5B"}

package/dist/core/embedder.js

@@ -1,6 +1,8 @@
 import { readFile, writeFile, mkdir } from "fs/promises";
 import { dirname } from "path";
 import { createHash } from "crypto";
+import { EmbedError } from "./errors.js";
+import { sleep, withRetry, withConcurrency } from "./utils.js";
 function chunkContentHash(chunk) {
     if (chunk.contentHash)
         return chunk.contentHash;
@@ -10,16 +12,17 @@ export class EmbedderProcessor {
     provider;
     rateLimitMs;
     batchSize;
+    retryOptions;
+    concurrency;
     constructor(provider, options = {}) {
         this.provider = provider;
         this.rateLimitMs = options.rateLimitMs ?? 500;
         this.batchSize = options.batchSize ?? 10;
-    }
-    async sleep(ms) {
-        return new Promise((resolve) => setTimeout(resolve, ms));
+        this.retryOptions = options.retry ?? {};
+        this.concurrency = options.concurrency ?? 1;
     }
     async embedChunk(chunk) {
-        const embedding = await this.provider.embed(chunk.content);
+        const embedding = await withRetry(() => this.provider.embed(chunk.content), this.retryOptions);
         return {
             ...chunk,
             embedding,
@@ -27,37 +30,35 @@ export class EmbedderProcessor {
         };
     }
     async embedBatch(chunks) {
-        const results = [];
         if (this.provider.embedBatch && chunks.length >= this.batchSize) {
             const texts = chunks.map((c) => c.content);
-            const embeddings = await this.provider.embedBatch(texts);
+            const embeddings = await withRetry(() => this.provider.embedBatch(texts), this.retryOptions);
             if (embeddings.length !== chunks.length) {
-                throw new Error(`embedBatch returned ${embeddings.length} embeddings for ${chunks.length} chunks`);
-            }
-            for (let i = 0; i < chunks.length; i++) {
-                results.push({
-                    ...chunks[i],
-                    embedding: embeddings[i],
-                    embeddedAt: Date.now() / 1000,
+                throw new EmbedError(`embedBatch returned ${embeddings.length} embeddings for ${chunks.length} chunks`, {
+                    suggestion: "Check that your EmbeddingProvider.embedBatch() returns one vector per input text.",
                 });
             }
+            return chunks.map((chunk, i) => ({
+                ...chunk,
+                embedding: embeddings[i],
+                embeddedAt: Date.now() / 1000,
+            }));
         }
-        else {
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                const eventType = chunk.metadata.event_type ||
-                    chunk.metadata.title ||
-                    chunk.sourceFile.split("/").pop() ||
-                    "unknown";
-                console.log(`  [${i + 1}/${chunks.length}] ${eventType}`);
-                const embedded = await this.embedChunk(chunk);
-                results.push(embedded);
-                if (this.rateLimitMs > 0 && i < chunks.length - 1) {
-                    await this.sleep(this.rateLimitMs);
-                }
+        let completed = 0;
+        const tasks = chunks.map((chunk) => async () => {
+            const label = chunk.metadata.event_type ||
+                chunk.metadata.title ||
+                chunk.sourceFile.split("/").pop() ||
+                "unknown";
+            const embedded = await this.embedChunk(chunk);
+            completed++;
+            console.log(`  [${completed}/${chunks.length}] ${label}`);
+            if (this.rateLimitMs > 0) {
+                await sleep(this.rateLimitMs);
             }
-        }
-        return results;
+            return embedded;
+        });
+        return withConcurrency(tasks, this.concurrency);
     }
     async getChunksToEmbed(chunksFile, force = false) {
         let chunks;
@@ -70,7 +71,10 @@ export class EmbedderProcessor {
             chunks = parsed;
         }
         catch (err) {
-            throw new Error(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
+            throw new EmbedError(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, {
+                suggestion: "Run the pipeline without --skip-upload to regenerate chunks first.",
+                cause: err,
+            });
         }
         console.log(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
         if (force) {

package/dist/core/embedder.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAE1B,YACE,QAA2B,EAC3B,UAAwD,EAAE;QAE1D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE3D,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,MAAM,OAAO,GAAoB,EAAE,CAAC;QAEpC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEzD,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CACb,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,CAClF,CAAC;YACJ,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,MAAM,CAAC,CAAC,CAAC;oBACZ,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;oBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;iBAC9B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GACZ,KAAK,CAAC,QAAQ,CAAC,UAAqB;oBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;oBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;oBACjC,SAAS,CAAC;gBAEZ,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC;gBAE1D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC9C,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEvB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAItB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QAEpE,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,IAAI,kBAAkB,GAAoB,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YACxD,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,IAAI,SAAS,CAAC,CAAC;QAEpE,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,QAAiB,KAAK;QAEtB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;gBACxD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,KAAK;YACjB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QAEP,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QAE7B,MAAM,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,WAAW,aAAa,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,CACpF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAE/D,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEjE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAE5D,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,eAAe,EAAgB,MAAM,YAAY,CAAC;AAE7E,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,YAAY,CAAe;IAC3B,WAAW,CAAS;IAE5B,YACE,QAA2B,EAC3B,UAKI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,IAAI,CAAC,YAAY,CAClB,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,IAAI,CAAC,YAAY,CAClB,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,MAAM,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAE1D,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAItB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QAEpE,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,IAAI,kBAAkB,GAAoB,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YACxD,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,2BAA2B,aAAa,CAAC,IAAI,SAAS,CAAC,CAAC;QAEpE,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,QAAiB,KAAK;QAEtB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;gBACxD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,KAAK;YACjB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QAEP,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QAE7B,MAAM,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,WAAW,aAAa,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,CACpF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAE/D,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEjE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAE5D,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}

package/dist/core/errors.d.ts

@@ -0,0 +1,16 @@
+export declare class RagError extends Error {
+    suggestion?: string;
+    constructor(message: string, options?: {
+        suggestion?: string;
+        cause?: unknown;
+    });
+}
+export declare class ConfigError extends RagError {
+}
+export declare class ChunkError extends RagError {
+}
+export declare class EmbedError extends RagError {
+}
+export declare class UploadError extends RagError {
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/core/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,qBAAa,QAAS,SAAQ,KAAK;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;gBAGlB,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CASrD;AAED,qBAAa,WAAY,SAAQ,QAAQ;CAAG;AAC5C,qBAAa,UAAW,SAAQ,QAAQ;CAAG;AAC3C,qBAAa,UAAW,SAAQ,QAAQ;CAAG;AAC3C,qBAAa,WAAY,SAAQ,QAAQ;CAAG"}

package/dist/core/errors.js

@@ -0,0 +1,17 @@
+export class RagError extends Error {
+    suggestion;
+    constructor(message, options) {
+        super(message, options?.cause !== undefined ? { cause: options.cause } : undefined);
+        this.name = this.constructor.name;
+        this.suggestion = options?.suggestion;
+    }
+}
+export class ConfigError extends RagError {
+}
+export class ChunkError extends RagError {
+}
+export class EmbedError extends RagError {
+}
+export class UploadError extends RagError {
+}
+//# sourceMappingURL=errors.js.map

package/dist/core/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,QAAS,SAAQ,KAAK;IACjC,UAAU,CAAU;IAEpB,YACE,OAAe,EACf,OAAkD;QAElD,KAAK,CACH,OAAO,EACP,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CACpE,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,CAAC;IACxC,CAAC;CACF;AAED,MAAM,OAAO,WAAY,SAAQ,QAAQ;CAAG;AAC5C,MAAM,OAAO,UAAW,SAAQ,QAAQ;CAAG;AAC3C,MAAM,OAAO,UAAW,SAAQ,QAAQ;CAAG;AAC3C,MAAM,OAAO,WAAY,SAAQ,QAAQ;CAAG"}

package/dist/core/orchestrator.d.ts

@@ -1,4 +1,5 @@
 import { FileChunker, EmbeddingProvider, VectorStore } from "../interfaces/index.js";
+import { RetryOptions } from "./utils.js";
 export interface RAGPipelineConfig {
     chunkers: FileChunker[];
     embedder: EmbeddingProvider;
@@ -8,8 +9,15 @@ export interface RAGPipelineConfig {
         embeddingsFile?: string;
         force?: boolean;
         skipUpload?: boolean;
+        dryRun?: boolean;
         rateLimitMs?: number;
         batchSize?: number;
+        retry?: RetryOptions;
+        concurrency?: number;
+        telemetry?: boolean;
+        notifications?: {
+            webhookUrl?: string;
+        };
     };
 }
 export declare class Orchestrator {

package/dist/core/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAGhC,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAqBD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA+D3B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACzC,CAAC;CACH;AA+CD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA+J3B"}

package/dist/core/orchestrator.js

@@ -2,6 +2,7 @@ import { GitTracker } from "./git-tracker.js";
 import { ChunkProcessor } from "./chunk-processor.js";
 import { EmbedderProcessor } from "./embedder.js";
 import { Uploader } from "./uploader.js";
+import { TelemetryCollector } from "./telemetry.js";
 import { readFile } from "fs/promises";
 async function loadPreviousState(chunksFile) {
     try {
@@ -21,6 +22,28 @@ async function loadPreviousState(chunksFile) {
         return new Map();
     }
 }
+async function loadExistingChunks(chunksFile) {
+    try {
+        const content = await readFile(chunksFile, "utf-8");
+        const parsed = JSON.parse(content);
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return [];
+    }
+}
+async function notifyWebhook(url, status, payload) {
+    try {
+        await fetch(url, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ status, ...payload }),
+        });
+    }
+    catch {
+        // Notification failures are non-fatal
+    }
+}
 export class Orchestrator {
     config;
     chunksFile;
@@ -32,47 +55,119 @@ export class Orchestrator {
             config.options?.embeddingsFile || "./docs/rag/embeddings.json";
     }
     async run() {
-        console.log("🚀 Starting RAG pipeline...\n");
-        console.log("📂 Step 1: Scanning for changes...");
-        const gitTracker = new GitTracker(this.config.chunkers);
-        const currentState = await gitTracker.getCurrentState();
-        const previousState = this.config.options?.force
-            ? new Map()
-            : await loadPreviousState(this.chunksFile);
-        const { toProcess, toDelete } = await gitTracker.getChangedFiles(previousState);
-        if (toProcess.length === 0 &&
-            toDelete.length === 0 &&
-            !this.config.options?.force) {
-            console.log("\n✨ No changes detected.");
-            return;
-        }
-        console.log(`\n📊 Changes: ${toProcess.length} to process, ${toDelete.length} to delete\n`);
-        console.log("🔪 Step 2: Generating chunks...");
-        const chunkProcessor = new ChunkProcessor(this.config.chunkers);
-        const fileState = new Map();
-        for (const file of toProcess) {
-            const info = currentState.get(file);
-            if (info)
-                fileState.set(file, info);
-        }
-        const chunks = await chunkProcessor.processFiles(toProcess, fileState);
-        await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
-        if (chunks.length === 0) {
-            console.log("\n⚠️ No chunks generated. Exiting.");
-            return;
+        const opts = this.config.options ?? {};
+        const telemetry = opts.telemetry ? new TelemetryCollector() : null;
+        telemetry?.start();
+        let uploadStats = { uploaded: 0, deleted: 0 };
+        try {
+            console.log("🚀 Starting RAG pipeline...\n");
+            // Step 1: Scan for changes
+            console.log("📂 Step 1: Scanning for changes...");
+            const t1 = Date.now();
+            const gitTracker = new GitTracker(this.config.chunkers);
+            const currentState = await gitTracker.getCurrentState();
+            const previousState = opts.force
+                ? new Map()
+                : await loadPreviousState(this.chunksFile);
+            const { toProcess, toDelete } = await gitTracker.getChangedFiles(previousState);
+            telemetry?.recordGitTracking({
+                durationMs: Date.now() - t1,
+                filesScanned: currentState.size,
+                toProcess: toProcess.length,
+                toDelete: toDelete.length,
+            });
+            if (toProcess.length === 0 && toDelete.length === 0 && !opts.force) {
+                console.log("\n✨ No changes detected.");
+                return;
+            }
+            console.log(`\n📊 Changes: ${toProcess.length} to process, ${toDelete.length} to delete\n`);
+            // Step 2: Generate chunks (with resume)
+            console.log("🔪 Step 2: Generating chunks...");
+            const t2 = Date.now();
+            const chunkProcessor = new ChunkProcessor(this.config.chunkers);
+            const fileState = new Map();
+            for (const file of toProcess) {
+                const info = currentState.get(file);
+                if (info)
+                    fileState.set(file, info);
+            }
+            const existingChunks = opts.force
+                ? []
+                : await loadExistingChunks(this.chunksFile);
+            const chunks = await chunkProcessor.processFiles(toProcess, fileState, existingChunks);
+            await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
+            telemetry?.recordChunking({
+                durationMs: Date.now() - t2,
+                filesProcessed: toProcess.length,
+                chunksGenerated: chunks.length,
+                errors: 0,
+            });
+            if (chunks.length === 0) {
+                console.log("\n⚠️ No chunks generated. Exiting.");
+                return;
+            }
+            // Step 3: Generate embeddings
+            console.log("\n🔢 Step 3: Generating embeddings...");
+            const t3 = Date.now();
+            const embedder = new EmbedderProcessor(this.config.embedder, {
+                rateLimitMs: opts.rateLimitMs,
+                batchSize: opts.batchSize,
+                retry: opts.retry,
+                concurrency: opts.concurrency,
+            });
+            const newEmbeddings = await embedder.run(this.chunksFile, opts.force || false);
+            telemetry?.recordEmbedding({
+                durationMs: Date.now() - t3,
+                chunksEmbedded: newEmbeddings.length,
+                chunksSkipped: chunks.length - newEmbeddings.length,
+            });
+            // Step 4: Upload
+            if (opts.dryRun) {
+                const uploader = new Uploader(this.config.vectorStore);
+                const { toUpload: dryUpload, toDelete: dryDelete } = await uploader.getItemsToUpload(this.embeddingsFile, opts.force || false);
+                console.log("\n📤 Step 4: Upload (dry-run — no changes written)");
+                console.log(`   Would upload: ${dryUpload.length} document(s)`);
+                console.log(`   Would delete: ${dryDelete.length} source file(s)`);
+            }
+            else if (!opts.skipUpload) {
+                console.log("\n📤 Step 4: Uploading to vector store...");
+                const t4 = Date.now();
+                const uploader = new Uploader(this.config.vectorStore, {
+                    retry: opts.retry,
+                });
+                uploadStats = await uploader.sync(this.embeddingsFile, opts.force || false);
+                telemetry?.recordUpload({
+                    durationMs: Date.now() - t4,
+                    uploaded: uploadStats.uploaded,
+                    deleted: uploadStats.deleted,
+                });
+            }
+            console.log("\n✨ RAG pipeline complete!");
+            if (telemetry) {
+                telemetry.finish();
+                telemetry.printSummary();
+                const telemetryFile = this.chunksFile.replace("chunks.json", "telemetry.json");
+                await telemetry.save(telemetryFile);
+            }
+            if (opts.notifications?.webhookUrl) {
+                await notifyWebhook(opts.notifications.webhookUrl, "success", {
+                    durationMs: telemetry?.getData().durationMs,
+                    stages: telemetry?.getData().stages,
+                    uploaded: uploadStats.uploaded,
+                    deleted: uploadStats.deleted,
+                });
+            }
         }
-        console.log("\n🔢 Step 3: Generating embeddings...");
-        const embedder = new EmbedderProcessor(this.config.embedder, {
-            rateLimitMs: this.config.options?.rateLimitMs,
-            batchSize: this.config.options?.batchSize,
-        });
-        await embedder.run(this.chunksFile, this.config.options?.force || false);
-        if (!this.config.options?.skipUpload) {
-            console.log("\n📤 Step 4: Uploading to vector store...");
-            const uploader = new Uploader(this.config.vectorStore);
-            await uploader.sync(this.embeddingsFile, this.config.options?.force || false);
+        catch (err) {
+            telemetry?.finish();
+            if (opts.notifications?.webhookUrl) {
+                await notifyWebhook(opts.notifications.webhookUrl, "error", {
+                    error: err instanceof Error ? err.message : String(err),
+                    durationMs: telemetry?.getData().durationMs,
+                });
+            }
+            throw err;
         }
-        console.log("\n✨ RAG pipeline complete!");
     }
 }
 //# sourceMappingURL=orchestrator.js.map