@vivantel/virage-core 0.2.2 → 0.2.4

package/dist/core/embedder.js

@@ -2,8 +2,7 @@ import { NullLogger } from "../logger/null-logger.js";
 import { readFile } from "fs/promises";
 import { createHash } from "crypto";
 import { EmbedError } from "./errors.js";
-import { sleep, withRetry, withConcurrency, batchBySize, } from "./utils.js";
-import { readEmbeddingsFile, writeEmbeddingsFile } from "./embeddings-io.js";
+import { sleep, withRetry, withConcurrency, batchBySize, defaultIsRetryable, } from "./utils.js";
 function chunkContentHash(chunk) {
     if (chunk.contentHash)
         return chunk.contentHash;
@@ -17,7 +16,10 @@ export class EmbedderProcessor {
     retryOptions;
     concurrency;
     vectorStoreName;
+    saveIntervalMs;
+    minIngestionBatchSize;
     logger;
+    onProgress;
     constructor(provider, options = {}) {
         this.provider = provider;
         this.rateLimitMs = options.rateLimitMs ?? 500;
@@ -26,10 +28,13 @@ export class EmbedderProcessor {
         this.retryOptions = options.retry ?? {};
         this.concurrency = options.concurrency ?? 1;
         this.vectorStoreName = options.vectorStoreName;
+        this.saveIntervalMs = options.saveIntervalMs ?? 30_000;
+        this.minIngestionBatchSize = options.minIngestionBatchSize ?? Infinity;
+        this.onProgress = options.onProgress;
         this.logger = (options.logger ?? new NullLogger()).withTag("embedder");
     }
     async embedChunk(chunk) {
-        const embedding = await withRetry(() => this.provider.embed(chunk.content), this.retryOptions, this.logger);
+        const embedding = await withRetry(() => this.provider.embed(chunk.content), { ...this.retryOptions, isRetryable: defaultIsRetryable }, this.logger);
         return {
             ...chunk,
             embedding,
@@ -39,7 +44,7 @@ export class EmbedderProcessor {
     async embedBatch(chunks) {
         if (this.provider.embedBatch && chunks.length > 0) {
             const texts = chunks.map((c) => c.content);
-            const embeddings = await withRetry(() => this.provider.embedBatch(texts), this.retryOptions, this.logger);
+            const embeddings = await withRetry(() => this.provider.embedBatch(texts), { ...this.retryOptions, isRetryable: defaultIsRetryable }, this.logger);
             if (embeddings.length !== chunks.length) {
                 throw new EmbedError(`embedBatch returned ${embeddings.length} embeddings for ${chunks.length} chunks`, {
                     suggestion: "Check that your EmbeddingProvider.embedBatch() returns one vector per input text.",
@@ -69,63 +74,36 @@ export class EmbedderProcessor {
         });
         return withConcurrency(tasks, this.concurrency);
     }
-    async getChunksToEmbed(chunksFile, force = false) {
-        let chunks;
-        try {
-            const content = await readFile(chunksFile, "utf-8");
-            const parsed = JSON.parse(content);
-            if (!Array.isArray(parsed)) {
-                throw new Error("chunks file does not contain a JSON array");
-            }
-            chunks = parsed;
-        }
-        catch (err) {
-            throw new EmbedError(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, {
-                suggestion: "Run the pipeline without --skip-upload to regenerate chunks first.",
-                cause: err,
-            });
-        }
-        this.logger.info(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
+    async getChunksToEmbed(db, chunks, force = false) {
+        this.logger.info(`📖 Loaded ${chunks.length} chunks`);
         this.logger.debug(`Provider: ${this.provider.name}, model: ${this.provider.model ?? "(none)"}, dims: ${this.provider.dimensions}`);
+        const existingMeta = db.getMeta();
         if (force) {
-            this.logger.verbose("Force mode: embedding all chunks");
-            return {
-                chunksToEmbed: chunks,
-                existingMeta: null,
-                isNewEmbeddingSpace: true,
-            };
+            this.logger.verbose("Force mode: clearing db and embedding all chunks");
+            db.clearAll();
+            return { chunksToEmbed: chunks, meta: null, isNewEmbeddingSpace: true };
         }
-        const embeddingsFile = chunksFile.replace("chunks", "embeddings");
-        const { meta, chunks: existingEmbeddings } = await readEmbeddingsFile(embeddingsFile);
-        // Check if the embedding model or dimensions changed — if so, all cached embeddings
-        // are invalid for the new model's vector space.
-        // Discriminators: model + dimensions. providerName is informational only (the same
-        // model served via OpenAI, Azure, or GitHub Models produces identical vectors).
-        const isNewEmbeddingSpace = false;
-        if (meta && existingEmbeddings.length > 0) {
-            const modelChanged = meta.model !== undefined &&
+        // Check if the embedding model or dimensions changed
+        if (existingMeta) {
+            const modelChanged = existingMeta.model !== undefined &&
                 this.provider.model !== undefined &&
-                meta.model !== this.provider.model;
-            const dimensionsChanged = meta.providerDimensions !== this.provider.dimensions;
+                existingMeta.model !== this.provider.model;
+            const dimensionsChanged = existingMeta.providerDimensions !== this.provider.dimensions;
             if (modelChanged || dimensionsChanged) {
-                const prevModel = meta.model ?? "(unknown)";
+                const prevModel = existingMeta.model ?? "(unknown)";
                 const curModel = this.provider.model ?? "(unknown)";
-                const prevDims = meta.providerDimensions;
+                const prevDims = existingMeta.providerDimensions;
                 const curDims = this.provider.dimensions;
                 this.logger.warn(`⚠️ Embedding model changed!`);
                 this.logger.warn(`   Previous: ${prevModel} (${prevDims}d)`);
                 this.logger.warn(`   Current:  ${curModel} (${curDims}d)`);
                 this.logger.warn(`   Discarding existing embeddings — all chunks will be re-embedded.`);
-                return {
-                    chunksToEmbed: chunks,
-                    existingMeta: null,
-                    isNewEmbeddingSpace: true,
-                };
+                db.clearAll();
+                return { chunksToEmbed: chunks, meta: null, isNewEmbeddingSpace: true };
             }
         }
-        // Two-level index:
-        //   (sourceFile::commitHash) → Set<contentHash>  — for file-level fast path
-        //   contentHash (global)                          — for per-chunk fallback
+        // Two-level index: (sourceFile::commitHash) → Set<contentHash> and global contentHash set
+        const existingEmbeddings = db.getAll();
         const embeddedFileVersions = new Map();
         const embeddedByContentHash = new Set();
         for (const emb of existingEmbeddings) {
@@ -142,7 +120,6 @@ export class EmbedderProcessor {
         }
         this.logger.verbose(`📊 Existing: ${existingEmbeddings.length} chunks across ${embeddedFileVersions.size} version(s)`);
         this.logger.debug(`Vector store: ${this.vectorStoreName ?? "(none)"}`);
-        // Group incoming chunks by (sourceFile, commitHash)
         const chunksByFileVersion = new Map();
         for (const chunk of chunks) {
             const key = `${chunk.sourceFile}::${chunk.commitHash}`;
@@ -159,7 +136,6 @@ export class EmbedderProcessor {
         let skippedChunks = 0;
         for (const [key, fileChunks] of chunksByFileVersion) {
             const fileVersionHashes = embeddedFileVersions.get(key);
-            // Fast path: every chunk for this (sourceFile, commitHash) is already embedded
             if (fileVersionHashes &&
                 fileVersionHashes.size === fileChunks.length &&
                 fileChunks.every((c) => fileVersionHashes.has(chunkContentHash(c)))) {
@@ -167,8 +143,6 @@ export class EmbedderProcessor {
                 skippedChunks += fileChunks.length;
                 continue;
             }
-            // Per-chunk fallback: check globally by contentHash (handles partial embeddings
-            // from interrupted runs, or content-identical chunks at a different commit)
             for (const chunk of fileChunks) {
                 if (!embeddedByContentHash.has(chunkContentHash(chunk))) {
                     chunksToEmbed.push(chunk);
@@ -178,67 +152,69 @@ export class EmbedderProcessor {
         if (skippedFiles > 0) {
             this.logger.verbose(`⏭️ Skipped ${skippedFiles} unchanged version(s) (${skippedChunks} chunks)`);
         }
-        return { chunksToEmbed, existingMeta: meta, isNewEmbeddingSpace };
+        return { chunksToEmbed, meta: existingMeta, isNewEmbeddingSpace: false };
     }
-    async saveEmbeddings(newEmbeddings, chunksFile, existingMeta, isFirstBatch, forceFirstBatch) {
-        const embeddingsFile = chunksFile.replace("chunks", "embeddings");
-        const now = Math.floor(Date.now() / 1000);
-        const newByHash = new Map();
-        for (const emb of newEmbeddings) {
-            const hash = emb.contentHash || chunkContentHash(emb);
-            newByHash.set(hash, emb);
+    async saveEmbeddings(db, newEmbeddings, meta) {
+        db.setMeta(meta);
+        db.insert(newEmbeddings);
+        this.logger.verbose(`Saved ${newEmbeddings.length} new embeddings to db`);
+    }
+    async run(db, chunksFile, force = false, onIntermediateBatch) {
+        this.logger.info("🔢 Starting incremental embedding generation...");
+        let chunks;
+        try {
+            const content = await readFile(chunksFile, "utf-8");
+            const parsed = JSON.parse(content);
+            if (!Array.isArray(parsed)) {
+                throw new Error("chunks file does not contain a JSON array");
+            }
+            chunks = parsed;
         }
-        // When force-clearing (force flag or model mismatch detected), discard existing chunks
-        // on the first batch; subsequent batches always merge incrementally.
-        const discardExisting = isFirstBatch && forceFirstBatch;
-        let keepChunks = [];
-        let createdAt = now;
-        if (!discardExisting) {
-            const { meta, chunks: existing } = await readEmbeddingsFile(embeddingsFile);
-            if (meta)
-                createdAt = meta.createdAt;
-            keepChunks = existing.filter((e) => {
-                const hash = e.contentHash || chunkContentHash(e);
-                return !newByHash.has(hash);
+        catch (err) {
+            throw new EmbedError(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, {
+                suggestion: "Run the pipeline without --skip-upload to regenerate chunks first.",
+                cause: err,
             });
         }
-        const final = [...keepChunks, ...newEmbeddings];
-        const meta = {
-            schemaVersion: 1,
-            providerName: this.provider.name,
-            providerDimensions: this.provider.dimensions,
-            ...(this.provider.model !== undefined
-                ? { model: this.provider.model }
-                : {}),
-            ...(this.vectorStoreName !== undefined
-                ? { vectorStoreName: this.vectorStoreName }
-                : {}),
-            // Preserve createdAt from the existing meta when available and not overwriting
-            createdAt: existingMeta && !discardExisting ? existingMeta.createdAt : createdAt,
-            updatedAt: now,
-        };
-        await writeEmbeddingsFile(embeddingsFile, meta, final);
-        this.logger.verbose(`Saved ${final.length} embeddings to ${embeddingsFile}`);
-        this.logger.verbose(`   New: ${newEmbeddings.length}, Existing: ${keepChunks.length}`);
-    }
-    async run(chunksFile, force = false) {
-        this.logger.info("🔢 Starting incremental embedding generation...");
-        const { chunksToEmbed, existingMeta, isNewEmbeddingSpace } = await this.getChunksToEmbed(chunksFile, force);
-        // Treat a model-space change the same as --force for the save step
-        const effectiveForce = force || isNewEmbeddingSpace;
+        const { chunksToEmbed, meta: existingMeta } = await this.getChunksToEmbed(db, chunks, force);
         if (chunksToEmbed.length === 0) {
             this.logger.info("✨ No chunks need embedding.");
             return [];
         }
         this.logger.info(`📝 Need to embed ${chunksToEmbed.length} chunks`);
+        const now = Math.floor(Date.now() / 1000);
         const batches = batchBySize(chunksToEmbed, this.batchSize, (c) => c.content.length, this.maxBatchChars);
         const newEmbeddings = [];
+        let lastSaveTime = Date.now();
         for (let i = 0; i < batches.length; i++) {
             const totalChars = batches[i].reduce((s, c) => s + c.content.length, 0);
             this.logger.debug(`Batch ${i + 1}/${batches.length}: ${batches[i].length} chunks, ~${totalChars} chars`);
             const embedded = await this.embedBatch(batches[i]);
             newEmbeddings.push(...embedded);
-            await this.saveEmbeddings(embedded, chunksFile, existingMeta, i === 0, i === 0 && effectiveForce);
+            this.onProgress?.(newEmbeddings.length, chunksToEmbed.length);
+            const isLastBatch = i === batches.length - 1;
+            const elapsed = Date.now() - lastSaveTime;
+            if (isLastBatch || elapsed >= this.saveIntervalMs) {
+                const meta = {
+                    schemaVersion: 1,
+                    providerName: this.provider.name,
+                    providerDimensions: this.provider.dimensions,
+                    ...(this.provider.model !== undefined
+                        ? { model: this.provider.model }
+                        : {}),
+                    ...(this.vectorStoreName !== undefined
+                        ? { vectorStoreName: this.vectorStoreName }
+                        : {}),
+                    createdAt: existingMeta?.createdAt ?? now,
+                    updatedAt: now,
+                };
+                await this.saveEmbeddings(db, embedded, meta);
+                lastSaveTime = Date.now();
+                if (onIntermediateBatch &&
+                    db.pendingCount() >= this.minIngestionBatchSize) {
+                    await onIntermediateBatch();
+                }
+            }
         }
         return newEmbeddings;
     }

package/dist/core/embedder.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,eAAe,EACf,WAAW,GAEZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,YAAY,CAAe;IAC3B,WAAW,CAAS;IACpB,eAAe,CAAU;IACzB,MAAM,CAAS;IAEvB,YACE,QAA2B,EAC3B,UAQI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,gBAAgB,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAClF,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChE,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAMtB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,aAAa,IAAI,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAChH,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YACxD,OAAO;gBACL,aAAa,EAAE,MAAM;gBACrB,YAAY,EAAE,IAAI;gBAClB,mBAAmB,EAAE,IAAI;aAC1B,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,GACxC,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAE3C,oFAAoF;QACpF,gDAAgD;QAChD,mFAAmF;QACnF,gFAAgF;QAChF,MAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAI,IAAI,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,YAAY,GAChB,IAAI,CAAC,KAAK,KAAK,SAAS;gBACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACjC,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,iBAAiB,GACrB,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAEvD,IAAI,YAAY,IAAI,iBAAiB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,WAAW,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBACzC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qEAAqE,CACtE,CAAC;gBACF,OAAO;oBACL,aAAa,EAAE,MAAM;oBACrB,YAAY,EAAE,IAAI;oBAClB,mBAAmB,EAAE,IAAI;iBAC1B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,4EAA4E;QAC5E,2EAA2E;QAC3E,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC5D,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,oBAAoB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gBAAgB,kBAAkB,CAAC,MAAM,kBAAkB,oBAAoB,CAAC,IAAI,aAAa,CAClG,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEvE,oDAAoD;QACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;YACpD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExD,+EAA+E;YAC/E,IACE,iBAAiB;gBACjB,iBAAiB,CAAC,IAAI,KAAK,UAAU,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,CAAC;gBACD,YAAY,EAAE,CAAC;gBACf,aAAa,IAAI,UAAU,CAAC,MAAM,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,gFAAgF;YAChF,4EAA4E;YAC5E,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,YAAY,0BAA0B,aAAa,UAAU,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,YAAmC,EACnC,YAAqB,EACrB,eAAwB;QAExB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAE1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,uFAAuF;QACvF,qEAAqE;QACrE,MAAM,eAAe,GAAG,YAAY,IAAI,eAAe,CAAC;QAExD,IAAI,UAAU,GAAoB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAG,GAAG,CAAC;QACpB,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAC9B,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAC3C,IAAI,IAAI;gBAAE,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACrC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjC,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAmB;YAC3B,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;YAChC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;YAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE;gBAC3C,CAAC,CAAC,EAAE,CAAC;YACP,+EAA+E;YAC/E,SAAS,EACP,YAAY,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACvE,SAAS,EAAE,GAAG;SACf,CAAC;QAEF,MAAM,mBAAmB,CAAC,cAAc,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,SAAS,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CACxD,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,WAAW,aAAa,CAAC,MAAM,eAAe,UAAU,CAAC,MAAM,EAAE,CAClE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,mBAAmB,EAAE,GACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjD,mEAAmE;QACnE,MAAM,cAAc,GAAG,KAAK,IAAI,mBAAmB,CAAC;QAEpD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAChD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAG,WAAW,CACzB,aAAa,EACb,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EACvB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAoB,EAAE,CAAC;QAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,QAAQ,CACtF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,cAAc,CACvB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,CAAC,KAAK,CAAC,EACP,CAAC,KAAK,CAAC,IAAI,cAAc,CAC1B,CAAC;QACJ,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,eAAe,EACf,WAAW,EACX,kBAAkB,GAEnB,MAAM,YAAY,CAAC;AAGpB,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,YAAY,CAAe;IAC3B,WAAW,CAAS;IACpB,eAAe,CAAU;IACzB,cAAc,CAAS;IACvB,qBAAqB,CAAS;IAC9B,MAAM,CAAS;IACvB,UAAU,CAA8C;IAExD,YACE,QAA2B,EAC3B,UAWI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,IAAI,QAAQ,CAAC;QACvE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,gBAAgB,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAClF,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChE,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,EAAgB,EAChB,MAAe,EACf,QAAiB,KAAK;QAMtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,aAAa,IAAI,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAChH,CAAC;QAEF,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAElC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kDAAkD,CAAC,CAAC;YACxE,EAAE,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC;QAC1E,CAAC;QAED,qDAAqD;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,YAAY,GAChB,YAAY,CAAC,KAAK,KAAK,SAAS;gBAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACjC,YAAY,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC7C,MAAM,iBAAiB,GACrB,YAAY,CAAC,kBAAkB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAE/D,IAAI,YAAY,IAAI,iBAAiB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,YAAY,CAAC,kBAAkB,CAAC;gBACjD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qEAAqE,CACtE,CAAC;gBACF,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,0FAA0F;QAC1F,MAAM,kBAAkB,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC5D,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,oBAAoB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gBAAgB,kBAAkB,CAAC,MAAM,kBAAkB,oBAAoB,CAAC,IAAI,aAAa,CAClG,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEvE,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;YACpD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExD,IACE,iBAAiB;gBACjB,iBAAiB,CAAC,IAAI,KAAK,UAAU,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,CAAC;gBACD,YAAY,EAAE,CAAC;gBACf,aAAa,IAAI,UAAU,CAAC,MAAM,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,YAAY,0BAA0B,aAAa,UAAU,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,EAAgB,EAChB,aAA8B,EAC9B,IAAoB;QAEpB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjB,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,aAAa,CAAC,MAAM,uBAAuB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,GAAG,CACP,EAAgB,EAChB,UAAkB,EAClB,QAAiB,KAAK,EACtB,mBAAyC;QAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpE,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACvE,EAAE,EACF,MAAM,EACN,KAAK,CACN,CAAC;QAEF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAChD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,WAAW,CACzB,aAAa,EACb,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EACvB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAoB,EAAE,CAAC;QAC1C,IAAI,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,QAAQ,CACtF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;YAC1C,IAAI,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAmB;oBAC3B,aAAa,EAAE,CAAC;oBAChB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBAChC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;oBAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;wBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;wBAChC,CAAC,CAAC,EAAE,CAAC;oBACP,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;wBACpC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE;wBAC3C,CAAC,CAAC,EAAE,CAAC;oBACP,SAAS,EAAE,YAAY,EAAE,SAAS,IAAI,GAAG;oBACzC,SAAS,EAAE,GAAG;iBACf,CAAC;gBACF,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC9C,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE1B,IACE,mBAAmB;oBACnB,EAAE,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,qBAAqB,EAC/C,CAAC;oBACD,MAAM,mBAAmB,EAAE,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}

package/dist/core/embeddings-db.d.ts

@@ -0,0 +1,20 @@
+import type { EmbeddedChunk, EmbeddingsMeta } from "../interfaces/index.js";
+export declare class EmbeddingsDb {
+    private db;
+    constructor(dbPath: string);
+    private isEmpty;
+    getMeta(): EmbeddingsMeta | null;
+    setMeta(meta: EmbeddingsMeta): void;
+    insert(chunks: EmbeddedChunk[]): void;
+    has(contentHash: string): boolean;
+    getAll(): EmbeddedChunk[];
+    getPending(): EmbeddedChunk[];
+    markUploaded(contentHashes: string[]): void;
+    pendingCount(): number;
+    clearAll(): void;
+    pruneUploaded(): void;
+    migrateFromJson(jsonPath: string): void;
+    close(): void;
+    private rows;
+}
+//# sourceMappingURL=embeddings-db.d.ts.map

package/dist/core/embeddings-db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings-db.d.ts","sourceRoot":"","sources":["../../src/core/embeddings-db.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AA2B5E,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAAoB;gBAElB,MAAM,EAAE,MAAM;IAW1B,OAAO,CAAC,OAAO;IAOf,OAAO,IAAI,cAAc,GAAG,IAAI;IAYhC,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,IAAI;IAMnC,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,IAAI;IAsBrC,GAAG,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAOjC,MAAM,IAAI,aAAa,EAAE;IAIzB,UAAU,IAAI,aAAa,EAAE;IAI7B,YAAY,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,IAAI;IAU3C,YAAY,IAAI,MAAM;IAOtB,QAAQ,IAAI,IAAI;IAKhB,aAAa,IAAI,IAAI;IAIrB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAyDvC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,IAAI;CAkCb"}

package/dist/core/embeddings-db.js

@@ -0,0 +1,181 @@
+import { createHash } from "crypto";
+import { existsSync, readFileSync } from "fs";
+import { rename } from "fs/promises";
+import Database from "better-sqlite3";
+function chunkContentHash(chunk) {
+    return (chunk.contentHash ??
+        createHash("sha256").update(chunk.content).digest("hex").slice(0, 16));
+}
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS meta (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS embeddings (
+  content_hash TEXT PRIMARY KEY,
+  source_file TEXT NOT NULL,
+  commit_hash TEXT NOT NULL,
+  content TEXT NOT NULL,
+  metadata_json TEXT NOT NULL,
+  embedding_json TEXT NOT NULL,
+  embedded_at INTEGER NOT NULL,
+  uploaded INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_source_file ON embeddings(source_file);
+`;
+export class EmbeddingsDb {
+    db;
+    constructor(dbPath) {
+        this.db = new Database(dbPath);
+        this.db.pragma("journal_mode = WAL");
+        this.db.exec(SCHEMA);
+        const jsonPath = dbPath.replace(/\.db$/, ".json");
+        if (this.isEmpty() && existsSync(jsonPath)) {
+            this.migrateFromJson(jsonPath);
+        }
+    }
+    isEmpty() {
+        const row = this.db
+            .prepare("SELECT COUNT(*) as cnt FROM embeddings")
+            .get();
+        return row.cnt === 0;
+    }
+    getMeta() {
+        const row = this.db
+            .prepare("SELECT value FROM meta WHERE key = 'meta'")
+            .get();
+        if (!row)
+            return null;
+        try {
+            return JSON.parse(row.value);
+        }
+        catch {
+            return null;
+        }
+    }
+    setMeta(meta) {
+        this.db
+            .prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('meta', ?)")
+            .run(JSON.stringify(meta));
+    }
+    insert(chunks) {
+        const stmt = this.db.prepare(`
+      INSERT OR IGNORE INTO embeddings
+        (content_hash, source_file, commit_hash, content, metadata_json, embedding_json, embedded_at, uploaded)
+      VALUES (?, ?, ?, ?, ?, ?, ?, 0)
+    `);
+        const insertMany = this.db.transaction((items) => {
+            for (const chunk of items) {
+                stmt.run(chunkContentHash(chunk), chunk.sourceFile, chunk.commitHash, chunk.content, JSON.stringify(chunk.metadata), JSON.stringify(chunk.embedding), chunk.embeddedAt);
+            }
+        });
+        insertMany(chunks);
+    }
+    has(contentHash) {
+        const row = this.db
+            .prepare("SELECT 1 FROM embeddings WHERE content_hash = ?")
+            .get(contentHash);
+        return row !== undefined;
+    }
+    getAll() {
+        return this.rows("SELECT * FROM embeddings");
+    }
+    getPending() {
+        return this.rows("SELECT * FROM embeddings WHERE uploaded = 0");
+    }
+    markUploaded(contentHashes) {
+        if (contentHashes.length === 0)
+            return;
+        const placeholders = contentHashes.map(() => "?").join(", ");
+        this.db
+            .prepare(`UPDATE embeddings SET uploaded = 1 WHERE content_hash IN (${placeholders})`)
+            .run(...contentHashes);
+    }
+    pendingCount() {
+        const row = this.db
+            .prepare("SELECT COUNT(*) as cnt FROM embeddings WHERE uploaded = 0")
+            .get();
+        return row.cnt;
+    }
+    clearAll() {
+        this.db.prepare("DELETE FROM embeddings").run();
+        this.db.prepare("DELETE FROM meta").run();
+    }
+    pruneUploaded() {
+        this.db.prepare("DELETE FROM embeddings WHERE uploaded = 1").run();
+    }
+    migrateFromJson(jsonPath) {
+        let raw;
+        try {
+            raw = readFileSync(jsonPath, "utf-8");
+        }
+        catch {
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        let chunks;
+        let meta = null;
+        if (Array.isArray(parsed)) {
+            chunks = parsed;
+        }
+        else {
+            const file = parsed;
+            chunks = file.chunks ?? [];
+            meta = file._meta ?? null;
+        }
+        if (meta)
+            this.setMeta(meta);
+        if (chunks.length > 0) {
+            // Mark migrated chunks as uploaded since they were previously written to JSON
+            // (implying they were already synced to the vector store)
+            const stmt = this.db.prepare(`
+        INSERT OR IGNORE INTO embeddings
+          (content_hash, source_file, commit_hash, content, metadata_json, embedding_json, embedded_at, uploaded)
+        VALUES (?, ?, ?, ?, ?, ?, ?, 1)
+      `);
+            const insertAll = this.db.transaction((items) => {
+                for (const chunk of items) {
+                    stmt.run(chunkContentHash(chunk), chunk.sourceFile, chunk.commitHash, chunk.content, JSON.stringify(chunk.metadata), JSON.stringify(chunk.embedding), chunk.embeddedAt);
+                }
+            });
+            insertAll(chunks);
+        }
+        rename(jsonPath, jsonPath + ".migrated").catch(() => { });
+    }
+    close() {
+        this.db.close();
+    }
+    rows(sql) {
+        const rows = this.db.prepare(sql).all();
+        return rows.map((row) => ({
+            contentHash: row.content_hash,
+            sourceFile: row.source_file,
+            commitHash: row.commit_hash,
+            content: row.content,
+            metadata: (() => {
+                try {
+                    return JSON.parse(row.metadata_json);
+                }
+                catch {
+                    return {};
+                }
+            })(),
+            embedding: (() => {
+                try {
+                    return JSON.parse(row.embedding_json);
+                }
+                catch {
+                    return [];
+                }
+            })(),
+            embeddedAt: row.embedded_at,
+        }));
+    }
+}
+//# sourceMappingURL=embeddings-db.js.map

package/dist/core/embeddings-db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings-db.js","sourceRoot":"","sources":["../../src/core/embeddings-db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC,SAAS,gBAAgB,CAAC,KAAoB;IAC5C,OAAO,CACL,KAAK,CAAC,WAAW;QACjB,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;CAgBd,CAAC;AAEF,MAAM,OAAO,YAAY;IACf,EAAE,CAAoB;IAE9B,YAAY,MAAc;QACxB,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAEO,OAAO;QACb,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,wCAAwC,CAAC;aACjD,GAAG,EAAqB,CAAC;QAC5B,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,2CAA2C,CAAC;aACpD,GAAG,EAAmC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAmB,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAoB;QAC1B,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,6DAA6D,CAAC;aACtE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,CAAC,MAAuB;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAsB,EAAE,EAAE;YAChE,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,CAAC,GAAG,CACN,gBAAgB,CAAC,KAAK,CAAC,EACvB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,OAAO,EACb,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,EAC/B,KAAK,CAAC,UAAU,CACjB,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,GAAG,CAAC,WAAmB;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,iDAAiD,CAAC;aAC1D,GAAG,CAAC,WAAW,CAAC,CAAC;QACpB,OAAO,GAAG,KAAK,SAAS,CAAC;IAC3B,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC/C,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IAClE,CAAC;IAED,YAAY,CAAC,aAAuB;QAClC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,EAAE;aACJ,OAAO,CACN,6DAA6D,YAAY,GAAG,CAC7E;aACA,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAC3B,CAAC;IAED,YAAY;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,2DAA2D,CAAC;aACpE,GAAG,EAAqB,CAAC;QAC5B,OAAO,GAAG,CAAC,GAAG,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,EAAE,CAAC;QAChD,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,CAAC;IAC5C,CAAC;IAED,aAAa;QACX,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC,GAAG,EAAE,CAAC;IACrE,CAAC;IAED,eAAe,CAAC,QAAgB;QAC9B,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,IAAI,MAAuB,CAAC;QAC5B,IAAI,IAAI,GAA0B,IAAI,CAAC;QAEvC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,MAAM,GAAG,MAAyB,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,MAGZ,CAAC;YACF,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YAC3B,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;QAC5B,CAAC;QAED,IAAI,IAAI;YAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,8EAA8E;YAC9E,0DAA0D;YAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAI5B,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAsB,EAAE,EAAE;gBAC/D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBAC1B,IAAI,CAAC,GAAG,CACN,gBAAgB,CAAC,KAAK,CAAC,EACvB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,OAAO,EACb,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,EAC/B,KAAK,CAAC,UAAU,CACjB,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;YACH,SAAS,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAED,MAAM,CAAC,QAAQ,EAAE,QAAQ,GAAG,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAEO,IAAI,CAAC,GAAW;QAWtB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAW,CAAC;QACjD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,WAAW,EAAE,GAAG,CAAC,YAAY;YAC7B,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAA4B,CAAC;gBAClE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,EAAE;YACJ,SAAS,EAAE,CAAC,GAAG,EAAE;gBACf,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAa,CAAC;gBACpD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,EAAE;YACJ,UAAU,EAAE,GAAG,CAAC,WAAW;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}

package/dist/core/orchestrator.d.ts

@@ -16,6 +16,7 @@ export interface RAGPipelineConfig {
         maxBatchChars?: number;
         retry?: RetryOptions;
         concurrency?: number;
+        minIngestionBatchSize?: number;
         telemetry?: boolean;
         notifications?: {
             webhookUrl?: string;

package/dist/core/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAyCD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAyL3B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAsCD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA4M3B"}