mcp-rag-memory 2.4.1 → 2.7.0

package/dist/rag/pipeline.js

@@ -4,20 +4,53 @@ import { contentHash, getDB, newId, nowMs, packVector } from "../db/database.js"
 import { chunkText } from "./chunker.js";
 import { embedText } from "./embeddings.js";
 import { searchChunks, vectorSearch, invalidateVectorCache } from "./vector-search.js";
-/** Insert a FTS row, tolerating an unavailable FTS5 backend. */
-function insertFtsRow(docId, rowid, text) {
+let ftsWarned = false;
+/** Insert a FTS row. Only a genuinely missing FTS5 backend degrades silently —
+ * any other failure (disk full, corruption, SQLITE_BUSY) must surface, because
+ * swallowing it left every chunk permanently invisible to the keyword leg with
+ * no diagnostic and no rebuild path. */
+function insertFtsRow(rowid, text) {
     try {
         getDB().prepare("INSERT INTO chunks_fts(rowid, content) VALUES (?, ?)").run(rowid, text);
     }
-    catch {
-        // FTS missing — hybrid degrades to vector-only.
+    catch (e) {
+        const msg = e.message;
+        // Match only "the FTS table is not there". A looser /chunks_fts/ test also
+        // swallowed UNIQUE and corruption errors, which are exactly the failures
+        // this change exists to surface.
+        if (!/no such table: chunks_fts|no such module: fts5/i.test(msg))
+            throw e;
+        if (!ftsWarned) {
+            ftsWarned = true;
+            process.stderr.write("rag: chunks_fts unavailable — hybrid search degraded to vector-only\n");
+        }
     }
 }
+/** SQLite UNIQUE violation, which node:sqlite surfaces as errcode 2067. */
+function isUniqueViolation(e, column) {
+    const err = e;
+    const msg = err?.message ?? "";
+    // "." must be escaped or it matches any character.
+    return err?.errcode === 2067 || msg.includes(`UNIQUE constraint failed: ${column}`);
+}
+/** Max content accepted from a tool call or CLI, so one payload cannot stall the server for a minute. */
+export const MAX_INGEST_CHARS = 1_000_000;
 export async function ingestText(content, title, opts = {}) {
     const db = getDB();
     const ts = nowMs();
-    // Content-hash dedup: identical documents are never re-embedded.
-    const hash = contentHash(content);
+    // trim(): whitespace-only content produced a permanent 0-chunk document that
+    // could never be searched yet still occupied the content_hash unique index.
+    if (typeof content !== "string" || content.trim().length === 0) {
+        throw new Error("Ingest content must be a non-empty string");
+    }
+    if (content.length > MAX_INGEST_CHARS) {
+        throw new Error(`Ingest content too large (${content.length} chars > limit ${MAX_INGEST_CHARS}). ` +
+            `Split it, or use ingest_file / ingest_dir for large sources.`);
+    }
+    // Content-hash dedup: identical documents are never re-embedded. `dedupScope`
+    // keeps sources that legitimately repeat the same text apart (e.g. "ok" typed
+    // in two different sessions).
+    const hash = contentHash(`${opts.dedupScope ? `${opts.dedupScope}\u0000` : ""}${content}`);
     const existing = db
         .prepare("SELECT id, title, chunk_count FROM documents WHERE content_hash = ?")
         .get(hash);
@@ -35,30 +68,102 @@ export async function ingestText(content, title, opts = {}) {
     }
     const docId = newId();
     const chunks = chunkText(content);
-    db.prepare(`INSERT INTO documents (id, title, source, content_type, metadata, content_hash, chunk_count, created_at, updated_at)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(docId, title, opts.source ?? null, opts.contentType ?? "text", JSON.stringify(opts.metadata ?? {}), hash, chunks.length, ts, ts);
+    if (chunks.length === 0) {
+        throw new Error("Ingest produced no indexable content (no non-whitespace text)");
+    }
     let tokens = 0;
-    const insertChunk = db.prepare(`INSERT INTO chunks (id, doc_id, idx, content, embedding, token_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+    // Embed BEFORE opening the transaction. The MCP SDK does not serialize tool
+    // calls, so awaiting inside a transaction let a concurrent request's writes
+    // join this one — and a failure here rolled back that other request's work.
+    // The yield keeps the event loop responsive: a 10 MB file is ~11k chunks and
+    // the local embedder is synchronous, so without it the stdio server could not
+    // answer any other tool call for the whole ingest.
+    const vectors = [];
     for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i];
-        const vec = await embedText(chunk.text);
-        tokens += chunk.tokenCount;
-        const res = insertChunk.run(newId(), docId, chunk.index, chunk.text, packVector(vec), chunk.tokenCount, ts);
-        insertFtsRow(docId, Number(res.lastInsertRowid), chunk.text);
+        vectors.push(await embedText(chunks[i].text));
+        if ((i & 15) === 15)
+            await new Promise((resolve) => setImmediate(resolve));
+    }
+    // All writes for one document are atomic, and the block below is fully
+    // synchronous, so no other request can interleave inside it.
+    db.exec("BEGIN IMMEDIATE;");
+    try {
+        db.prepare(`INSERT INTO documents (id, title, source, content_type, metadata, content_hash, chunk_count, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(docId, title, opts.source ?? null, opts.contentType ?? "text", JSON.stringify(opts.metadata ?? {}), hash, chunks.length, ts, ts);
+        const insertChunk = db.prepare(`INSERT INTO chunks (id, doc_id, idx, content, embedding, token_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`);
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            tokens += chunk.tokenCount;
+            const res = insertChunk.run(newId(), docId, chunk.index, chunk.text, packVector(vectors[i]), chunk.tokenCount, ts);
+            insertFtsRow(Number(res.lastInsertRowid), chunk.text);
+        }
+        db.exec("COMMIT;");
+    }
+    catch (e) {
+        try {
+            db.exec("ROLLBACK;");
+        }
+        catch {
+            /* transaction already closed */
+        }
+        // Two identical ingests can both pass the dedup SELECT above: the check and
+        // this INSERT are separated by chunking + embedding, and MCP tool calls are
+        // not serialized. Let the unique index arbitrate and report a dedup hit
+        // rather than failing the caller with a raw driver error.
+        if (isUniqueViolation(e, "documents.content_hash")) {
+            const again = getDB()
+                .prepare("SELECT id, title, chunk_count FROM documents WHERE content_hash = ?")
+                .get(hash);
+            if (again) {
+                return {
+                    docId: again.id,
+                    title: again.title,
+                    chunks: again.chunk_count,
+                    tokens: 0,
+                    deduplicated: true,
+                };
+            }
+        }
+        throw e;
     }
     invalidateVectorCache();
     return { docId, title, chunks: chunks.length, tokens, deduplicated: false };
 }
-function assertAllowedPath(abs) {
+/** Roots are realpath'd too, so a junction/symlinked allowlist entry still matches. */
+function allowedRoots() {
     const raw = process.env.RAG_ALLOWED_DIRS;
     if (!raw)
-        return;
-    const roots = raw
+        return [];
+    return raw
         .split(/[;|,]/)
-        .map((p) => path.resolve(p.trim()))
-        .filter(Boolean);
-    if (roots.length === 0)
+        .map((p) => p.trim())
+        .filter(Boolean)
+        .map((p) => {
+        const abs = path.resolve(p);
+        try {
+            return fs.realpathSync(abs);
+        }
+        catch {
+            return abs;
+        }
+    });
+}
+let warnedOpenAllowlist = false;
+/**
+ * RAG_ALLOWED_DIRS is opt-in, so by default any readable path can be ingested
+ * and then stays retrievable from every future session sharing this store. Warn
+ * once instead of silently widening the blast radius of an agent-driven tool.
+ */
+function assertAllowedPath(abs) {
+    const roots = allowedRoots();
+    if (roots.length === 0) {
+        if (!warnedOpenAllowlist) {
+            warnedOpenAllowlist = true;
+            process.stderr.write("rag: RAG_ALLOWED_DIRS is not set — any readable path can be ingested and stays " +
+                `retrievable from this store. Set RAG_ALLOWED_DIRS to restrict it.\n`);
+        }
         return;
+    }
     const real = fs.realpathSync(abs);
     if (!roots.some((r) => real === r || real.startsWith(r + path.sep))) {
         throw new Error(`Path not allowed by RAG_ALLOWED_DIRS: ${abs}`);
@@ -71,7 +176,8 @@ export async function ingestFile(filePath, opts = {}) {
     const st = fs.statSync(abs);
     if (!st.isFile())
         throw new Error(`Not a file: ${abs}`);
-    const maxMb = Number(process.env.RAG_MAX_FILE_MB ?? 10);
+    const maxMbRaw = Number(process.env.RAG_MAX_FILE_MB ?? 10);
+    const maxMb = Number.isFinite(maxMbRaw) && maxMbRaw > 0 ? maxMbRaw : 10;
     const maxBytes = Math.max(1024, Math.floor(maxMb * 1024 * 1024));
     if (st.size > maxBytes) {
         throw new Error(`File too large (${st.size} bytes > limit ${maxBytes}): ${abs}. Raise RAG_MAX_FILE_MB to allow bigger files.`);
@@ -79,25 +185,53 @@ export async function ingestFile(filePath, opts = {}) {
     assertAllowedPath(abs);
     const content = fs.readFileSync(abs, "utf-8");
     const title = path.basename(abs);
-    return ingestText(content, title, {
+    return ingestText(stripBom(content), title, {
         source: abs,
         contentType: opts.contentType ?? guessContentType(abs),
         metadata: opts.metadata ?? { path: abs },
     });
 }
+/**
+ * Drop a UTF-8 BOM. It survives readFileSync as U+FEFF, so a Windows-authored
+ * .md file's first heading indexed as "\ufeff# Title" — the FTS trigram tokenizer
+ * keyed on it and the title never matched a plain "Title" query.
+ */
+function stripBom(text) {
+    return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+}
 /** Let the event loop breathe every N files during large directory ingests. */
 function yieldLoop() {
     return new Promise((resolve) => setImmediate(resolve));
 }
+/** Cap the payload: a big directory must not return one line per file. */
+const DIR_REPORT_LIMIT = 50;
+/**
+ * Hard ceiling on files actually ingested. DIR_REPORT_LIMIT only truncated the
+ * response, so pointing this at a large tree still embedded every file and held
+ * the server hostage; the cap is reported explicitly instead of silently
+ * dropping the remainder.
+ */
+const DIR_MAX_FILES = 500;
 export async function ingestDirectory(dirPath, opts = {}) {
     const root = path.resolve(dirPath);
     if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
         throw new Error(`Not a directory: ${dirPath}`);
     }
     assertAllowedPath(root);
-    const exts = opts.extensions?.map((e) => (e.startsWith(".") ? e : `.${e}`)) ?? [
+    if (opts.extensions !== undefined) {
+        if (opts.extensions.length === 0)
+            throw new Error("extensions must not be empty");
+        for (const e of opts.extensions) {
+            if (typeof e !== "string" || e.trim() === "")
+                throw new Error(`Invalid extension: ${JSON.stringify(e)}`);
+        }
+    }
+    // Lowercase once: `full.endsWith(e)` is case-sensitive, so an explicit
+    // extensions: [".TS"] silently matched nothing on a Windows-authored tree
+    // (or a request for ".Md" against README.MD).
+    const exts = (opts.extensions?.map((e) => (e.startsWith(".") ? e : `.${e}`)) ?? [
         ".ts", ".tsx", ".js", ".jsx", ".json", ".md", ".txt", ".py", ".go", ".rs", ".java", ".c", ".h", ".cpp", ".html", ".css",
-    ];
+    ]).map((e) => e.toLowerCase());
     const skipDirs = new Set([".git", "dist", "node_modules", ".rag-data", ".test-data", ".venv"]);
     const files = [];
     const walk = (dir) => {
@@ -107,30 +241,48 @@ export async function ingestDirectory(dirPath, opts = {}) {
                 if (opts.recursive && !skipDirs.has(entry.name) && !entry.name.startsWith("."))
                     walk(full);
             }
-            else if (entry.isFile() && exts.some((e) => full.endsWith(e))) {
+            else if (entry.isFile() && exts.some((e) => full.toLowerCase().endsWith(e))) {
                 files.push(full);
             }
         }
     };
     walk(root);
+    // Deterministic slice before any work: walk() order is filesystem-dependent,
+    // so capping after sorting keeps "which 500" reproducible across machines.
+    const sorted = files.sort();
+    const truncated = Math.max(0, sorted.length - DIR_MAX_FILES);
+    const targets = truncated > 0 ? sorted.slice(0, DIR_MAX_FILES) : sorted;
     const ingested = [];
     const skipped = [];
-    for (let i = 0; i < files.sort().length; i++) {
-        const f = files[i];
+    let ingestedCount = 0;
+    let skippedCount = 0;
+    for (let i = 0; i < targets.length; i++) {
+        const f = targets[i];
         try {
-            ingested.push(await ingestFile(f, { metadata: opts.metadata ?? { ingestDir: root } }));
+            const res = await ingestFile(f, { metadata: opts.metadata ?? { ingestDir: root } });
+            ingestedCount++;
+            if (ingested.length < DIR_REPORT_LIMIT)
+                ingested.push(res);
         }
         catch (e) {
-            skipped.push(`${f} (${e.message})`);
+            skippedCount++;
+            if (skipped.length < DIR_REPORT_LIMIT)
+                skipped.push(`${f} (${e.message})`);
         }
         if (i % 10 === 9)
             await yieldLoop();
     }
-    return { ingested, skipped };
+    if (truncated > 0) {
+        skippedCount += truncated;
+        if (skipped.length < DIR_REPORT_LIMIT) {
+            skipped.push(`${truncated} more file(s) matched but were not ingested: directory ingest is capped at ${DIR_MAX_FILES} per call`);
+        }
+    }
+    return { ingested, skipped, ingestedCount, skippedCount };
 }
-export async function searchDocs(query, topK = 10, minScore = 0.08, filters) {
+export async function searchDocs(query, topK = 10, minScore = 0.08, filters, explain = false) {
     const q = await embedText(query);
-    return searchChunks(q, { queryText: query, topK, minScore, filters });
+    return searchChunks(q, { queryText: query, topK, minScore, filters, explain });
 }
 export async function retrieve(query, topK = 6, minScore = 0.08, filters) {
     const hits = await searchDocs(query, topK, minScore, filters);
@@ -146,19 +298,40 @@ export async function retrieve(query, topK = 6, minScore = 0.08, filters) {
         totalTokens: withTokens.reduce((acc, c) => acc + c.tokenCount, 0),
     };
 }
-export function listDocuments() {
+/** Bounded by default: a big store would otherwise dump every row into one tool result. */
+export const LIST_DOCS_DEFAULT = 100;
+export const LIST_DOCS_MAX = 1000;
+export function listDocuments(opts = {}) {
     const db = getDB();
+    const limit = Math.min(Math.max(Math.trunc(opts.limit ?? LIST_DOCS_DEFAULT), 1), LIST_DOCS_MAX);
+    const offset = Math.max(Math.trunc(opts.offset ?? 0), 0);
+    const total = db.prepare("SELECT COUNT(*) AS c FROM documents").get().c;
     const rows = db
-        .prepare(`SELECT id, title, source, content_type, chunk_count, created_at, updated_at, metadata FROM documents ORDER BY updated_at DESC`)
-        .all();
-    return rows;
+        .prepare(`SELECT id, title, source, content_type, chunk_count, created_at, updated_at, metadata
+         FROM documents ORDER BY updated_at DESC LIMIT ? OFFSET ?`)
+        .all(limit, offset);
+    return { documents: rows, total, limit, offset };
 }
 export function deleteDocument(docId) {
     const db = getDB();
-    db.prepare("DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks WHERE doc_id = ?)").run(docId);
-    const res = db.prepare("DELETE FROM documents WHERE id = ?").run(docId);
+    db.exec("BEGIN IMMEDIATE;");
+    let changes = 0;
+    try {
+        db.prepare("DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks WHERE doc_id = ?)").run(docId);
+        changes = Number(db.prepare("DELETE FROM documents WHERE id = ?").run(docId).changes);
+        db.exec("COMMIT;");
+    }
+    catch (e) {
+        try {
+            db.exec("ROLLBACK;");
+        }
+        catch {
+            /* transaction already closed */
+        }
+        throw e;
+    }
     invalidateVectorCache();
-    return { deleted: Number(res.changes) > 0 };
+    return { deleted: changes > 0 };
 }
 export function documentStats() {
     const db = getDB();

package/dist/rag/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAkCvF,gEAAgE;AAChE,SAAS,YAAY,CAAC,KAAa,EAAE,KAAa,EAAE,IAAY;IAC9D,IAAI,CAAC;QACH,KAAK,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAAe,EACf,KAAa,EACb,OAA0B,EAAE;IAE5B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IAEnB,iEAAiE;IACjE,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CAAC,qEAAqE,CAAC;SAC9E,GAAG,CAAC,IAAI,CAAmE,CAAC;IAC/E,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CAAC,wEAAwE,CAAC;aACjF,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAkB,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,EAAE;YAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,MAAM,EAAE,MAAM,CAAC,CAAC;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAElC,EAAE,CAAC,OAAO,CACR;wCACoC,CACrC,CAAC,GAAG,CACH,KAAK,EACL,KAAK,EACL,IAAI,CAAC,MAAM,IAAI,IAAI,EACnB,IAAI,CAAC,WAAW,IAAI,MAAM,EAC1B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,EACnC,IAAI,EACJ,MAAM,CAAC,MAAM,EACb,EAAE,EACF,EAAE,CACH,CAAC;IAEF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAC5B,gHAAgH,CACjH,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC5G,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IACD,qBAAqB,EAAE,CAAC;IAExB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,KAAK,GAAG,GAAG;SACd,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC/B,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB,EAAE,OAA0B,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IACnE,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IAExD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IACjE,IAAI,EAAE,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,mBAAmB,EAAE,CAAC,IAAI,kBAAkB,QAAQ,MAAM,GAAG,gDAAgD,CAC9G,CAAC;IACJ,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE;QAChC,MAAM,EAAE,GAAG;QACX,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC;QACtD,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE;KACzC,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,SAAS,SAAS;IAChB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAExB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI;QAC7E,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;KACxH,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/F,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEX,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QACzF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAM,CAAW,CAAC,OAAO,GAAG,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,MAAM,SAAS,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,GAAG,IAAI,EAAE,OAA4B;IACtG,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,YAAY,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,IAAI,GAAG,CAAC,EACR,QAAQ,GAAG,IAAI,EACf,OAA4B;IAE5B,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;KACzB,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN,+HAA+H,CAChI;SACA,GAAG,EAA+C,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,EAAE,CAAC,OAAO,CAAC,mFAAmF,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3G,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACxE,qBAAqB,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACtF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACrF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,EAAmB,CAAC;IACzG,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,GAAG,GAA2B;QAClC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;QACvF,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;QAC/F,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;KACjH,CAAC;IACF,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;AAC5B,CAAC;AAED,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAoCvF,IAAI,SAAS,GAAG,KAAK,CAAC;AAEtB;;;wCAGwC;AACxC,SAAS,YAAY,CAAC,KAAa,EAAE,IAAY;IAC/C,IAAI,CAAC;QACH,KAAK,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3F,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,GAAG,GAAI,CAAW,CAAC,OAAO,CAAC;QACjC,2EAA2E;QAC3E,yEAAyE;QACzE,iCAAiC;QACjC,IAAI,CAAC,iDAAiD,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,IAAI,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,SAAS,iBAAiB,CAAC,CAAU,EAAE,MAAc;IACnD,MAAM,GAAG,GAAG,CAA2C,CAAC;IACxD,MAAM,GAAG,GAAG,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;IAC/B,mDAAmD;IACnD,OAAO,GAAG,EAAE,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,6BAA6B,MAAM,EAAE,CAAC,CAAC;AACtF,CAAC;AAED,yGAAyG;AACzG,MAAM,CAAC,MAAM,gBAAgB,GAAG,SAAS,CAAC;AAE1C,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAAe,EACf,KAAa,EACb,OAA0B,EAAE;IAE5B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,6EAA6E;IAC7E,4EAA4E;IAC5E,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,6BAA6B,OAAO,CAAC,MAAM,kBAAkB,gBAAgB,KAAK;YAChF,8DAA8D,CACjE,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,8EAA8E;IAC9E,8BAA8B;IAC9B,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CAAC,qEAAqE,CAAC;SAC9E,GAAG,CAAC,IAAI,CAAmE,CAAC;IAC/E,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CAAC,wEAAwE,CAAC;aACjF,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAkB,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,EAAE;YAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,MAAM,EAAE,MAAM,CAAC,CAAC;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,4EAA4E;IAC5E,4EAA4E;IAC5E,4EAA4E;IAC5E,6EAA6E;IAC7E,8EAA8E;IAC9E,mDAAmD;IACnD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,uEAAuE;IACvE,6DAA6D;IAC7D,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CACR;0CACoC,CACrC,CAAC,GAAG,CACH,KAAK,EACL,KAAK,EACL,IAAI,CAAC,MAAM,IAAI,IAAI,EACnB,IAAI,CAAC,WAAW,IAAI,MAAM,EAC1B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,EACnC,IAAI,EACJ,MAAM,CAAC,MAAM,EACb,EAAE,EACF,EAAE,CACH,CAAC;QAEF,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAC5B,gHAAgH,CACjH,CAAC;QACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;YAC3B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACnH,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;QACD,4EAA4E;QAC5E,4EAA4E;QAC5E,wEAAwE;QACxE,0DAA0D;QAC1D,IAAI,iBAAiB,CAAC,CAAC,EAAE,wBAAwB,CAAC,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,KAAK,EAAE;iBAClB,OAAO,CAAC,qEAAqE,CAAC;iBAC9E,GAAG,CAAC,IAAI,CAAmE,CAAC;YAC/E,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO;oBACL,KAAK,EAAE,KAAK,CAAC,EAAE;oBACf,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK,CAAC,WAAW;oBACzB,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,IAAI;iBACnB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,qBAAqB,EAAE,CAAC;IAExB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC9E,CAAC;AAED,uFAAuF;AACvF,SAAS,YAAY;IACnB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,OAAO,GAAG;SACP,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,OAAO,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;AAEhC;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,KAAK,GAAG,YAAY,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACzB,mBAAmB,GAAG,IAAI,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iFAAiF;gBAC/E,qEAAqE,CACxE,CAAC;QACJ,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB,EAAE,OAA0B,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IACnE,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IACjE,IAAI,EAAE,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,mBAAmB,EAAE,CAAC,IAAI,kBAAkB,QAAQ,MAAM,GAAG,gDAAgD,CAC9G,CAAC;IACJ,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE;QAC1C,MAAM,EAAE,GAAG;QACX,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC;QACtD,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE;KACzC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC;AAED,+EAA+E;AAC/E,SAAS,SAAS;IAChB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,0EAA0E;AAC1E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B;;;;;GAKG;AACH,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAExB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IACD,uEAAuE;IACvE,0EAA0E;IAC1E,8CAA8C;IAC9C,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI;QAC9E,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;KACxH,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/F,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEX,6EAA6E;IAC7E,2EAA2E;IAC3E,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACpF,aAAa,EAAE,CAAC;YAChB,IAAI,QAAQ,CAAC,MAAM,GAAG,gBAAgB;gBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,YAAY,EAAE,CAAC;YACf,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB;gBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAM,CAAW,CAAC,OAAO,GAAG,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,MAAM,SAAS,EAAE,CAAC;IACtC,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,YAAY,IAAI,SAAS,CAAC;QAC1B,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CACV,GAAG,SAAS,8EAA8E,aAAa,WAAW,CACnH,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAa,EACb,IAAI,GAAG,EAAE,EACT,QAAQ,GAAG,IAAI,EACf,OAA4B,EAC5B,OAAO,GAAG,KAAK;IAEf,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,YAAY,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,IAAI,GAAG,CAAC,EACR,QAAQ,GAAG,IAAI,EACf,OAA4B;IAE5B,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;KACzB,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AACrC,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC;AAElC,MAAM,UAAU,aAAa,CAAC,OAA4C,EAAE;IAM1E,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC,EACxD,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAI,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IAC3F,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN;kEAC4D,CAC7D;SACA,GAAG,CAAC,KAAK,EAAE,MAAM,CAA8C,CAAC;IACnE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CAAC,mFAAmF,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3G,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QACtF,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,qBAAqB,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACtF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACrF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,EAAmB,CAAC;IACzG,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,GAAG,GAA2B;QAClC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;QACvF,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;QAC/F,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;KACjH,CAAC;IACF,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;AAC5B,CAAC;AAED,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}

package/dist/rag/reindex.d.ts

@@ -0,0 +1,27 @@
+export declare const REINDEX_BATCH = 32;
+export type ReindexPhase = "chunks" | "memories" | "finalize";
+export interface ReindexProgress {
+    phase: ReindexPhase;
+    done: number;
+    total: number;
+}
+export interface ReindexResult {
+    chunks: number;
+    memories: number;
+    dimBefore: number;
+    dimAfter: number;
+    modelBefore: string;
+    modelAfter: string;
+    ms: number;
+    changed: boolean;
+}
+export interface ReindexOptions {
+    batchSize?: number;
+    onProgress?: (p: ReindexProgress) => void;
+}
+/**
+ * Migrate the store to the currently configured `EMBEDDING_PROVIDER` /
+ * `EMBEDDING_MODEL`. Embedding is deterministic per model, so this is idempotent.
+ */
+export declare function reindex(opts?: ReindexOptions): Promise<ReindexResult>;
+//# sourceMappingURL=reindex.d.ts.map

package/dist/rag/reindex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../../src/rag/reindex.ts"],"names":[],"mappings":"AAgCA,eAAO,MAAM,aAAa,KAAK,CAAC;AAIhC,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;CAC3C;AAgED;;;GAGG;AACH,wBAAsB,OAAO,CAAC,IAAI,GAAE,cAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CA4D/E"}

package/dist/rag/reindex.js

@@ -0,0 +1,138 @@
+/**
+ * Re-embed every stored vector in place.
+ *
+ * The embedding guard (`ensureDim`) treats mixing two vector spaces as a hard
+ * error, which is correct but leaves the operator with a manual chore: point
+ * `RAG_DB_DIR` at an empty directory and re-ingest every document from scratch.
+ * For a memory store holding months of session history that is not a migration,
+ * it is data loss with extra steps.
+ *
+ * `reindex` performs the migration in place instead:
+ *   1. Mark the store `reindexing`, which makes every search and ingest fail
+ *      loudly (`assertEmbedStateReady`). A partially migrated store would
+ *      otherwise mix old and new vectors and return confident nonsense.
+ *   2. Re-embed chunks, then memories, in batches, committing as it goes so an
+ *      interrupted run leaves a resumable store rather than a lost one.
+ *   3. Rewrite `embed_dim` / `embed_model` and clear the flag.
+ *
+ * Running it again after an interruption is always safe: the whole store is
+ * rewritten from the documents' text, which is the authoritative copy.
+ */
+import { getDB, packVector, unpackVector, getEmbedDim, getEmbedModel, setEmbedMeta, setEmbedState, } from "../db/database.js";
+import { embedTextUnguarded, modelName } from "./embeddings.js";
+import { invalidateVectorCache } from "./vector-search.js";
+export const REINDEX_BATCH = 32;
+/** Yield to the event loop this often so a stdio server stays responsive mid-run. */
+const YIELD_EVERY = 4;
+function resolveBatch(raw) {
+    if (raw === undefined || !Number.isFinite(raw) || raw < 1)
+        return REINDEX_BATCH;
+    return Math.min(200, Math.max(1, Math.trunc(raw)));
+}
+const tick = () => new Promise((r) => setImmediate(r));
+/**
+ * Re-embed one table. Uses `iterate()` so content is streamed from disk in
+ * batches rather than materialising every row, and the UPDATE is parameterized
+ * so document text can never be interpreted as SQL.
+ */
+async function reembedTable(table, batchSize, onProgress) {
+    const db = getDB();
+    const total = db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n;
+    const reader = db
+        .prepare(`SELECT id, content FROM ${table}`)
+        .iterate();
+    const update = db.prepare(`UPDATE ${table} SET embedding = ? WHERE id = ?`);
+    let done = 0;
+    let pending = [];
+    let sinceYield = 0;
+    const flush = () => {
+        if (pending.length === 0)
+            return;
+        // One transaction per batch: a crash mid-run loses at most a batch, and the
+        // `reindexing` flag stays set so nothing can read the half-updated store.
+        db.exec("BEGIN");
+        try {
+            for (const row of pending)
+                update.run(packVector(row.vec), row.id);
+            db.exec("COMMIT");
+        }
+        catch (err) {
+            db.exec("ROLLBACK");
+            throw err;
+        }
+        done += pending.length;
+        pending = [];
+        onProgress?.({ phase: table, done, total });
+    };
+    for (const row of reader) {
+        pending.push({ id: row.id, vec: await embedTextUnguarded(row.content) });
+        if (pending.length >= batchSize) {
+            flush();
+            if (++sinceYield >= YIELD_EVERY) {
+                sinceYield = 0;
+                await tick();
+            }
+        }
+    }
+    flush();
+    onProgress?.({ phase: table, done, total });
+    return done;
+}
+/**
+ * Migrate the store to the currently configured `EMBEDDING_PROVIDER` /
+ * `EMBEDDING_MODEL`. Embedding is deterministic per model, so this is idempotent.
+ */
+export async function reindex(opts = {}) {
+    const started = Date.now();
+    const batchSize = resolveBatch(opts.batchSize);
+    const dimBefore = getEmbedDim();
+    const modelBefore = getEmbedModel();
+    const modelAfter = modelName();
+    const chunksRow = getDB().prepare("SELECT COUNT(*) AS n FROM chunks").get();
+    const memoriesRow = getDB().prepare("SELECT COUNT(*) AS n FROM memories").get();
+    if (chunksRow.n === 0 && memoriesRow.n === 0) {
+        // Nothing to migrate. Still normalise the metadata so a fresh store records
+        // the model it is about to start writing.
+        setEmbedMeta(dimBefore || 0, modelAfter);
+        return {
+            chunks: 0,
+            memories: 0,
+            dimBefore,
+            dimAfter: dimBefore,
+            modelBefore,
+            modelAfter,
+            ms: Date.now() - started,
+            changed: false,
+        };
+    }
+    setEmbedState("reindexing");
+    try {
+        const chunks = await reembedTable("chunks", batchSize, opts.onProgress);
+        const memories = await reembedTable("memories", batchSize, opts.onProgress);
+        // Read the dimension the provider actually produced rather than trusting
+        // expectedDim(), which is 0 for an unrecognised model name.
+        const first = getDB().prepare("SELECT embedding FROM chunks WHERE embedding IS NOT NULL LIMIT 1").get();
+        const dimAfter = (unpackVector(first?.embedding ?? null)?.length ?? dimBefore) || dimBefore;
+        opts.onProgress?.({ phase: "finalize", done: chunks + memories, total: chunks + memories });
+        setEmbedMeta(dimAfter, modelAfter);
+        invalidateVectorCache();
+        return {
+            chunks,
+            memories,
+            dimBefore,
+            dimAfter,
+            modelBefore,
+            modelAfter,
+            ms: Date.now() - started,
+            changed: modelBefore !== modelAfter || dimBefore !== dimAfter,
+        };
+    }
+    catch (err) {
+        // Leave the flag set on purpose: the store is genuinely inconsistent now and
+        // must not look usable. The message tells the operator to simply re-run.
+        throw new Error(`Reindex failed partway (${chunksRow.n} chunks / ${memoriesRow.n} memories pending): ` +
+            `${err instanceof Error ? err.message : String(err)}. ` +
+            `The store is still flagged as reindexing and will refuse searches until you re-run 'reindex'.`);
+    }
+}
+//# sourceMappingURL=reindex.js.map

package/dist/rag/reindex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../../src/rag/reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,EACL,KAAK,EACL,UAAU,EACV,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,aAAa,GACd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAChC,qFAAqF;AACrF,MAAM,WAAW,GAAG,CAAC,CAAC;AA0BtB,SAAS,YAAY,CAAC,GAAuB;IAC3C,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,aAAa,CAAC;IAChF,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,IAAI,GAAG,GAAkB,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;AAEtE;;;;GAIG;AACH,KAAK,UAAU,YAAY,CACzB,KAA4B,EAC5B,SAAiB,EACjB,UAAyC;IAEzC,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,KAAK,GACT,EAAE,CAAC,OAAO,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC,GAAG,EACrD,CAAC,CAAC,CAAC;IACJ,MAAM,MAAM,GAAG,EAAE;SACd,OAAO,CAAC,2BAA2B,KAAK,EAAE,CAAC;SAC3C,OAAO,EAAuD,CAAC;IAClE,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,iCAAiC,CAAC,CAAC;IAE5E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAA6C,EAAE,CAAC;IAC3D,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,4EAA4E;QAC5E,0EAA0E;QAC1E,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,OAAO;gBAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACnE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACpB,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;QACvB,OAAO,GAAG,EAAE,CAAC;QACb,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAChC,KAAK,EAAE,CAAC;YACR,IAAI,EAAE,UAAU,IAAI,WAAW,EAAE,CAAC;gBAChC,UAAU,GAAG,CAAC,CAAC;gBACf,MAAM,IAAI,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,EAAE,CAAC;IACR,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAuB,EAAE;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,WAAW,EAAE,CAAC;IAChC,MAAM,WAAW,GAAG,aAAa,EAAE,CAAC;IACpC,MAAM,UAAU,GAAG,SAAS,EAAE,CAAC;IAE/B,MAAM,SAAS,GAAG,KAAK,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;IAC7F,MAAM,WAAW,GAAG,KAAK,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACjG,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7C,4EAA4E;QAC5E,0CAA0C;QAC1C,YAAY,CAAC,SAAS,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;QACzC,OAAO;YACL,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;YACX,SAAS;YACT,QAAQ,EAAE,SAAS;YACnB,WAAW;YACX,UAAU;YACV,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;YACxB,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,YAAY,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAE5E,yEAAyE;QACzE,4DAA4D;QAC5D,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC,OAAO,CAAC,kEAAkE,CAAC,CAAC,GAAG,EAExF,CAAC;QACd,MAAM,QAAQ,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,SAAS,IAAI,IAAI,CAAC,EAAE,MAAM,IAAI,SAAS,CAAC,IAAI,SAAS,CAAC;QAE5F,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC;QAC5F,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACnC,qBAAqB,EAAE,CAAC;QAExB,OAAO;YACL,MAAM;YACN,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,WAAW;YACX,UAAU;YACV,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;YACxB,OAAO,EAAE,WAAW,KAAK,UAAU,IAAI,SAAS,KAAK,QAAQ;SAC9D,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,6EAA6E;QAC7E,yEAAyE;QACzE,MAAM,IAAI,KAAK,CACb,2BAA2B,SAAS,CAAC,CAAC,aAAa,WAAW,CAAC,CAAC,sBAAsB;YACpF,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI;YACvD,+FAA+F,CAClG,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/rag/vector-search.d.ts

@@ -14,11 +14,30 @@ export interface SearchHit {
     content: string;
     score: number;
     tokenCount: number;
+    /** Which retrieval legs surfaced this chunk. Always populated; cheap and useful. */
+    legs: SearchLeg[];
+    /** Per-leg diagnostics. Populated only when `explain` is requested. */
+    explain?: SearchExplain;
+}
+export type SearchLeg = "vector" | "keyword";
+export interface SearchExplain {
+    /** Cosine similarity in [-1, 1]; absent when the chunk is keyword-only. */
+    vectorScore?: number;
+    /** BM25 relevance mapped to 0..1; absent when the chunk is vector-only. */
+    keywordScore?: number;
+    /** Reciprocal-rank-fusion contribution used to order the fused list. */
+    rrfScore: number;
+    /** Zero-based rank within each leg that contributed, for tie diagnosis. */
+    vectorRank?: number;
+    keywordRank?: number;
 }
 export interface SearchOptions {
     topK?: number;
     minScore?: number;
     collection?: "chunks" | "memories";
+    onlyKeys?: Set<string>;
+    /** Restrict to chunks of these documents (cheap: doc ids, not chunk ids). */
+    onlyDocIds?: Set<string>;
 }
 export declare function invalidateVectorCache(): void;
 /** Brute-force vector search over stored vectors (uses the in-memory cache). */
@@ -35,10 +54,16 @@ export interface SearchChunkOptions {
     topK: number;
     minScore: number;
     filters?: SearchChunkFilters;
+    /** Attach per-leg scores to every hit. Costs nothing extra, but only populated on request. */
+    explain?: boolean;
 }
 /**
- * Hybrid search: FTS5 BM25 keyword + vector, fused with RRF.
+ * Hybrid search: FTS5 BM25 keyword + vector, fused with RRF for ranking.
  * Controlled by SEARCH_MODE=hybrid|vector|keyword (default hybrid).
+ *
+ * The reported `score` is always on a 0..1 relevance scale — the vector cosine
+ * when the chunk has one, otherwise a rank-decayed keyword score — so `minScore`
+ * means the same thing in every mode (RRF only decides the order).
  */
 export declare function searchChunks(queryVec: Float64Array, opts: SearchChunkOptions): SearchHit[];
 //# sourceMappingURL=vector-search.d.ts.map

package/dist/rag/vector-search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../src/rag/vector-search.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;CACpC;AAeD,wBAAgB,qBAAqB,IAAI,IAAI,CAG5C;AAuCD,gFAAgF;AAChF,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,aAAkB,GAC1B,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAcvC;AA0CD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,SAAS,EAAE,CAkF1F"}
+{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../src/rag/vector-search.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,uEAAuE;IACvE,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,MAAM,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE7C,MAAM,WAAW,aAAa;IAC5B,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IACnC,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CAC1B;AAmBD,wBAAgB,qBAAqB,IAAI,IAAI,CAM5C;AAiGD,gFAAgF;AAChF,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,aAAkB,GAC1B,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAuBvC;AAyED,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAC7B,8FAA8F;IAC9F,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,SAAS,EAAE,CA6H1F"}