mcp-rag-memory 2.4.1 → 2.7.0

package/dist/rag/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAmED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,KAAK,EAAE,CA6C3E;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAInD"}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAwHD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,KAAK,EAAE,CAiF3E;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAInD;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAG7D"}

package/dist/rag/chunker.js

@@ -18,7 +18,9 @@ function splitIntoParagraphs(text) {
     let current = "";
     let inCode = false;
     for (const line of text.split(/\r?\n/)) {
-        const codeBoundary = /^\s*(```|~~~|---)/.test(line.trim());
+        // Only real fences toggle code mode — `---` is a thematic break / front
+        // matter delimiter, and one stray `---` used to swallow the whole document.
+        const codeBoundary = /^\s*(```|~~~)/.test(line.trim());
         if (codeBoundary)
             inCode = !inCode;
         if (inCode || isCodeLine(line)) {
@@ -38,9 +40,59 @@ function splitIntoParagraphs(text) {
         blocks.push(current.trimEnd());
     return blocks.filter((b) => b.trim().length > 0);
 }
+/**
+ * Sentence split that keeps decimals and version numbers intact
+ * ("pip 24.0.1" stays one sentence) and keeps fenced code blocks whole,
+ * because newlines are never treated as terminators.
+ */
 function splitIntoSentences(block) {
-    const sentences = block.match(/[^.!?。！？\n]+[.!?。！？]?/g) ?? [];
-    return sentences.map((s) => s.trim()).filter((s) => s.length > 0);
+    const out = [];
+    const TERMINATORS = ".!?。！？";
+    let start = 0;
+    for (let i = 0; i < block.length; i++) {
+        const ch = block[i];
+        if (!TERMINATORS.includes(ch))
+            continue;
+        // A period between two digits is a decimal point, not a sentence end.
+        if (ch === "." && /\d/.test(block[i - 1] ?? "") && /\d/.test(block[i + 1] ?? ""))
+            continue;
+        let end = i + 1;
+        while (end < block.length && /["')\]}»”]/.test(block[end]))
+            end++;
+        const piece = block.slice(start, end).trim();
+        if (piece)
+            out.push(piece);
+        start = end;
+        i = end - 1;
+    }
+    const tail = block.slice(start).trim();
+    if (tail)
+        out.push(tail);
+    return out;
+}
+/**
+ * Trim an overlap tail so a chunk never starts with a partial word: when the cut
+ * landed inside a word (last char before the cut and first char of the tail are
+ * both word characters), the leading fragment is dropped.
+ */
+function snapToWordStart(tail, charBeforeCut) {
+    const at = tail.search(/\S/);
+    const body = at <= 0 ? tail : tail.slice(at);
+    if (!body)
+        return body;
+    const startsMidWord = /[\p{L}\p{N}_]/u.test(charBeforeCut) && /[\p{L}\p{N}_]/u.test(body[0]);
+    if (!startsMidWord)
+        return body;
+    const space = body.search(/\s/);
+    return space === -1 ? body : body.slice(space).trimStart();
+}
+/** Last whitespace break inside the budget, so a hard cut avoids mid-word cuts. */
+function lastBreak(candidate, max) {
+    for (let i = Math.min(candidate.length, max) - 1; i > Math.floor(max * 0.6); i--) {
+        if (/\s/.test(candidate[i]))
+            return i + 1;
+    }
+    return max;
 }
 function splitLongText(text, max) {
     const parts = [];
@@ -48,7 +100,7 @@ function splitLongText(text, max) {
     while (rest.length > max) {
         const candidate = rest.slice(0, max);
         const cut = Math.max(candidate.lastIndexOf(". "), candidate.lastIndexOf(".\n"), candidate.lastIndexOf("。"), candidate.lastIndexOf("\n"), candidate.lastIndexOf(", "));
-        const pos = cut > Math.floor(max * 0.6) ? cut + 1 : max;
+        const pos = cut > Math.floor(max * 0.6) ? cut + 1 : lastBreak(candidate, max);
         parts.push(rest.slice(0, pos).trim());
         rest = rest.slice(pos);
     }
@@ -56,15 +108,40 @@ function splitLongText(text, max) {
         parts.push(rest.trim());
     return parts;
 }
+/** Separator that never mangles code: newline if either side has one. */
+function joiner(left, right) {
+    if (!left)
+        return "";
+    if (left.endsWith("\n") || right.startsWith("\n"))
+        return "";
+    return left.includes("\n") || right.includes("\n") ? "\n" : " ";
+}
 export function chunkText(text, options = {}) {
     const maxChars = options.maxChars ?? DEFAULT_MAX;
     const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP;
+    if (!Number.isFinite(maxChars) || maxChars < 1)
+        throw new RangeError("chunkText: maxChars must be >= 1");
+    if (!Number.isFinite(overlapChars) || overlapChars < 0)
+        throw new RangeError("chunkText: overlapChars must be >= 0");
     const chunks = [];
     let cursor = 0;
     const emit = (content) => {
         const trimmed = content.trim();
         if (!trimmed)
             return;
+        // Backstop for maxChars. The window is normally kept under budget, but
+        // carry() (up to maxChars/2) plus a full-size piece could still land at
+        // ~1.5x, and an over-budget chunk blows the embedder's context for no
+        // benefit. Split rather than silently shipping an oversized chunk.
+        if (trimmed.length > maxChars) {
+            for (const part of splitLongText(trimmed, maxChars)) {
+                const t = part.trim();
+                if (!t)
+                    continue;
+                chunks.push({ text: t, index: cursor++, tokenCount: estimateTokens(t) });
+            }
+            return;
+        }
         chunks.push({
             text: trimmed,
             index: cursor++,
@@ -72,9 +149,12 @@ export function chunkText(text, options = {}) {
         });
     };
     const paragraphs = splitIntoParagraphs(text);
+    const overlap = Math.min(overlapChars, Math.floor(maxChars / 2));
+    /** The tail carried into the next chunk; "" when overlapChars is 0. */
+    const carry = (emitted) => overlap > 0 ? snapToWordStart(emitted.slice(-overlap), emitted.slice(-overlap - 1, -overlap)) : "";
     let window = "";
     for (const para of paragraphs) {
-        let pieces = [];
+        let pieces;
         if (para.length > maxChars) {
             pieces = splitLongText(para, maxChars);
         }
@@ -84,16 +164,33 @@ export function chunkText(text, options = {}) {
         }
         for (const piece of pieces) {
             if (piece.length > maxChars) {
-                // Rare pathological case: keep as-is, hard split.
-                for (const hard of splitLongText(piece, maxChars))
-                    emit(hard);
+                // A single sentence longer than maxChars: hard-split it. The pending
+                // window must be flushed first, otherwise the text collected so far is
+                // emitted twice (once here, once with the next piece appended).
+                const hards = splitLongText(piece, maxChars);
+                if (window) {
+                    emit(window);
+                    window = carry(window);
+                }
+                for (const hard of hards) {
+                    const projectedHard = window.length + joiner(window, hard).length + hard.length;
+                    if (window && projectedHard > maxChars) {
+                        emit(window);
+                        window = carry(window);
+                    }
+                    window += joiner(window, hard) + hard;
+                }
                 continue;
             }
-            if (window.length + piece.length + 1 > maxChars && window.length >= overlapChars) {
+            const projected = window.length + joiner(window, piece).length + piece.length;
+            if (window && projected > maxChars) {
                 emit(window);
-                window = window.slice(-overlapChars);
+                // Overlap carries the tail of the previous chunk. overlap === 0 must
+                // reset the window: slice(-0) returns the whole string, which used to
+                // duplicate the previous chunk and grow it without bound.
+                window = carry(window);
             }
-            window += (window ? " " : "") + piece;
+            window += joiner(window, piece) + piece;
         }
     }
     if (window.trim())
@@ -106,4 +203,15 @@ export function estimateTokens(text) {
     const rest = text.length - cjk;
     return Math.ceil(cjk * 1.5) + Math.ceil(rest / 4);
 }
+/**
+ * Same estimate for an already-known character count. Lets a caller total
+ * LENGTH() across many rows in SQL instead of materialising each content
+ * blob just to re-count its characters. Assumes no CJK in the total, which
+ * under-counts CJK-heavy stores — the same tradeoff the SQL path already made.
+ */
+export function estimateTokensFromChars(chars) {
+    if (!Number.isFinite(chars) || chars <= 0)
+        return 0;
+    return Math.ceil(chars / 4);
+}
 //# sourceMappingURL=chunker.js.map

package/dist/rag/chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,wCAAwC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,+FAA+F,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,YAAY,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,IAAI,YAAY;YAAE,MAAM,GAAG,CAAC,MAAM,CAAC;QAEnC,IAAI,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;YACvB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/B,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAAI,EAAE,CAAC;IAC9D,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,GAAW;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAC5B,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,EAC1B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAC5B,CAAC;QACF,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC;IAC7D,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,EAAE;YACf,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC;SACpC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,MAAM,GAAa,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC5B,kDAAkD;gBAClD,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC9D,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;gBACjF,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;YACvC,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,4CAA4C,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;AACpD,CAAC"}
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,wCAAwC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,+FAA+F,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACvD,IAAI,YAAY;YAAE,MAAM,GAAG,CAAC,MAAM,CAAC;QAEnC,IAAI,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;YACvB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/B,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,KAAa;IACvC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,QAAQ,CAAC;IAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,SAAS;QACxC,sEAAsE;QACtE,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,SAAS;QAC3F,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,GAAG,GAAG,CAAC;QACZ,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,aAAqB;IAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,IAAI,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;AAC7D,CAAC;AAED,mFAAmF;AACnF,SAAS,SAAS,CAAC,SAAiB,EAAE,GAAW;IAC/C,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjF,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,GAAW;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAC5B,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,EAC1B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAC5B,CAAC;QACF,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,yEAAyE;AACzE,SAAS,MAAM,CAAC,IAAY,EAAE,KAAa;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC;IAC7D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,kCAAkC,CAAC,CAAC;IACzG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,sCAAsC,CAAC,CAAC;IACrH,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,uEAAuE;QACvE,wEAAwE;QACxE,sEAAsE;QACtE,mEAAmE;QACnE,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACpD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3E,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,EAAE;YACf,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC;SACpC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,uEAAuE;IACvE,MAAM,KAAK,GAAG,CAAC,OAAe,EAAE,EAAE,CAChC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErG,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC5B,qEAAqE;gBACrE,uEAAuE;gBACvE,gEAAgE;gBAChE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,CAAC;oBACb,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzB,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;oBAChF,IAAI,MAAM,IAAI,aAAa,GAAG,QAAQ,EAAE,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACb,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBACzB,CAAC;oBACD,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC;gBACxC,CAAC;gBACD,SAAS;YACX,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC9E,IAAI,MAAM,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,qEAAqE;gBACrE,sEAAsE;gBACtE,0DAA0D;gBAC1D,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,4CAA4C,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa;IACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AAC9B,CAAC"}

package/dist/rag/embeddings.d.ts

@@ -4,11 +4,36 @@ export declare const EMBEDDING_PROVIDER: EmbeddingProvider;
 export declare const EMBEDDING_MODEL: string;
 export declare function expectedDim(): number;
 export declare function modelName(): string;
+/**
+ * What the vector leg actually measures, reported honestly.
+ *
+ * `local` is a hashing embedder over character n-grams: it scores lexical and
+ * sub-word overlap, not meaning. A query and a document that use entirely
+ * different words for the same idea score near zero, no matter how well written
+ * either is. `transformers` produces true semantic vectors.
+ *
+ * Callers should not describe the `local` vector leg as "semantic"; the hybrid
+ * score is dominated by the FTS leg in that configuration, and an agent that
+ * believes otherwise will trust synonym-blind recall that is not there.
+ */
+export declare function vectorLegKind(): "lexical" | "semantic";
+export declare function vectorLegNote(): string;
 export declare function embeddingInfo(): {
     provider: EmbeddingProvider;
     dim: number;
     model: string;
+    vectorLeg: "lexical" | "semantic";
+    note: string;
 };
 /** Embed text into a normalized Float64Array. Async for provider parity. */
 export declare function embedText(text: string): Promise<Float64Array>;
+/**
+ * Embed without the store-dimension guard.
+ *
+ * `reindex` is the one operation whose entire purpose is to *replace* the stored
+ * vector space, so requiring the new vectors to already match the old ones would
+ * make the migration impossible to express. Every other caller must use
+ * `embedText`, which enforces the guard.
+ */
+export declare function embedTextUnguarded(text: string): Promise<Float64Array>;
 //# sourceMappingURL=embeddings.d.ts.map

package/dist/rag/embeddings.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzD,eAAO,MAAM,kBAAkB,EAAE,iBAC6C,CAAC;AAE/E,eAAO,MAAM,eAAe,QAA2D,CAAC;AAYxF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,aAAa,IAAI;IAAE,QAAQ,EAAE,iBAAiB,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM3F;AAoBD,4EAA4E;AAC5E,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAenE"}
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzD,eAAO,MAAM,kBAAkB,EAAE,iBAC6C,CAAC;AAE/E,eAAO,MAAM,eAAe,QAA2D,CAAC;AAYxF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,IAAI,SAAS,GAAG,UAAU,CAEtD;AAED,wBAAgB,aAAa,IAAI,MAAM,CAMtC;AAED,wBAAgB,aAAa,IAAI;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,SAAS,GAAG,UAAU,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd,CAQA;AAyBD,4EAA4E;AAC5E,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAInE;AAED;;;;;;;GAOG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAE5E"}

package/dist/rag/embeddings.js

@@ -28,17 +28,43 @@ export function expectedDim() {
 export function modelName() {
     return EMBEDDING_PROVIDER === "local" ? "local-hash-1024" : EMBEDDING_MODEL;
 }
+/**
+ * What the vector leg actually measures, reported honestly.
+ *
+ * `local` is a hashing embedder over character n-grams: it scores lexical and
+ * sub-word overlap, not meaning. A query and a document that use entirely
+ * different words for the same idea score near zero, no matter how well written
+ * either is. `transformers` produces true semantic vectors.
+ *
+ * Callers should not describe the `local` vector leg as "semantic"; the hybrid
+ * score is dominated by the FTS leg in that configuration, and an agent that
+ * believes otherwise will trust synonym-blind recall that is not there.
+ */
+export function vectorLegKind() {
+    return EMBEDDING_PROVIDER === "local" ? "lexical" : "semantic";
+}
+export function vectorLegNote() {
+    return EMBEDDING_PROVIDER === "local"
+        ? "Local hashing embedder: the vector leg scores lexical/sub-word overlap, not meaning. " +
+            "Synonym-only queries depend on the FTS5 trigram leg. Set EMBEDDING_PROVIDER=transformers " +
+            "(npm i @huggingface/transformers) then run 'rag_reindex' for true semantic recall."
+        : `Semantic embeddings from ${EMBEDDING_MODEL}; the vector leg captures meaning, not just overlap.`;
+}
 export function embeddingInfo() {
     return {
         provider: EMBEDDING_PROVIDER,
         dim: getEmbedDim() || expectedDim() || EMBED_DIM_LOCAL,
         model: modelName(),
+        vectorLeg: vectorLegKind(),
+        note: vectorLegNote(),
     };
 }
 /* eslint-disable @typescript-eslint/no-explicit-any */
 let extractorPromise = null;
 async function getExtractor() {
     if (!extractorPromise) {
+        // A failed model load must not be cached for the process lifetime: reset on
+        // rejection so a transient network/first-download error can be retried.
         extractorPromise = (async () => {
             // @ts-expect-error — @huggingface/transformers is an optional dependency
             const mod = await import("@huggingface/transformers");
@@ -48,15 +74,33 @@ async function getExtractor() {
             }
             return pipeline("feature-extraction", EMBEDDING_MODEL, { quantized: true });
         })();
+        extractorPromise.catch(() => {
+            extractorPromise = null;
+        });
     }
     return extractorPromise;
 }
 /** Embed text into a normalized Float64Array. Async for provider parity. */
 export async function embedText(text) {
+    const vec = await embedRaw(text);
+    ensureDim(vec, modelName());
+    return vec;
+}
+/**
+ * Embed without the store-dimension guard.
+ *
+ * `reindex` is the one operation whose entire purpose is to *replace* the stored
+ * vector space, so requiring the new vectors to already match the old ones would
+ * make the migration impossible to express. Every other caller must use
+ * `embedText`, which enforces the guard.
+ */
+export async function embedTextUnguarded(text) {
+    return embedRaw(text);
+}
+/** Provider call plus the finite-value check, shared by both entry points. */
+async function embedRaw(text) {
     if (EMBEDDING_PROVIDER === "local") {
-        const vec = embed(text);
-        ensureDim(vec, modelName());
-        return vec;
+        return embed(text);
     }
     const extractor = await getExtractor();
     const out = await extractor(text, { pooling: "mean", normalize: true });
@@ -65,7 +109,15 @@ export async function embedText(text) {
         throw new Error("transformers embed returned an unexpected shape");
     }
     const vec = Float64Array.from(data);
-    ensureDim(vec, modelName());
+    // A NaN/Inf from the model poisons every score it touches: dot products become
+    // NaN, comparisons go false, and the row silently stops matching anything —
+    // an invisible, permanent retrieval hole. Reject it at the boundary instead.
+    for (let i = 0; i < vec.length; i++) {
+        if (!Number.isFinite(vec[i])) {
+            throw new Error(`Embedding for this input contains a non-finite value at index ${i}. ` +
+                `The input is likely empty or degenerate; retry with more text or set EMBEDDING_PROVIDER=local.`);
+        }
+    }
     return vec;
 }
 //# sourceMappingURL=embeddings.js.map

package/dist/rag/embeddings.js.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAIpC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;AAE/E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AAExF,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,yBAAyB,EAAE,GAAG;IAC9B,8CAA8C,EAAE,GAAG;IACnD,8BAA8B,EAAE,GAAG;IACnC,kBAAkB,EAAE,GAAG;IACvB,0BAA0B,EAAE,GAAG;IAC/B,6BAA6B,EAAE,GAAG;CACnC,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,IAAI,eAAe;QACtD,KAAK,EAAE,SAAS,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,yEAAyE;YACzE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAmG,CAAC;YACzH,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,IAAI,kBAAkB,KAAK,OAAO,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1E,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,IAAyB,CAAC,CAAC;IACzD,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IAC5B,OAAO,GAAG,CAAC;AACb,CAAC"}
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAIpC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;AAE/E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AAExF,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,yBAAyB,EAAE,GAAG;IAC9B,8CAA8C,EAAE,GAAG;IACnD,8BAA8B,EAAE,GAAG;IACnC,kBAAkB,EAAE,GAAG;IACvB,0BAA0B,EAAE,GAAG;IAC/B,6BAA6B,EAAE,GAAG;CACnC,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;AAC9E,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa;IAC3B,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO,kBAAkB,KAAK,OAAO;QACnC,CAAC,CAAC,uFAAuF;YACvF,2FAA2F;YAC3F,oFAAoF;QACtF,CAAC,CAAC,4BAA4B,eAAe,sDAAsD,CAAC;AACxG,CAAC;AAED,MAAM,UAAU,aAAa;IAO3B,OAAO;QACL,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,IAAI,eAAe;QACtD,KAAK,EAAE,SAAS,EAAE;QAClB,SAAS,EAAE,aAAa,EAAE;QAC1B,IAAI,EAAE,aAAa,EAAE;KACtB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,4EAA4E;QAC5E,wEAAwE;QACxE,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,yEAAyE;YACzE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAmG,CAAC;YACzH,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAC;QACL,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1B,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjC,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IAC5B,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED,8EAA8E;AAC9E,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,IAAI,kBAAkB,KAAK,OAAO,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1E,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,IAAyB,CAAC,CAAC;IACzD,+EAA+E;IAC/E,4EAA4E;IAC5E,6EAA6E;IAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CACb,iEAAiE,CAAC,IAAI;gBACpE,gGAAgG,CACnG,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/rag/pipeline.d.ts

@@ -26,21 +26,38 @@ export interface IngestFileOptions {
     source?: string;
     contentType?: string;
     metadata?: Record<string, unknown>;
+    /** Namespace the dedup hash, so equal text from different scopes is kept apart. */
+    dedupScope?: string;
 }
 export interface IngestDirectoryOptions {
     extensions?: string[];
     recursive?: boolean;
     metadata?: Record<string, unknown>;
 }
+/** Max content accepted from a tool call or CLI, so one payload cannot stall the server for a minute. */
+export declare const MAX_INGEST_CHARS = 1000000;
 export declare function ingestText(content: string, title: string, opts?: IngestFileOptions): Promise<IngestResult>;
 export declare function ingestFile(filePath: string, opts?: IngestFileOptions): Promise<IngestResult>;
 export declare function ingestDirectory(dirPath: string, opts?: IngestDirectoryOptions): Promise<{
     ingested: IngestResult[];
     skipped: string[];
+    ingestedCount: number;
+    skippedCount: number;
 }>;
-export declare function searchDocs(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters): Promise<SearchHit[]>;
+export declare function searchDocs(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters, explain?: boolean): Promise<SearchHit[]>;
 export declare function retrieve(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters): Promise<RetrievedContext>;
-export declare function listDocuments(): Array<Record<string, unknown>>;
+/** Bounded by default: a big store would otherwise dump every row into one tool result. */
+export declare const LIST_DOCS_DEFAULT = 100;
+export declare const LIST_DOCS_MAX = 1000;
+export declare function listDocuments(opts?: {
+    limit?: number;
+    offset?: number;
+}): {
+    documents: Array<Record<string, unknown>>;
+    total: number;
+    limit: number;
+    offset: number;
+};
 export declare function deleteDocument(docId: string): {
     deleted: boolean;
 };

package/dist/rag/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5G,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAWD,wBAAsB,UAAU,CAC9B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,iBAAsB,GAC3B,OAAO,CAAC,YAAY,CAAC,CAsDvB;AAgBD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAE,iBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAsBtG;AAOD,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC;IAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAoC1D;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,EAAE,QAAQ,SAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG9H;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,SAAI,EACR,QAAQ,SAAO,EACf,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAa3B;AAED,wBAAgB,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAQ9D;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAMlE;AAED,wBAAgB,aAAa,IAAI;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAMrF;AAYD,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5G,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAgCD,yGAAyG;AACzG,eAAO,MAAM,gBAAgB,UAAY,CAAC;AAE1C,wBAAsB,UAAU,CAC9B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,iBAAsB,GAC3B,OAAO,CAAC,YAAY,CAAC,CAgHvB;AA6CD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAE,iBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAuBtG;AA2BD,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC;IAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CA+DvG;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,IAAI,SAAK,EACT,QAAQ,SAAO,EACf,OAAO,CAAC,EAAE,kBAAkB,EAC5B,OAAO,UAAQ,GACd,OAAO,CAAC,SAAS,EAAE,CAAC,CAGtB;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,SAAI,EACR,QAAQ,SAAO,EACf,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAa3B;AAED,2FAA2F;AAC3F,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,aAAa,OAAO,CAAC;AAElC,wBAAgB,aAAa,CAAC,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG;IAC7E,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAeA;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAkBlE;AAED,wBAAgB,aAAa,IAAI;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAMrF;AAYD,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}