@stablemodels/qmd-cf 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 stablemodels
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/chunker.d.ts

@@ -0,0 +1,11 @@
+import type { Chunk } from "./types.js";
+/**
+ * Chunk a document into overlapping segments, seeking intelligent break points.
+ *
+ * Uses a scored break point system (from qmd) that pre-scans the entire document
+ * for structural markers (headings, code fences, paragraphs, etc.) and picks the
+ * highest-scoring break point within a window around the target cut position.
+ * Avoids splitting inside fenced code blocks.
+ */
+export declare function chunkText(docId: string, content: string, maxChars?: number, overlapChars?: number): Chunk[];
+//# sourceMappingURL=chunker.d.ts.map

package/dist/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAyBxC;;;;;;;GAOG;AACH,wBAAgB,SAAS,CACxB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,QAAQ,GAAE,MAA2B,EACrC,YAAY,GAAE,MAA8B,GAC1C,KAAK,EAAE,CA2DT"}

package/dist/chunker.js

@@ -0,0 +1,199 @@
+const DEFAULT_CHUNK_SIZE = 3200; // ~800 tokens at ~4 chars/token
+const DEFAULT_CHUNK_OVERLAP = 480; // 15% overlap
+/** Break point scores — spread wide so headings decisively win over paragraphs. */
+const BREAK_SCORES = {
+    h1: 100,
+    h2: 90,
+    h3: 80,
+    h4: 70,
+    h5: 60,
+    h6: 50,
+    code_fence: 80,
+    hr: 60,
+    paragraph: 20,
+    list_item: 5,
+    newline: 1,
+};
+/**
+ * Chunk a document into overlapping segments, seeking intelligent break points.
+ *
+ * Uses a scored break point system (from qmd) that pre-scans the entire document
+ * for structural markers (headings, code fences, paragraphs, etc.) and picks the
+ * highest-scoring break point within a window around the target cut position.
+ * Avoids splitting inside fenced code blocks.
+ */
+export function chunkText(docId, content, maxChars = DEFAULT_CHUNK_SIZE, overlapChars = DEFAULT_CHUNK_OVERLAP) {
+    if (content.length === 0) {
+        return [];
+    }
+    // Short content: single chunk, no splitting needed
+    if (content.length <= maxChars) {
+        return [{ docId, seq: 0, text: content, charOffset: 0 }];
+    }
+    const breakPoints = scanBreakPoints(content);
+    const codeFences = findCodeFences(content);
+    const chunks = [];
+    let pos = 0;
+    let seq = 0;
+    while (pos < content.length) {
+        const remaining = content.length - pos;
+        if (remaining <= maxChars) {
+            chunks.push({ docId, seq, text: content.slice(pos), charOffset: pos });
+            break;
+        }
+        const targetEnd = pos + maxChars;
+        const cutoff = findBestCutoff(content, breakPoints, codeFences, targetEnd, maxChars);
+        // Ensure we make forward progress
+        const endPos = cutoff > pos ? cutoff : pos + maxChars;
+        chunks.push({
+            docId,
+            seq,
+            text: content.slice(pos, endPos),
+            charOffset: pos,
+        });
+        // Advance position, subtracting overlap
+        const advance = endPos - pos - overlapChars;
+        pos += Math.max(advance, 1);
+        // Don't start the next chunk inside a code fence — skip to fence end
+        for (const [fStart, fEnd] of codeFences) {
+            if (pos > fStart && pos < fEnd) {
+                pos = fEnd;
+                break;
+            }
+        }
+        seq++;
+    }
+    return chunks;
+}
+/**
+ * Pre-scan the document for structural break points with scores.
+ *
+ * Returns break points sorted by offset. Each offset points to the first
+ * character of the new section (i.e., right after the structural marker).
+ */
+function scanBreakPoints(text) {
+    const points = [];
+    const lines = text.split("\n");
+    let offset = 0;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const lineStart = offset;
+        const nextLineStart = offset + line.length + 1; // +1 for the \n
+        // Headings (must be at start of line)
+        if (line.startsWith("###### ")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.h6 });
+        }
+        else if (line.startsWith("##### ")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.h5 });
+        }
+        else if (line.startsWith("#### ")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.h4 });
+        }
+        else if (line.startsWith("### ")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.h3 });
+        }
+        else if (line.startsWith("## ")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.h2 });
+        }
+        else if (line.startsWith("# ")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.h1 });
+        }
+        // Code fences (``` at start of line) — break before the fence
+        if (line.startsWith("```")) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.code_fence });
+        }
+        // Horizontal rules (---, ***, ___ with optional spaces)
+        if (/^(\s*[-*_]\s*){3,}$/.test(line)) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.hr });
+        }
+        // Paragraph boundary (empty line followed by content)
+        if (line === "" && i > 0) {
+            points.push({ offset: nextLineStart, score: BREAK_SCORES.paragraph });
+        }
+        // List items
+        if (/^(\s*[-*+]\s|\s*\d+\.\s)/.test(line)) {
+            points.push({ offset: lineStart, score: BREAK_SCORES.list_item });
+        }
+        // Every newline is a minimal break point
+        if (i < lines.length - 1) {
+            points.push({ offset: nextLineStart, score: BREAK_SCORES.newline });
+        }
+        offset = nextLineStart;
+    }
+    return points;
+}
+/**
+ * Find matched code fence (```) ranges. Returns [start, end] pairs
+ * where start is the offset of the opening fence and end is the offset
+ * just after the closing fence line's newline.
+ */
+function findCodeFences(text) {
+    const ranges = [];
+    const lines = text.split("\n");
+    let offset = 0;
+    let fenceStart = null;
+    for (const line of lines) {
+        if (line.startsWith("```")) {
+            if (fenceStart === null) {
+                fenceStart = offset;
+            }
+            else {
+                // Close the fence — end is after this line
+                ranges.push([fenceStart, offset + line.length + 1]);
+                fenceStart = null;
+            }
+        }
+        offset += line.length + 1;
+    }
+    return ranges;
+}
+/**
+ * Check if an offset falls inside any code fence range.
+ */
+function isInsideCodeFence(offset, codeFences) {
+    for (const [start, end] of codeFences) {
+        // Inside means strictly between the opening and closing fence lines.
+        // Breaking AT the start of a fence (before it) is fine.
+        if (offset > start && offset < end)
+            return true;
+    }
+    return false;
+}
+/**
+ * Find the best break point near the target cut position.
+ *
+ * Searches a window from 50% to 100% of maxChars around the chunk start.
+ * Applies squared distance decay so breaks closer to the target are preferred.
+ * Rejects candidates inside code fences.
+ */
+function findBestCutoff(text, breakPoints, codeFences, targetEnd, maxChars) {
+    const windowStart = targetEnd - Math.floor(maxChars * 0.5);
+    const windowEnd = targetEnd;
+    const windowSize = windowEnd - windowStart;
+    let bestScore = -1;
+    let bestOffset = targetEnd;
+    for (const bp of breakPoints) {
+        if (bp.offset < windowStart || bp.offset > windowEnd)
+            continue;
+        if (isInsideCodeFence(bp.offset, codeFences))
+            continue;
+        // Squared distance decay: prefer breaks closer to targetEnd
+        const dist = Math.abs(bp.offset - targetEnd);
+        const normalizedDist = dist / windowSize;
+        const multiplier = 1.0 - normalizedDist * normalizedDist * 0.7;
+        const weightedScore = bp.score * multiplier;
+        if (weightedScore > bestScore) {
+            bestScore = weightedScore;
+            bestOffset = bp.offset;
+        }
+    }
+    // Fallback: if no structural break points found, try word boundary (last space)
+    if (bestScore < 0) {
+        const lastSpace = text.lastIndexOf(" ", targetEnd);
+        if (lastSpace >= windowStart) {
+            return lastSpace + 1;
+        }
+    }
+    return bestOffset;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAEA,MAAM,kBAAkB,GAAG,IAAI,CAAC,CAAC,gCAAgC;AACjE,MAAM,qBAAqB,GAAG,GAAG,CAAC,CAAC,cAAc;AAEjD,mFAAmF;AACnF,MAAM,YAAY,GAA2B;IAC5C,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,EAAE;IACN,EAAE,EAAE,EAAE;IACN,EAAE,EAAE,EAAE;IACN,EAAE,EAAE,EAAE;IACN,EAAE,EAAE,EAAE;IACN,UAAU,EAAE,EAAE;IACd,EAAE,EAAE,EAAE;IACN,SAAS,EAAE,EAAE;IACb,SAAS,EAAE,CAAC;IACZ,OAAO,EAAE,CAAC;CACV,CAAC;AAOF;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CACxB,KAAa,EACb,OAAe,EACf,WAAmB,kBAAkB,EACrC,eAAuB,qBAAqB;IAE5C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACX,CAAC;IAED,mDAAmD;IACnD,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QAChC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,CAAC,CAAC;IAEZ,OAAO,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC;QAEvC,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YACvE,MAAM;QACP,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,GAAG,QAAQ,CAAC;QACjC,MAAM,MAAM,GAAG,cAAc,CAC5B,OAAO,EACP,WAAW,EACX,UAAU,EACV,SAAS,EACT,QAAQ,CACR,CAAC;QAEF,kCAAkC;QAClC,MAAM,MAAM,GAAG,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAC;QAEtD,MAAM,CAAC,IAAI,CAAC;YACX,KAAK;YACL,GAAG;YACH,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;YAChC,UAAU,EAAE,GAAG;SACf,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,OAAO,GAAG,MAAM,GAAG,GAAG,GAAG,YAAY,CAAC;QAC5C,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAE5B,qEAAqE;QACrE,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;YACzC,IAAI,GAAG,GAAG,MAAM,IAAI,GAAG,GAAG,IAAI,EAAE,CAAC;gBAChC,GAAG,GAAG,IAAI,CAAC;gBACX,MAAM;YACP,CAAC;QACF,CAAC;QAED,GAAG,EAAE,CAAC;IACP,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CAAC,IAAY;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,MAAM,CAAC;QACzB,MAAM,aAAa,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,gBAAgB;QAEhE,sCAAsC;QACtC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,8DAA8D;QAC9D,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,wDAAwD;QACxD,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,sDAAsD;QACtD,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,aAAa;QACb,IAAI,0BAA0B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,yCAAyC;QACzC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,GAAG,aAAa,CAAC;IACxB,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,UAAU,GAAkB,IAAI,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACzB,UAAU,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACP,2CAA2C;gBAC3C,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;gBACpD,UAAU,GAAG,IAAI,CAAC;YACnB,CAAC;QACF,CAAC;QACD,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CACzB,MAAc,EACd,UAAmC;IAEnC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;QACvC,qEAAqE;QACrE,wDAAwD;QACxD,IAAI,MAAM,GAAG,KAAK,IAAI,MAAM,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CACtB,IAAY,EACZ,WAAyB,EACzB,UAAmC,EACnC,SAAiB,EACjB,QAAgB;IAEhB,MAAM,WAAW,GAAG,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,SAAS,CAAC;IAC5B,MAAM,UAAU,GAAG,SAAS,GAAG,WAAW,CAAC;IAE3C,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,SAAS,CAAC;IAE3B,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC9B,IAAI,EAAE,CAAC,MAAM,GAAG,WAAW,IAAI,EAAE,CAAC,MAAM,GAAG,SAAS;YAAE,SAAS;QAC/D,IAAI,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;YAAE,SAAS;QAEvD,4DAA4D;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QAC7C,MAAM,cAAc,GAAG,IAAI,GAAG,UAAU,CAAC;QACzC,MAAM,UAAU,GAAG,GAAG,GAAG,cAAc,GAAG,cAAc,GAAG,GAAG,CAAC;QAC/D,MAAM,aAAa,GAAG,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC;QAE5C,IAAI,aAAa,GAAG,SAAS,EAAE,CAAC;YAC/B,SAAS,GAAG,aAAa,CAAC;YAC1B,UAAU,GAAG,EAAE,CAAC,MAAM,CAAC;QACxB,CAAC;IACF,CAAC;IAED,gFAAgF;IAChF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACnD,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;YAC9B,OAAO,SAAS,GAAG,CAAC,CAAC;QACtB,CAAC;IACF,CAAC;IAED,OAAO,UAAU,CAAC;AACnB,CAAC"}

package/dist/fts.d.ts

@@ -0,0 +1,19 @@
+import type { FtsResult, SearchOptions } from "./types.js";
+/**
+ * Build an FTS5 query string from a natural language query.
+ *
+ * Strategy (from qmd):
+ * - Single term: prefix match ("term"*)
+ * - Multi-term: exact phrase OR NEAR(terms, 10) OR individual terms ORed
+ *
+ * This gives good recall while still ranking exact matches highest via BM25.
+ */
+export declare function buildFts5Query(query: string): string;
+/**
+ * Execute a full-text search using FTS5 BM25 ranking.
+ *
+ * BM25 weights: title gets 10x boost, content gets 1x.
+ * Results are deduplicated by document (keeping the best-scoring chunk).
+ */
+export declare function searchFts(sql: SqlStorage, query: string, options?: SearchOptions): FtsResult[];
+//# sourceMappingURL=fts.d.ts.map

package/dist/fts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3D;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAqBpD;AAsBD;;;;;GAKG;AACH,wBAAgB,SAAS,CACxB,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAkB,GACzB,SAAS,EAAE,CA2Eb"}

package/dist/fts.js

@@ -0,0 +1,109 @@
+/**
+ * Build an FTS5 query string from a natural language query.
+ *
+ * Strategy (from qmd):
+ * - Single term: prefix match ("term"*)
+ * - Multi-term: exact phrase OR NEAR(terms, 10) OR individual terms ORed
+ *
+ * This gives good recall while still ranking exact matches highest via BM25.
+ */
+export function buildFts5Query(query) {
+    const terms = query
+        .trim()
+        .split(/\s+/)
+        .filter((t) => t.length > 0)
+        // Strip characters that break FTS5 syntax
+        .map((t) => t.replace(/['"(){}[\]*^~:]/g, ""));
+    if (terms.length === 0)
+        return '""';
+    if (terms.length === 1) {
+        // Single term: prefix match
+        return `"${terms[0]}" *`;
+    }
+    // Multi-term: combine strategies for best recall
+    const phrase = `"${terms.join(" ")}"`;
+    const near = `NEAR(${terms.map((t) => `"${t}"`).join(" ")}, 10)`;
+    const orTerms = terms.map((t) => `"${t}"`).join(" OR ");
+    return `(${phrase}) OR (${near}) OR (${orTerms})`;
+}
+/**
+ * Normalize a raw BM25 score to (0, 1].
+ * SQLite FTS5 bm25() returns negative values where lower (more negative) = better match.
+ * We convert to: score = 1 / (1 + abs(raw))
+ */
+function normalizeBm25(raw) {
+    return Math.abs(raw) / (1 + Math.abs(raw));
+}
+/**
+ * Execute a full-text search using FTS5 BM25 ranking.
+ *
+ * BM25 weights: title gets 10x boost, content gets 1x.
+ * Results are deduplicated by document (keeping the best-scoring chunk).
+ */
+export function searchFts(sql, query, options = {}) {
+    const ftsQuery = buildFts5Query(query);
+    const limit = options.limit ?? 10;
+    // Build WHERE clauses for optional filters
+    const filters = [];
+    const bindings = [ftsQuery];
+    if (options.docType) {
+        filters.push("d.doc_type = ?");
+        bindings.push(options.docType);
+    }
+    if (options.namespace) {
+        if (options.namespace.includes("*")) {
+            const prefix = options.namespace.replace(/\*+$/, "").replace(/\/+$/, "");
+            filters.push("d.namespace LIKE ?");
+            bindings.push(`${prefix}/%`);
+        }
+        else {
+            filters.push("d.namespace = ?");
+            bindings.push(options.namespace);
+        }
+    }
+    const whereClause = filters.length > 0 ? `AND ${filters.join(" AND ")}` : "";
+    // BM25 weights: doc_id (unindexed, 0), seq (unindexed, 0), title (10.0), content (1.0)
+    const rows = sql
+        .exec(`
+			SELECT
+				f.doc_id,
+				f.seq,
+				f.content,
+				bm25(qmd_chunks_fts, 0, 0, 10.0, 1.0) as rank,
+				d.title,
+				d.doc_type,
+				d.namespace,
+				d.metadata
+			FROM qmd_chunks_fts f
+			JOIN qmd_documents d ON d.id = f.doc_id
+			WHERE qmd_chunks_fts MATCH ?
+			${whereClause}
+			ORDER BY rank
+			LIMIT ?
+		`, ...bindings, 
+    // Fetch extra to allow dedup
+    limit * 3)
+        .toArray();
+    // Deduplicate by doc_id, keeping the best-scoring chunk
+    const seen = new Map();
+    for (const row of rows) {
+        const score = normalizeBm25(row.rank);
+        const existing = seen.get(row.doc_id);
+        if (!existing || score > existing.score) {
+            seen.set(row.doc_id, {
+                docId: row.doc_id,
+                score,
+                snippet: row.content,
+                seq: row.seq,
+                title: row.title,
+                docType: row.doc_type,
+                namespace: row.namespace,
+                metadata: row.metadata ? JSON.parse(row.metadata) : null,
+            });
+        }
+    }
+    return Array.from(seen.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+}
+//# sourceMappingURL=fts.js.map

package/dist/fts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.js","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAEA;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC3C,MAAM,KAAK,GAAG,KAAK;SACjB,IAAI,EAAE;SACN,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC5B,0CAA0C;SACzC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC;IAEhD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,4BAA4B;QAC5B,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,iDAAiD;IACjD,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACtC,MAAM,IAAI,GAAG,QAAQ,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;IACjE,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAExD,OAAO,IAAI,MAAM,SAAS,IAAI,SAAS,OAAO,GAAG,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,GAAW;IACjC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5C,CAAC;AAaD;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CACxB,GAAe,EACf,KAAa,EACb,UAAyB,EAAE;IAE3B,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAElC,2CAA2C;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAc,CAAC,QAAQ,CAAC,CAAC;IAEvC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,IAAI,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;QAC9B,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;IACF,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAE7E,uFAAuF;IACvF,MAAM,IAAI,GAAG,GAAG;SACd,IAAI,CACJ;;;;;;;;;;;;;KAaE,WAAW;;;GAGb,EACA,GAAG,QAAQ;IACX,6BAA6B;IAC7B,KAAK,GAAG,CAAC,CACT;SACA,OAAO,EAAE,CAAC;IAEZ,wDAAwD;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAC;IAE1C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEtC,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE;gBACpB,KAAK,EAAE,GAAG,CAAC,MAAM;gBACjB,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,GAAG,EAAE,GAAG,CAAC,GAAa;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,QAAQ;gBACrB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;aAClE,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;SAC9B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACnB,CAAC"}

package/dist/hash.d.ts

@@ -0,0 +1,7 @@
+/**
+ * FNV-1a 32-bit hash — fast, deterministic, non-cryptographic.
+ * Used to detect content changes for skip-on-unchanged indexing.
+ * Returns an 8-character lowercase hex string.
+ */
+export declare function fnv1a32(input: string): string;
+//# sourceMappingURL=hash.d.ts.map

package/dist/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAO7C"}

package/dist/hash.js

@@ -0,0 +1,14 @@
+/**
+ * FNV-1a 32-bit hash — fast, deterministic, non-cryptographic.
+ * Used to detect content changes for skip-on-unchanged indexing.
+ * Returns an 8-character lowercase hex string.
+ */
+export function fnv1a32(input) {
+    let hash = 0x811c9dc5; // FNV offset basis
+    for (let i = 0; i < input.length; i++) {
+        hash ^= input.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193); // FNV prime
+    }
+    return (hash >>> 0).toString(16).padStart(8, "0");
+}
+//# sourceMappingURL=hash.js.map

package/dist/hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,KAAa;IACpC,IAAI,IAAI,GAAG,UAAU,CAAC,CAAC,mBAAmB;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,YAAY;IACjD,CAAC;IACD,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACnD,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,56 @@
+/**
+ * @stablemodels/qmd-cf — Hybrid full-text + vector search for Cloudflare Durable Objects.
+ *
+ * A DO-native reimagination of qmd (https://github.com/tobi/qmd).
+ *
+ * FTS5 runs co-located in the Durable Object's SQLite for zero-latency BM25 keyword search.
+ * Optionally, Cloudflare Vectorize adds semantic vector search, fused via Reciprocal Rank Fusion.
+ *
+ * Cloudflare platform types (SqlStorage, Vectorize, VectorizeVector, etc.) are
+ * ambient from @cloudflare/workers-types — consumers access them directly.
+ *
+ * @example FTS-only (zero external dependencies)
+ * ```ts
+ * import { Qmd } from "@stablemodels/qmd-cf";
+ *
+ * export class MyDurableObject extends DurableObject {
+ *   qmd: Qmd;
+ *
+ *   constructor(ctx: DurableObjectState, env: Env) {
+ *     super(ctx, env);
+ *     this.qmd = new Qmd(ctx.storage.sql);
+ *   }
+ *
+ *   async search(query: string) {
+ *     return this.qmd.search(query);
+ *   }
+ * }
+ * ```
+ *
+ * @example Hybrid FTS + Vector search
+ * ```ts
+ * import { Qmd } from "@stablemodels/qmd-cf";
+ *
+ * export class MyDurableObject extends DurableObject {
+ *   qmd: Qmd;
+ *
+ *   constructor(ctx: DurableObjectState, env: Env) {
+ *     super(ctx, env);
+ *     this.qmd = new Qmd(ctx.storage.sql, {
+ *       vectorize: env.VECTORIZE,
+ *       embedFn: (texts) =>
+ *         env.AI.run("@cf/baai/bge-m3", { text: texts })
+ *           .then(r => r.data),
+ *     });
+ *   }
+ * }
+ * ```
+ */
+export { Qmd } from "./qmd.js";
+export type { Document, Chunk, FtsResult, VectorResult, SearchResult, SearchOptions, HybridSearchOptions, QmdConfig, IndexStats, EmbedFn, } from "./types.js";
+export { chunkText } from "./chunker.js";
+export { buildFts5Query } from "./fts.js";
+export { fnv1a32 } from "./hash.js";
+export { reciprocalRankFusion } from "./rrf.js";
+export { formatDocForEmbedding, formatQueryForEmbedding } from "./vector.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAGH,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,YAAY,EACX,QAAQ,EACR,KAAK,EACL,SAAS,EACT,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,mBAAmB,EACnB,SAAS,EACT,UAAU,EACV,OAAO,GACP,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC"}

package/dist/index.js

@@ -0,0 +1,57 @@
+/**
+ * @stablemodels/qmd-cf — Hybrid full-text + vector search for Cloudflare Durable Objects.
+ *
+ * A DO-native reimagination of qmd (https://github.com/tobi/qmd).
+ *
+ * FTS5 runs co-located in the Durable Object's SQLite for zero-latency BM25 keyword search.
+ * Optionally, Cloudflare Vectorize adds semantic vector search, fused via Reciprocal Rank Fusion.
+ *
+ * Cloudflare platform types (SqlStorage, Vectorize, VectorizeVector, etc.) are
+ * ambient from @cloudflare/workers-types — consumers access them directly.
+ *
+ * @example FTS-only (zero external dependencies)
+ * ```ts
+ * import { Qmd } from "@stablemodels/qmd-cf";
+ *
+ * export class MyDurableObject extends DurableObject {
+ *   qmd: Qmd;
+ *
+ *   constructor(ctx: DurableObjectState, env: Env) {
+ *     super(ctx, env);
+ *     this.qmd = new Qmd(ctx.storage.sql);
+ *   }
+ *
+ *   async search(query: string) {
+ *     return this.qmd.search(query);
+ *   }
+ * }
+ * ```
+ *
+ * @example Hybrid FTS + Vector search
+ * ```ts
+ * import { Qmd } from "@stablemodels/qmd-cf";
+ *
+ * export class MyDurableObject extends DurableObject {
+ *   qmd: Qmd;
+ *
+ *   constructor(ctx: DurableObjectState, env: Env) {
+ *     super(ctx, env);
+ *     this.qmd = new Qmd(ctx.storage.sql, {
+ *       vectorize: env.VECTORIZE,
+ *       embedFn: (texts) =>
+ *         env.AI.run("@cf/baai/bge-m3", { text: texts })
+ *           .then(r => r.data),
+ *     });
+ *   }
+ * }
+ * ```
+ */
+// Main class
+export { Qmd } from "./qmd.js";
+// Utilities (useful for custom pipelines)
+export { chunkText } from "./chunker.js";
+export { buildFts5Query } from "./fts.js";
+export { fnv1a32 } from "./hash.js";
+export { reciprocalRankFusion } from "./rrf.js";
+export { formatDocForEmbedding, formatQueryForEmbedding } from "./vector.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,aAAa;AACb,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAgB/B,0CAA0C;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC"}