@monoes/monomindcli 2.14.1 → 2.15.0

package/dist/src/knowledge/document-chunking.js

@@ -0,0 +1,288 @@
+/**
+ * Document chunking — turning a document's extracted text into the enriched
+ * chunks that get embedded and stored.
+ *
+ * Two passes, in this order:
+ *  1. CHUNK — split on heading/paragraph boundaries (shared `@monoes/memory`
+ *     chunker, with an inline fallback kept byte-identical to it).
+ *  2. ENRICH — replace each chunk's leaf-heading prefix with a situating blurb
+ *     (doc title + full heading path + summary) so the embedding model can tell
+ *     two identically-titled sections of different documents apart.
+ *
+ * Split out of document-pipeline.ts. Pure string work: no fs, no network, no
+ * store access — which is what makes chunk spans reproducible at citation time
+ * (`chunkSpans`) without re-reading anything.
+ *
+ * @module v1/cli/knowledge/document-chunking
+ */
+import * as path from 'node:path';
+const DEFAULT_CHUNK_SIZE = 3200;
+const DEFAULT_OVERLAP = 400;
+// Inline fallback identical to @monoes/memory's knowledge/document-chunker.ts —
+// used only if the dynamic import below fails (package not installed/built).
+// Keep in sync if the shared chunker's boundary-snapping logic changes.
+const HEADING_LINE_RE = /^#{1,6} /;
+const FENCE_LINE_RE = /^\s{0,3}(`{3,}|~{3,})/;
+function fenceTogglesInline(text) {
+    const toggles = [];
+    let lineStart = 0;
+    while (lineStart <= text.length) {
+        const eol = text.indexOf('\n', lineStart);
+        const line = text.slice(lineStart, eol === -1 ? undefined : eol);
+        if (FENCE_LINE_RE.test(line))
+            toggles.push(lineStart);
+        if (eol === -1)
+            break;
+        lineStart = eol + 1;
+    }
+    return toggles;
+}
+function inFenceInline(toggles, pos) {
+    let lo = 0, hi = toggles.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (toggles[mid] <= pos)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    return (lo & 1) === 1;
+}
+function lastHeadingBefore(text, pos, toggles) {
+    let i = text.lastIndexOf('\n#', pos - 1);
+    while (i !== -1) {
+        const eol = text.indexOf('\n', i + 1);
+        const line = text.slice(i + 1, eol === -1 ? undefined : eol);
+        if (HEADING_LINE_RE.test(line) && !inFenceInline(toggles, i + 1))
+            return line.replace(/^#+ /, '').trim();
+        i = i > 0 ? text.lastIndexOf('\n#', i - 1) : -1; // fromIndex -1 clamps to 0 — would loop on a match at 0
+    }
+    const firstEol = text.indexOf('\n');
+    const firstLine = firstEol === -1 ? text : text.slice(0, firstEol);
+    return HEADING_LINE_RE.test(firstLine) &&
+        !inFenceInline(toggles, 0) &&
+        firstEol !== -1 &&
+        firstEol < pos
+        ? firstLine.replace(/^#+ /, '').trim()
+        : null;
+}
+function chunkDocumentInline(docId, text) {
+    if (text.includes('\r\n'))
+        text = text.replace(/\r\n/g, '\n');
+    if (text.length === 0)
+        return [];
+    const toggles = fenceTogglesInline(text);
+    const chunks = [];
+    let startChar = 0;
+    let chunkIndex = 0;
+    while (startChar < text.length) {
+        let endChar = Math.min(startChar + DEFAULT_CHUNK_SIZE, text.length);
+        let brokeAtHeading = false;
+        if (endChar < text.length) {
+            const windowStart = Math.max(startChar, endChar - Math.floor(DEFAULT_CHUNK_SIZE * 0.2));
+            const window = text.slice(windowStart, endChar);
+            let h = window.lastIndexOf('\n#');
+            while (h !== -1) {
+                const eol = window.indexOf('\n', h + 1);
+                const line = window.slice(h + 1, eol === -1 ? undefined : eol);
+                if (HEADING_LINE_RE.test(line) &&
+                    windowStart + h > startChar &&
+                    !inFenceInline(toggles, windowStart + h + 1))
+                    break;
+                h = h > 0 ? window.lastIndexOf('\n#', h - 1) : -1;
+            }
+            if (h !== -1 && windowStart + h > startChar) {
+                endChar = windowStart + h + 1;
+                brokeAtHeading = true;
+            }
+            else {
+                let lastParagraph = window.lastIndexOf('\n\n');
+                while (lastParagraph > 0 && inFenceInline(toggles, windowStart + lastParagraph + 1)) {
+                    lastParagraph = window.lastIndexOf('\n\n', lastParagraph - 1);
+                }
+                if (lastParagraph === 0 && inFenceInline(toggles, windowStart + 1))
+                    lastParagraph = -1;
+                if (lastParagraph !== -1)
+                    endChar = windowStart + lastParagraph + 2;
+            }
+        }
+        let chunkText = text.slice(startChar, endChar);
+        const heading = lastHeadingBefore(text, startChar + 1, toggles);
+        if (heading && !HEADING_LINE_RE.test(chunkText.trimStart()))
+            chunkText = `§ ${heading}\n${chunkText}`;
+        chunks.push({
+            chunkId: `${docId}:${chunkIndex}`,
+            docId,
+            text: chunkText,
+            startChar,
+            endChar,
+            chunkIndex,
+        });
+        chunkIndex++;
+        if (endChar >= text.length)
+            break;
+        startChar += brokeAtHeading
+            ? Math.max(1, endChar - startChar)
+            : Math.max(1, endChar - startChar - DEFAULT_OVERLAP);
+    }
+    return chunks;
+}
+export async function chunkDocument(docId, text) {
+    try {
+        const mod = await import('@monoes/memory');
+        return mod.chunkDocument(docId, text, DEFAULT_CHUNK_SIZE, DEFAULT_OVERLAP);
+    }
+    catch {
+        return chunkDocumentInline(docId, text);
+    }
+}
+// ── Contextual chunk enrichment (item 6a) ─────────────────────────
+// Prepend a situating blurb per chunk before embedding: full heading
+// path + doc title + doc summary. No LLM, no network.
+//
+// The `§ heading` prefix `chunkDocumentInline` writes provides only the nearest
+// leaf heading. This replaces it with doc-level context so the embedding
+// model can distinguish "Memory Coordination" in a hooks doc from
+// "Memory Coordination" in a concepts doc.
+//
+// Applied at INGEST TIME (after chunking, before embedding), so:
+//  - Works identically regardless of which chunker ran (inline or @monoes/memory)
+//  - Works identically for both better-sqlite3 and sql.js (pure string ops)
+//  - Zero dependencies, zero network
+const SECTION_PREFIX_RE = /^§ [^\n]+\n/;
+/** Cap on the situating summary prepended to each chunk, ellipsis included. */
+const SUMMARY_MAX_CHARS = 120;
+function extractDocTitle(text, filePath) {
+    const eol = text.indexOf('\n');
+    const first = eol === -1 ? text : text.slice(0, eol);
+    return HEADING_LINE_RE.test(first)
+        ? first.replace(/^#+ /, '').trim()
+        : path.basename(filePath, path.extname(filePath)).replace(/[-_]/g, ' ');
+}
+function extractDocSummary(text) {
+    const lines = text.split('\n');
+    let inFence = false;
+    const parts = [];
+    for (const line of lines) {
+        if (FENCE_LINE_RE.test(line)) {
+            inFence = !inFence;
+            continue;
+        }
+        if (inFence)
+            continue;
+        if (HEADING_LINE_RE.test(line)) {
+            if (parts.length > 0)
+                break;
+            continue;
+        }
+        const t = line.trim();
+        if (!t || /^[|=-]/.test(t)) {
+            if (parts.length > 0)
+                break;
+            continue;
+        }
+        parts.push(t.startsWith('>') ? t.replace(/^>\s*/, '') : t);
+    }
+    const joined = parts.join(' ');
+    // Truncate with an ellipsis so a clipped summary is visibly clipped. The bare
+    // 150-char slice this replaces gave no signal that anything was cut, which
+    // read as a complete sentence to both a human and the embedding model.
+    return joined.length > SUMMARY_MAX_CHARS
+        ? `${joined.slice(0, SUMMARY_MAX_CHARS - 3).trimEnd()}...`
+        : joined;
+}
+function buildHeadingHierarchy(text, toggles) {
+    const out = [];
+    const eol0 = text.indexOf('\n');
+    const line0 = eol0 === -1 ? text : text.slice(0, eol0);
+    const firstLevel = line0.match(/^(#{1,6}) /)?.[1]?.length;
+    if (firstLevel !== undefined && !inFenceInline(toggles, 0)) {
+        out.push({
+            level: firstLevel,
+            text: line0.replace(/^#+ /, '').trim(),
+            offset: 0,
+        });
+    }
+    let i = text.indexOf('\n#', 0);
+    while (i !== -1) {
+        const ls = i + 1;
+        const e = text.indexOf('\n', ls);
+        const line = text.slice(ls, e === -1 ? undefined : e);
+        const level = line.match(/^(#{1,6}) /)?.[1]?.length;
+        if (level !== undefined && !inFenceInline(toggles, ls)) {
+            out.push({
+                level,
+                text: line.replace(/^#+ /, '').trim(),
+                offset: ls,
+            });
+        }
+        i = text.indexOf('\n#', ls);
+    }
+    return out;
+}
+function headingPathAt(hierarchy, pos) {
+    const stack = [];
+    for (const h of hierarchy) {
+        if (h.offset >= pos)
+            break;
+        while (stack.length > 0 && stack[stack.length - 1].level >= h.level)
+            stack.pop();
+        stack.push(h);
+    }
+    return stack.map((s) => s.text);
+}
+/**
+ * Replace each chunk's `§ heading` prefix with a richer situating blurb:
+ *   § <doc title> · <full heading path>
+ *   <doc summary for non-first chunks>
+ *
+ * First chunks that start with their own heading are left untouched (the
+ * heading IS the context). The summary line is omitted for the first
+ * chunk since it is adjacent to the summary text anyway.
+ */
+export function enrichChunks(chunks, fullText, filePath) {
+    if (chunks.length === 0)
+        return chunks;
+    const toggles = fenceTogglesInline(fullText);
+    const hierarchy = buildHeadingHierarchy(fullText, toggles);
+    const title = extractDocTitle(fullText, filePath);
+    const summary = extractDocSummary(fullText);
+    return chunks.map((c) => {
+        let text = c.text;
+        // First chunk starting with its own heading — the heading IS the context
+        if (c.chunkIndex === 0 && HEADING_LINE_RE.test(text.trimStart()))
+            return c;
+        // Strip the old § leaf-heading prefix; we replace it with a richer one
+        text = text.replace(SECTION_PREFIX_RE, '');
+        const hpath = headingPathAt(hierarchy, c.startChar + 1);
+        const parts = [];
+        // Title + full heading path
+        if (hpath.length > 0 && hpath[0] !== title) {
+            parts.push(`§ ${title} · ${hpath.join(' > ')}`);
+        }
+        else if (hpath.length > 1) {
+            parts.push(`§ ${hpath.join(' > ')}`);
+        }
+        else {
+            parts.push(`§ ${title}`);
+        }
+        // Summary for non-first chunks — they are far from the doc intro
+        if (c.chunkIndex > 0 && summary) {
+            const snip = summary.length > 120 ? `${summary.slice(0, 117)}...` : summary;
+            parts.push(snip);
+        }
+        return { ...c, text: `${parts.join('\n')}\n${text}` };
+    });
+}
+/** Chunk spans for a text, using the SAME chunker the ingest used — this is
+ *  what makes a chunk index resolvable back to a character range without
+ *  storing the text twice. */
+export async function chunkSpans(text) {
+    const chunks = await chunkDocument('cite', text);
+    return chunks.map((c) => ({
+        chunkIndex: c.chunkIndex,
+        startChar: c.startChar,
+        endChar: c.endChar,
+    }));
+}
+//# sourceMappingURL=document-chunking.js.map

package/dist/src/knowledge/document-chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-chunking.js","sourceRoot":"","sources":["../../../src/knowledge/document-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAYlC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,gFAAgF;AAChF,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,eAAe,GAAG,UAAU,CAAC;AACnC,MAAM,aAAa,GAAG,uBAAuB,CAAC;AAC9C,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjE,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM;QACtB,SAAS,GAAG,GAAG,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,SAAS,aAAa,CAAC,OAAiB,EAAE,GAAW;IACnD,IAAI,EAAE,GAAG,CAAC,EACR,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IACtB,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG;YAAE,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;;YACjC,EAAE,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AACD,SAAS,iBAAiB,CAAC,IAAY,EAAE,GAAW,EAAE,OAAiB;IACrE,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,wDAAwD;IAC3G,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,SAAS,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,OAAO,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1B,QAAQ,KAAK,CAAC,CAAC;QACf,QAAQ,GAAG,GAAG;QACd,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AACD,SAAS,mBAAmB,CAAC,KAAa,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC9D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACpE,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,CAAC;YACxF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAChD,IAAI,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC/D,IACE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC1B,WAAW,GAAG,CAAC,GAAG,SAAS;oBAC3B,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;oBAE5C,MAAM;gBACR,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;gBAC5C,OAAO,GAAG,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9B,cAAc,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,IAAI,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC/C,OAAO,aAAa,GAAG,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;oBACpF,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC;gBAChE,CAAC;gBACD,IAAI,aAAa,KAAK,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,CAAC,CAAC;oBAAE,aAAa,GAAG,CAAC,CAAC,CAAC;gBACvF,IAAI,aAAa,KAAK,CAAC,CAAC;oBAAE,OAAO,GAAG,WAAW,GAAG,aAAa,GAAG,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QACD,IAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACzD,SAAS,GAAG,KAAK,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC;YACV,OAAO,EAAE,GAAG,KAAK,IAAI,UAAU,EAAE;YACjC,KAAK;YACL,IAAI,EAAE,SAAS;YACf,SAAS;YACT,OAAO;YACP,UAAU;SACX,CAAC,CAAC;QACH,UAAU,EAAE,CAAC;QACb,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM;QAClC,SAAS,IAAI,cAAc;YACzB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;YAClC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,GAAG,eAAe,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,IAAY;IAC7D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,gBAA0B,CAAC,CAAC;QACrD,OAAO,GAAG,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,kBAAkB,EAAE,eAAe,CAAC,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,qEAAqE;AACrE,sDAAsD;AACtD,EAAE;AACF,gFAAgF;AAChF,yEAAyE;AACzE,kEAAkE;AAClE,2CAA2C;AAC3C,EAAE;AACF,iEAAiE;AACjE,kFAAkF;AAClF,4EAA4E;AAC5E,qCAAqC;AAErC,MAAM,iBAAiB,GAAG,aAAa,CAAC;AACxC,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,SAAS,eAAe,CAAC,IAAY,EAAE,QAAgB;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACrD,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;QAClC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO,GAAG,CAAC,OAAO,CAAC;YACnB,SAAS;QACX,CAAC;QACD,IAAI,OAAO;YAAE,SAAS;QACtB,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM;YAC5B,SAAS;QACX,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM;YAC5B,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/B,8EAA8E;IAC9E,2EAA2E;IAC3E,uEAAuE;IACvE,OAAO,MAAM,CAAC,MAAM,GAAG,iBAAiB;QACtC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK;QAC1D,CAAC,CAAC,MAAM,CAAC;AACb,CAAC;AAED,SAAS,qBAAqB,CAC5B,IAAY,EACZ,OAAiB;IAEjB,MAAM,GAAG,GAA2D,EAAE,CAAC;IACvE,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IAC1D,IAAI,UAAU,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,UAAU;YACjB,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;YACtC,MAAM,EAAE,CAAC;SACV,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QACpD,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK;gBACL,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;gBACrC,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;QACL,CAAC;QACD,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CACpB,SAAiE,EACjE,GAAW;IAEX,MAAM,KAAK,GAA2C,EAAE,CAAC;IACzD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM;QAC3B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK;YAAE,KAAK,CAAC,GAAG,EAAE,CAAC;QACjF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,MAAmB,EAAE,QAAgB,EAAE,QAAgB;IAClF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACvC,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,qBAAqB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAE5C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACtB,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;QAElB,yEAAyE;QACzE,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAAE,OAAO,CAAC,CAAC;QAE3E,uEAAuE;QACvE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QACxD,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,4BAA4B;QAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,iEAAiE;QACjE,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;YAC5E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;QAED,OAAO,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;IACxD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;8BAE8B;AAC9B,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/src/knowledge/document-index.d.ts

@@ -0,0 +1,124 @@
+/**
+ * The document index — the append-only metadata log that records which
+ * documents are indexed, at which version, and whether that is still true.
+ *
+ * The log (`.monomind/knowledge/doc-metadata.jsonl`) is the pipeline's source
+ * of truth. Everything here reads or writes it:
+ *
+ *  - WRITE: append a version record, tombstone a removed one.
+ *  - READ:  the last-wins live view, the full history, and lookups by path,
+ *           canonical URL or `<scope>:<path>` docId.
+ *  - JUDGE: which content hashes are current, which chunk keys are therefore
+ *           superseded, and which indexed files no longer exist on disk.
+ *
+ * Nothing here touches the vector store. Removal tombstones metadata and lets
+ * the superseded filter hide the chunks — mark, don't destroy.
+ *
+ * Split out of document-pipeline.ts.
+ *
+ * @module v1/cli/knowledge/document-index
+ */
+import type { DocumentMeta, ReconcileReport } from './document-types.js';
+export declare const METADATA_FILE = "doc-metadata.jsonl";
+export declare function metadataPath(rootDir: string): string;
+/** Every record ever appended, in order, including superseded versions and
+ *  removal tombstones. Reads the path directly instead of via `metadataPath`,
+ *  which mkdir's. */
+export declare function readMetadataLog(rootDir: string): DocumentMeta[];
+export declare function readMetadata(rootDir: string): DocumentMeta[];
+export declare function appendMetadata(rootDir: string, meta: DocumentMeta): void;
+export declare function removeMetadataEntry(rootDir: string, filePath: string, scope: string): void;
+/** Content hashes of the documents currently indexed under `rootDir`. */
+export declare function liveContentHashes(rootDir: string): Set<string>;
+/** True when a metadata log exists under `rootDir`.
+ *
+ * An empty live-hash set has two very different causes: the log is missing (we
+ * cannot judge what is current) or the log exists and every document has been
+ * removed (nothing is current). Collapsing them made `doc remove` of the LAST
+ * document a no-op — the tombstoned chunks came straight back in search.
+ *
+ * Reads the path directly instead of via `metadataPath`, which mkdir's. */
+export declare function hasKnowledgeMetadata(rootDir: string): boolean;
+/**
+ * True when `key` is a document chunk whose version is no longer current.
+ * Non-`doc:` keys are never superseded. When no metadata is available nothing
+ * is filtered, because "no metadata" must not read as "everything is stale".
+ *
+ * `metadataPresent` defaults to the old `live.size > 0` heuristic so existing
+ * two-argument callers keep their exact behaviour; pass `hasKnowledgeMetadata`
+ * to also filter correctly once the last document has been removed.
+ */
+export declare function isSupersededKey(key: string, live: Set<string>, metadataPresent?: boolean): boolean;
+export declare function supersededOverfetchLimit(limit: number, live: Set<string>): number;
+export declare function listDocuments(rootDir?: string, scope?: string): DocumentMeta[];
+/**
+ * Every recorded version of one document, oldest first (RCL-06).
+ *
+ * `target` is either an indexed file path or a capture's `canonicalUrl`. Read
+ * from the append-only log rather than the last-wins view, which is how a
+ * superseded version stays addressable after its record has been replaced or
+ * tombstoned.
+ *
+ * Best-effort by design: `readMetadata` compacts the log once it passes 1MB
+ * and keeps only live records, so history older than a compaction is gone.
+ * The `supersedes` pointer on each record is the durable part.
+ */
+export declare function listDocumentVersions(rootDir?: string, target?: string, scope?: string): DocumentMeta[];
+/**
+ * The indexed record for whatever a caller wrote down: a file path, a
+ * capture's `canonicalUrl` (fragment ignored), or the `<scope>:<path>` docId.
+ *
+ * Live records win. A superseded version is only reached through the
+ * append-only log, and only when nothing live matches — otherwise citing a
+ * re-captured page would resolve against the version it replaced.
+ */
+export declare function findDocumentRecord(rootDir?: string, target?: string, scope?: string): DocumentMeta | undefined;
+export declare function removeDocument(filePath: string, scope?: string, rootDir?: string): Promise<void>;
+/**
+ * True for macOS AppleDouble sidecars (`._name`).
+ *
+ * Matches on the BASENAME PREFIX only. A legitimate document may contain `._`
+ * elsewhere in its name (`v1._2-release.md`), or live under a dot-directory
+ * that is deliberately indexed (`.monodesign/` critique snapshots), and
+ * neither may be rejected.
+ */
+export declare function isResourceFork(filePath: string): boolean;
+/**
+ * Reconcile the document index against the filesystem: find index entries whose
+ * source file no longer exists and, only when explicitly asked, tombstone them.
+ *
+ * WHY — `removeDocument` only ever tombstoned metadata, and nothing has ever
+ * compared the index against the disk, so a deleted file stayed searchable
+ * forever. Measured 2026-07-28: 109 of 257 live entries (42.4%) had no file
+ * behind them, including `docs/concepts/memory.md`. The Second Brain was
+ * answering questions from documents the user had deleted.
+ *
+ * WHY IT IS THIS CAUTIOUS — "drop the index entry when the file is missing" is
+ * a rule with a known catastrophic reading. A missing file is also an unmounted
+ * volume, a checked-out branch, a partial clone, or a permissions failure. Two
+ * guards were tried against real data and REJECTED; they are recorded here so
+ * they are not re-proposed:
+ *
+ *   - "abort if >50% of entries are missing" — the real, legitimate missing
+ *     fraction was 42.4%, so the threshold never fires in the one case we have.
+ *     Any threshold that would have blocked this reconcile is fitted to nothing.
+ *   - "only reconcile when the parent directory still exists" — 26 of the 109
+ *     missing files had no parent directory, because `docs/concepts`,
+ *     `docs/adrs` and `docs/commands` were legitimately deleted wholesale. A
+ *     deleted directory and an unmounted volume are indistinguishable there.
+ *
+ * What does discriminate is the ROOT. An intact, readable root carrying a
+ * metadata log means the tree is genuinely present, so a missing file is
+ * genuinely gone. A missing root means nothing beneath it is knowable and
+ * nothing may be removed — hence throw rather than reconcile.
+ *
+ * Removal tombstones metadata; it does not delete store rows. Chunks stay on
+ * disk and fall out of search through the existing superseded filter, which
+ * keeps this consistent with the mark-don't-destroy rule and leaves the whole
+ * operation reversible from the archive.
+ */
+export declare function reconcileIndex(rootDir?: string, opts?: {
+    scope?: string;
+    apply?: boolean;
+}): Promise<ReconcileReport>;
+//# sourceMappingURL=document-index.d.ts.map

package/dist/src/knowledge/document-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-index.d.ts","sourceRoot":"","sources":["../../../src/knowledge/document-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AASH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEzE,eAAO,MAAM,aAAa,uBAAuB,CAAC;AAIlD,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED;;qBAEqB;AACrB,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,EAAE,CAa/D;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,EAAE,CAoC5D;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,IAAI,CAExE;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAa1F;AAkBD,yEAAyE;AACzE,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAI9D;AAED;;;;;;;2EAO2E;AAC3E,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAE7D;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,eAAe,UAAgB,GAC9B,OAAO,CAIT;AAQD,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAGjF;AAID,wBAAgB,aAAa,CAAC,OAAO,SAAmB,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE,CAGxF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,SAAmB,EAC1B,MAAM,CAAC,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GACb,YAAY,EAAE,CAUhB;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,SAAmB,EAC1B,MAAM,SAAK,EACX,KAAK,CAAC,EAAE,MAAM,GACb,YAAY,GAAG,SAAS,CA0B1B;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,KAAK,SAAW,EAChB,OAAO,SAAmB,GACzB,OAAO,CAAC,IAAI,CAAC,CAIf;AAID;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAExD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAsB,cAAc,CAClC,OAAO,SAAmB,EAC1B,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,GACzC,OAAO,CAAC,eAAe,CAAC,CAwC1B"}