@monoes/monomindcli 2.14.1 → 2.15.1

package/dist/src/knowledge/document-ingest.js

@@ -0,0 +1,368 @@
+/**
+ * Document ingest — the write side of the pipeline: a file on disk (or a
+ * capture envelope) becomes an indexed, versioned, searchable document.
+ *
+ * One pass per document: guard the input, resolve the capture envelope's
+ * identity, extract and hash the text, chunk and enrich it, store every chunk,
+ * and only then commit the version record. An incomplete store commits
+ * nothing, which is what makes a re-ingest repair it.
+ *
+ * Split out of document-pipeline.ts.
+ *
+ * @module v1/cli/knowledge/document-ingest
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { DOC_EXTENSIONS, extractText } from '../capabilities/cap-documents.js';
+// Static import is safe and deliberate: memory-bridge imports only node builtins
+// at module scope (everything heavy is lazy), and the project-root rule must not
+// be duplicated — two copies of "which directory is this project" is exactly the
+// bug this default exists to fix.
+import { getProjectRoot } from '../memory/memory-bridge.js';
+import { captureIdentityUrl, ENVELOPE_DOCUMENTS, ENVELOPE_READABLE_FILE, envelopePrimaryDocument, readCaptureProvenance, } from './capture-envelope.js';
+import { spanTag } from './citation.js';
+import { chunkDocument, enrichChunks } from './document-chunking.js';
+import { appendMetadata, isResourceFork, readMetadata, removeMetadataEntry, } from './document-index.js';
+import { contentHash, effectiveRoot, getBridge, namespace, storeDbPath } from './document-store.js';
+import { captureScope } from './profile-store.js';
+const IGNORE_DIRS = new Set([
+    'node_modules',
+    '.git',
+    'dist',
+    '.monomind',
+    '.claude',
+    '.next',
+    '__pycache__',
+    '.venv',
+    'vendor',
+]);
+const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
+/** @internal — shared with okf-bundle, which extracts text the same way. */
+export function toFileEntry(filePath) {
+    const stat = fs.statSync(filePath);
+    return {
+        path: filePath,
+        absolutePath: path.resolve(filePath),
+        extension: path.extname(filePath).toLowerCase(),
+        size: stat.size,
+        modified: stat.mtime,
+        created: stat.birthtime,
+    };
+}
+export async function ingestDocument(filePath, scope = 'shared', rootDir = getProjectRoot(), _metadataCache) {
+    const resolved = path.resolve(filePath);
+    const ext = path.extname(resolved).toLowerCase();
+    // AppleDouble resource forks (`._name.md`) are binary macOS sidecars, not
+    // documents. The directory walk has skipped dotfiles since 3e429194
+    // (2026-07-19), but that walk is only ONE of six callers that reach this
+    // function — the CLI `doc ingest`, the MCP `knowledge_ingest` tool, the
+    // dashboard's live fs.watch and its polling sweep, the eval harness, and
+    // `ingestDirectory` all land here, and four of them had no guard at all.
+    //
+    // Guarding at the boundary covers every caller at once, including callers
+    // added later. Guarding at each call site covers only the ones we thought to
+    // enumerate — which is how two `._` files reached the live index despite a
+    // working guard in the walk.
+    //
+    // Measured on this repo 2026-07-28: 96 `._` entries in the live index, 91 of
+    // them shadowing a real document of the same name and competing with it for
+    // top-k slots. That is a direct Recall@5/MRR@10 loss, not wasted storage.
+    if (isResourceFork(resolved)) {
+        return {
+            filePath: resolved,
+            chunksIndexed: 0,
+            scope,
+            skipped: true,
+            error: 'AppleDouble resource fork',
+        };
+    }
+    if (!DOC_EXTENSIONS.has(ext)) {
+        return {
+            filePath: resolved,
+            chunksIndexed: 0,
+            scope,
+            skipped: true,
+            error: `unsupported extension: ${ext}`,
+        };
+    }
+    if (!fs.existsSync(resolved)) {
+        return { filePath: resolved, chunksIndexed: 0, scope, skipped: true, error: 'file not found' };
+    }
+    const stat = fs.statSync(resolved);
+    if (stat.size > MAX_FILE_SIZE) {
+        return {
+            filePath: resolved,
+            chunksIndexed: 0,
+            scope,
+            skipped: true,
+            error: 'file too large (>50MB)',
+        };
+    }
+    // RCL-01/RCL-06: one capture envelope is ONE document, whichever of its
+    // members the caller points at. `page.html`/`page.mhtml` need no guard —
+    // extraction already redirects them to `readable.md`, so they hash
+    // identically and fall out as "unchanged" below. `page.pdf` does not: its
+    // text differs from the readable pass, so ingesting it alongside would
+    // version-flip the same page back and forth on every sweep.
+    const envelopePrimary = envelopePrimaryDocument(path.dirname(resolved));
+    if (envelopePrimary && envelopePrimary !== resolved) {
+        const isMember = ENVELOPE_DOCUMENTS.includes(path.basename(resolved));
+        const redirectsToPrimary = path.basename(envelopePrimary) === ENVELOPE_READABLE_FILE &&
+            (ext === '.html' || ext === '.htm' || ext === '.xhtml' || ext === '.mhtml' || ext === '.mht');
+        if (isMember && !redirectsToPrimary) {
+            return {
+                filePath: resolved,
+                chunksIndexed: 0,
+                scope,
+                skipped: true,
+                error: `capture envelope: ${path.basename(envelopePrimary)} is this capture's document`,
+            };
+        }
+    }
+    // A capture that names a profile belongs to that profile's store, whoever
+    // is ingesting it and whatever scope they reached for — see
+    // knowledge/profile-store.ts. A no-op for everything else.
+    scope = captureScope(scope, resolved);
+    rootDir = effectiveRoot(scope, rootDir);
+    const meta = _metadataCache ?? readMetadata(rootDir);
+    // RCL-07: provenance is read before extraction so it is recorded even when a
+    // later step degrades. Absent, truncated or wrong-typed `meta.json` yields
+    // null and never throws — see capture-envelope.
+    const provenance = readCaptureProvenance(resolved);
+    const canonicalUrl = captureIdentityUrl(provenance);
+    // RCL-06: identity is the PAGE, not the path. A re-capture lands in a new
+    // timestamped directory, so matching on filePath alone would file every
+    // capture of one article as a separate document.
+    const existing = meta.find((m) => m.filePath === resolved && m.scope === scope) ??
+        (canonicalUrl
+            ? meta.find((m) => m.scope === scope && m.canonicalUrl === canonicalUrl)
+            : undefined);
+    let fullContent;
+    try {
+        const entry = toFileEntry(resolved);
+        fullContent = await extractText(entry);
+    }
+    catch (err) {
+        return { filePath: resolved, chunksIndexed: 0, scope, skipped: false, error: String(err) };
+    }
+    if (!fullContent || fullContent.trim().length === 0) {
+        return {
+            filePath: resolved,
+            chunksIndexed: 0,
+            scope,
+            skipped: true,
+            error: 'no text extracted',
+        };
+    }
+    const hash = contentHash(fullContent);
+    // RCL-06: same page, same extracted text — a no-op, not a duplicate row and
+    // not an error. `unchanged` is what a caller reports to the user.
+    if (existing && existing.contentHash === hash) {
+        return {
+            filePath: resolved,
+            chunksIndexed: existing.chunkCount,
+            scope,
+            skipped: true,
+            unchanged: true,
+            ...(existing.version ? { version: existing.version } : {}),
+            ...(provenance ? { provenance } : {}),
+        };
+    }
+    const version = (existing?.version ?? (existing ? 1 : 0)) + 1;
+    const supersedes = existing?.contentHash || undefined;
+    // NOTE: the previous version's metadata record is deliberately NOT tombstoned
+    // here. `readMetadata` is last-wins per (filePath, scope), so appending the
+    // new record below already supersedes the old one — the tombstone was a no-op
+    // on the success path and destructive on the failure path: it retired a
+    // perfectly good previous index before knowing whether the replacement would
+    // land, so a failed re-ingest left the document with NO live version at all.
+    const docId = `${scope}:${resolved}`;
+    const rawChunks = await chunkDocument(docId, fullContent);
+    // monolean: [re-enabled] item 2 shipped 768d gte-modernbert-base — capacity handles enrichment
+    const chunks = enrichChunks(rawChunks, fullContent, resolved);
+    const bridge = await getBridge();
+    let indexed = 0;
+    let highlights;
+    for (const chunk of chunks) {
+        const key = `doc:${hash}:${chunk.chunkIndex}`;
+        if (bridge) {
+            try {
+                const storeResult = await bridge.bridgeStoreEntry({
+                    key,
+                    value: chunk.text,
+                    namespace: namespace(scope),
+                    generateEmbeddingFlag: true,
+                    tags: [
+                        'document',
+                        ext,
+                        `src:${resolved}`,
+                        // RCL-10: the chunk's span against the extracted text, so a search
+                        // hit can cite a passage without re-reading the document.
+                        spanTag(chunk.startChar, chunk.endChar),
+                        ...(canonicalUrl ? [`url:${canonicalUrl}`] : []),
+                    ],
+                    upsert: true,
+                    dbPath: storeDbPath(scope),
+                });
+                if (storeResult?.success)
+                    indexed++;
+            }
+            catch (e) {
+                if (process.env.DEBUG || process.env.MONOMIND_DEBUG)
+                    console.error(`[ingestDocument] failed to store chunk ${chunk.chunkIndex} of ${resolved}:`, e);
+            }
+        }
+    }
+    // Commit the document version ONLY when EVERY chunk stored. Recording the
+    // content hash after a partial store was the worse half of this bug: the
+    // hash check above then skipped the file on every future ingest, so the
+    // chunks that failed were never retried — a permanently, silently
+    // half-indexed document feeding knowledge retrieval with no signal at all.
+    // (Total failure was already handled; partial success was not.)
+    //
+    // Not committing is what makes a retry work: chunk keys are
+    // `doc:<contentHash>:<index>` and stores are upserts, so re-ingesting the
+    // same bytes rewrites the same keys and fills the gaps. Until it succeeds the
+    // partially-written chunks sit under a hash that is not live, and superseded
+    // filtering keeps them out of search (see `liveContentHashes`).
+    const complete = indexed === chunks.length;
+    if (complete) {
+        appendMetadata(rootDir, {
+            filePath: resolved,
+            contentHash: hash,
+            chunkCount: indexed,
+            indexedAt: new Date().toISOString(),
+            scope,
+            size: stat.size,
+            version,
+            ...(supersedes ? { supersedes } : {}),
+            ...(canonicalUrl ? { canonicalUrl } : {}),
+            ...(provenance ? { provenance } : {}),
+        });
+        // A re-capture of the same page arrives at a NEW path, so the previous
+        // version's record is a different (filePath, scope) key and survives
+        // last-wins — leaving its contentHash live and its chunks answering
+        // searches forever. Tombstone it so it leaves the live-hash set, which is
+        // exactly how a same-path re-ingest already retires its predecessor.
+        //
+        // AFTER the append, never before: retiring the old version before the
+        // replacement is known to have landed is the failure mode the partial-store
+        // fix above exists to prevent.
+        if (existing && existing.filePath !== resolved) {
+            removeMetadataEntry(rootDir, existing.filePath, scope);
+        }
+        // RCL-04: the reader's own highlights, stored as notes linked to this
+        // document — see knowledge/highlights.ts. AFTER the version is committed,
+        // because a note pointing at an uncommitted version would be a citation
+        // into something search cannot return.
+        //
+        // Wrapped, and never fatal: a malformed highlight, a failed note store,
+        // anything at all here must not cost someone their capture. The page is
+        // already committed above; the annotations are a bonus that reports its
+        // own shortfall through `IngestResult.highlights`.
+        try {
+            const { ingestHighlights, readHighlightsFor } = await import('./highlights.js');
+            const found = readHighlightsFor(resolved);
+            if (found.length) {
+                const result = await ingestHighlights({
+                    filePath: resolved,
+                    scope,
+                    contentHash: hash,
+                    text: fullContent,
+                    ...(canonicalUrl ? { canonicalUrl } : {}),
+                    ...(provenance ? { provenance } : {}),
+                }, { highlights: found, ...(storeDbPath(scope) ? { dbPath: storeDbPath(scope) } : {}) });
+                highlights = {
+                    found: result.found,
+                    stored: result.stored,
+                    anchored: result.anchored,
+                    ...(result.error ? { error: result.error } : {}),
+                };
+            }
+        }
+        catch (e) {
+            if (process.env.DEBUG || process.env.MONOMIND_DEBUG)
+                console.error(`[ingestDocument] highlights for ${resolved}:`, e);
+            highlights = { found: 0, stored: 0, anchored: 0, error: String(e) };
+        }
+    }
+    return {
+        filePath: resolved,
+        chunksIndexed: indexed,
+        scope,
+        skipped: false,
+        ...(complete ? { version, ...(supersedes ? { supersedes } : {}) } : {}),
+        ...(provenance ? { provenance } : {}),
+        ...(highlights ? { highlights } : {}),
+        ...(complete
+            ? {}
+            : indexed > 0
+                ? {
+                    partial: true,
+                    error: `partial store: ${indexed}/${chunks.length} chunks — version not committed, re-ingest to repair`,
+                }
+                : {
+                    error: bridge
+                        ? 'all chunk stores failed'
+                        : 'memory bridge unavailable — nothing indexed',
+                }),
+    };
+}
+export async function ingestDirectory(dirPath, scope = 'shared', opts) {
+    const scanDir = path.resolve(dirPath);
+    const rootDir = path.resolve(opts?.rootDir ?? getProjectRoot());
+    const files = [];
+    function walk(dir, depth = 0) {
+        if (depth > 10)
+            return;
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            // Skip dotfiles/dot-dirs (incl. exFAT `._*` junk) — except `.monodesign`,
+            // whose critique snapshots are markdown worth surfacing in the Second Brain.
+            if (entry.name.startsWith('.') && entry.name !== '.monodesign')
+                continue;
+            const full = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (!IGNORE_DIRS.has(entry.name))
+                    walk(full, depth + 1);
+            }
+            else if (entry.isFile()) {
+                const ext = path.extname(entry.name).toLowerCase();
+                if (DOC_EXTENSIONS.has(ext))
+                    files.push(full);
+            }
+        }
+    }
+    walk(scanDir);
+    const metadataCache = readMetadata(rootDir);
+    const result = {
+        filesProcessed: 0,
+        filesSkipped: 0,
+        totalChunks: 0,
+        errors: [],
+        results: [],
+    };
+    for (let i = 0; i < files.length; i++) {
+        opts?.onProgress?.(files[i], i, files.length);
+        const r = await ingestDocument(files[i], scope, rootDir, metadataCache);
+        result.results.push(r);
+        if (r.skipped) {
+            result.filesSkipped++;
+        }
+        else {
+            result.filesProcessed++;
+            result.totalChunks += r.chunksIndexed;
+        }
+        if (r.error && !r.skipped) {
+            result.errors.push(`${r.filePath}: ${r.error}`);
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=document-ingest.js.map

package/dist/src/knowledge/document-ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-ingest.js","sourceRoot":"","sources":["../../../src/knowledge/document-ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAE/E,iFAAiF;AACjF,iFAAiF;AACjF,iFAAiF;AACjF,kCAAkC;AAClC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAkB,MAAM,wBAAwB,CAAC;AACrF,OAAO,EACL,cAAc,EACd,cAAc,EACd,YAAY,EACZ,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEpG,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IAC1B,cAAc;IACd,MAAM;IACN,MAAM;IACN,WAAW;IACX,SAAS;IACT,OAAO;IACP,aAAa;IACb,OAAO;IACP,QAAQ;CACT,CAAC,CAAC;AACH,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;AAE/C,4EAA4E;AAC5E,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,IAAI,CAAC,KAAK;QACpB,OAAO,EAAE,IAAI,CAAC,SAAS;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,KAAK,GAAG,QAAQ,EAChB,OAAO,GAAG,cAAc,EAAE,EAC1B,cAA+B;IAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAEjD,0EAA0E;IAC1E,oEAAoE;IACpE,yEAAyE;IACzE,wEAAwE;IACxE,yEAAyE;IACzE,yEAAyE;IACzE,EAAE;IACF,0EAA0E;IAC1E,6EAA6E;IAC7E,2EAA2E;IAC3E,6BAA6B;IAC7B,EAAE;IACF,6EAA6E;IAC7E,4EAA4E;IAC5E,0EAA0E;IAC1E,IAAI,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,aAAa,EAAE,CAAC;YAChB,KAAK;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,2BAA2B;SACnC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,aAAa,EAAE,CAAC;YAChB,KAAK;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,0BAA0B,GAAG,EAAE;SACvC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;QAC9B,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,aAAa,EAAE,CAAC;YAChB,KAAK;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,wBAAwB;SAChC,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,yEAAyE;IACzE,mEAAmE;IACnE,0EAA0E;IAC1E,uEAAuE;IACvE,4DAA4D;IAC5D,MAAM,eAAe,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxE,IAAI,eAAe,IAAI,eAAe,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAI,kBAAwC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7F,MAAM,kBAAkB,GACtB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,KAAK,sBAAsB;YACzD,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC;QAChG,IAAI,QAAQ,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACpC,OAAO;gBACL,QAAQ,EAAE,QAAQ;gBAClB,aAAa,EAAE,CAAC;gBAChB,KAAK;gBACL,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,qBAAqB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,6BAA6B;aACxF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,4DAA4D;IAC5D,2DAA2D;IAC3D,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACtC,OAAO,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,cAAc,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;IAErD,6EAA6E;IAC7E,2EAA2E;IAC3E,gDAAgD;IAChD,MAAM,UAAU,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAEpD,0EAA0E;IAC1E,wEAAwE;IACxE,iDAAiD;IACjD,MAAM,QAAQ,GACZ,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;QAC9D,CAAC,YAAY;YACX,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,YAAY,KAAK,YAAY,CAAC;YACxE,CAAC,CAAC,SAAS,CAAC,CAAC;IACjB,IAAI,WAAmB,CAAC;IAExB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACpC,WAAW,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IAC7F,CAAC;IAED,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,aAAa,EAAE,CAAC;YAChB,KAAK;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,mBAAmB;SAC3B,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IAEtC,4EAA4E;IAC5E,kEAAkE;IAClE,IAAI,QAAQ,IAAI,QAAQ,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;QAC9C,OAAO;YACL,QAAQ,EAAE,QAAQ;YAClB,aAAa,EAAE,QAAQ,CAAC,UAAU;YAClC,KAAK;YACL,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI;YACf,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtC,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,QAAQ,EAAE,WAAW,IAAI,SAAS,CAAC;IAEtD,8EAA8E;IAC9E,4EAA4E;IAC5E,8EAA8E;IAC9E,wEAAwE;IACxE,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,KAAK,GAAG,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC;IACrC,MAAM,SAAS,GAAgB,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IACvE,+FAA+F;IAC/F,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,UAAsC,CAAC;IAE3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAE9C,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC;oBAChD,GAAG;oBACH,KAAK,EAAE,KAAK,CAAC,IAAI;oBACjB,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC;oBAC3B,qBAAqB,EAAE,IAAI;oBAC3B,IAAI,EAAE;wBACJ,UAAU;wBACV,GAAG;wBACH,OAAO,QAAQ,EAAE;wBACjB,mEAAmE;wBACnE,0DAA0D;wBAC1D,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC;wBACvC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;qBACjD;oBACD,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC;iBAC3B,CAAC,CAAC;gBACH,IAAI,WAAW,EAAE,OAAO;oBAAE,OAAO,EAAE,CAAC;YACtC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;oBACjD,OAAO,CAAC,KAAK,CACX,0CAA0C,KAAK,CAAC,UAAU,OAAO,QAAQ,GAAG,EAC5E,CAAC,CACF,CAAC;YACN,CAAC;QACH,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,yEAAyE;IACzE,wEAAwE;IACxE,kEAAkE;IAClE,2EAA2E;IAC3E,gEAAgE;IAChE,EAAE;IACF,4DAA4D;IAC5D,0EAA0E;IAC1E,8EAA8E;IAC9E,6EAA6E;IAC7E,gEAAgE;IAChE,MAAM,QAAQ,GAAG,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC;IAC3C,IAAI,QAAQ,EAAE,CAAC;QACb,cAAc,CAAC,OAAO,EAAE;YACtB,QAAQ,EAAE,QAAQ;YAClB,WAAW,EAAE,IAAI;YACjB,UAAU,EAAE,OAAO;YACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO;YACP,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtC,CAAC,CAAC;QAEH,uEAAuE;QACvE,qEAAqE;QACrE,oEAAoE;QACpE,0EAA0E;QAC1E,qEAAqE;QACrE,EAAE;QACF,sEAAsE;QACtE,4EAA4E;QAC5E,+BAA+B;QAC/B,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/C,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC;QAED,sEAAsE;QACtE,0EAA0E;QAC1E,wEAAwE;QACxE,uCAAuC;QACvC,EAAE;QACF,wEAAwE;QACxE,wEAAwE;QACxE,wEAAwE;QACxE,mDAAmD;QACnD,IAAI,CAAC;YACH,MAAM,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAChF,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAC1C,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,MAAM,MAAM,GAAG,MAAM,gBAAgB,CACnC;oBACE,QAAQ,EAAE,QAAQ;oBAClB,KAAK;oBACL,WAAW,EAAE,IAAI;oBACjB,IAAI,EAAE,WAAW;oBACjB,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACtC,EACD,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CACrF,CAAC;gBACF,UAAU,GAAG;oBACX,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACjD,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;gBACjD,OAAO,CAAC,KAAK,CAAC,mCAAmC,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC;YACnE,UAAU,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,QAAQ;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK;QACL,OAAO,EAAE,KAAK;QACd,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,GAAG,CAAC,QAAQ;YACV,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,OAAO,GAAG,CAAC;gBACX,CAAC,CAAC;oBACE,OAAO,EAAE,IAAI;oBACb,KAAK,EAAE,kBAAkB,OAAO,IAAI,MAAM,CAAC,MAAM,sDAAsD;iBACxG;gBACH,CAAC,CAAC;oBACE,KAAK,EAAE,MAAM;wBACX,CAAC,CAAC,yBAAyB;wBAC3B,CAAC,CAAC,6CAA6C;iBAClD,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,KAAK,GAAG,QAAQ,EAChB,IAA6F;IAE7F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,IAAI,cAAc,EAAE,CAAC,CAAC;IAChE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,SAAS,IAAI,CAAC,GAAW,EAAE,KAAK,GAAG,CAAC;QAClC,IAAI,KAAK,GAAG,EAAE;YAAE,OAAO;QACvB,IAAI,OAAoB,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,0EAA0E;YAC1E,6EAA6E;YAC7E,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;gBAAE,SAAS;YACzE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;oBAAE,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC1D,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnD,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,CAAC;IAEd,MAAM,aAAa,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAsB;QAChC,cAAc,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;QACxE,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEvB,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,aAAa,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/knowledge/document-pipeline.d.ts

@@ -1,213 +1,31 @@
 /**
- * Document Pipeline — wires text extraction, chunking, embedding, and SQLite storage
- * into an end-to-end ingest/search/export pipeline for the Second Brain.
+ * Document Pipeline — the Second Brain's end-to-end ingest/search/export path,
+ * and the single import point for it.
+ *
+ * The implementation lives in focused modules; this file is the public surface
+ * every caller (CLI commands, MCP tools, the dashboard, the eval harness)
+ * imports from, so the map is here:
+ *
+ *  - `document-types.ts`    the shapes: IngestResult, KnowledgeExcerpt,
+ *                           DocumentMeta, ReconcileReport.
+ *  - `document-store.ts`    which store a scope writes to, and the lazy memory
+ *                           bridge that reaches it.
+ *  - `document-chunking.ts` text → heading-anchored, context-enriched chunks
+ *                           (and the spans that make one citable).
+ *  - `document-index.ts`    the append-only metadata log: versions,
+ *                           tombstones, lookups, superseded filtering,
+ *                           filesystem reconciliation.
+ *  - `document-ingest.ts`   file → extracted → chunked → stored → committed.
+ *  - `document-search.ts`   query → live chunks, decorated for citation.
+ *  - `okf-bundle.ts`        export to / import from a portable OKF bundle.
  *
  * @module v1/cli/knowledge/document-pipeline
  */
-import { type CaptureProvenance } from './capture-envelope.js';
-import { type ChunkSpan } from './citation.js';
-export interface IngestResult {
-    filePath: string;
-    chunksIndexed: number;
-    scope: string;
-    skipped: boolean;
-    error?: string;
-    /** True when SOME but not all chunks stored. The new version was NOT
-     *  committed: the document keeps whatever version it had before (possibly
-     *  none), and re-ingesting repairs it. Absent means the ingest was complete —
-     *  a caller must not read `chunksIndexed > 0` alone as success. */
-    partial?: boolean;
-    /** RCL-06: the document was already indexed at this exact extracted-text
-     *  hash, so nothing was written. Reported as "unchanged" rather than as a
-     *  skip with an error, because nothing went wrong. */
-    unchanged?: boolean;
-    /** RCL-06: 1 for a first ingest, incremented for each stored revision. */
-    version?: number;
-    /** RCL-06: contentHash of the version this one replaced, when there was one. */
-    supersedes?: string;
-    /** RCL-07: provenance read from the capture envelope's `meta.json`. */
-    provenance?: CaptureProvenance;
-}
-export interface BatchIngestResult {
-    filesProcessed: number;
-    filesSkipped: number;
-    totalChunks: number;
-    errors: string[];
-    results: IngestResult[];
-}
-export interface KnowledgeExcerpt {
-    /** Memory entry id — pass back to memory_feedback/bridgeApplyFeedback to rate usefulness. */
-    id: string;
-    filePath: string;
-    text: string;
-    similarity: number;
-    chunkIndex: number;
-    scope: string;
-    /** True when this chunk belongs to a document version that has since been
-     *  re-ingested (its contentHash is no longer the file's current one). Only
-     *  ever set when the caller opted into `includeSuperseded`. */
-    superseded?: boolean;
-    /** RCL-07: capture provenance, when the document came from a capture
-     *  envelope — this is what lets a result cite the page it came from
-     *  instead of a path under `~/.monomind/inbox`. */
-    provenance?: CaptureProvenance;
-    /** RCL-10: the chunk's character span against the extracted text. Absent
-     *  for chunks stored before span tags existed — re-ingest to get them. */
-    startChar?: number;
-    endChar?: number;
-    /** RCL-10: `<hash12>#<start>-<end>`, resolvable with `monomind doc cite`. */
-    anchor?: string;
-}
-export interface DocumentMeta {
-    filePath: string;
-    contentHash: string;
-    chunkCount: number;
-    indexedAt: string;
-    scope: string;
-    size: number;
-    /** RCL-06: identity URL for a captured page (`canonicalUrl`, else `url`,
-     *  fragment stripped). Absent for ordinary files on disk. */
-    canonicalUrl?: string;
-    /** RCL-06: 1 for a first ingest, +1 per stored revision. */
-    version?: number;
-    /** RCL-06: contentHash of the version this record replaced. Walk it back
-     *  through `listDocumentVersions` to reach the older ones. */
-    supersedes?: string;
-    /** RCL-07: the capture envelope's `meta.json`, normalized. */
-    provenance?: CaptureProvenance;
-}
-export declare function ingestDocument(filePath: string, scope?: string, rootDir?: string, _metadataCache?: DocumentMeta[]): Promise<IngestResult>;
-export declare function ingestDirectory(dirPath: string, scope?: string, opts?: {
-    rootDir?: string;
-    onProgress?: (file: string, done: number, total: number) => void;
-}): Promise<BatchIngestResult>;
-/** Content hashes of the documents currently indexed under `rootDir`. */
-export declare function liveContentHashes(rootDir: string): Set<string>;
-/** True when a metadata log exists under `rootDir`.
- *
- * An empty live-hash set has two very different causes: the log is missing (we
- * cannot judge what is current) or the log exists and every document has been
- * removed (nothing is current). Collapsing them made `doc remove` of the LAST
- * document a no-op — the tombstoned chunks came straight back in search.
- *
- * Reads the path directly instead of via `metadataPath`, which mkdir's. */
-export declare function hasKnowledgeMetadata(rootDir: string): boolean;
-/**
- * True when `key` is a document chunk whose version is no longer current.
- * Non-`doc:` keys are never superseded. When no metadata is available nothing
- * is filtered, because "no metadata" must not read as "everything is stale".
- *
- * `metadataPresent` defaults to the old `live.size > 0` heuristic so existing
- * two-argument callers keep their exact behaviour; pass `hasKnowledgeMetadata`
- * to also filter correctly once the last document has been removed.
- */
-export declare function isSupersededKey(key: string, live: Set<string>, metadataPresent?: boolean): boolean;
-export declare function supersededOverfetchLimit(limit: number, live: Set<string>): number;
-export declare function searchKnowledge(query: string, opts?: {
-    scope?: string;
-    limit?: number;
-    minScore?: number;
-    rootDir?: string;
-    /** which store(s): project-only, global-only, or both (default). */
-    store?: 'project' | 'global' | 'all';
-    /** Return chunks from superseded document versions too, flagged
-     *  `superseded: true`. Default false — see the note above `liveContentHashes`. */
-    includeSuperseded?: boolean;
-    /** Skip cross-encoder reranking. Default false. */
-    skipRerank?: boolean;
-}): Promise<KnowledgeExcerpt[]>;
-export declare function listDocuments(rootDir?: string, scope?: string): DocumentMeta[];
-/**
- * Every recorded version of one document, oldest first (RCL-06).
- *
- * `target` is either an indexed file path or a capture's `canonicalUrl`. Read
- * from the append-only log rather than the last-wins view, which is how a
- * superseded version stays addressable after its record has been replaced or
- * tombstoned.
- *
- * Best-effort by design: `readMetadata` compacts the log once it passes 1MB
- * and keeps only live records, so history older than a compaction is gone.
- * The `supersedes` pointer on each record is the durable part.
- */
-export declare function listDocumentVersions(rootDir?: string, target?: string, scope?: string): DocumentMeta[];
-/** The store root for a scope: the global brain for `global`, else the project. */
-export declare function getKnowledgeRoot(scope?: string, rootDir?: string): string;
-/** Chunk spans for a text, using the SAME chunker the ingest used — this is
- *  what makes a chunk index resolvable back to a character range without
- *  storing the text twice. */
-export declare function chunkSpans(text: string): Promise<ChunkSpan[]>;
-/**
- * The indexed record for whatever a caller wrote down: a file path, a
- * capture's `canonicalUrl` (fragment ignored), or the `<scope>:<path>` docId.
- *
- * Live records win. A superseded version is only reached through the
- * append-only log, and only when nothing live matches — otherwise citing a
- * re-captured page would resolve against the version it replaced.
- */
-export declare function findDocumentRecord(rootDir?: string, target?: string, scope?: string): DocumentMeta | undefined;
-export declare function removeDocument(filePath: string, scope?: string, rootDir?: string): Promise<void>;
-/**
- * True for macOS AppleDouble sidecars (`._name`).
- *
- * Matches on the BASENAME PREFIX only. A legitimate document may contain `._`
- * elsewhere in its name (`v1._2-release.md`), or live under a dot-directory
- * that is deliberately indexed (`.monodesign/` critique snapshots), and
- * neither may be rejected.
- */
-export declare function isResourceFork(filePath: string): boolean;
-export interface ReconcileReport {
-    /** Indexed documents whose source file is no longer on disk. */
-    missing: DocumentMeta[];
-    /** Total index entries examined. */
-    scanned: number;
-    /** False for a dry run — the default. */
-    applied: boolean;
-    /** Entries actually tombstoned. Always 0 when `applied` is false. */
-    removed: number;
-    /** Where removed records were archived, when anything was removed. */
-    archivePath?: string;
-}
-/**
- * Reconcile the document index against the filesystem: find index entries whose
- * source file no longer exists and, only when explicitly asked, tombstone them.
- *
- * WHY — `removeDocument` only ever tombstoned metadata, and nothing has ever
- * compared the index against the disk, so a deleted file stayed searchable
- * forever. Measured 2026-07-28: 109 of 257 live entries (42.4%) had no file
- * behind them, including `docs/concepts/memory.md`. The Second Brain was
- * answering questions from documents the user had deleted.
- *
- * WHY IT IS THIS CAUTIOUS — "drop the index entry when the file is missing" is
- * a rule with a known catastrophic reading. A missing file is also an unmounted
- * volume, a checked-out branch, a partial clone, or a permissions failure. Two
- * guards were tried against real data and REJECTED; they are recorded here so
- * they are not re-proposed:
- *
- *   - "abort if >50% of entries are missing" — the real, legitimate missing
- *     fraction was 42.4%, so the threshold never fires in the one case we have.
- *     Any threshold that would have blocked this reconcile is fitted to nothing.
- *   - "only reconcile when the parent directory still exists" — 26 of the 109
- *     missing files had no parent directory, because `docs/concepts`,
- *     `docs/adrs` and `docs/commands` were legitimately deleted wholesale. A
- *     deleted directory and an unmounted volume are indistinguishable there.
- *
- * What does discriminate is the ROOT. An intact, readable root carrying a
- * metadata log means the tree is genuinely present, so a missing file is
- * genuinely gone. A missing root means nothing beneath it is knowable and
- * nothing may be removed — hence throw rather than reconcile.
- *
- * Removal tombstones metadata; it does not delete store rows. Chunks stay on
- * disk and fall out of search through the existing superseded filter, which
- * keeps this consistent with the mark-don't-destroy rule and leaves the whole
- * operation reversible from the archive.
- */
-export declare function reconcileIndex(rootDir?: string, opts?: {
-    scope?: string;
-    apply?: boolean;
-}): Promise<ReconcileReport>;
-export declare function exportToOKF(outputDir: string, rootDir?: string, scope?: string): Promise<{
-    exported: number;
-    outputDir: string;
-}>;
-export declare function importFromOKF(bundleDir: string, scope?: string, rootDir?: string): Promise<BatchIngestResult>;
+export { chunkSpans } from './document-chunking.js';
+export { findDocumentRecord, hasKnowledgeMetadata, isResourceFork, isSupersededKey, listDocuments, listDocumentVersions, liveContentHashes, reconcileIndex, removeDocument, supersededOverfetchLimit, } from './document-index.js';
+export { ingestDirectory, ingestDocument } from './document-ingest.js';
+export { searchKnowledge } from './document-search.js';
+export { getKnowledgeRoot } from './document-store.js';
+export type { BatchIngestResult, DocumentMeta, IngestResult, KnowledgeExcerpt, ReconcileReport, } from './document-types.js';
+export { exportToOKF, importFromOKF } from './okf-bundle.js';
 //# sourceMappingURL=document-pipeline.d.ts.map

package/dist/src/knowledge/document-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"document-pipeline.d.ts","sourceRoot":"","sources":["../../../src/knowledge/document-pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAaH,OAAO,EACL,KAAK,iBAAiB,EAMvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,SAAS,EAAyC,MAAM,eAAe,CAAC;AAyRtF,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;uEAGmE;IACnE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;0DAEsD;IACtD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0EAA0E;IAC1E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,6FAA6F;IAC7F,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd;;mEAE+D;IAC/D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;uDAEmD;IACnD,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B;8EAC0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb;iEAC6D;IAC7D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;kEAC8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAuJD,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,KAAK,SAAW,EAChB,OAAO,SAAmB,EAC1B,cAAc,CAAC,EAAE,YAAY,EAAE,GAC9B,OAAO,CAAC,YAAY,CAAC,CA+OvB;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,KAAK,SAAW,EAChB,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,GAC5F,OAAO,CAAC,iBAAiB,CAAC,CAyD5B;AAwBD,yEAAyE;AACzE,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAI9D;AAED;;;;;;;2EAO2E;AAC3E,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAE7D;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EACjB,eAAe,UAAgB,GAC9B,OAAO,CAIT;AAQD,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAGjF;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;IACL,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,KAAK,CAAC,EAAE,SAAS,GAAG,QAAQ,GAAG,KAAK,CAAC;IACrC;sFACkF;IAClF,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,mDAAmD;IACnD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,GACA,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAkH7B;AAID,wBAAgB,aAAa,CAAC,OAAO,SAAmB,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE,CAGxF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,SAAmB,EAC1B,MAAM,CAAC,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GACb,YAAY,EAAE,CAUhB;AAID,mFAAmF;AACnF,wBAAgB,gBAAgB,CAAC,KAAK,SAAW,EAAE,OAAO,SAAmB,GAAG,MAAM,CAErF;AAED;;8BAE8B;AAC9B,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAOnE;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,SAAmB,EAC1B,MAAM,SAAK,EACX,KAAK,CAAC,EAAE,MAAM,GACb,YAAY,GAAG,SAAS,CA0B1B;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,KAAK,SAAW,EAChB,OAAO,SAAmB,GACzB,OAAO,CAAC,IAAI,CAAC,CAIf;AAID;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,OAAO,EAAE,OAAO,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAsB,cAAc,CAClC,OAAO,SAAmB,EAC1B,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,GACzC,OAAO,CAAC,eAAe,CAAC,CAwC1B;AAID,wBAAsB,WAAW,CAC/B,SAAS,EAAE,MAAM,EACjB,OAAO,SAAmB,EAC1B,KAAK,SAAW,GACf,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CA8DlD;AAID,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,KAAK,SAAW,EAChB,OAAO,SAAmB,GACzB,OAAO,CAAC,iBAAiB,CAAC,CA4B5B"}
+{"version":3,"file":"document-pipeline.d.ts","sourceRoot":"","sources":["../../../src/knowledge/document-pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,eAAe,EACf,aAAa,EACb,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,wBAAwB,GACzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,YAAY,EACV,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC"}