@monoes/monomindcli 2.14.1 → 2.15.0

package/dist/src/knowledge/document-index.js

@@ -0,0 +1,314 @@
+/**
+ * The document index — the append-only metadata log that records which
+ * documents are indexed, at which version, and whether that is still true.
+ *
+ * The log (`.monomind/knowledge/doc-metadata.jsonl`) is the pipeline's source
+ * of truth. Everything here reads or writes it:
+ *
+ *  - WRITE: append a version record, tombstone a removed one.
+ *  - READ:  the last-wins live view, the full history, and lookups by path,
+ *           canonical URL or `<scope>:<path>` docId.
+ *  - JUDGE: which content hashes are current, which chunk keys are therefore
+ *           superseded, and which indexed files no longer exist on disk.
+ *
+ * Nothing here touches the vector store. Removal tombstones metadata and lets
+ * the superseded filter hide the chunks — mark, don't destroy.
+ *
+ * Split out of document-pipeline.ts.
+ *
+ * @module v1/cli/knowledge/document-index
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+// Static import is safe and deliberate: memory-bridge imports only node builtins
+// at module scope (everything heavy is lazy), and the project-root rule must not
+// be duplicated — two copies of "which directory is this project" is exactly the
+// bug this default exists to fix.
+import { getProjectRoot } from '../memory/memory-bridge.js';
+export const METADATA_FILE = 'doc-metadata.jsonl';
+// ── Metadata log ───────────────────────────────────────────────────
+export function metadataPath(rootDir) {
+    const dir = path.join(rootDir, '.monomind', 'knowledge');
+    fs.mkdirSync(dir, { recursive: true });
+    return path.join(dir, METADATA_FILE);
+}
+/** Every record ever appended, in order, including superseded versions and
+ *  removal tombstones. Reads the path directly instead of via `metadataPath`,
+ *  which mkdir's. */
+export function readMetadataLog(rootDir) {
+    const file = path.join(rootDir, '.monomind', 'knowledge', METADATA_FILE);
+    if (!fs.existsSync(file))
+        return [];
+    const out = [];
+    for (const line of fs.readFileSync(file, 'utf-8').split('\n')) {
+        if (!line.trim())
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            /* torn line */
+        }
+    }
+    return out;
+}
+export function readMetadata(rootDir) {
+    const file = metadataPath(rootDir);
+    if (!fs.existsSync(file))
+        return [];
+    // Last-wins per (filePath, scope): the file is append-only under concurrent
+    // ingests (session-start detached reindex + a manual `doc ingest` can
+    // overlap), so duplicates are expected and the newest record is truth.
+    // Corrupt lines (torn concurrent writes) are skipped, not fatal.
+    const latest = new Map();
+    for (const l of fs.readFileSync(file, 'utf-8').split('\n')) {
+        if (!l.trim())
+            continue;
+        try {
+            const m = JSON.parse(l);
+            latest.set(`${m.filePath} ${m.scope}`, m);
+        }
+        catch {
+            /* torn line */
+        }
+    }
+    // chunkCount -1 records are removal tombstones (see removeMetadataEntry)
+    const live = [...latest.values()].filter((m) => m.chunkCount >= 0);
+    // Occasional compaction: append-only + tombstones grow without bound; when
+    // the log gets big, rewrite it deduped (atomic rename — a concurrent append
+    // in the tiny window loses only its own record and self-heals on re-ingest).
+    try {
+        if (fs.statSync(file).size > 1024 * 1024) {
+            const tmp = `${file}.${process.pid}.compact`;
+            fs.writeFileSync(tmp, live.map((r) => JSON.stringify(r)).join('\n') + (live.length ? '\n' : ''), 'utf-8');
+            fs.renameSync(tmp, file);
+        }
+    }
+    catch {
+        /* compaction is best-effort */
+    }
+    return live;
+}
+export function appendMetadata(rootDir, meta) {
+    fs.appendFileSync(metadataPath(rootDir), `${JSON.stringify(meta)}\n`, 'utf-8');
+}
+export function removeMetadataEntry(rootDir, filePath, scope) {
+    const file = metadataPath(rootDir);
+    if (!fs.existsSync(file))
+        return;
+    // Tombstone by APPEND (chunkCount -1) instead of read-filter-rewrite — the
+    // rewrite raced concurrent appends and silently dropped them.
+    appendMetadata(rootDir, {
+        filePath,
+        scope,
+        contentHash: '',
+        chunkCount: -1,
+        indexedAt: new Date().toISOString(),
+        size: 0,
+    });
+}
+// ── Superseded-version filtering ───────────────────────────────────
+//
+// Chunk keys are `doc:<contentHash>:<chunkIndex>`. Re-ingesting a changed file
+// produces a NEW contentHash, so its chunks land under new keys — the previous
+// version's rows are never touched (`removeDocument` only tombstones metadata;
+// the bridge exposes no delete-by-prefix). The store therefore accumulates every
+// version a document has ever had, and all of them stay searchable.
+//
+// Measured on this repo's own store (2026-07-26): 9,067 `doc:`-keyed rows in
+// `knowledge:shared` spanning 798 distinct content hashes, of which only 139
+// are current — 8,542 rows (94.2%) are orphaned older versions.
+//
+// Nothing is deleted here. The current-hash set from doc-metadata.jsonl is used
+// to decide what search RETURNS; `includeSuperseded` puts the old versions back
+// (flagged `superseded: true`) for anyone who wants document history.
+/** Content hashes of the documents currently indexed under `rootDir`. */
+export function liveContentHashes(rootDir) {
+    const live = new Set();
+    for (const m of readMetadata(rootDir))
+        if (m.contentHash)
+            live.add(m.contentHash);
+    return live;
+}
+/** True when a metadata log exists under `rootDir`.
+ *
+ * An empty live-hash set has two very different causes: the log is missing (we
+ * cannot judge what is current) or the log exists and every document has been
+ * removed (nothing is current). Collapsing them made `doc remove` of the LAST
+ * document a no-op — the tombstoned chunks came straight back in search.
+ *
+ * Reads the path directly instead of via `metadataPath`, which mkdir's. */
+export function hasKnowledgeMetadata(rootDir) {
+    return fs.existsSync(path.join(rootDir, '.monomind', 'knowledge', METADATA_FILE));
+}
+/**
+ * True when `key` is a document chunk whose version is no longer current.
+ * Non-`doc:` keys are never superseded. When no metadata is available nothing
+ * is filtered, because "no metadata" must not read as "everything is stale".
+ *
+ * `metadataPresent` defaults to the old `live.size > 0` heuristic so existing
+ * two-argument callers keep their exact behaviour; pass `hasKnowledgeMetadata`
+ * to also filter correctly once the last document has been removed.
+ */
+export function isSupersededKey(key, live, metadataPresent = live.size > 0) {
+    if (!key?.startsWith('doc:'))
+        return false;
+    if (!metadataPresent)
+        return false;
+    return !live.has(key.split(':')[1] ?? '');
+}
+/** How many rows to ask the backend for per requested result when superseded
+ *  filtering is active — most rows in a long-lived store are old versions, so
+ *  a 1:1 fetch would return an almost-empty page. */
+const SUPERSEDED_OVERFETCH = 20;
+const SUPERSEDED_OVERFETCH_CAP = 300;
+export function supersededOverfetchLimit(limit, live) {
+    if (live.size === 0)
+        return limit;
+    return Math.min(Math.max(limit * SUPERSEDED_OVERFETCH, limit), SUPERSEDED_OVERFETCH_CAP);
+}
+// ── List / lookup ──────────────────────────────────────────────────
+export function listDocuments(rootDir = getProjectRoot(), scope) {
+    const all = readMetadata(rootDir);
+    return scope ? all.filter((m) => m.scope === scope) : all;
+}
+/**
+ * Every recorded version of one document, oldest first (RCL-06).
+ *
+ * `target` is either an indexed file path or a capture's `canonicalUrl`. Read
+ * from the append-only log rather than the last-wins view, which is how a
+ * superseded version stays addressable after its record has been replaced or
+ * tombstoned.
+ *
+ * Best-effort by design: `readMetadata` compacts the log once it passes 1MB
+ * and keeps only live records, so history older than a compaction is gone.
+ * The `supersedes` pointer on each record is the durable part.
+ */
+export function listDocumentVersions(rootDir = getProjectRoot(), target, scope) {
+    const resolved = target ? path.resolve(target) : undefined;
+    return readMetadataLog(rootDir)
+        .filter((m) => m.chunkCount >= 0)
+        .filter((m) => !scope || m.scope === scope)
+        .filter((m) => !target || m.filePath === resolved || m.filePath === target || m.canonicalUrl === target)
+        .sort((a, b) => (a.version ?? 0) - (b.version ?? 0) || a.indexedAt.localeCompare(b.indexedAt));
+}
+/**
+ * The indexed record for whatever a caller wrote down: a file path, a
+ * capture's `canonicalUrl` (fragment ignored), or the `<scope>:<path>` docId.
+ *
+ * Live records win. A superseded version is only reached through the
+ * append-only log, and only when nothing live matches — otherwise citing a
+ * re-captured page would resolve against the version it replaced.
+ */
+export function findDocumentRecord(rootDir = getProjectRoot(), target = '', scope) {
+    const raw = target.trim();
+    if (!raw)
+        return undefined;
+    // `<scope>:<path>` — but not `https://…`, whose colon is a URL scheme.
+    const scoped = /^([a-z][a-z0-9_-]*):(?!\/\/)(.+)$/i.exec(raw);
+    const candidates = scoped ? [raw, scoped[2]] : [raw];
+    const hash = raw.indexOf('#');
+    if (hash > 0)
+        candidates.push(raw.slice(0, hash));
+    const resolvedPaths = new Set(candidates.map((c) => path.resolve(c)));
+    const wanted = new Set(candidates);
+    const scopeFilter = scope ?? (scoped ? scoped[1] : undefined);
+    const matches = (m) => {
+        if (scopeFilter && m.scope !== scopeFilter && !wanted.has(m.filePath))
+            return false;
+        if (wanted.has(m.filePath) || resolvedPaths.has(path.resolve(m.filePath)))
+            return true;
+        return !!m.canonicalUrl && wanted.has(m.canonicalUrl);
+    };
+    const live = readMetadata(rootDir).filter(matches);
+    if (live.length) {
+        return live.sort((a, b) => (b.version ?? 0) - (a.version ?? 0))[0];
+    }
+    const historical = readMetadataLog(rootDir)
+        .filter((m) => m.chunkCount >= 0)
+        .filter(matches);
+    return historical.sort((a, b) => (b.version ?? 0) - (a.version ?? 0))[0];
+}
+export async function removeDocument(filePath, scope = 'shared', rootDir = getProjectRoot()) {
+    removeMetadataEntry(rootDir, path.resolve(filePath), scope);
+    // SQLite cleanup: bridge doesn't expose delete-by-key, so metadata removal is sufficient.
+    // Orphaned SQLite entries get swept on next full re-index or TTL expiry.
+}
+// ── Filesystem reconciliation (item 4b-i) ──────────────────────────
+/**
+ * True for macOS AppleDouble sidecars (`._name`).
+ *
+ * Matches on the BASENAME PREFIX only. A legitimate document may contain `._`
+ * elsewhere in its name (`v1._2-release.md`), or live under a dot-directory
+ * that is deliberately indexed (`.monodesign/` critique snapshots), and
+ * neither may be rejected.
+ */
+export function isResourceFork(filePath) {
+    return path.basename(filePath).startsWith('._');
+}
+/**
+ * Reconcile the document index against the filesystem: find index entries whose
+ * source file no longer exists and, only when explicitly asked, tombstone them.
+ *
+ * WHY — `removeDocument` only ever tombstoned metadata, and nothing has ever
+ * compared the index against the disk, so a deleted file stayed searchable
+ * forever. Measured 2026-07-28: 109 of 257 live entries (42.4%) had no file
+ * behind them, including `docs/concepts/memory.md`. The Second Brain was
+ * answering questions from documents the user had deleted.
+ *
+ * WHY IT IS THIS CAUTIOUS — "drop the index entry when the file is missing" is
+ * a rule with a known catastrophic reading. A missing file is also an unmounted
+ * volume, a checked-out branch, a partial clone, or a permissions failure. Two
+ * guards were tried against real data and REJECTED; they are recorded here so
+ * they are not re-proposed:
+ *
+ *   - "abort if >50% of entries are missing" — the real, legitimate missing
+ *     fraction was 42.4%, so the threshold never fires in the one case we have.
+ *     Any threshold that would have blocked this reconcile is fitted to nothing.
+ *   - "only reconcile when the parent directory still exists" — 26 of the 109
+ *     missing files had no parent directory, because `docs/concepts`,
+ *     `docs/adrs` and `docs/commands` were legitimately deleted wholesale. A
+ *     deleted directory and an unmounted volume are indistinguishable there.
+ *
+ * What does discriminate is the ROOT. An intact, readable root carrying a
+ * metadata log means the tree is genuinely present, so a missing file is
+ * genuinely gone. A missing root means nothing beneath it is knowable and
+ * nothing may be removed — hence throw rather than reconcile.
+ *
+ * Removal tombstones metadata; it does not delete store rows. Chunks stay on
+ * disk and fall out of search through the existing superseded filter, which
+ * keeps this consistent with the mark-don't-destroy rule and leaves the whole
+ * operation reversible from the archive.
+ */
+export async function reconcileIndex(rootDir = getProjectRoot(), opts) {
+    const apply = opts?.apply === true;
+    // Root guard — the unmounted-volume case. Every file below a missing root
+    // looks deleted, so this must abort rather than reconcile.
+    if (!rootDir || !fs.existsSync(rootDir)) {
+        throw new Error(`reconcileIndex: project root does not exist: ${rootDir} — refusing to reconcile ` +
+            `(an unmounted volume makes every indexed file look deleted)`);
+    }
+    if (!hasKnowledgeMetadata(rootDir)) {
+        throw new Error(`reconcileIndex: no knowledge metadata log under ${rootDir} — refusing to reconcile ` +
+            `("no metadata" must not read as "everything is stale")`);
+    }
+    const records = readMetadata(rootDir).filter((m) => !opts?.scope || m.scope === opts.scope);
+    const missing = records.filter((m) => !fs.existsSync(m.filePath));
+    if (!apply || missing.length === 0) {
+        return { missing, scanned: records.length, applied: apply, removed: 0 };
+    }
+    // Archive BEFORE removing, inside the operation so no caller can bypass it
+    // by forgetting — the same precondition rule the delete path uses.
+    const dir = path.join(rootDir, '.monomind', 'knowledge', 'archive');
+    fs.mkdirSync(dir, { recursive: true });
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const archivePath = path.join(dir, `reconcile-${stamp}.jsonl`);
+    fs.writeFileSync(archivePath, `${missing.map((m) => JSON.stringify(m)).join('\n')}\n`, 'utf-8');
+    let removed = 0;
+    for (const m of missing) {
+        removeMetadataEntry(rootDir, m.filePath, m.scope);
+        removed++;
+    }
+    return { missing, scanned: records.length, applied: true, removed, archivePath };
+}
+//# sourceMappingURL=document-index.js.map

package/dist/src/knowledge/document-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-index.js","sourceRoot":"","sources":["../../../src/knowledge/document-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,iFAAiF;AACjF,iFAAiF;AACjF,iFAAiF;AACjF,kCAAkC;AAClC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAG5D,MAAM,CAAC,MAAM,aAAa,GAAG,oBAAoB,CAAC;AAElD,sEAAsE;AAEtE,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;IACzD,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;AACvC,CAAC;AAED;;qBAEqB;AACrB,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IACzE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,IAAI,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,4EAA4E;IAC5E,sEAAsE;IACtE,uEAAuE;IACvE,iEAAiE;IACjE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;YAAE,SAAS;QACxB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAiB,CAAC;YACxC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;IACnE,2EAA2E;IAC3E,4EAA4E;IAC5E,6EAA6E;IAC7E,IAAI,CAAC;QACH,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACzC,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,UAAU,CAAC;YAC7C,EAAE,CAAC,aAAa,CACd,GAAG,EACH,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EACzE,OAAO,CACR,CAAC;YACF,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAe,EAAE,IAAkB;IAChE,EAAE,CAAC,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAe,EAAE,QAAgB,EAAE,KAAa;IAClF,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO;IACjC,2EAA2E;IAC3E,8DAA8D;IAC9D,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ;QACR,KAAK;QACL,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,CAAC;QACd,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,IAAI,EAAE,CAAC;KACR,CAAC,CAAC;AACL,CAAC;AAED,sEAAsE;AACtE,EAAE;AACF,+EAA+E;AAC/E,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,oEAAoE;AACpE,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,gEAAgE;AAChE,EAAE;AACF,gFAAgF;AAChF,gFAAgF;AAChF,sEAAsE;AAEtE,yEAAyE;AACzE,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC;QAAE,IAAI,CAAC,CAAC,WAAW;YAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;2EAO2E;AAC3E,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;AACpF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,IAAiB,EACjB,eAAe,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC;IAE/B,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3C,IAAI,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED;;qDAEqD;AACrD,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAErC,MAAM,UAAU,wBAAwB,CAAC,KAAa,EAAE,IAAiB;IACvE,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,oBAAoB,EAAE,KAAK,CAAC,EAAE,wBAAwB,CAAC,CAAC;AAC3F,CAAC;AAED,sEAAsE;AAEtE,MAAM,UAAU,aAAa,CAAC,OAAO,GAAG,cAAc,EAAE,EAAE,KAAc;IACtE,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAO,GAAG,cAAc,EAAE,EAC1B,MAAe,EACf,KAAc;IAEd,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,OAAO,eAAe,CAAC,OAAO,CAAC;SAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;SAC1C,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,MAAM,IAAI,CAAC,CAAC,YAAY,KAAK,MAAM,CAC3F;SACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACnG,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAO,GAAG,cAAc,EAAE,EAC1B,MAAM,GAAG,EAAE,EACX,KAAc;IAEd,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC1B,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,uEAAuE;IACvE,MAAM,MAAM,GAAG,oCAAoC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,IAAI,GAAG,CAAC;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,WAAW,GAAG,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,CAAC,CAAe,EAAW,EAAE;QAC3C,IAAI,WAAW,IAAI,CAAC,CAAC,KAAK,KAAK,WAAW,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YAAE,OAAO,KAAK,CAAC;QACpF,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACvF,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACxD,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC;SACxC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;SAChC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,KAAK,GAAG,QAAQ,EAChB,OAAO,GAAG,cAAc,EAAE;IAE1B,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5D,0FAA0F;IAC1F,yEAAyE;AAC3E,CAAC;AAED,sEAAsE;AAEtE;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAO,GAAG,cAAc,EAAE,EAC1B,IAA0C;IAE1C,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC;IAEnC,0EAA0E;IAC1E,2DAA2D;IAC3D,IAAI,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CACb,gDAAgD,OAAO,2BAA2B;YAChF,6DAA6D,CAChE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,mDAAmD,OAAO,2BAA2B;YACnF,wDAAwD,CAC3D,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAElE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IACpE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,KAAK,QAAQ,CAAC,CAAC;IAC/D,EAAE,CAAC,aAAa,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEhG,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,mBAAmB,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AACnF,CAAC"}

package/dist/src/knowledge/document-ingest.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Document ingest — the write side of the pipeline: a file on disk (or a
+ * capture envelope) becomes an indexed, versioned, searchable document.
+ *
+ * One pass per document: guard the input, resolve the capture envelope's
+ * identity, extract and hash the text, chunk and enrich it, store every chunk,
+ * and only then commit the version record. An incomplete store commits
+ * nothing, which is what makes a re-ingest repair it.
+ *
+ * Split out of document-pipeline.ts.
+ *
+ * @module v1/cli/knowledge/document-ingest
+ */
+import type { FileEntry } from '../capabilities/types.js';
+import type { BatchIngestResult, DocumentMeta, IngestResult } from './document-types.js';
+/** @internal — shared with okf-bundle, which extracts text the same way. */
+export declare function toFileEntry(filePath: string): FileEntry;
+export declare function ingestDocument(filePath: string, scope?: string, rootDir?: string, _metadataCache?: DocumentMeta[]): Promise<IngestResult>;
+export declare function ingestDirectory(dirPath: string, scope?: string, opts?: {
+    rootDir?: string;
+    onProgress?: (file: string, done: number, total: number) => void;
+}): Promise<BatchIngestResult>;
+//# sourceMappingURL=document-ingest.d.ts.map

package/dist/src/knowledge/document-ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-ingest.d.ts","sourceRoot":"","sources":["../../../src/knowledge/document-ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAsB1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAgBzF,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAUvD;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,KAAK,SAAW,EAChB,OAAO,SAAmB,EAC1B,cAAc,CAAC,EAAE,YAAY,EAAE,GAC9B,OAAO,CAAC,YAAY,CAAC,CA0RvB;AAED,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,KAAK,SAAW,EAChB,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,GAC5F,OAAO,CAAC,iBAAiB,CAAC,CAyD5B"}