onadit 0.0.1 → 1.0.0

package/dist/src/storage/chunk-backfill.js

@@ -0,0 +1,295 @@
+/**
+ * Owner-only historical conversation chunk backfill.
+ *
+ * Index-time chunking (behind `RIFT_CHUNKING`) only splits NEW conversations as
+ * they are saved/ingested; conversations indexed before the flag was on are still
+ * stored as one whole row each. This module re-chunks those historical
+ * conversations from their raw artifacts so the owner can dogfood retrieval over a
+ * fully-chunked corpus and gather OFF-vs-ON eval evidence before any product
+ * default flip.
+ *
+ * Deliberately CRUDE and owner-grade — NOT the productized beta-user migration:
+ *   - explicit, one-shot (run from the offline CLI; no background daemon job),
+ *   - `dryRun` reports what would change without embedding or writing,
+ *   - lightly throttleable, and safe to Ctrl-C (an `AbortSignal` is checked
+ *     between conversations, so the in-flight conversation's atomic write always
+ *     finishes before we stop — never a half-written set on interrupt),
+ *   - no resumable status UI, pause/cancel, or rollback tooling.
+ *
+ * Data-safety mirrors the save/ingest chunk-write path: every chunk of a
+ * conversation is embedded BEFORE any destructive delete, and the raw artifact
+ * that drove the re-chunk is left on disk, so an interrupt between delete and
+ * insert is fully repairable by re-running (or by reconcile/rebuild).
+ *
+ * Reads are compatible with the `expand` stitching slice: a backfilled set uses
+ * the same `parent_id` / `chunk_index` / `chunk_count` markers and `convChunkId`
+ * scheme, so `detail="full"` reassembles a backfilled conversation through the
+ * exact same path as a natively-chunked one.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { CONVERSATION_SOURCES } from "../providers/types.js";
+import { RIFT_NONE_EMBEDDING_PROVIDER, isPlaceholderEmbeddingProvider, } from "../providers/placeholder-embed.js";
+import { recordEmbed } from "../observability/embedding-events.js";
+import { recordIndexWrite } from "../observability/index-events.js";
+import { eqFilter } from "./filter.js";
+import { getTable } from "./tables.js";
+import { chunkConversation } from "../ingestion/chunk-text.js";
+import { convChunkId, conversationChunkColumns, chunkSetComplete, expectedChunkCount, } from "../ingestion/chunk-meta.js";
+import { conversationContentFingerprint } from "../ingestion/inbox-core/conversation-fingerprint.js";
+import { extractIdFromFilename, compareRawRecencyDesc } from "./rebuild.js";
+const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
+const CONV_TABLES = [
+    "conversations_hot",
+    "conversations_cold",
+];
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Collect the most-recent raw artifact per conversation id across every source.
+ * Same recency policy as reconcile/rebuild (`compareRawRecencyDesc`) so all three
+ * tools canonicalise a duplicated id to the same winner.
+ */
+function collectRawConversations(dataDir) {
+    const byId = new Map();
+    for (const source of ALL_CONV_SOURCES) {
+        const rawDir = path.join(dataDir, "raw", "conversations", source);
+        if (!fs.existsSync(rawDir))
+            continue;
+        for (const file of fs.readdirSync(rawDir)) {
+            if (!file.endsWith(".json"))
+                continue;
+            const id = extractIdFromFilename(file);
+            if (!id)
+                continue;
+            const rawPath = path.join(rawDir, file);
+            const prior = byId.get(id);
+            if (!prior || compareRawRecencyDesc(rawPath, prior) < 0) {
+                byId.set(id, rawPath);
+            }
+        }
+    }
+    return [...byId.entries()].map(([id, rawPath]) => ({ id, rawPath }));
+}
+/**
+ * All currently-indexed rows of a conversation (the whole row OR its chunk set),
+ * gathered across BOTH tiers. The same id can appear in hot AND cold after a
+ * compaction crash ("copied to cold but not yet removed from hot"); we dedupe by
+ * id with the cold copy winning, and pick cold as the write target so the
+ * rechunk collapses the duplicate rather than leaving a stale copy in the other
+ * tier.
+ */
+async function fetchConversationRows(id) {
+    const idFilter = eqFilter("id", id, { validateAsRowId: true });
+    const parentFilter = eqFilter("parent_id", id, { validateAsRowId: true });
+    const uniq = new Map();
+    let hasCold = false;
+    let hasCrossTierRows = false;
+    // Iterate hot → cold so the canonical cold row overwrites its hot duplicate.
+    for (const table of CONV_TABLES) {
+        const handle = getTable(table);
+        // Schema-aware filter. A legacy pre-chunk table (e.g. beta.21) has NO
+        // `parent_id` column; filtering on it would throw. When the column is
+        // absent we match by id only and the rows are necessarily legacy unchunked
+        // (no chunk set can exist without the column). This lets dry-run recon run
+        // against the exact incident shape WITHOUT migrating the columns on — which
+        // is the whole point of the read-only path.
+        const hasParentId = (await handle.schema()).fields.some((f) => f.name === "parent_id");
+        const setFilter = hasParentId
+            ? `(${idFilter}) OR (${parentFilter})`
+            : idFilter;
+        const rows = (await handle
+            .query()
+            .where(setFilter)
+            .toArray());
+        if (rows.length === 0)
+            continue;
+        if (table === "conversations_cold")
+            hasCold = true;
+        for (const r of rows) {
+            // Already seen this id in an earlier tier → it lives in both → duplicate.
+            if (uniq.has(r.id))
+                hasCrossTierRows = true;
+            uniq.set(r.id, r);
+        }
+    }
+    if (uniq.size === 0)
+        return null;
+    return {
+        targetTable: hasCold ? "conversations_cold" : "conversations_hot",
+        rows: [...uniq.values()],
+        hasCrossTierRows,
+    };
+}
+/**
+ * Re-chunk historical conversations from raw into chunk sets. Returns a summary
+ * of what changed. Idempotent: a conversation already in its desired shape (a
+ * complete set of the right count, or a single unmarked row for a short
+ * conversation) is skipped, so re-running only touches the remainder.
+ */
+export async function backfillConversationChunks(opts) {
+    const { embedding, dataDir } = opts;
+    const dryRun = opts.dryRun ?? false;
+    const throttleMs = opts.throttleMs ?? 0;
+    const limit = opts.limit ?? 0;
+    const log = opts.log ?? (() => { });
+    const summary = {
+        scanned: 0,
+        rechunked: 0,
+        alreadyChunked: 0,
+        singleChunk: 0,
+        orphaned: 0,
+        errored: 0,
+        aborted: false,
+    };
+    const raws = collectRawConversations(dataDir);
+    log(`Found ${raws.length} historical conversation(s) to examine.`);
+    for (const { id, rawPath } of raws) {
+        if (opts.signal?.aborted) {
+            summary.aborted = true;
+            log("Aborted — stopping before the next conversation.");
+            break;
+        }
+        if (limit > 0 && summary.rechunked >= limit) {
+            log(`Reached --limit ${limit}; stopping.`);
+            break;
+        }
+        summary.scanned++;
+        let raw;
+        try {
+            raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+        }
+        catch {
+            summary.errored++;
+            log(`! ${id}: unreadable raw artifact — skipped.`);
+            continue;
+        }
+        // Treat whitespace-only as empty: a raw with `"content": "   "` is truthy
+        // but `chunkConversation` trims it to zero chunks, which would otherwise
+        // drive an empty replacement set straight to the delete path below.
+        const pickContent = (v) => typeof v === "string" && v.trim().length > 0 ? v : "";
+        const content = pickContent(raw.content) || pickContent(raw.summary);
+        if (content.trim().length === 0) {
+            summary.errored++;
+            log(`! ${id}: raw has no content — skipped.`);
+            continue;
+        }
+        const existing = await fetchConversationRows(id);
+        if (!existing) {
+            // Raw with no indexed row: an orphan (failed save / pruned row). Do NOT
+            // resurrect it here — that's reconcile/rebuild's job, with their full
+            // dedup machinery. Backfill only re-shapes already-indexed conversations.
+            summary.orphaned++;
+            continue;
+        }
+        const chunks = chunkConversation(content);
+        const desiredCount = chunks.length;
+        // Hard guard against ever reaching the delete path with an empty
+        // replacement set. The content check above already rejects blank raws (the
+        // only input for which `chunkConversation` returns []), so this is purely
+        // defensive — but a zero-chunk set means `rechunkConversation` would delete
+        // the existing rows from both tiers and insert nothing, silently dropping an
+        // indexed conversation. Skip and count it as an error instead.
+        if (desiredCount === 0) {
+            summary.errored++;
+            log(`! ${id}: produced no chunks — skipped (existing rows untouched).`);
+            continue;
+        }
+        // Already in the desired shape? (complete set of the right count, or a
+        // single unmarked row for a short conversation.) Skip — keeps re-runs cheap
+        // and avoids needless re-embedding. But NEVER skip a cross-tier duplicate:
+        // the deduped `existing.rows` can look complete while a stale copy survives
+        // in the other tier, so it still needs the delete-both-tiers repair below.
+        if (!existing.hasCrossTierRows &&
+            chunkSetComplete(existing.rows) &&
+            expectedChunkCount(existing.rows) === desiredCount) {
+            if (desiredCount > 1)
+                summary.alreadyChunked++;
+            else
+                summary.singleChunk++;
+            continue;
+        }
+        if (dryRun) {
+            summary.rechunked++;
+            log(`~ ${id}: would re-chunk into ${desiredCount} section(s) [${existing.targetTable}] (dry-run).`);
+            if (throttleMs > 0)
+                await sleep(throttleMs);
+            continue;
+        }
+        await rechunkConversation(embedding, dataDir, id, content, chunks, existing);
+        summary.rechunked++;
+        log(`+ ${id}: re-chunked into ${desiredCount} section(s) [${existing.targetTable}].`);
+        if (throttleMs > 0)
+            await sleep(throttleMs);
+    }
+    return summary;
+}
+/**
+ * Embed the full chunk set, then atomically replace the conversation's rows in
+ * its current tier. Metadata (domain/intent/quality/topics/decisions/key_outputs/
+ * summary/idempotency_key/fingerprint/metadata_provider) is preserved from the
+ * existing indexed row so an enriched conversation is NOT downgraded to its raw
+ * artifact's basic metadata — only `content`/`embedding`/the chunk markers change.
+ *
+ * Ordering = embed-all → delete-old → insert-new (same invariant as reconcile):
+ * the embeddings exist before any delete, and the raw artifact survives, so an
+ * interrupt in the brief delete→insert window is repaired by re-running.
+ */
+async function rechunkConversation(embedding, dataDir, id, content, chunks, existing) {
+    const count = chunks.length;
+    const embeddings = await recordEmbed(dataDir, embedding, {
+        pipeline: "backfill",
+        operation: "rechunk_conversation",
+        input_count: count,
+    }, () => embedding.embedBatch(chunks));
+    // Representative = lowest chunk_index (or the lone row) — its metadata is the
+    // conversation-level metadata shared by every chunk.
+    const rep = [...existing.rows].sort((a, b) => Number(a.chunk_index ?? 0) - Number(b.chunk_index ?? 0))[0];
+    const fingerprint = typeof rep.conversation_fingerprint === "string" &&
+        rep.conversation_fingerprint.length > 0
+        ? rep.conversation_fingerprint
+        : conversationContentFingerprint(content);
+    const embeddingProviderMarker = isPlaceholderEmbeddingProvider(embedding)
+        ? RIFT_NONE_EMBEDDING_PROVIDER
+        : "";
+    const indexedAt = new Date().toISOString();
+    const rows = chunks.map((chunk, i) => ({
+        id: count > 1 ? convChunkId(id, i) : id,
+        content: chunk,
+        summary: rep.summary,
+        embedding: embeddings[i],
+        source: rep.source,
+        domain: rep.domain,
+        intent: rep.intent,
+        quality: rep.quality,
+        topics: rep.topics,
+        decisions: rep.decisions,
+        key_outputs: rep.key_outputs,
+        indexed_at: indexedAt,
+        idempotency_key: rep.idempotency_key,
+        conversation_fingerprint: fingerprint,
+        metadata_provider: typeof rep.metadata_provider === "string" ? rep.metadata_provider : "",
+        embedding_provider: embeddingProviderMarker,
+        ...conversationChunkColumns(id, i, count),
+    }));
+    // Delete every prior row for this conversation (the whole row keyed by id and
+    // any chunk set keyed by parent_id) from BOTH tiers — a compaction crash can
+    // leave the same id in hot AND cold, and deleting only one tier would leave a
+    // stale duplicate behind. Then insert the new set into the canonical target
+    // tier (cold if the conversation lives in cold, else hot), preserving the
+    // hot/cold placement rather than promoting old conversations into hot.
+    const idFilter = eqFilter("id", id, { validateAsRowId: true });
+    const parentFilter = eqFilter("parent_id", id, { validateAsRowId: true });
+    for (const table of CONV_TABLES) {
+        await getTable(table).delete(idFilter);
+        await getTable(table).delete(parentFilter);
+    }
+    await recordIndexWrite(dataDir, {
+        table: existing.targetTable,
+        pipeline: "backfill",
+        operation: "rechunk_conversation",
+        row_count: rows.length,
+    }, () => getTable(existing.targetTable).add(rows));
+}
+//# sourceMappingURL=chunk-backfill.js.map

package/dist/src/storage/chunk-backfill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-backfill.js","sourceRoot":"","sources":["../../../src/storage/chunk-backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EACL,4BAA4B,EAC5B,8BAA8B,GAC/B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAwC,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EACL,WAAW,EACX,wBAAwB,EACxB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,8BAA8B,EAAE,MAAM,qDAAqD,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE5E,MAAM,gBAAgB,GAAG,CAAC,GAAG,oBAAoB,EAAE,OAAO,CAAU,CAAC;AACrE,MAAM,WAAW,GAAyB;IACxC,mBAAmB;IACnB,oBAAoB;CACrB,CAAC;AAuCF,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,OAAe;IAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAClE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,MAAM,EAAE,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC3B,IAAI,CAAC,KAAK,IAAI,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AACvE,CAAC;AAqBD;;;;;;;GAOG;AACH,KAAK,UAAU,qBAAqB,CAClC,EAAU;IAEV,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,QAAQ,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;IAChD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,6EAA6E;IAC7E,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC/B,sEAAsE;QACtE,sEAAsE;QACtE,2EAA2E;QAC3E,2EAA2E;QAC3E,4EAA4E;QAC5E,4CAA4C;QAC5C,MAAM,WAAW,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CACrD,CAAC,CAAmB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAChD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW;YAC3B,CAAC,CAAC,IAAI,QAAQ,SAAS,YAAY,GAAG;YACtC,CAAC,CAAC,QAAQ,CAAC;QACb,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM;aACvB,KAAK,EAAE;aACP,KAAK,CAAC,SAAS,CAAC;aAChB,OAAO,EAAE,CAAsB,CAAC;QACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,IAAI,KAAK,KAAK,oBAAoB;YAAE,OAAO,GAAG,IAAI,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,0EAA0E;YAC1E,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,gBAAgB,GAAG,IAAI,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,OAAO;QACL,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,mBAAmB;QACjE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACxB,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,IAA0B;IAE1B,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAEnC,MAAM,OAAO,GAAyB;QACpC,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,CAAC;QACZ,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,KAAK;KACf,CAAC;IAEF,MAAM,IAAI,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC9C,GAAG,CAAC,SAAS,IAAI,CAAC,MAAM,yCAAyC,CAAC,CAAC;IAEnE,KAAK,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YACzB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,GAAG,CAAC,kDAAkD,CAAC,CAAC;YACxD,MAAM;QACR,CAAC;QACD,IAAI,KAAK,GAAG,CAAC,IAAI,OAAO,CAAC,SAAS,IAAI,KAAK,EAAE,CAAC;YAC5C,GAAG,CAAC,mBAAmB,KAAK,aAAa,CAAC,CAAC;YAC3C,MAAM;QACR,CAAC;QACD,OAAO,CAAC,OAAO,EAAE,CAAC;QAElB,IAAI,GAA4B,CAAC;QACjC,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAGjD,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,OAAO,EAAE,CAAC;YAClB,GAAG,CAAC,KAAK,EAAE,sCAAsC,CAAC,CAAC;YACnD,SAAS;QACX,CAAC;QAED,0EAA0E;QAC1E,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,WAAW,GAAG,CAAC,CAAU,EAAU,EAAE,CACzC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,OAAO,EAAE,CAAC;YAClB,GAAG,CAAC,KAAK,EAAE,iCAAiC,CAAC,CAAC;YAC9C,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,wEAAwE;YACxE,sEAAsE;YACtE,0EAA0E;YAC1E,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnB,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QAEnC,iEAAiE;QACjE,2EAA2E;QAC3E,0EAA0E;QAC1E,4EAA4E;QAC5E,6EAA6E;QAC7E,+DAA+D;QAC/D,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,OAAO,EAAE,CAAC;YAClB,GAAG,CAAC,KAAK,EAAE,2DAA2D,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QAED,uEAAuE;QACvE,4EAA4E;QAC5E,2EAA2E;QAC3E,4EAA4E;QAC5E,2EAA2E;QAC3E,IACE,CAAC,QAAQ,CAAC,gBAAgB;YAC1B,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC;YAC/B,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,YAAY,EAClD,CAAC;YACD,IAAI,YAAY,GAAG,CAAC;gBAAE,OAAO,CAAC,cAAc,EAAE,CAAC;;gBAC1C,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,GAAG,CACD,KAAK,EAAE,yBAAyB,YAAY,gBAAgB,QAAQ,CAAC,WAAW,cAAc,CAC/F,CAAC;YACF,IAAI,UAAU,GAAG,CAAC;gBAAE,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;YAC5C,SAAS;QACX,CAAC;QAED,MAAM,mBAAmB,CACvB,SAAS,EACT,OAAO,EACP,EAAE,EACF,OAAO,EACP,MAAM,EACN,QAAQ,CACT,CAAC;QACF,OAAO,CAAC,SAAS,EAAE,CAAC;QACpB,GAAG,CAAC,KAAK,EAAE,qBAAqB,YAAY,gBAAgB,QAAQ,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtF,IAAI,UAAU,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,mBAAmB,CAChC,SAA4B,EAC5B,OAAe,EACf,EAAU,EACV,OAAe,EACf,MAAgB,EAChB,QAA8B;IAE9B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAE5B,MAAM,UAAU,GAAG,MAAM,WAAW,CAClC,OAAO,EACP,SAAS,EACT;QACE,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,sBAAsB;QACjC,WAAW,EAAE,KAAK;KACnB,EACD,GAAG,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CACnC,CAAC;IAEF,8EAA8E;IAC9E,qDAAqD;IACrD,MAAM,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CACjC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,CAC1D,CAAC,CAAC,CAAE,CAAC;IACN,MAAM,WAAW,GACf,OAAO,GAAG,CAAC,wBAAwB,KAAK,QAAQ;QAChD,GAAG,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC;QACrC,CAAC,CAAC,GAAG,CAAC,wBAAwB;QAC9B,CAAC,CAAC,8BAA8B,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,SAAS,CAAC;QACvE,CAAC,CAAC,4BAA4B;QAC9B,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,MAAM,IAAI,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACxD,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QACvC,OAAO,EAAE,KAAK;QACd,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAE;QACzB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,UAAU,EAAE,SAAS;QACrB,eAAe,EAAE,GAAG,CAAC,eAAe;QACpC,wBAAwB,EAAE,WAAW;QACrC,iBAAiB,EACf,OAAO,GAAG,CAAC,iBAAiB,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE;QACxE,kBAAkB,EAAE,uBAAuB;QAC3C,GAAG,wBAAwB,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC;KAC1C,CAAC,CAAC,CAAC;IAEJ,8EAA8E;IAC9E,6EAA6E;IAC7E,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,uEAAuE;IACvE,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,QAAQ,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,gBAAgB,CACpB,OAAO,EACP;QACE,KAAK,EAAE,QAAQ,CAAC,WAAW;QAC3B,QAAQ,EAAE,UAAU;QACpB,SAAS,EAAE,sBAAsB;QACjC,SAAS,EAAE,IAAI,CAAC,MAAM;KACvB,EACD,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAC/C,CAAC;AACJ,CAAC"}

package/dist/src/storage/db.d.ts

@@ -0,0 +1,15 @@
+import type { Connection, Table } from "@lancedb/lancedb";
+export type { Connection, Table };
+/**
+ * Open (or reuse) a LanceDB connection at the given directory.
+ *
+ * If the existing connection was opened for a different path,
+ * closes it first and opens a new one — never silently reuses
+ * the wrong database.
+ */
+export declare function getConnection(dbPath: string): Promise<Connection>;
+/** Close the connection. Safe to call multiple times or if never opened. */
+export declare function closeConnection(): void;
+/** Reset connection reference (for tests). Does not close. */
+export declare function resetConnectionRef(): void;
+//# sourceMappingURL=db.d.ts.map

package/dist/src/storage/db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../../src/storage/db.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE1D,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAKlC;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAevE;AAED,4EAA4E;AAC5E,wBAAgB,eAAe,IAAI,IAAI,CAMtC;AAED,8DAA8D;AAC9D,wBAAgB,kBAAkB,IAAI,IAAI,CAGzC"}

package/dist/src/storage/db.js

@@ -0,0 +1,47 @@
+/**
+ * LanceDB connection management.
+ *
+ * Provides a singleton-style connection factory that is
+ * initialized once and shared across the process lifetime.
+ */
+import * as lancedb from "@lancedb/lancedb";
+let connection = null;
+let connectionPath = null;
+/**
+ * Open (or reuse) a LanceDB connection at the given directory.
+ *
+ * If the existing connection was opened for a different path,
+ * closes it first and opens a new one — never silently reuses
+ * the wrong database.
+ */
+export async function getConnection(dbPath) {
+    if (connection && connection.isOpen()) {
+        if (connectionPath === dbPath)
+            return connection;
+        // Path mismatch — close stale connection before opening new one.
+        connection.close();
+        connection = null;
+        connectionPath = null;
+    }
+    // LanceDB defaults to 6 GiB of index cache and 1 GiB of metadata cache.
+    // Share a laptop-sized budget across all tables in this connection. These
+    // caches are disposable read accelerators, not a limit on stored data.
+    const session = new lancedb.Session(64n * 1024n * 1024n, 32n * 1024n * 1024n);
+    connection = await lancedb.connect(dbPath, { session });
+    connectionPath = dbPath;
+    return connection;
+}
+/** Close the connection. Safe to call multiple times or if never opened. */
+export function closeConnection() {
+    if (connection) {
+        connection.close();
+        connection = null;
+        connectionPath = null;
+    }
+}
+/** Reset connection reference (for tests). Does not close. */
+export function resetConnectionRef() {
+    connection = null;
+    connectionPath = null;
+}
+//# sourceMappingURL=db.js.map

package/dist/src/storage/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/db.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAK5C,IAAI,UAAU,GAAsB,IAAI,CAAC;AACzC,IAAI,cAAc,GAAkB,IAAI,CAAC;AAEzC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,IAAI,cAAc,KAAK,MAAM;YAAE,OAAO,UAAU,CAAC;QACjD,iEAAiE;QACjE,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC;QAClB,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;IACD,wEAAwE;IACxE,0EAA0E;IAC1E,uEAAuE;IACvE,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,KAAK,GAAG,KAAK,EAAE,GAAG,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC;IAC9E,UAAU,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IACxD,cAAc,GAAG,MAAM,CAAC;IACxB,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,eAAe;IAC7B,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC;QAClB,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;AACH,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,kBAAkB;IAChC,UAAU,GAAG,IAAI,CAAC;IAClB,cAAc,GAAG,IAAI,CAAC;AACxB,CAAC"}

package/dist/src/storage/filter.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Shared LanceDB filter-construction helpers.
+ *
+ * LanceDB exposes `.where()` / `.delete()` as SQL-ish filter STRINGS rather than
+ * a bind-parameter API, so every interpolated value must be escaped here — one
+ * contract, used by every callsite that builds a filter from a `source_path`,
+ * `parent_id`, row `id`, or `idempotency_key`. No write path may hand-roll its
+ * own escaping.
+ *
+ * (Also advances the tracked "LanceDB filter hardening" follow-up: centralize
+ * id/idempotency-key filter construction + a charset guard before interpolation.)
+ */
+export { escapeLike } from "../retrieval/cwd.js";
+/**
+ * Escape a value for use inside a single-quoted SQL string literal by doubling
+ * embedded single quotes. The canonical escaper for LanceDB `.where()` /
+ * `.delete()` equality filters.
+ */
+export declare function escapeSql(value: string): string;
+export declare function assertValidRowId(value: string, label?: string): string;
+/**
+ * Build an equality filter `column = '<escaped>'`. For id-shaped columns pass
+ * `validateAsRowId` so the value is charset-guarded as well as escaped.
+ */
+export declare function eqFilter(column: string, value: string, opts?: {
+    validateAsRowId?: boolean;
+}): string;
+/**
+ * Build an `IN (...)` filter — `column IN ('a', 'b', ...)` — from a list of
+ * values, centralizing escaping for the batch-fetch READ paths the same way
+ * {@link eqFilter} does for single-value writes. No callsite should hand-roll
+ * `id IN (...)` with manual quote replacement.
+ *
+ * Pass `validateAsRowId` for id-shaped columns to charset-guard every value in
+ * addition to escaping. Throws on an empty list: `IN ()` is invalid SQL and
+ * every caller already guards against the empty case, so an empty list here is
+ * a bug, not a recoverable condition.
+ */
+export declare function inFilter(column: string, values: readonly string[], opts?: {
+    validateAsRowId?: boolean;
+}): string;
+//# sourceMappingURL=filter.d.ts.map

package/dist/src/storage/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../../src/storage/filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AAcD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,SAAW,GAAG,MAAM,CAOxE;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CACtB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAO,GACvC,MAAM,CAGR;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,QAAQ,CACtB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,IAAI,GAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAO,GACvC,MAAM,CAWR"}

package/dist/src/storage/filter.js

@@ -0,0 +1,70 @@
+/**
+ * Shared LanceDB filter-construction helpers.
+ *
+ * LanceDB exposes `.where()` / `.delete()` as SQL-ish filter STRINGS rather than
+ * a bind-parameter API, so every interpolated value must be escaped here — one
+ * contract, used by every callsite that builds a filter from a `source_path`,
+ * `parent_id`, row `id`, or `idempotency_key`. No write path may hand-roll its
+ * own escaping.
+ *
+ * (Also advances the tracked "LanceDB filter hardening" follow-up: centralize
+ * id/idempotency-key filter construction + a charset guard before interpolation.)
+ */
+export { escapeLike } from "../retrieval/cwd.js";
+/**
+ * Escape a value for use inside a single-quoted SQL string literal by doubling
+ * embedded single quotes. The canonical escaper for LanceDB `.where()` /
+ * `.delete()` equality filters.
+ */
+export function escapeSql(value) {
+    return value.replace(/'/g, "''");
+}
+/**
+ * Charset guard for internally-generated identifiers (`id`, `parent_id`).
+ *
+ * These are never user-authored — they are sha256 hex (docs / web-import rows),
+ * UUIDs (save chain roots), or those shapes with a chunk suffix (`:N` / `::N`).
+ * Defense-in-depth: reject anything outside that alphabet BEFORE it reaches an
+ * interpolated filter, so a malformed id can never smuggle SQL even though the
+ * value should already be escaped. Throws on violation; callers treat a throw as
+ * a bug, not a recoverable condition.
+ */
+const ROW_ID_CHARSET = /^[A-Za-z0-9_:.-]+$/;
+export function assertValidRowId(value, label = "row id") {
+    if (!value || !ROW_ID_CHARSET.test(value)) {
+        throw new Error(`Invalid ${label} for filter (expected [A-Za-z0-9_:.-]): ${JSON.stringify(value)}`);
+    }
+    return value;
+}
+/**
+ * Build an equality filter `column = '<escaped>'`. For id-shaped columns pass
+ * `validateAsRowId` so the value is charset-guarded as well as escaped.
+ */
+export function eqFilter(column, value, opts = {}) {
+    const checked = opts.validateAsRowId ? assertValidRowId(value, column) : value;
+    return `${column} = '${escapeSql(checked)}'`;
+}
+/**
+ * Build an `IN (...)` filter — `column IN ('a', 'b', ...)` — from a list of
+ * values, centralizing escaping for the batch-fetch READ paths the same way
+ * {@link eqFilter} does for single-value writes. No callsite should hand-roll
+ * `id IN (...)` with manual quote replacement.
+ *
+ * Pass `validateAsRowId` for id-shaped columns to charset-guard every value in
+ * addition to escaping. Throws on an empty list: `IN ()` is invalid SQL and
+ * every caller already guards against the empty case, so an empty list here is
+ * a bug, not a recoverable condition.
+ */
+export function inFilter(column, values, opts = {}) {
+    if (values.length === 0) {
+        throw new Error(`inFilter requires at least one value for column ${column}`);
+    }
+    const quoted = values.map((value) => {
+        const checked = opts.validateAsRowId
+            ? assertValidRowId(value, column)
+            : value;
+        return `'${escapeSql(checked)}'`;
+    });
+    return `${column} IN (${quoted.join(", ")})`;
+}
+//# sourceMappingURL=filter.js.map

package/dist/src/storage/filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.js","sourceRoot":"","sources":["../../../src/storage/filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,KAAa;IACrC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,cAAc,GAAG,oBAAoB,CAAC;AAE5C,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,KAAK,GAAG,QAAQ;IAC9D,IAAI,CAAC,KAAK,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,WAAW,KAAK,2CAA2C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CACnF,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CACtB,MAAc,EACd,KAAa,EACb,OAAsC,EAAE;IAExC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/E,OAAO,GAAG,MAAM,OAAO,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC;AAC/C,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,QAAQ,CACtB,MAAc,EACd,MAAyB,EACzB,OAAsC,EAAE;IAExC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,mDAAmD,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe;YAClC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC;YACjC,CAAC,CAAC,KAAK,CAAC;QACV,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,MAAM,QAAQ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;AAC/C,CAAC"}

package/dist/src/storage/fts.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Full-text (BM25) indexes for keyword retrieval.
+ *
+ * One LanceDB inverted index per searchable text column — the same text the
+ * old query-time scan tokenized (content, summary, JSON metadata arrays,
+ * conversation source and domain, doc paths). `domain` is a facet column:
+ * lexical search queries it separately so the no-AI placeholder domain of
+ * basic rows stays unsearchable (see FTS_FACET_COLUMNS). Rows written after an index was built are still searched: LanceDB
+ * flat-searches unindexed fragments unless `fastSearch()` is set, and
+ * `optimize()` folds them into the index later. Deletes and updates are
+ * honoured immediately (verified against @lancedb/lancedb 0.27.2).
+ *
+ * Tokenizer: lowercase + ASCII folding, so "decide" matches "décidé" and the
+ * reverse. No stemming or stop-word removal: identifiers and mixed
+ * French/English text must match as written.
+ */
+import { type Table } from "@lancedb/lancedb";
+import type { TableName } from "./tables.js";
+export declare const FTS_COLUMNS: Record<TableName, readonly string[]>;
+/**
+ * Indexed columns searched only on enriched rows: basic imports stamp a
+ * placeholder there (e.g. domain "tech") that must not match the literal word.
+ */
+export declare const FTS_FACET_COLUMNS: readonly string[];
+export declare const FTS_OPTIONS: {
+    readonly baseTokenizer: "simple";
+    readonly lowercase: true;
+    readonly asciiFolding: true;
+    readonly stem: false;
+    readonly removeStopWords: false;
+    readonly withPosition: false;
+};
+/** Human-readable gaps for the maintenance scheduler. Empty tables need no index. */
+export declare function fullTextIndexGaps(name: TableName, table: Table): Promise<string[]>;
+/**
+ * Create any missing FTS indexes. Each `createIndex` commits atomically, so an
+ * interrupted run leaves either no index for that column (keyword search falls
+ * back to the scan) or a complete one; the next run finishes the rest.
+ */
+export declare function ensureFullTextIndexes(name: TableName, table: Table): Promise<string[]>;
+/**
+ * Columns of `name` that are fully covered by an FTS index, in FTS_COLUMNS
+ * order. Callers use the indexed path only when every column is indexed, so
+ * a partially built index set never silently narrows what keyword search sees.
+ */
+export declare function fullTextReady(name: TableName, table: Table): Promise<boolean>;
+//# sourceMappingURL=fts.d.ts.map

package/dist/src/storage/fts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../../../src/storage/fts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAS,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAM5D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,MAAM,EAAe,CAAC;AAE/D,eAAO,MAAM,WAAW;;;;;;;CASd,CAAC;AAkBX,qFAAqF;AACrF,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAIxF;AAED;;;;GAIG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQ5F;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAEnF"}

package/dist/src/storage/fts.js

@@ -0,0 +1,86 @@
+/**
+ * Full-text (BM25) indexes for keyword retrieval.
+ *
+ * One LanceDB inverted index per searchable text column — the same text the
+ * old query-time scan tokenized (content, summary, JSON metadata arrays,
+ * conversation source and domain, doc paths). `domain` is a facet column:
+ * lexical search queries it separately so the no-AI placeholder domain of
+ * basic rows stays unsearchable (see FTS_FACET_COLUMNS). Rows written after an index was built are still searched: LanceDB
+ * flat-searches unindexed fragments unless `fastSearch()` is set, and
+ * `optimize()` folds them into the index later. Deletes and updates are
+ * honoured immediately (verified against @lancedb/lancedb 0.27.2).
+ *
+ * Tokenizer: lowercase + ASCII folding, so "decide" matches "décidé" and the
+ * reverse. No stemming or stop-word removal: identifiers and mixed
+ * French/English text must match as written.
+ */
+import { Index } from "@lancedb/lancedb";
+export const FTS_COLUMNS = {
+    conversations_hot: ["content", "summary", "topics", "decisions", "key_outputs", "source", "domain"],
+    conversations_cold: ["content", "summary", "topics", "decisions", "key_outputs", "source", "domain"],
+    digests: ["content", "summary"],
+    structured_docs: ["content", "source_path", "client_name", "metadata"],
+    structured_docs_local: ["content", "source_path", "client_name", "metadata"],
+};
+/**
+ * Indexed columns searched only on enriched rows: basic imports stamp a
+ * placeholder there (e.g. domain "tech") that must not match the literal word.
+ */
+export const FTS_FACET_COLUMNS = ["domain"];
+export const FTS_OPTIONS = {
+    baseTokenizer: "simple",
+    lowercase: true,
+    asciiFolding: true,
+    stem: false,
+    removeStopWords: false,
+    // No phrase queries are issued, and LanceDB exposes no match offsets, so
+    // positions would only cost index size and build time.
+    withPosition: false,
+};
+async function indexedColumns(table) {
+    const indices = await table.listIndices();
+    const cols = new Set();
+    for (const idx of indices) {
+        if (/fts|inverted/i.test(idx.indexType))
+            for (const c of idx.columns)
+                cols.add(c);
+    }
+    return cols;
+}
+/** Columns that exist in the table schema but have no FTS index yet. */
+async function missingColumns(name, table) {
+    const schemaCols = new Set((await table.schema()).fields.map((f) => f.name));
+    const have = await indexedColumns(table);
+    return FTS_COLUMNS[name].filter((c) => schemaCols.has(c) && !have.has(c));
+}
+/** Human-readable gaps for the maintenance scheduler. Empty tables need no index. */
+export async function fullTextIndexGaps(name, table) {
+    if ((await table.countRows()) === 0)
+        return [];
+    const missing = await missingColumns(name, table);
+    return missing.length ? [`missing full-text index on ${missing.join(", ")}`] : [];
+}
+/**
+ * Create any missing FTS indexes. Each `createIndex` commits atomically, so an
+ * interrupted run leaves either no index for that column (keyword search falls
+ * back to the scan) or a complete one; the next run finishes the rest.
+ */
+export async function ensureFullTextIndexes(name, table) {
+    if ((await table.countRows()) === 0)
+        return [];
+    const created = [];
+    for (const column of await missingColumns(name, table)) {
+        await table.createIndex(column, { config: Index.fts(FTS_OPTIONS), replace: false });
+        created.push(column);
+    }
+    return created;
+}
+/**
+ * Columns of `name` that are fully covered by an FTS index, in FTS_COLUMNS
+ * order. Callers use the indexed path only when every column is indexed, so
+ * a partially built index set never silently narrows what keyword search sees.
+ */
+export async function fullTextReady(name, table) {
+    return (await missingColumns(name, table)).length === 0;
+}
+//# sourceMappingURL=fts.js.map

package/dist/src/storage/fts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts.js","sourceRoot":"","sources":["../../../src/storage/fts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,KAAK,EAAc,MAAM,kBAAkB,CAAC;AAGrD,MAAM,CAAC,MAAM,WAAW,GAAyC;IAC/D,iBAAiB,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,CAAC;IACnG,kBAAkB,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,CAAC;IACpG,OAAO,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;IAC/B,eAAe,EAAE,CAAC,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,CAAC;IACtE,qBAAqB,EAAE,CAAC,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,CAAC;CAC7E,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB,CAAC,QAAQ,CAAC,CAAC;AAE/D,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,aAAa,EAAE,QAAQ;IACvB,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,IAAI;IAClB,IAAI,EAAE,KAAK;IACX,eAAe,EAAE,KAAK;IACtB,yEAAyE;IACzE,uDAAuD;IACvD,YAAY,EAAE,KAAK;CACX,CAAC;AAEX,KAAK,UAAU,cAAc,CAAC,KAAY;IACxC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO;gBAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,wEAAwE;AACxE,KAAK,UAAU,cAAc,CAAC,IAAe,EAAE,KAAY;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAe,EAAE,KAAY;IACnE,IAAI,CAAC,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACpF,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAe,EAAE,KAAY;IACvE,IAAI,CAAC,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAe,EAAE,KAAY;IAC/D,OAAO,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;AAC1D,CAAC"}