onadit 0.0.1 → 1.0.0

package/dist/src/storage/report-card-backfill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-card-backfill.d.ts","sourceRoot":"","sources":["../../../src/storage/report-card-backfill.ts"],"names":[],"mappings":"AA0BA,OAAO,EAKL,KAAK,UAAU,EAChB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAU/D,MAAM,WAAW,wBAAwB;IACvC,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,iBAAiB,CAAC;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAgBD,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,yBAAyB,CAAC,CAuEpC"}

package/dist/src/storage/report-card-backfill.js

@@ -0,0 +1,114 @@
+/**
+ * Offline backfill of write-time index cards for Onadit-owned reports
+ * (beta.28 retrieval coverage, docs/10).
+ *
+ * Scans `selfReportsRoot` (= `<repoRoot>/reports`), builds a deterministic card
+ * per report (see report-cards.ts), embeds the curated card text, and upserts a
+ * `source_type: "report_card"` row into `structured_docs`. Idempotent on the
+ * card's stable id (`report_card:<slug>`) — re-running replaces, never dupes.
+ *
+ * SAFETY: this is an EXPLICIT, offline backfill. Callers point `dataDir` at a
+ * COPY of the live data dir (never the live one) per the standing
+ * chunk-backfill rule. The ONLY external call is the embedding batch.
+ *
+ * Why a REPO-RELATIVE `source_path` (`reports/<rel>`), not the absolute path
+ * the watcher uses: it (a) equals the eval's repo-relative `target_path` so
+ * `exactTargetMatch` resolves the card, and (b) is NOT under the *absolute*
+ * `selfReportsRoot`, so the report-demotion snapshot classifier
+ * (`isSnapshotPath` → `isUnderRoot`, a literal prefix check) does NOT treat a
+ * card as a generated report — so cards surface for locator queries WITHOUT any
+ * change to the proven demotion path. (A card is a pointer, not a report body;
+ * not demoting it is the correct semantics.)
+ */
+import path from "node:path";
+import fs from "node:fs";
+import { initTables, getTable } from "./tables.js";
+import { eqFilter } from "./filter.js";
+import { buildReportCard, dedupeCardSlugs, reportCardRowId, reportCardEmbedText, } from "../ingestion/report-cards.js";
+const REPORT_CARD_TABLE = "structured_docs";
+/** The columns a card row sets explicitly; any others are schema padding. */
+const CARD_CORE_KEYS = new Set([
+    "id", "source_path", "content", "embedding", "source_type",
+    "source_scope", "client_name", "indexed_at", "metadata",
+]);
+/** Recursively list `*.md` under `dir` (sorted for deterministic batches). */
+function listMarkdown(dir) {
+    const out = [];
+    const walk = (d) => {
+        for (const e of fs.readdirSync(d, { withFileTypes: true })) {
+            const p = path.join(d, e.name);
+            if (e.isDirectory())
+                walk(p);
+            else if (e.isFile() && e.name.toLowerCase().endsWith(".md"))
+                out.push(p);
+        }
+    };
+    if (fs.existsSync(dir))
+        walk(dir);
+    return out.sort();
+}
+export async function backfillReportCards(input) {
+    const { repoRoot, reportsRoot, dataDir, embedding, dryRun = false, log = () => { } } = input;
+    const files = listMarkdown(reportsRoot);
+    log(`scanning ${reportsRoot}: ${files.length} markdown report(s)`);
+    // Build cards (pure — no I/O beyond the file read), then make slugs globally
+    // unique so two same-stem reports can't share a `report_card:<slug>` row id.
+    const cards = dedupeCardSlugs(files.map((abs) => {
+        const repoRel = path.relative(repoRoot, abs).split(path.sep).join("/");
+        return buildReportCard({
+            path: repoRel,
+            content: fs.readFileSync(abs, "utf8"),
+            mtimeISO: fs.statSync(abs).mtime.toISOString(),
+        });
+    }));
+    if (dryRun) {
+        for (const c of cards)
+            log(`~ ${reportCardRowId(c.slug)} ← ${c.path} (dry-run)`);
+        return { scanned: files.length, written: 0, cards };
+    }
+    if (cards.length === 0)
+        return { scanned: 0, written: 0, cards };
+    // Embed the curated card text — the only external call.
+    const vectors = await embedding.embedBatch(cards.map(reportCardEmbedText));
+    await initTables(dataDir);
+    const table = getTable(REPORT_CARD_TABLE);
+    // Match the live table's exact column set: a copy that was chunked carries
+    // extra columns (parent_id/…), and lancedb `.add()` is schema-strict. Pad
+    // any non-core column with "" (the unchunked/legacy default).
+    const sample = (await table.query().limit(1).toArray());
+    const extraKeys = sample.length > 0
+        ? Object.keys(sample[0]).filter((k) => !CARD_CORE_KEYS.has(k) && k !== "embedding")
+        : [];
+    const now = new Date().toISOString();
+    const rows = cards.map((c, i) => {
+        const row = {
+            id: reportCardRowId(c.slug),
+            source_path: c.path, // REPO-RELATIVE on purpose (see module doc)
+            content: reportCardEmbedText(c),
+            embedding: vectors[i],
+            source_type: "report_card",
+            source_scope: "project",
+            client_name: "",
+            indexed_at: now,
+            metadata: JSON.stringify({
+                card: true,
+                slug: c.slug,
+                title: c.title,
+                summary: c.summary,
+                topics: c.topics,
+                date: c.date,
+                path: c.path,
+            }),
+        };
+        for (const k of extraKeys)
+            row[k] = "";
+        return row;
+    });
+    // Idempotent: drop any prior card row by its stable id, then add.
+    for (const r of rows)
+        await table.delete(eqFilter("id", String(r.id)));
+    await table.add(rows);
+    log(`wrote ${rows.length} report card(s) into ${path.join(dataDir, "structured_docs")}`);
+    return { scanned: files.length, written: rows.length, cards };
+}
+//# sourceMappingURL=report-card-backfill.js.map

package/dist/src/storage/report-card-backfill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-card-backfill.js","sourceRoot":"","sources":["../../../src/storage/report-card-backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAyB,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EACL,eAAe,EACf,eAAe,EACf,eAAe,EACf,mBAAmB,GAEpB,MAAM,8BAA8B,CAAC;AAGtC,MAAM,iBAAiB,GAAG,iBAA0B,CAAC;AAErD,6EAA6E;AAC7E,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa;IAC1D,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU;CACxD,CAAC,CAAC;AAoBH,8EAA8E;AAC9E,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,CAAC,CAAS,EAAQ,EAAE;QAC/B,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,CAAC,WAAW,EAAE;gBAAE,IAAI,CAAC,CAAC,CAAC,CAAC;iBACxB,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC,CAAC;IACF,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA+B;IAE/B,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,GAAE,CAAC,EAAE,GAAG,KAAK,CAAC;IAE5F,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACxC,GAAG,CAAC,YAAY,WAAW,KAAK,KAAK,CAAC,MAAM,qBAAqB,CAAC,CAAC;IAEnE,6EAA6E;IAC7E,6EAA6E;IAC7E,MAAM,KAAK,GAAiB,eAAe,CACzC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvE,OAAO,eAAe,CAAC;YACrB,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC;YACrC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE;SAC/C,CAAC,CAAC;IACL,CAAC,CAAC,CACH,CAAC;IAEF,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,GAAG,CAAC,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC;QACjF,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IAEjE,wDAAwD;IACxD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAE3E,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;IAC1B,MAAM,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAE1C,2EAA2E;IAC3E,0EAA0E;IAC1E,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAmC,CAAC;IAC1F,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC;QACpF,CAAC,CAAC,EAAE,CAAC;IAET,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,IAAI,GAAuB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAClD,MAAM,GAAG,GAA4B;YACnC,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3B,WAAW,EAAE,CAAC,CAAC,IAAI,EAAE,4CAA4C;YACjE,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC;YAC/B,SAAS,EAAE,OAAO,CAAC,CAAC,CAAE;YACtB,WAAW,EAAE,aAAa;YAC1B,YAAY,EAAE,SAAS;YACvB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC;gBACvB,IAAI,EAAE,IAAI;gBACV,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;aACb,CAAC;SACH,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,SAAS;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACvC,OAAO,GAAuB,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,kEAAkE;IAClE,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtB,GAAG,CAAC,SAAS,IAAI,CAAC,MAAM,wBAAwB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC;IAEzF,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;AAChE,CAAC"}

package/dist/src/storage/shadow-swap.d.ts

@@ -0,0 +1,20 @@
+import type { Table } from "./db.js";
+import type { TableName } from "./tables.js";
+export interface ShadowSwap {
+    /** Create a shadow (temporary) table for writing rebuilt data. */
+    createShadow(tableName: TableName): Promise<Table>;
+    /**
+     * Commit all pending shadow swaps atomically.
+     * Closes all handles, renames shadow dirs to live, reopens connection,
+     * and updates the table handle cache.
+     *
+     * @param swaps - Map of tableName -> shadowTable handle to commit.
+     */
+    commitAllSwaps(swaps: Map<TableName, Table>): Promise<void>;
+}
+/**
+ * Create a ShadowSwap bound to a specific data directory.
+ * The dbPath is derived as `<dataDir>/lancedb`.
+ */
+export declare function createShadowSwap(dataDir: string): ShadowSwap;
+//# sourceMappingURL=shadow-swap.d.ts.map

package/dist/src/storage/shadow-swap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shadow-swap.d.ts","sourceRoot":"","sources":["../../../src/storage/shadow-swap.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAG7C,MAAM,WAAW,UAAU;IACzB,kEAAkE;IAClE,YAAY,CAAC,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAEnD;;;;;;OAMG;IACH,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAMD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,CA0H5D"}

package/dist/src/storage/shadow-swap.js

@@ -0,0 +1,163 @@
+/**
+ * Shadow-table swap — Slice 15 implementation.
+ *
+ * Used by reindex/rebuild: build a new table in the background,
+ * then swap it in place of the current table.
+ *
+ * ## Two-phase swap pattern
+ *
+ * To ensure four-table coherence (all tables reflect the same generation),
+ * the swap uses a two-phase approach with rollback:
+ *
+ * 1. `conn.createTable("_shadow_" + name, seedData)` — create shadow table
+ * 2. Populate shadow with rebuilt data
+ * 3. Close all handles and the connection
+ * 4. **Phase A (Evacuate):** rename all `live.lance` → `_old_live.lance`
+ *    If this fails, rollback restores any already-evacuated dirs.
+ * 5. **Phase B (Install):** rename all `_shadow_live.lance` → `live.lance`
+ *    If this fails, rollback removes installed shadows and restores `_old_`.
+ * 6. **Phase C (Cleanup):** delete all `_old_` dirs (harmless if it fails).
+ * 7. Reopen connection and all tables.
+ *
+ * This guarantees that on any failure, ALL tables revert to the pre-swap
+ * generation — no mixed old/new state.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { getConnection, closeConnection, resetConnectionRef } from "./db.js";
+import { SEED_ID, seedForTable, setTable, setSwapInProgress, closeAllHandles, TABLE_NAMES } from "./tables.js";
+function shadowName(tableName) {
+    return `_shadow_${tableName}`;
+}
+/**
+ * Create a ShadowSwap bound to a specific data directory.
+ * The dbPath is derived as `<dataDir>/lancedb`.
+ */
+export function createShadowSwap(dataDir) {
+    const dbPath = path.join(dataDir, "lancedb");
+    return {
+        async createShadow(tableName) {
+            const conn = await getConnection(dbPath);
+            const sName = shadowName(tableName);
+            // Clean up any leftover shadow from a previous failed attempt.
+            const existing = new Set(await conn.tableNames());
+            if (existing.has(sName)) {
+                await conn.dropTable(sName);
+            }
+            const seed = seedForTable(tableName);
+            const table = await conn.createTable(sName, seed);
+            await table.delete(`id = '${SEED_ID}'`);
+            return table;
+        },
+        async commitAllSwaps(swaps) {
+            // Acquire swap lock — readers will get a clear error during this window.
+            setSwapInProgress(true);
+            try {
+                // Step 1: Close all shadow handles (flush writes to disk).
+                for (const shadowTable of swaps.values()) {
+                    shadowTable.close();
+                }
+                // Step 2: Close all live table handles (bypass swap lock via closeAllHandles).
+                closeAllHandles();
+                // Step 3: Close the connection (clear LanceDB internal state).
+                closeConnection();
+                resetConnectionRef();
+                // Step 4: Two-phase filesystem swap for four-table coherence.
+                const tableNames = [...swaps.keys()];
+                const oldDirs = new Map();
+                // Phase A: Evacuate — rename all live dirs to _old_ (backup).
+                try {
+                    for (const tableName of tableNames) {
+                        const liveDir = path.join(dbPath, `${tableName}.lance`);
+                        const oldDir = path.join(dbPath, `_old_${tableName}.lance`);
+                        if (fs.existsSync(liveDir)) {
+                            fs.renameSync(liveDir, oldDir);
+                            oldDirs.set(tableName, oldDir);
+                        }
+                    }
+                }
+                catch {
+                    // Evacuate failed — restore any already-evacuated tables.
+                    for (const [tn, od] of oldDirs) {
+                        try {
+                            fs.renameSync(od, path.join(dbPath, `${tn}.lance`));
+                        }
+                        catch { /* best effort */ }
+                    }
+                    throw new Error("shadow-swap: evacuate phase failed");
+                }
+                // Phase B: Install — rename all shadow dirs to live names.
+                try {
+                    for (const tableName of tableNames) {
+                        const liveDir = path.join(dbPath, `${tableName}.lance`);
+                        const sDir = path.join(dbPath, `${shadowName(tableName)}.lance`);
+                        fs.renameSync(sDir, liveDir);
+                    }
+                }
+                catch {
+                    // Install failed — rollback: remove installed shadows, restore _old_.
+                    for (const tableName of tableNames) {
+                        const liveDir = path.join(dbPath, `${tableName}.lance`);
+                        try {
+                            if (fs.existsSync(liveDir))
+                                fs.rmSync(liveDir, { recursive: true, force: true });
+                        }
+                        catch { /* best effort */ }
+                    }
+                    for (const [tn, od] of oldDirs) {
+                        try {
+                            fs.renameSync(od, path.join(dbPath, `${tn}.lance`));
+                        }
+                        catch { /* best effort */ }
+                    }
+                    throw new Error("shadow-swap: install phase failed, rolled back to previous state");
+                }
+                // Phase C: Cleanup — remove _old_ dirs (harmless if this fails).
+                for (const od of oldDirs.values()) {
+                    try {
+                        fs.rmSync(od, { recursive: true, force: true });
+                    }
+                    catch { /* stale cruft */ }
+                }
+            }
+            finally {
+                // Step 5: Safety net — if rollback itself partially failed, ensure
+                // every table has a live directory from whatever source remains.
+                for (const tableName of swaps.keys()) {
+                    const liveDir = path.join(dbPath, `${tableName}.lance`);
+                    const oldDir = path.join(dbPath, `_old_${tableName}.lance`);
+                    const sDir = path.join(dbPath, `${shadowName(tableName)}.lance`);
+                    try {
+                        if (!fs.existsSync(liveDir)) {
+                            if (fs.existsSync(oldDir))
+                                fs.renameSync(oldDir, liveDir);
+                            else if (fs.existsSync(sDir))
+                                fs.renameSync(sDir, liveDir);
+                        }
+                        if (fs.existsSync(oldDir))
+                            fs.rmSync(oldDir, { recursive: true, force: true });
+                        if (fs.existsSync(sDir))
+                            fs.rmSync(sDir, { recursive: true, force: true });
+                    }
+                    catch { /* integrity check will catch remaining damage */ }
+                }
+                // Step 6: Reopen connection and all tables from whatever survived.
+                try {
+                    const conn = await getConnection(dbPath);
+                    const existing = new Set(await conn.tableNames());
+                    for (const tableName of TABLE_NAMES) {
+                        if (existing.has(tableName)) {
+                            setTable(tableName, await conn.openTable(tableName));
+                        }
+                    }
+                }
+                catch (err) {
+                    process.stderr.write(`shadow-swap: recovery reopen failed: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+                // Release swap lock — readers can proceed again.
+                setSwapInProgress(false);
+            }
+        },
+    };
+}
+//# sourceMappingURL=shadow-swap.js.map

package/dist/src/storage/shadow-swap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shadow-swap.js","sourceRoot":"","sources":["../../../src/storage/shadow-swap.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,iBAAiB,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAgB/G,SAAS,UAAU,CAAC,SAAiB;IACnC,OAAO,WAAW,SAAS,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE7C,OAAO;QACL,KAAK,CAAC,YAAY,CAAC,SAAoB;YACrC,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;YAEpC,+DAA+D;YAC/D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YAClD,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;YAED,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAClD,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,OAAO,GAAG,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,KAAK,CAAC,cAAc,CAAC,KAA4B;YAC/C,yEAAyE;YACzE,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAExB,IAAI,CAAC;gBACH,2DAA2D;gBAC3D,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;oBACzC,WAAW,CAAC,KAAK,EAAE,CAAC;gBACtB,CAAC;gBAED,+EAA+E;gBAC/E,eAAe,EAAE,CAAC;gBAElB,+DAA+D;gBAC/D,eAAe,EAAE,CAAC;gBAClB,kBAAkB,EAAE,CAAC;gBAErB,8DAA8D;gBAC9D,MAAM,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;gBACrC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;gBAE7C,8DAA8D;gBAC9D,IAAI,CAAC;oBACH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;wBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,QAAQ,CAAC,CAAC;wBACxD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,SAAS,QAAQ,CAAC,CAAC;wBAC5D,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;4BAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;wBACjC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,0DAA0D;oBAC1D,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;wBAC/B,IAAI,CAAC;4BAAC,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAAC,CAAC;wBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBAC1F,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACxD,CAAC;gBAED,2DAA2D;gBAC3D,IAAI,CAAC;oBACH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;wBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,QAAQ,CAAC,CAAC;wBACxD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;wBACjE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBAC/B,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,sEAAsE;oBACtE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;wBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,QAAQ,CAAC,CAAC;wBACxD,IAAI,CAAC;4BACH,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;gCAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;wBACnF,CAAC;wBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBAC/B,CAAC;oBACD,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;wBAC/B,IAAI,CAAC;4BAAC,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAAC,CAAC;wBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBAC1F,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACtF,CAAC;gBAED,iEAAiE;gBACjE,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;oBAClC,IAAI,CAAC;wBAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,mEAAmE;gBACnE,iEAAiE;gBACjE,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;oBACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,QAAQ,CAAC,CAAC;oBACxD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,SAAS,QAAQ,CAAC,CAAC;oBAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;oBAEjE,IAAI,CAAC;wBACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC5B,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;gCAAE,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;iCACrD,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;gCAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;wBAC7D,CAAC;wBACD,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC/E,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;4BAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7E,CAAC;oBAAC,MAAM,CAAC,CAAC,iDAAiD,CAAC,CAAC;gBAC/D,CAAC;gBAED,mEAAmE;gBACnE,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;oBACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;oBAClD,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;wBACpC,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;4BAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;wBACvD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,wCAAwC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAC7F,CAAC;gBACJ,CAAC;gBAED,iDAAiD;gBACjD,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/src/storage/tables.d.ts

@@ -0,0 +1,147 @@
+import type { Table } from "./db.js";
+import { type ConversationSource, type Domain, type Intent, type ConversationQuality, type SourceType, type SourceScope } from "../providers/types.js";
+/** Core tables that must exist after first boot. */
+export declare const CORE_TABLE_NAMES: readonly ["structured_docs", "conversations_hot", "conversations_cold", "digests"];
+/** All tables including optional ones created by initTables. */
+export declare const TABLE_NAMES: readonly ["structured_docs", "conversations_hot", "conversations_cold", "digests", "structured_docs_local"];
+export type TableName = (typeof TABLE_NAMES)[number];
+export interface StructuredDocRow {
+    [key: string]: unknown;
+    id: string;
+    source_path: string;
+    content: string;
+    embedding: number[];
+    source_type: SourceType;
+    source_scope: SourceScope;
+    client_name: string;
+    indexed_at: string;
+    metadata: string;
+}
+export interface ConversationRow {
+    [key: string]: unknown;
+    id: string;
+    content: string;
+    summary: string;
+    embedding: number[];
+    source: ConversationSource;
+    domain: Domain;
+    intent: Intent;
+    quality: ConversationQuality;
+    topics: string;
+    decisions: string;
+    key_outputs: string;
+    indexed_at: string;
+    idempotency_key: string;
+    /**
+     * Canonical content fingerprint (sha256 of `conversationContentFingerprint`
+     * input). Used by ingest refresh-on-grow to decide skip vs refresh without
+     * re-deriving from `content` on every replay, and to give rebuild a single
+     * source of truth even when the original raw artifact has been pruned.
+     * Empty string for legacy rows written before this column existed and
+     * for rows where no fingerprint was computable at write time.
+     */
+    conversation_fingerprint: string;
+    /**
+     * Which extractor produced this row's metadata.
+     *   - "rift-basic": deterministic basic metadata written by a no-AI fast
+     *     import (BasicMetadataExtractor). The ONLY metadata-enrichment backfill
+     *     target. Its default domain/intent/quality must NOT be treated as
+     *     authoritative facets/filters/ranking boosts.
+     *   - "" (empty): enriched, or provider-unknown (legacy rows written before
+     *     this column existed). NOT a backfill target.
+     *   - any other value (e.g. "codex-cli"): enriched by a named AI provider.
+     */
+    metadata_provider: string;
+    /**
+     * Which provider produced this row's embedding vector.
+     *   - "rift-none": placeholder zero-vector written by a no-key import. The
+     *     ONLY embedding backfill target; vector search must exclude these once
+     *     a real provider is connected (exclusion + backfill are deferred).
+     *   - "" (empty): a valid embedding, or provider-unknown (legacy rows).
+     *     NOT a backfill target.
+     */
+    embedding_provider: string;
+    /**
+     * Chunk markers (index-time chunking). All three are empty strings for an
+     * unchunked conversation or a legacy row written before these columns
+     * existed. When a long conversation is split, every chunk row shares the same
+     * `parent_id` and conversation-level metadata; only `content`/`embedding`/
+     * `chunk_index` differ.
+     *   - `parent_id`: stable conversation key shared by all chunks (the save
+     *     chain-root id, or the web-import stable row id). Empty = unchunked.
+     *   - `chunk_index`: 0-based ordinal within the set ("" = unchunked).
+     *   - `chunk_count`: total chunks; `> 1` is the explicit marker retrieval
+     *     grouping + replay completeness checks key on. "" / "1" = single
+     *     unmarked row that is never grouped.
+     *
+     * Optional in the type (legacy/null rows read as unchunked), but ALWAYS
+     * written by production producers via `conversationChunkColumns(...)`. The
+     * columns themselves always exist on disk (seed + `migrateConversationColumns`).
+     */
+    parent_id?: string;
+    chunk_index?: string;
+    chunk_count?: string;
+}
+export interface DigestRow {
+    [key: string]: unknown;
+    id: string;
+    content: string;
+    summary: string;
+    embedding: number[];
+    period_start: string;
+    period_end: string;
+    digest_type: string;
+    created_at: string;
+}
+export declare const SEED_ID = "__seed__";
+export declare function setSwapInProgress(v: boolean): void;
+export declare function isSwapInProgress(): boolean;
+/**
+ * Initialize all four LanceDB tables. Idempotent — safe to call on every startup.
+ * Creates tables that don't exist; opens tables that already do.
+ */
+export declare function initTables(dataDir: string): Promise<void>;
+/**
+ * Open EXISTING tables into the handle cache WITHOUT creating or migrating any.
+ *
+ * The read-only counterpart to {@link initTables}, for recon/dry-run/diagnostic
+ * paths that must never mutate the live schema. `initTables` adds missing
+ * columns (`migrateConversationColumns`) and creates missing tables — both are
+ * schema writes. Running it from a chunk-aware tool against a dir served by a
+ * pre-chunk daemon is what added the chunk columns and broke that daemon's
+ * writes (the `chunk-backfill --dry-run` incident). A read-only consumer has no
+ * business doing either, so it opens tables as-is and leaves the schema and the
+ * on-disk version untouched.
+ *
+ * Opens every table in {@link TABLE_NAMES} that exists; silently skips ones that
+ * don't (it will NOT create them). Any name in `require` that is missing throws
+ * a clear error rather than being created — so a caller that needs a specific
+ * table fails loudly instead of operating on a table that a later
+ * schema-migrating open would have silently created.
+ */
+export declare function openTablesReadOnly(dataDir: string, opts?: {
+    require?: readonly TableName[];
+}): Promise<void>;
+/**
+ * Get a typed table handle. Throws if tables haven't been initialized
+ * or if a shadow swap is in progress.
+ */
+export declare function getTable(name: TableName): Table;
+/** Replace the cached handle for a table (used by shadow-swap after commit). */
+export declare function setTable(name: TableName, table: Table): void;
+/**
+ * Close all live table handles without checking the swap lock.
+ * Used internally by shadow-swap during commitAllSwaps.
+ */
+export declare function closeAllHandles(): void;
+/**
+ * Refresh all cached table handles to see the latest data.
+ * Called after worker processes write to the same LanceDB files.
+ */
+export declare function refreshTables(): Promise<void>;
+/** Close all table handles and the connection. */
+export declare function closeTables(): void;
+/** Reset all state (for tests). */
+export declare function resetTables(): void;
+export declare function seedForTable(name: TableName): Record<string, unknown>[];
+//# sourceMappingURL=tables.d.ts.map

package/dist/src/storage/tables.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../src/storage/tables.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAc,KAAK,EAAE,MAAM,SAAS,CAAC;AAEjD,OAAO,EAGL,KAAK,kBAAkB,EACvB,KAAK,MAAM,EACX,KAAK,MAAM,EACX,KAAK,mBAAmB,EACxB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,uBAAuB,CAAC;AAI/B,oDAAoD;AACpD,eAAO,MAAM,gBAAgB,oFAKnB,CAAC;AAEX,gEAAgE;AAChE,eAAO,MAAM,WAAW,6GAGd,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAIrD,MAAM,WAAW,gBAAgB;IAC/B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,UAAU,CAAC;IACxB,YAAY,EAAE,WAAW,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;;;;;;;OASG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,SAAS;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAID,eAAO,MAAM,OAAO,aAAa,CAAC;AAgFlC,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI,CAElD;AAED,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,SAAS,SAAS,EAAE,CAAA;CAAO,GAC5C,OAAO,CAAC,IAAI,CAAC,CAkCf;AAwCD;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,KAAK,CAS/C;AAED,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAE5D;AAED;;;GAGG;AACH,wBAAgB,eAAe,IAAI,IAAI,CAStC;AAED;;;GAGG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CASnD;AAED,kDAAkD;AAClD,wBAAgB,WAAW,IAAI,IAAI,CAMlC;AAED,mCAAmC;AACnC,wBAAgB,WAAW,IAAI,IAAI,CAGlC;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAYvE"}