onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/jobs/handlers/reconcile.js

@@ -0,0 +1,685 @@
+/**
+ * Reconcile job handler — Slice 10.
+ *
+ * Enforces the Source Recoverability Contract: scans every canonical
+ * raw-state location and repairs index drift.
+ *
+ * - filesystem_watched / filesystem_scheduled_scan: source roots -> structured_docs
+ * - conversation sources: data/raw/conversations/{source}/ -> conversations_hot + conversations_cold
+ * - digests: data/raw/digests/ -> digests table
+ *
+ * Missing (raw exists, not indexed) -> re-index.
+ * Orphaned (indexed, no raw) -> remove.
+ *
+ * Implements Live-Indexer / Repair-Job Coordination Rule:
+ * 1. Pause watchers + scanner
+ * 2. Perform reconcile work
+ * 3. Post-pass: resume watchers (drains buffered events)
+ * 4. Resume scanner
+ * 5. Only then mark job complete
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { CONVERSATION_SOURCES } from "../../providers/types.js";
+import { RIFT_NONE_EMBEDDING_PROVIDER, isPlaceholderEmbeddingProvider, } from "../../providers/placeholder-embed.js";
+import { isWatcherPathIgnored } from "../../ingestion/watcher.js";
+import { walkDirectory } from "../../ingestion/scanner.js";
+import { buildIgnoredPaths } from "../../ingestion/ignored-paths.js";
+import { recordEmbed } from "../../observability/embedding-events.js";
+import { recordIndexWrite } from "../../observability/index-events.js";
+import { Indexer, fileId } from "../../ingestion/indexer.js";
+import { eqFilter } from "../../storage/filter.js";
+import { chunkSetComplete } from "../../ingestion/chunk-meta.js";
+import { isSupported } from "../../ingestion/extractor.js";
+import { getTable } from "../../storage/tables.js";
+import { conversationContentFingerprint } from "../../ingestion/inbox-core/conversation-fingerprint.js";
+import { dedupeConversations } from "./dedupe-conversations.js";
+import { compareRawRecencyDesc } from "../../storage/rebuild.js";
+/** All conversation source directories to reconcile. */
+const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
+/** Compute the union of ignored roots across all configured sources. */
+function computeIgnoredPaths(deps) {
+    const all = deps.config.sources.flatMap((s) => buildIgnoredPaths(s, deps.config.data_paths));
+    return Array.from(new Set(all));
+}
+export function createReconcileHandler(deps) {
+    return async (job) => {
+        // Read the job's input args before we overwrite `payload` with the report.
+        const args = job.payload ?? {};
+        const dedupeMode = args.dedupe_conversations === true;
+        const apply = args.apply === true;
+        // --- 1. Pause all live indexers ---
+        for (const w of deps.watchers)
+            w.pause();
+        if (deps.scanner)
+            deps.scanner.pause();
+        const report = {
+            files_checked: 0,
+            missing_reindexed: 0,
+            orphans_removed: 0,
+            engine_owned_purged: 0,
+            missing_skipped_duplicate_key: 0,
+            duplicate_rows_removed: 0,
+        };
+        try {
+            if (dedupeMode) {
+                // Dedupe-conversations repair pass. Skips the normal structured-doc
+                // / conversation / digest reconciliation — this is a focused
+                // one-shot cleanup for pre-stable-id legacy duplicates.
+                const dedupeReport = await dedupeConversations({
+                    apply,
+                    dataDir: deps.dataDir,
+                    reportsDir: path.resolve(deps.dataDir, "..", "reports"),
+                });
+                job.payload = dedupeReport;
+                return;
+            }
+            // --- 2. Reconcile structured_docs ---
+            await reconcileStructuredDocs(deps, report);
+            // --- 3. Reconcile conversations ---
+            await reconcileConversations(deps, report);
+            // --- 4. Reconcile digests ---
+            await reconcileDigests(deps, report);
+            // Store report on job for inspection
+            job.payload = report;
+        }
+        finally {
+            // --- 5. Post-pass ---
+            // Watcher post-pass: resume drains the event buffer (events that
+            // arrived via chokidar while paused).
+            for (const w of deps.watchers)
+                await w.resume();
+            // Scanner post-pass: scheduled_scan dirs have no filesystem watcher,
+            // so files written during reconcile are invisible to the buffer.
+            // Re-scan those directories and index anything new. This runs after
+            // the main reconcile mutations are done, so no conflict.
+            // Always run if there are scheduled_scan sources, regardless of
+            // whether a Scanner instance exists (the cron scanner is optional;
+            // the post-pass is mandatory).
+            // Scanner stays paused until the post-pass completes so a cron-
+            // triggered scan cannot overlap recovery work.
+            await scannerPostPass(deps);
+            if (deps.scanner)
+                deps.scanner.resume();
+        }
+    };
+}
+/**
+ * Scanner post-pass: re-scan all scheduled_scan directories to pick up
+ * changes that happened during the reconcile pause window. Handles all
+ * three drift cases:
+ * - New file on disk, not indexed -> add
+ * - Indexed file modified on disk -> re-index
+ * - Indexed file deleted from disk -> remove orphan
+ */
+async function scannerPostPass(deps) {
+    const scanSources = deps.config.sources.filter((s) => s.mode === "scheduled_scan");
+    if (scanSources.length === 0)
+        return;
+    const cloudTable = getTable("structured_docs");
+    const localTable = getTable("structured_docs_local");
+    const ignoredPaths = computeIgnoredPaths(deps);
+    for (const source of scanSources) {
+        // Build set of current disk files for this source.
+        const diskIds = new Map(); // id -> canonical path
+        const files = walkDirectory(source.path, { ignoredPaths });
+        for (const rawPath of files) {
+            if (!isSupported(rawPath))
+                continue;
+            let canonical;
+            try {
+                canonical = fs.realpathSync(rawPath);
+            }
+            catch {
+                continue;
+            }
+            diskIds.set(fileId(canonical), canonical);
+        }
+        // Query all indexed rows for this source root (both tables).
+        const table = source.extraction === "local" ? localTable : cloudTable;
+        const allRows = (await table.query().toArray());
+        // Group indexed rows by their PARENT file id. A chunked doc is N rows that
+        // share one `source_path` but carry ids `<fileId>:<i>`, while the on-disk
+        // file is keyed by `fileId(source_path)`. Keying membership on `row.id`
+        // would make every chunk read as an orphan — the file would be re-indexed
+        // and then its freshly written chunk set deleted on every scan. Group by
+        // `fileId(source_path)` so a chunk set is treated as exactly one file.
+        const rowsByFileId = new Map();
+        for (const r of allRows) {
+            if (r.source_type !== "filesystem_scheduled_scan")
+                continue;
+            if (!r.source_path.startsWith(source.path))
+                continue;
+            const key = fileId(r.source_path);
+            const list = rowsByFileId.get(key) ?? [];
+            list.push(r);
+            rowsByFileId.set(key, list);
+        }
+        // Resolve provider per file via longest-prefix match so nested
+        // local roots under a cloud parent are correctly attributed.
+        const resolveProvider = (filePath) => {
+            let best = source;
+            for (const s of deps.config.sources) {
+                if (filePath.startsWith(s.path) && s.path.length > best.path.length) {
+                    best = s;
+                }
+            }
+            return best.extraction === "local" && deps.localEmbeddingProvider
+                ? deps.localEmbeddingProvider
+                : deps.embeddingProvider;
+        };
+        const resolveIsLocal = (filePath) => {
+            let best = source;
+            for (const s of deps.config.sources) {
+                if (filePath.startsWith(s.path) && s.path.length > best.path.length) {
+                    best = s;
+                }
+            }
+            return best.extraction === "local" && !!deps.localEmbeddingProvider;
+        };
+        const makeIndexer = (filePath) => {
+            const owner = deps.config.sources.reduce((best, s) => filePath.startsWith(s.path) && s.path.length > best.path.length
+                ? s
+                : best, source);
+            const isLocal = resolveIsLocal(filePath);
+            return new Indexer(resolveProvider(filePath), {
+                sourceType: "filesystem_scheduled_scan",
+                sourceScope: owner.scope,
+                clientName: owner.client_name ?? "",
+                allowedRoots: [owner.path],
+                dataDir: deps.dataDir,
+                ...(isLocal ? { tableName: "structured_docs_local" } : {}),
+            });
+        };
+        // New or modified files -> add/re-index.
+        for (const [id, canonical] of diskIds) {
+            const existing = rowsByFileId.get(id) ?? [];
+            if (existing.length === 0) {
+                // New file
+                try {
+                    await makeIndexer(canonical).handleEvent({
+                        type: "add",
+                        path: canonical,
+                    });
+                }
+                catch (err) {
+                    process.stderr.write(`reconcile post-pass: failed to index ${canonical}: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+            }
+            else {
+                // Check if modified since last index. All chunks of one file share an
+                // `indexed_at`, but take the most recent defensively.
+                try {
+                    const stats = fs.statSync(canonical);
+                    const indexedAt = new Date(existing.reduce((latest, r) => (r.indexed_at > latest ? r.indexed_at : latest), existing[0].indexed_at));
+                    if (stats.mtime > indexedAt) {
+                        await makeIndexer(canonical).handleEvent({
+                            type: "change",
+                            path: canonical,
+                        });
+                    }
+                }
+                catch {
+                    // stat failed — skip
+                }
+            }
+        }
+        // Deleted files -> remove orphans. Delete by `source_path` so the whole
+        // chunk set for a vanished file is removed in one shot (ids would only
+        // remove a single chunk).
+        for (const [id, rows] of rowsByFileId) {
+            if (diskIds.has(id))
+                continue;
+            await table.delete(eqFilter("source_path", rows[0].source_path));
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Structured docs: source roots -> structured_docs table
+// ---------------------------------------------------------------------------
+async function reconcileStructuredDocs(deps, report) {
+    const cloudTable = getTable("structured_docs");
+    const localTable = getTable("structured_docs_local");
+    const ignoredPaths = computeIgnoredPaths(deps);
+    // Build set of all supported files on disk across configured source roots.
+    // Key = fileId(canonical_path), value = { canonical path, source config }.
+    // When roots overlap or nest, keep the most-specific (longest path)
+    // owner so local files under a cloud parent root are correctly attributed.
+    const diskFiles = new Map();
+    for (let i = 0; i < deps.config.sources.length; i++) {
+        const source = deps.config.sources[i];
+        const files = walkDirectory(source.path, { ignoredPaths });
+        for (const rawPath of files) {
+            if (!isSupported(rawPath))
+                continue;
+            let canonical;
+            try {
+                canonical = fs.realpathSync(rawPath);
+            }
+            catch {
+                continue; // inaccessible
+            }
+            const id = fileId(canonical);
+            const existing = diskFiles.get(id);
+            if (existing) {
+                // Keep the most-specific (longest path) source root.
+                const existingPath = deps.config.sources[existing.sourceIdx].path;
+                if (source.path.length > existingPath.length) {
+                    diskFiles.set(id, { filePath: canonical, sourceIdx: i });
+                }
+                // Don't double-count files_checked for duplicates.
+            }
+            else {
+                diskFiles.set(id, { filePath: canonical, sourceIdx: i });
+                report.files_checked++;
+            }
+        }
+    }
+    // Query all indexed structured_docs rows (both cloud and local tables).
+    const cloudRows = (await cloudTable.query().toArray());
+    const localRows = (await localTable.query().toArray());
+    const allRows = [...cloudRows, ...localRows];
+    // Group the indexed side by source_path: a chunked document is N rows that
+    // share one path, so membership must be path-keyed, not id-keyed (a chunk
+    // id is `${fileId}:${i}`, never `fileId`). Disk membership is likewise by
+    // canonical path.
+    const indexedPaths = new Set();
+    for (const row of allRows)
+        indexedPaths.add(row.source_path);
+    const diskPaths = new Set();
+    for (const { filePath } of diskFiles.values())
+        diskPaths.add(filePath);
+    // Missing: on disk but no indexed row for that path -> re-index
+    for (const [, { filePath, sourceIdx }] of diskFiles) {
+        if (indexedPaths.has(filePath))
+            continue;
+        const source = deps.config.sources[sourceIdx];
+        const isLocal = source.extraction === "local" && !!deps.localEmbeddingProvider;
+        const embedding = isLocal
+            ? deps.localEmbeddingProvider
+            : deps.embeddingProvider;
+        const indexer = new Indexer(embedding, {
+            sourceType: source.mode === "watch"
+                ? "filesystem_watched"
+                : "filesystem_scheduled_scan",
+            sourceScope: source.scope,
+            clientName: source.client_name ?? "",
+            allowedRoots: [source.path],
+            dataDir: deps.dataDir,
+            ...(isLocal ? { tableName: "structured_docs_local" } : {}),
+        });
+        try {
+            await indexer.handleEvent({ type: "add", path: filePath });
+            report.missing_reindexed++;
+        }
+        catch (err) {
+            process.stderr.write(`reconcile: failed to re-index ${filePath}: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+    // Orphaned: indexed but not on disk -> remove from whichever table has it.
+    // Only consider filesystem source types to avoid removing rows from
+    // a hypothetical future source type.
+    // Rows whose `source_path` lies under an engine-owned root (dist/, logs/,
+    // data_dir, jobs_dir, .codex-worktrees/) are classified as retroactive
+    // pollution and counted in `engine_owned_purged` rather than
+    // `orphans_removed` — same delete, different meaning in the report.
+    // Collect orphaned source_paths per table (deduped, so a chunk set counts
+    // and deletes once), then delete the whole set for each by source_path.
+    let purgedAny = false;
+    const collectOrphanPaths = (rows) => {
+        const paths = new Set();
+        for (const row of rows) {
+            if ((row.source_type === "filesystem_watched" ||
+                row.source_type === "filesystem_scheduled_scan") &&
+                !diskPaths.has(row.source_path)) {
+                paths.add(row.source_path);
+            }
+        }
+        return paths;
+    };
+    const purgeOrphans = async (table, rows) => {
+        for (const sourcePath of collectOrphanPaths(rows)) {
+            await table.delete(eqFilter("source_path", sourcePath));
+            purgedAny = true;
+            if (isWatcherPathIgnored(sourcePath, ignoredPaths)) {
+                report.engine_owned_purged++;
+            }
+            else {
+                report.orphans_removed++;
+            }
+        }
+    };
+    await purgeOrphans(cloudTable, cloudRows);
+    await purgeOrphans(localTable, localRows);
+    // Reclaim disk: LanceDB keeps old versions until optimize runs. Set
+    // `cleanupOlderThan: new Date()` so every version except the current one
+    // is pruned — safe because reconcile has no concurrent in-progress
+    // transactions here (watchers + scanner are paused by the caller).
+    if (purgedAny) {
+        try {
+            await cloudTable.optimize({ cleanupOlderThan: new Date() });
+            await localTable.optimize({ cleanupOlderThan: new Date() });
+        }
+        catch (err) {
+            process.stderr.write(`reconcile: structured_docs optimize failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Conversations: data/raw/conversations/{source}/ -> hot + cold tables
+// ---------------------------------------------------------------------------
+async function reconcileConversations(deps, report) {
+    const hotTable = getTable("conversations_hot");
+    const coldTable = getTable("conversations_cold");
+    // Collect all raw conversation files.
+    // Key = row ID (extracted from filename), value = raw file path.
+    const rawFiles = new Map();
+    for (const source of ALL_CONV_SOURCES) {
+        const rawDir = path.join(deps.dataDir, "raw", "conversations", source);
+        if (!fs.existsSync(rawDir))
+            continue;
+        const files = fs.readdirSync(rawDir).filter((f) => f.endsWith(".json"));
+        for (const file of files) {
+            const id = extractIdFromFilename(file);
+            if (!id)
+                continue;
+            report.files_checked++;
+            // If >1 raw artifact exists for the same id (interrupted prune), keep the
+            // most-recent one — same recency policy rebuild uses, so both tools agree.
+            const rawPath = path.join(rawDir, file);
+            const prior = rawFiles.get(id);
+            if (!prior || compareRawRecencyDesc(rawPath, prior) < 0) {
+                rawFiles.set(id, rawPath);
+            }
+        }
+    }
+    const hotRows = (await hotTable.query().toArray());
+    const coldRows = (await coldTable.query().toArray());
+    const indexedIds = new Set();
+    // A chunked conversation is N rows (`<parent>::<i>`) that share one raw
+    // artifact keyed by the parent id. Track every chunk row per parent so the
+    // raw's id counts as "indexed" only when its chunk set is COMPLETE —
+    // otherwise the missing-check would either resurrect a duplicate whole row
+    // beside the chunks, or freeze a partial set (crash mid-append) as if it were
+    // whole. A partial set reads as not-indexed → its winner is reindexed/repaired.
+    const parentRows = new Map();
+    for (const row of [...hotRows, ...coldRows]) {
+        indexedIds.add(row.id);
+        if (typeof row.parent_id === "string" && row.parent_id) {
+            const list = parentRows.get(row.parent_id) ?? [];
+            list.push(row);
+            parentRows.set(row.parent_id, list);
+        }
+    }
+    const rawIdIndexed = (id) => {
+        if (indexedIds.has(id))
+            return true;
+        const chunks = parentRows.get(id);
+        if (!chunks)
+            return false;
+        // Dedupe by row id before the completeness check: the same chunk can
+        // legitimately appear in BOTH tiers after a compaction crash (that is the
+        // cross-tier pass's job, not a "missing chunk"), so completeness is about
+        // the distinct chunk indexes present, not raw row count.
+        const uniq = new Map();
+        for (const r of chunks)
+            uniq.set(r.id, r);
+        return chunkSetComplete([...uniq.values()]);
+    };
+    const byKey = new Map();
+    const keylessMissing = [];
+    // One pass over the most-recent raw per id. The key always comes from the raw
+    // file (see comment above) — for indexed and missing rows alike. Each id is
+    // unique in `rawFiles`, so cross-tier duplicates (same id in hot+cold) appear
+    // once here and are reconciled by the dedicated cross-tier pass below.
+    for (const [id, rawPath] of rawFiles) {
+        const indexed = rawIdIndexed(id);
+        let key = "";
+        try {
+            const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+            key = typeof raw.idempotency_key === "string" ? raw.idempotency_key : "";
+        }
+        catch {
+            // Unreadable/invalid raw — treat as keyless; a missing-raw reindex below
+            // surfaces its own error if the file is genuinely broken.
+        }
+        if (!key) {
+            if (!indexed)
+                keylessMissing.push({ id, rawPath });
+            // keyless indexed row -> leave as-is
+            continue;
+        }
+        const list = byKey.get(key) ?? [];
+        list.push({ id, rawPath, indexed });
+        byKey.set(key, list);
+    }
+    // Keyless missing raws: no dedup possible -> always reindex into hot tier.
+    for (const { id, rawPath } of keylessMissing) {
+        try {
+            await reindexConversation(deps.embeddingProvider, id, rawPath, deps.dataDir);
+            report.missing_reindexed++;
+        }
+        catch (err) {
+            process.stderr.write(`reconcile: failed to re-index conversation ${id}: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+    // Fully remove a row id from BOTH tiers — a key-race loser must not survive in
+    // either, regardless of which tier it currently sits in.
+    const deleteIndexedRow = async (id) => {
+        const idFilter = eqFilter("id", id, { validateAsRowId: true });
+        await hotTable.delete(idFilter);
+        await coldTable.delete(idFilter);
+        // Also drop any chunk set that hangs off this id as its parent.
+        const parentFilter = eqFilter("parent_id", id, { validateAsRowId: true });
+        await hotTable.delete(parentFilter);
+        await coldTable.delete(parentFilter);
+    };
+    // Per key: most-recent raw wins. Reindex the winner if not yet indexed; skip
+    // missing losers as duplicates; DELETE an already-indexed loser (the round-6
+    // flip: an older indexed row beside a newer same-key raw). Reindex the winner
+    // FIRST and bail on failure so we never drop the conversation entirely.
+    for (const cands of byKey.values()) {
+        cands.sort((a, b) => compareRawRecencyDesc(a.rawPath, b.rawPath));
+        const [winner, ...losers] = cands;
+        if (!winner)
+            continue;
+        if (!winner.indexed) {
+            try {
+                await reindexConversation(deps.embeddingProvider, winner.id, winner.rawPath, deps.dataDir);
+                report.missing_reindexed++;
+            }
+            catch (err) {
+                process.stderr.write(`reconcile: failed to re-index conversation ${winner.id}: ${err instanceof Error ? err.message : String(err)}\n`);
+                continue; // leave the existing index untouched; don't delete losers
+            }
+        }
+        for (const loser of losers) {
+            if (loser.indexed) {
+                await deleteIndexedRow(loser.id);
+                report.duplicate_rows_removed++;
+            }
+            else {
+                report.missing_skipped_duplicate_key++;
+            }
+        }
+    }
+    // Orphaned: indexed but no raw -> remove. For a chunk row the backing raw is
+    // keyed by its parent id, so check that; every chunk of a parent with no raw
+    // is removed (one delete per chunk row).
+    const rawKeyOf = (row) => typeof row.parent_id === "string" && row.parent_id ? row.parent_id : row.id;
+    for (const row of hotRows) {
+        if (!rawFiles.has(rawKeyOf(row))) {
+            await hotTable.delete(eqFilter("id", row.id, { validateAsRowId: true }));
+            report.orphans_removed++;
+        }
+    }
+    for (const row of coldRows) {
+        if (!rawFiles.has(rawKeyOf(row))) {
+            await coldTable.delete(eqFilter("id", row.id, { validateAsRowId: true }));
+            report.orphans_removed++;
+        }
+    }
+    // Cross-tier duplicates: same ID in both hot and cold.
+    // Cold is canonical (compaction moved it there), so remove from hot.
+    // This repairs the main compaction crash mode: "copied to cold but
+    // not yet removed from hot." A chunk row's backing raw is keyed by its
+    // PARENT id (`rawKeyOf`), not its own `<parent>::<i>` id, so gate on that —
+    // otherwise a chunk duplicated across tiers after a crash is never deduped.
+    const coldIds = new Set();
+    for (const row of coldRows) {
+        if (rawFiles.has(rawKeyOf(row)))
+            coldIds.add(row.id);
+    }
+    for (const row of hotRows) {
+        if (coldIds.has(row.id)) {
+            await hotTable.delete(eqFilter("id", row.id, { validateAsRowId: true }));
+            report.orphans_removed++;
+        }
+    }
+}
+/**
+ * Extract the row ID from a raw conversation filename.
+ * Filename format: {timestamp}_{id}.json  (timestamp has no underscores).
+ */
+function extractIdFromFilename(filename) {
+    const match = filename.match(/^[^_]+_(.+)\.json$/);
+    return match?.[1];
+}
+async function reindexConversation(embedding, id, rawPath, dataDir) {
+    const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+    const summary = raw.summary ?? "";
+    // Ingest raw files store full content separately; save raw files use summary as content.
+    const content = raw.content ?? summary;
+    const vec = await recordEmbed(dataDir, embedding, {
+        pipeline: "reconcile",
+        operation: "reindex_conversation",
+        input_count: 1,
+    }, () => embedding.embed(content || summary));
+    const fingerprint = typeof raw.conversation_fingerprint === "string" &&
+        raw.conversation_fingerprint.length > 0
+        ? raw.conversation_fingerprint
+        : content
+            ? conversationContentFingerprint(content)
+            : "";
+    const row = {
+        id,
+        content,
+        summary,
+        embedding: vec,
+        source: raw.source ?? "claude_code",
+        domain: raw.domain ?? "tech",
+        intent: raw.intent ?? "build",
+        quality: raw.quality ?? "medium",
+        topics: JSON.stringify(raw.topics ?? []),
+        decisions: JSON.stringify(raw.decisions ?? []),
+        key_outputs: JSON.stringify(raw.key_outputs ?? []),
+        indexed_at: new Date().toISOString(),
+        idempotency_key: raw.idempotency_key ?? "",
+        conversation_fingerprint: fingerprint,
+        // Metadata restored from raw → preserve marker. Embedding recomputed → marker
+        // reflects the provider that just ran.
+        metadata_provider: typeof raw.metadata_provider === "string" ? raw.metadata_provider : "",
+        embedding_provider: isPlaceholderEmbeddingProvider(embedding)
+            ? RIFT_NONE_EMBEDDING_PROVIDER
+            : "",
+        // PR1 reconstructs a single whole row from the raw artifact (conversation
+        // chunking on rebuild/reconcile is the deferred historical-backfill path).
+        parent_id: "",
+        chunk_index: "",
+        chunk_count: "",
+    };
+    // Replace any prior rows for this conversation before writing the
+    // reconstructed single row. This covers two cases: (a) a stale/partial chunk
+    // set keyed by `parent_id = id` left behind by an interrupted chunked write
+    // (which `rawIdIndexed` now reports as not-indexed so we land here), and
+    // (b) a pre-existing whole row with the same id. Deleting after the embed
+    // (above) preserves the embed-before-delete invariant; the brief window is
+    // recoverable from the raw artifact that drove this reindex.
+    const idFilter = eqFilter("id", id, { validateAsRowId: true });
+    const parentFilter = eqFilter("parent_id", id, { validateAsRowId: true });
+    for (const t of ["conversations_hot", "conversations_cold"]) {
+        await getTable(t).delete(idFilter);
+        await getTable(t).delete(parentFilter);
+    }
+    await recordIndexWrite(dataDir, {
+        table: "conversations_hot",
+        pipeline: "reconcile",
+        operation: "reindex_conversation",
+        row_count: 1,
+    }, () => getTable("conversations_hot").add([row]));
+}
+// ---------------------------------------------------------------------------
+// Digests: data/raw/digests/ -> digests table
+// ---------------------------------------------------------------------------
+async function reconcileDigests(deps, report) {
+    const digestTable = getTable("digests");
+    const digestDir = path.join(deps.dataDir, "raw", "digests");
+    // Collect raw digest files. Each must contain an `id` field.
+    const rawDigests = new Map();
+    if (fs.existsSync(digestDir)) {
+        const files = fs.readdirSync(digestDir).filter((f) => f.endsWith(".json"));
+        for (const file of files) {
+            const rawPath = path.join(digestDir, file);
+            try {
+                const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+                if (typeof raw.id === "string" && raw.id) {
+                    rawDigests.set(raw.id, rawPath);
+                    report.files_checked++;
+                }
+            }
+            catch {
+                // Skip malformed files
+            }
+        }
+    }
+    // Collect indexed digest IDs.
+    const allDigests = (await digestTable.query().toArray());
+    const indexedIds = new Set();
+    for (const row of allDigests)
+        indexedIds.add(row.id);
+    // Missing: raw exists, not indexed -> re-index.
+    for (const [id, rawPath] of rawDigests) {
+        if (indexedIds.has(id))
+            continue;
+        try {
+            await reindexDigest(deps.embeddingProvider, rawPath, deps.dataDir);
+            report.missing_reindexed++;
+        }
+        catch (err) {
+            process.stderr.write(`reconcile: failed to re-index digest ${id}: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+    // Orphaned: indexed but no raw -> remove.
+    for (const row of allDigests) {
+        if (!rawDigests.has(row.id)) {
+            await digestTable.delete(`id = '${row.id}'`);
+            report.orphans_removed++;
+        }
+    }
+}
+async function reindexDigest(embedding, rawPath, dataDir) {
+    const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+    const summary = raw.summary ?? raw.content ?? "";
+    const vec = await recordEmbed(dataDir, embedding, {
+        pipeline: "reconcile",
+        operation: "reindex_digest",
+        input_count: 1,
+    }, () => embedding.embed(summary));
+    const row = {
+        id: raw.id,
+        content: raw.content ?? "",
+        summary: raw.summary ?? "",
+        embedding: vec,
+        period_start: raw.period_start ?? "",
+        period_end: raw.period_end ?? "",
+        digest_type: raw.digest_type ?? "weekly",
+        created_at: raw.created_at ?? new Date().toISOString(),
+    };
+    await recordIndexWrite(dataDir, {
+        table: "digests",
+        pipeline: "reconcile",
+        operation: "reindex_digest",
+        row_count: 1,
+    }, () => getTable("digests").add([row]));
+}
+//# sourceMappingURL=reconcile.js.map