@getrift/rift 0.1.0-beta.11 → 0.1.0-beta.13

package/dist/src/jobs/handlers/dedupe-conversations.js

@@ -0,0 +1,371 @@
+/**
+ * Dedupe-conversations repair pass — opt-in tool.
+ *
+ * Closes the residual gap from the 2026-05-13 stable-row-id upgrade:
+ * rows written before that slice carry a legacy row id derived from
+ * `(conversationId, jobRootId)` instead of the new
+ * `stableConversationRowId(source, conversationId)`. A re-import after
+ * the upgrade lands the new stable-id row alongside the surviving legacy
+ * row instead of collapsing onto it, leaving two rows for the same
+ * `(source, conversation_id)`.
+ *
+ * Strategy: scan `conversations_hot` + `conversations_cold`, match each
+ * row to its raw artifact (by row id embedded in the filename), read
+ * `(source, conversation_id)` from the raw, and group rows by that
+ * canonical key. For groups with more than one member, pick a keeper
+ * and delete the rest (plus the matching raw files).
+ *
+ * Decision rule:
+ *  1. If any group member's row id matches
+ *     `stableConversationRowId(source, conversation_id)`, that member
+ *     wins ("stable-id is canonical post-upgrade"). Ties between
+ *     multiple stable members (degenerate — same row id would be one
+ *     row) fall through to the same secondary tie-breakers.
+ *  2. Otherwise (all members are legacy), prefer the newest
+ *     `indexed_at`, then prefer the `conversations_cold` member
+ *     (compaction promotes cold to canonical), then prefer larger
+ *     `content_length` (avoid silently shrinking the indexed snapshot),
+ *     then the lexicographically smaller row id (deterministic).
+ *
+ * Safety guard on the stable rule: a stable-id row may have been
+ * imported from an older export, so it can legitimately be older or
+ * shorter than a surviving legacy row from a more recent import. If
+ * any legacy member of the group is newer OR has more content than
+ * the chosen stable winner, the group is classified `suspicious` and
+ * `--apply` never mutates it. The operator triages by hand.
+ *
+ * Dry-run by default. Only `--apply` mutates rows or deletes raw files.
+ *
+ * CLI capture sources (`claude_code`, `codex_cli`, `gemini_cli`) opt out
+ * of `(source, conversation_id)` dedupe: they have no provider-side
+ * conversation id and their row id is the canonical identifier. The
+ * planner skips them entirely — they are neither grouped nor counted in
+ * `rows_without_dedupe_key`.
+ *
+ * For web-source rows whose raw file is missing, OR exists but lacks
+ * `conversation_id`, the planner reports them in `rows_without_dedupe_key`
+ * for operator triage. Truly orphan rows (no raw on disk) are also
+ * handled by the normal reconcile pass — that's where any deletion lives.
+ */
+import { stableConversationRowId } from "../../ingestion/inbox-core/conversation-key.js";
+/**
+ * CLI capture sources. These hit `POST /save` and have no provider-side
+ * conversation id — the row id (job chain root) IS the canonical
+ * identifier, so they cannot participate in stable-vs-legacy dedupe.
+ * The planner skips them entirely: they are neither grouped nor flagged
+ * in `rows_without_dedupe_key`.
+ */
+const CLI_CONVERSATION_SOURCES = new Set([
+    "claude_code",
+    "codex_cli",
+    "gemini_cli",
+]);
+/**
+ * Pure planner. Given a snapshot of rows + raw files, produces the
+ * dedupe plan with no I/O. Caller decides whether to apply.
+ */
+export function planDedupe(input) {
+    // Group raw files by the row id in the filename. A row id can have
+    // multiple raws (refresh-on-grow before the prune-prior-raw slice).
+    const rawsByRowId = new Map();
+    for (const raw of input.raw_files) {
+        const arr = rawsByRowId.get(raw.row_id_from_filename) ?? [];
+        arr.push(raw);
+        rawsByRowId.set(raw.row_id_from_filename, arr);
+    }
+    // Derive (source, conversation_id) per row id from its latest raw
+    // (lexicographically greatest filename). All raws for the same row
+    // id share the same provider conversation id, so this is just
+    // picking the most recent metadata snapshot.
+    const sourceAndConvByRowId = new Map();
+    for (const [rowId, raws] of rawsByRowId) {
+        const latest = raws.reduce((a, b) => (a.filename > b.filename ? a : b));
+        sourceAndConvByRowId.set(rowId, {
+            source: latest.source,
+            conversation_id: latest.conversation_id,
+        });
+    }
+    // Group rows by (source, conversation_id). Nested map (source ->
+    // conversation_id -> members) avoids string-key delimiter ambiguity
+    // since provider conversation ids can in principle contain any
+    // character.
+    const groups = new Map();
+    const rowsWithoutDedupeKey = [];
+    for (const row of input.rows) {
+        // CLI sources opt out of (source, conversation_id) dedupe: their
+        // row id IS the canonical key, so they cannot have stable-vs-legacy
+        // duplicates and should not be flagged when their raw lacks
+        // `conversation_id` (that's the expected shape).
+        if (CLI_CONVERSATION_SOURCES.has(row.source))
+            continue;
+        const lookup = sourceAndConvByRowId.get(row.row_id);
+        if (!lookup) {
+            rowsWithoutDedupeKey.push({
+                row_id: row.row_id,
+                table: row.table,
+                source: row.source,
+            });
+            continue;
+        }
+        const stableId = stableConversationRowId(lookup.source, lookup.conversation_id);
+        const member = {
+            row_id: row.row_id,
+            table: row.table,
+            indexed_at: row.indexed_at,
+            content_length: row.content_length,
+            is_stable_id: row.row_id === stableId,
+            raw_files: (rawsByRowId.get(row.row_id) ?? []).map((r) => r.path),
+        };
+        let bySource = groups.get(lookup.source);
+        if (!bySource) {
+            bySource = new Map();
+            groups.set(lookup.source, bySource);
+        }
+        const arr = bySource.get(lookup.conversation_id) ?? [];
+        arr.push(member);
+        bySource.set(lookup.conversation_id, arr);
+    }
+    // Build group plans for actual duplicates (size > 1).
+    const actionable = [];
+    const suspicious = [];
+    for (const [source, bySource] of groups) {
+        for (const [conversationId, members] of bySource) {
+            if (members.length < 2)
+                continue;
+            const stableId = stableConversationRowId(source, conversationId);
+            const decision = pickKeeper(members);
+            const deleted = members.filter((m) => m !== decision.kept);
+            const reasons = detectSuspicion(decision.kept, deleted);
+            const plan = {
+                source,
+                conversation_id: conversationId,
+                stable_id: stableId,
+                members,
+                kept_member: decision.kept,
+                deleted_members: deleted,
+                raw_files_to_delete: deleted.flatMap((m) => m.raw_files),
+                decision_reason: decision.reason,
+                classification: reasons.length > 0 ? "suspicious" : "actionable",
+                suspicious_reasons: reasons,
+            };
+            if (reasons.length > 0)
+                suspicious.push(plan);
+            else
+                actionable.push(plan);
+        }
+    }
+    // Deterministic ordering within each bucket: by source, then conversation_id.
+    const orderGroups = (a, b) => {
+        if (a.source !== b.source)
+            return a.source < b.source ? -1 : 1;
+        return a.conversation_id < b.conversation_id ? -1 : 1;
+    };
+    actionable.sort(orderGroups);
+    suspicious.sort(orderGroups);
+    return {
+        actionable_groups: actionable,
+        suspicious_groups: suspicious,
+        rows_without_dedupe_key: rowsWithoutDedupeKey,
+    };
+}
+/**
+ * A stable winner is only safe to delete legacy members when it is
+ * at least as new AND at least as long as every loser. Returns the
+ * list of suspicion reasons; empty list means the group is actionable.
+ * Legacy-only groups (where the keeper is itself legacy) are always
+ * actionable — the keeper rule already picked the safest legacy
+ * member (newest, then cold-preferred, then longer content).
+ */
+function detectSuspicion(kept, deleted) {
+    if (!kept.is_stable_id)
+        return [];
+    const reasons = new Set();
+    for (const loser of deleted) {
+        if (loser.indexed_at > kept.indexed_at)
+            reasons.add("legacy_member_is_newer");
+        if (loser.content_length > kept.content_length)
+            reasons.add("legacy_member_has_more_content");
+    }
+    return [...reasons];
+}
+function pickKeeper(members) {
+    const stable = members.filter((m) => m.is_stable_id);
+    if (stable.length > 0) {
+        const kept = stable.reduce(secondaryTieBreak);
+        return { kept, reason: "stable-id row is canonical post-upgrade" };
+    }
+    const kept = members.reduce(legacyPick);
+    return {
+        kept,
+        reason: "no stable-id row in group; kept newest legacy (cold-preferred, then longer content)",
+    };
+}
+/** Newest indexed_at wins; then prefer cold; then longer content; then lex-smaller id. */
+function legacyPick(a, b) {
+    if (a.indexed_at !== b.indexed_at)
+        return a.indexed_at > b.indexed_at ? a : b;
+    return secondaryTieBreak(a, b);
+}
+/** Prefer cold; then longer content; then lex-smaller row id. */
+function secondaryTieBreak(a, b) {
+    if (a.table !== b.table) {
+        return a.table === "conversations_cold" ? a : b;
+    }
+    if (a.content_length !== b.content_length) {
+        return a.content_length > b.content_length ? a : b;
+    }
+    return a.row_id < b.row_id ? a : b;
+}
+// ---------------------------------------------------------------------------
+// Side-effecting wrapper: reads tables + raw dirs, plans, optionally applies.
+// ---------------------------------------------------------------------------
+import fs from "node:fs";
+import path from "node:path";
+import { getTable } from "../../storage/tables.js";
+import { CONVERSATION_SOURCES } from "../../providers/types.js";
+const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
+/**
+ * Scan `conversations_hot` + `conversations_cold`, plan the dedupe,
+ * optionally apply. Caller (the reconcile handler) wraps this in the
+ * watcher pause/resume protocol so live writes can't race.
+ */
+export async function dedupeConversations(opts) {
+    const hotTable = getTable("conversations_hot");
+    const coldTable = getTable("conversations_cold");
+    const hotRows = (await hotTable.query().toArray());
+    const coldRows = (await coldTable.query().toArray());
+    const rows = [
+        ...hotRows.map((r) => rowView(r, "conversations_hot")),
+        ...coldRows.map((r) => rowView(r, "conversations_cold")),
+    ];
+    const errors = [];
+    const rawFiles = [];
+    // Total raw JSON files visited on disk, regardless of whether the
+    // planner could derive a `(source, conversation_id)` key from them.
+    // CLI raws (no `conversation_id`) count here even though they don't
+    // land in `rawFiles`, so the metric matches its name.
+    let rawFilesScanned = 0;
+    for (const source of ALL_CONV_SOURCES) {
+        const rawDir = path.join(opts.dataDir, "raw", "conversations", source);
+        if (!fs.existsSync(rawDir))
+            continue;
+        for (const filename of fs.readdirSync(rawDir)) {
+            if (!filename.endsWith(".json"))
+                continue;
+            const rowIdFromFilename = extractIdFromFilename(filename);
+            if (!rowIdFromFilename)
+                continue;
+            const fullPath = path.join(rawDir, filename);
+            rawFilesScanned++;
+            try {
+                const raw = JSON.parse(fs.readFileSync(fullPath, "utf-8"));
+                const rawSource = typeof raw.source === "string" ? raw.source : source;
+                const conversationId = typeof raw.conversation_id === "string" ? raw.conversation_id : "";
+                if (!conversationId) {
+                    // Pre-stable-id raws and CLI captures don't carry
+                    // conversation_id. Without it we can't group, so skip silently
+                    // — but the raw was still scanned (counted above).
+                    continue;
+                }
+                rawFiles.push({
+                    path: fullPath,
+                    filename,
+                    row_id_from_filename: rowIdFromFilename,
+                    source: rawSource,
+                    conversation_id: conversationId,
+                });
+            }
+            catch (err) {
+                errors.push(`read raw failed: ${fullPath}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    const plan = planDedupe({ rows, raw_files: rawFiles });
+    const rowsToDelete = plan.actionable_groups.reduce((n, g) => n + g.deleted_members.length, 0);
+    const rawFilesToDelete = plan.actionable_groups.reduce((n, g) => n + g.raw_files_to_delete.length, 0);
+    let rowsDeleted = 0;
+    let rawFilesDeleted = 0;
+    if (opts.apply) {
+        // Suspicious groups are deliberately NOT touched, even with --apply.
+        for (const group of plan.actionable_groups) {
+            for (const member of group.deleted_members) {
+                const table = member.table === "conversations_cold"
+                    ? coldTable
+                    : hotTable;
+                try {
+                    await table.delete(`id = '${member.row_id}'`);
+                    rowsDeleted++;
+                }
+                catch (err) {
+                    errors.push(`delete row failed: ${member.table}/${member.row_id}: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            for (const rawPath of group.raw_files_to_delete) {
+                try {
+                    fs.unlinkSync(rawPath);
+                    rawFilesDeleted++;
+                }
+                catch (err) {
+                    if (err && typeof err === "object" && "code" in err
+                        && err.code === "ENOENT") {
+                        // Already gone — count as deleted, not an error.
+                        rawFilesDeleted++;
+                    }
+                    else {
+                        errors.push(`delete raw failed: ${rawPath}: ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                }
+            }
+        }
+    }
+    const report = {
+        apply: opts.apply,
+        rows_scanned: rows.length,
+        raw_files_scanned: rawFilesScanned,
+        duplicate_groups_found: plan.actionable_groups.length + plan.suspicious_groups.length,
+        rows_to_delete: rowsToDelete,
+        raw_files_to_delete: rawFilesToDelete,
+        rows_deleted: rowsDeleted,
+        raw_files_deleted: rawFilesDeleted,
+        rows_without_dedupe_key: plan.rows_without_dedupe_key.length,
+        suspicious_groups_skipped: plan.suspicious_groups.length,
+        actionable_groups: plan.actionable_groups,
+        suspicious_groups: plan.suspicious_groups,
+        errors,
+    };
+    if (opts.reportsDir) {
+        try {
+            fs.mkdirSync(opts.reportsDir, { recursive: true });
+            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+            const suffix = opts.apply ? "apply" : "dry-run";
+            const reportPath = path.join(opts.reportsDir, `dedupe-conversations-${stamp}-${suffix}.json`);
+            // Set report_path BEFORE serializing so the persisted file is
+            // self-identifying — `report_path` on disk equals its own location.
+            report.report_path = reportPath;
+            fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+        }
+        catch (err) {
+            report.errors.push(`write report failed: ${opts.reportsDir}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return report;
+}
+function rowView(row, table) {
+    return {
+        row_id: row.id,
+        table,
+        indexed_at: row.indexed_at,
+        content_length: (row.content ?? "").length,
+        source: row.source,
+    };
+}
+/**
+ * Filename format: `{ISO-timestamp}_{rowId}.json`. ISO timestamps in
+ * the writer use `:` and `.` replaced by `-`, so the first `_`
+ * is the unambiguous separator.
+ */
+function extractIdFromFilename(filename) {
+    const match = filename.match(/^[^_]+_(.+)\.json$/);
+    return match?.[1];
+}
+//# sourceMappingURL=dedupe-conversations.js.map

package/dist/src/jobs/handlers/dedupe-conversations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedupe-conversations.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/dedupe-conversations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AAKzF;;;;;;GAMG;AACH,MAAM,wBAAwB,GAAoC,IAAI,GAAG,CAAC;IACxE,aAAa;IACb,WAAW;IACX,YAAY;CACb,CAAC,CAAC;AA0FH;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAAsB;IAC/C,mEAAmE;IACnE,oEAAoE;IACpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;QAC5D,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACd,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,kEAAkE;IAClE,mEAAmE;IACnE,8DAA8D;IAC9D,6CAA6C;IAC7C,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAGjC,CAAC;IACJ,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,oBAAoB,CAAC,GAAG,CAAC,KAAK,EAAE;YAC9B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,eAAe;SACxC,CAAC,CAAC;IACL,CAAC;IAED,iEAAiE;IACjE,oEAAoE;IACpE,+DAA+D;IAC/D,aAAa;IACb,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuC,CAAC;IAC9D,MAAM,oBAAoB,GAIpB,EAAE,CAAC;IAET,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,iEAAiE;QACjE,oEAAoE;QACpE,4DAA4D;QAC5D,iDAAiD;QACjD,IAAI,wBAAwB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QACvD,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,oBAAoB,CAAC,IAAI,CAAC;gBACxB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,uBAAuB,CACtC,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,eAAe,CACvB,CAAC;QACF,MAAM,MAAM,GAAiB;YAC3B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,YAAY,EAAE,GAAG,CAAC,MAAM,KAAK,QAAQ;YACrC,SAAS,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SAClE,CAAC;QACF,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjB,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,sDAAsD;IACtD,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YACjC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,IAAI,GAAoB;gBAC5B,MAAM;gBACN,eAAe,EAAE,cAAc;gBAC/B,SAAS,EAAE,QAAQ;gBACnB,OAAO;gBACP,WAAW,EAAE,QAAQ,CAAC,IAAI;gBAC1B,eAAe,EAAE,OAAO;gBACxB,mBAAmB,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBACxD,eAAe,EAAE,QAAQ,CAAC,MAAM;gBAChC,cAAc,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY;gBAChE,kBAAkB,EAAE,OAAO;aAC5B,CAAC;YACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBACzC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,MAAM,WAAW,GAAG,CAAC,CAAkB,EAAE,CAAkB,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC;IACF,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7B,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAE7B,OAAO;QACL,iBAAiB,EAAE,UAAU;QAC7B,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,oBAAoB;KAC9C,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CACtB,IAAkB,EAClB,OAAuB;IAEvB,IAAI,CAAC,IAAI,CAAC,YAAY;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC9E,IAAI,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc;YAC5C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,UAAU,CACjB,OAAuB;IAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,OAAO;QACL,IAAI;QACJ,MAAM,EACJ,qFAAqF;KACxF,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,SAAS,UAAU,CAAC,CAAe,EAAE,CAAe;IAClD,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,OAAO,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,iEAAiE;AACjE,SAAS,iBAAiB,CAAC,CAAe,EAAE,CAAe;IACzD,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC,KAAK,KAAK,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,cAAc,KAAK,CAAC,CAAC,cAAc,EAAE,CAAC;QAC1C,OAAO,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAE9E,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAEhE,MAAM,gBAAgB,GAAG,CAAC,GAAG,oBAAoB,EAAE,OAAO,CAAU,CAAC;AA6CrE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAA6B;IAE7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAEjD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAsB,CAAC;IACxE,MAAM,QAAQ,GAAG,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAsB,CAAC;IAE1E,MAAM,IAAI,GAAoB;QAC5B,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACtD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;KACzD,CAAC;IAEF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,kEAAkE;IAClE,oEAAoE;IACpE,oEAAoE;IACpE,sDAAsD;IACtD,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,KAAK,MAAM,QAAQ,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAC1C,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YAC1D,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC7C,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAGxD,CAAC;gBACF,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBACvE,MAAM,cAAc,GAClB,OAAO,GAAG,CAAC,eAAe,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,kDAAkD;oBAClD,+DAA+D;oBAC/D,mDAAmD;oBACnD,SAAS;gBACX,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,QAAQ;oBACd,QAAQ;oBACR,oBAAoB,EAAE,iBAAiB;oBACvC,MAAM,EAAE,SAAS;oBACjB,eAAe,EAAE,cAAc;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CACT,oBAAoB,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACpF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEvD,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,EACtC,CAAC,CACF,CAAC;IACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAC1C,CAAC,CACF,CAAC;IAEF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,qEAAqE;QACrE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,KAAK,oBAAoB;oBACjD,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;oBAC9C,WAAW,EAAE,CAAC;gBAChB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,sBAAsB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;oBACvB,eAAe,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IACE,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG;2BAC3C,GAAwB,CAAC,IAAI,KAAK,QAAQ,EAC9C,CAAC;wBACD,iDAAiD;wBACjD,eAAe,EAAE,CAAC;oBACpB,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrF,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAA8B;QACxC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,YAAY,EAAE,IAAI,CAAC,MAAM;QACzB,iBAAiB,EAAE,eAAe;QAClC,sBAAsB,EACpB,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAC/D,cAAc,EAAE,YAAY;QAC5B,mBAAmB,EAAE,gBAAgB;QACrC,YAAY,EAAE,WAAW;QACzB,iBAAiB,EAAE,eAAe;QAClC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,CAAC,MAAM;QAC5D,yBAAyB,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;QACxD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,MAAM;KACP,CAAC;IAEF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;YAChD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAC1B,IAAI,CAAC,UAAU,EACf,wBAAwB,KAAK,IAAI,MAAM,OAAO,CAC/C,CAAC;YACF,8DAA8D;YAC9D,oEAAoE;YACpE,MAAM,CAAC,WAAW,GAAG,UAAU,CAAC;YAChC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,wBAAwB,IAAI,CAAC,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,GAAoB,EAAE,KAAwB;IAC7D,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,EAAE;QACd,KAAK;QACL,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,cAAc,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM;QAC1C,MAAM,EAAE,GAAG,CAAC,MAAM;KACnB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC"}

package/dist/src/jobs/handlers/ingest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAsDA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EACV,iBAAiB,EACjB,iBAAiB,EAElB,MAAM,0BAA0B,CAAC;AAalC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAKnE,MAAM,WAAW,iBAAiB;IAChC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;IAChB,wBAAwB,CAAC,EAAE,CACzB,OAAO,EAAE,aAAa,EACtB,IAAI,EAAE,iBAAiB,KACpB,iBAAiB,CAAC;CACxB;AAiHD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,UAAU,CA4PvE"}
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAsDA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EACV,iBAAiB,EACjB,iBAAiB,EAElB,MAAM,0BAA0B,CAAC;AAalC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAKnE,MAAM,WAAW,iBAAiB;IAChC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;IAChB,wBAAwB,CAAC,EAAE,CACzB,OAAO,EAAE,aAAa,EACtB,IAAI,EAAE,iBAAiB,KACpB,iBAAiB,CAAC;CACxB;AA4HD,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,UAAU,CAqQvE"}

package/dist/src/jobs/handlers/ingest.js

@@ -40,14 +40,14 @@
  *   row replacement, prior raw artifacts for that row id are pruned
  *   best-effort so rebuild does not insert duplicates.
  *
- * Schema caveat (deferred):
- * - `conversation_fingerprint` is persisted in raw artifacts only.
- *   Adding it as a LanceDB column would require a schema migration
- *   across both `conversations_hot` and `conversations_cold` plus the
- *   rebuild / shadow-swap paths, which is non-trivial and out of scope
- *   for this slice. The row's stored `content` remains the comparison
- *   source until that migration lands; the raw fingerprint is kept
- *   for forward-compat and external tooling.
+ * Persisted fingerprint:
+ * - `conversation_fingerprint` is written both into the raw artifact and
+ *   into the LanceDB row. Refresh-on-grow prefers the row column when it
+ *   is non-empty (avoids re-deriving from `content` on every replay), and
+ *   falls back to deriving from row `content` for legacy rows written
+ *   before the column existed. The migration in `initTables` adds the
+ *   column to existing tables filled with empty strings, so those rows
+ *   continue to go through the derivation fallback until next refresh.
  */
 import crypto from "node:crypto";
 import fs from "node:fs";
@@ -105,19 +105,31 @@ function findRawFile(rawDir, rowId) {
     return path.join(rawDir, matches[matches.length - 1]);
 }
 /**
- * Compute the canonical content fingerprint of an existing indexed row.
+ * Read (or derive) the canonical content fingerprint of an existing
+ * indexed row.
  *
- * The row's own `content` is the source of truth for "what is currently
- * indexed". Using the row content (rather than the raw artifact on disk)
- * avoids a raw-vs-row drift bug: if Phase 2 crashes between writing the
- * new raw and replacing the row, the raw on disk is ahead of the table,
- * and a subsequent re-import comparing against raw would mistakenly
- * decide "unchanged" while the row is still stale.
+ * Preference order:
+ *  1. The row's persisted `conversation_fingerprint` column (when
+ *     non-empty) — this is the source of truth written at last ingest.
+ *  2. Derive from the row's own `content` — used for legacy rows
+ *     written before the column existed (post-migration the column is
+ *     present but empty for those rows).
  *
- * Returns `undefined` if the row has no readable content; callers
+ * Using the row content (rather than the raw artifact on disk) as the
+ * derivation fallback avoids a raw-vs-row drift bug: if Phase 2 crashes
+ * between writing the new raw and replacing the row, the raw on disk is
+ * ahead of the table, and a subsequent re-import comparing against raw
+ * would mistakenly decide "unchanged" while the row is still stale.
+ *
+ * Returns `undefined` if neither source yields a fingerprint; callers
  * treat that as "cannot prove unchanged" and refresh.
  */
 function existingRowFingerprint(row) {
+    const stored = typeof row.conversation_fingerprint === "string"
+        ? row.conversation_fingerprint
+        : "";
+    if (stored)
+        return stored;
     const content = typeof row.content === "string" ? row.content : undefined;
     if (!content)
         return undefined;
@@ -173,8 +185,14 @@ export function createIngestHandler(deps) {
         }
         const parsed = parser(files);
         if (parsed.length === 0) {
-            // Nothing to ingest — not an error, just empty export
-            return;
+            // A user-visible inbox drop that produced zero parseable
+            // conversations is an export the friend will want to know about
+            // and re-drop — not a silent no-op. Surfacing this as a failed
+            // job lets /status/friend show `last_error_reason: "empty_export"`
+            // and the existing Next: line route the user to re-drop.
+            // The phrase "No conversations" is what classifyJobError maps to
+            // the `empty_export` class; do not rephrase without updating it.
+            throw new Error(`No conversations found in export "${payload.filename}"`);
         }
         const rawDir = path.join(deps.dataDir, "raw", "conversations", payload.rawSource ?? payload.source);
         const table = getTable("conversations_hot");
@@ -334,6 +352,7 @@ export function createIngestHandler(deps) {
                 key_outputs: JSON.stringify(item.metadata.key_outputs),
                 indexed_at: new Date().toISOString(),
                 idempotency_key: payload.idempotency_key ?? "",
+                conversation_fingerprint: item.fingerprint,
             };
             await recordIndexWrite(deps.dataDir, {
                 table: targetTableName,
@@ -387,6 +406,12 @@ async function reindexFromRaw(rawPath, rowId, payload, tables, embeddingProvider
         operation: "reindex_from_raw",
         input_count: 1,
     }, () => embeddingProvider.embed(content || summary));
+    const fingerprint = typeof raw.conversation_fingerprint === "string"
+        && raw.conversation_fingerprint.length > 0
+        ? raw.conversation_fingerprint
+        : content
+            ? conversationContentFingerprint(content)
+            : "";
     const row = {
         id: rowId,
         content,
@@ -401,6 +426,7 @@ async function reindexFromRaw(rawPath, rowId, payload, tables, embeddingProvider
         key_outputs: JSON.stringify(raw.key_outputs ?? []),
         indexed_at: new Date().toISOString(),
         idempotency_key: payload.idempotency_key ?? "",
+        conversation_fingerprint: fingerprint,
     };
     await recordIndexWrite(dataDir, {
         table: targetTableName,

package/dist/src/jobs/handlers/ingest.js.map

@@ -1 +1 @@
-{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAS7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wCAAwC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,8BAA8B,EAAE,MAAM,wDAAwD,CAAC;AAGxG,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAavE,2EAA2E;AAC3E,MAAM,OAAO,GAAuC;IAClD,WAAW,EAAE,eAAe;IAC5B,UAAU,EAAE,cAAc;IAC1B,UAAU,EAAE,cAAc;IAC1B,QAAQ,EAAE,YAAY;CACvB,CAAC;AAuBF;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAQ,EAAE,KAAe;IACjD,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,OAAO,CAAC,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,OAAO,GAAG,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,MAAc,EAAE,KAAa;IAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,CAAC,IAAI,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,sBAAsB,CAAC,GAA4B;IAC1D,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,OAAO,8BAA8B,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,sBAAsB,CAC7B,MAAc,EACd,KAAa,EACb,YAAoB;IAEpB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO;IACnC,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,IAAI,IAAI,KAAK,YAAY;YAAE,SAAS;QACpC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,WAAW,GAAI,OAAO,CAAC,OAAqC;YAC/D,GAAG,CAAC,OAAyB,CAAC;QAEjC,MAAM,OAAO,GAAG,WAAW,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,iBAAiB,GAAG,CAAC,IAAI,CAAC,wBAAwB,IAAI,wBAAwB,CAAC,CACnF,OAAO,EACP,IAAI,CACL,CAAC;QAEF,qCAAqC;QACrC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE/D,iCAAiC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,yBAAyB,OAAO,CAAC,MAAM,0BAA0B,CAClE,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,sDAAsD;YACtD,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,eAAe,EACf,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CACpC,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAEjD,+DAA+D;QAC/D,iEAAiE;QACjE,kEAAkE;QAClE,6DAA6D;QAC7D,8DAA8D;QAC9D,wCAAwC;QACxC,MAAM,SAAS,GAA4B,EAAE,CAAC;QAC9C,MAAM,WAAW,GAA8B,EAAE,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YAClC,6DAA6D;YAC7D,+DAA+D;YAC/D,+DAA+D;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,8BAA8B,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAEzE,0BAA0B;YAC1B,+DAA+D;YAC/D,kEAAkE;YAClE,uEAAuE;YACvE,gEAAgE;YAChE,gDAAgD;YAChD,MAAM,WAAW,GAAG,MAAM,KAAK;iBAC5B,KAAK,EAAE;iBACP,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC;iBACxB,OAAO,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,KAAK,CAAC;gBAC3C,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,OAAO,EAAE;gBAC5D,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,aAAa,GAA0B,WAAW,CAAC,MAAM,GAAG,CAAC;gBACjE,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;oBACvB,CAAC,CAAC,oBAAoB;oBACtB,CAAC,CAAC,SAAS,CAAC;YAEhB,IAAI,aAAa,EAAE,CAAC;gBAClB,8DAA8D;gBAC9D,yDAAyD;gBACzD,6DAA6D;gBAC7D,4DAA4D;gBAC5D,mCAAmC;gBACnC,MAAM,WAAW,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAE,CAAC;gBACzD,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,WAAW,CAAC,CAAC;gBAEhE,IAAI,mBAAmB,IAAI,mBAAmB,KAAK,WAAW,EAAE,CAAC;oBAC/D,8CAA8C;oBAC9C,SAAS;gBACX,CAAC;gBAED,6DAA6D;gBAC7D,6DAA6D;gBAC7D,qDAAqD;gBACrD,0DAA0D;gBAC1D,iDAAiD;YACnD,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACnD,IAAI,eAAe,EAAE,CAAC;oBACpB,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;oBACtD,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,QAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,eAAe,CAChD,YAAY,CAAC,OAAO,CACrB,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,IAAI,SAAmB,CAAC;YACxB,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,WAAW,CAC3B,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,iBAAiB,EACtB;oBACE,QAAQ,EAAE,YAAY;oBACtB,SAAS,EAAE,wBAAwB;oBACnC,WAAW,EAAE,CAAC;iBACf,EACD,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CACzD,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,YAAY;gBACZ,KAAK;gBACL,WAAW;gBACX,QAAQ;gBACR,SAAS;gBACT,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,+BAA+B;QACzC,CAAC;QAED,8DAA8D;QAC9D,0DAA0D;QAC1D,+DAA+D;QAC/D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEjE,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC;YACjE,mEAAmE;YACnE,kEAAkE;YAClE,+DAA+D;YAC/D,0DAA0D;YAC1D,MAAM,WAAW,GAAc,IAAI,CAAC,YAAY,IAAI,mBAAmB,CAAC;YACxE,MAAM,OAAO,GAAG;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,eAAe,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;gBACrC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAC9B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS;gBACvC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,wBAAwB,EAAE,IAAI,CAAC,WAAW;gBAC1C,YAAY,EAAE,WAAW;gBACzB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;gBAClC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;gBACtC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,GAAG,CAAC,oBAAoB;oBACtB,CAAC,CAAC,EAAE,qBAAqB,EAAE,oBAAoB,EAAE;oBACjD,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;YACF,MAAM,WAAW,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,KAAK,OAAO,CAAC;YACtD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAC/C,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,kEAAkE;YAClE,mEAAmE;YACnE,iEAAiE;YACjE,gEAAgE;YAChE,iEAAiE;YACjE,mDAAmD;YACnD,MAAM,eAAe,GAAc,IAAI,CAAC,YAAY,IAAI,mBAAmB,CAAC;YAC5E,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,KAAK,oBAAoB;gBAC5D,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK,CAAC;YAEV,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YACnD,CAAC;YAED,MAAM,GAAG,GAAoB;gBAC3B,EAAE,EAAE,IAAI,CAAC,KAAK;gBACd,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAClD,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACtD,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;aAC/C,CAAC;YAEF,MAAM,gBAAgB,CACpB,IAAI,CAAC,OAAO,EACZ;gBACE,KAAK,EAAE,eAAe;gBACtB,QAAQ,EAAE,YAAY;gBACtB,SAAS,EAAE,IAAI,CAAC,YAAY;oBAC1B,CAAC,CAAC,sBAAsB;oBACxB,CAAC,CAAC,qBAAqB;gBACzB,SAAS,EAAE,CAAC;aACb,EACD,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAC7B,CAAC;YAEF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,8DAA8D;gBAC9D,2DAA2D;gBAC3D,8DAA8D;gBAC9D,oDAAoD;gBACpD,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;gBAC9C,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,yDAAyD;QACzD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,CAClB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,OAAO,EACP,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAC/B,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,OAAO,CACb,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAAsB,EACtB,IAAuB;IAEvB,IAAI,OAAO,CAAC,4BAA4B,KAAK,WAAW,EAAE,CAAC;QACzD,OAAO,IAAI,yBAAyB,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,iBAAiB,CAAC;AAChC,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,KAAa,EACb,OAAsB,EACtB,MAA+E,EAC/E,iBAAoC,EACpC,OAAe;IAEf,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CACP,CAAC;IAE7B,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC;IACnC,MAAM,eAAe,GACnB,SAAS,KAAK,oBAAoB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAClF,MAAM,WAAW,GACf,eAAe,KAAK,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IAEtE,MAAM,SAAS,GAAG,MAAM,WAAW,CACjC,OAAO,EACP,iBAAiB,EACjB;QACE,QAAQ,EAAE,YAAY;QACtB,SAAS,EAAE,kBAAkB;QAC7B,WAAW,EAAE,CAAC;KACf,EACD,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,CAClD,CAAC;IAEF,MAAM,GAAG,GAAoB;QAC3B,EAAE,EAAE,KAAK;QACT,OAAO;QACP,OAAO;QACP,SAAS;QACT,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,MAAM;QAC3D,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,OAAO;QAC5D,OAAO,EAAG,GAAG,CAAC,OAAsC,IAAI,QAAQ;QAChE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,MAAmB,IAAI,EAAE,CAAC;QACtD,SAAS,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,SAAsB,IAAI,EAAE,CAAC;QAC5D,WAAW,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,WAAwB,IAAI,EAAE,CAAC;QAChE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;KAC/C,CAAC;IAEF,MAAM,gBAAgB,CACpB,OAAO,EACP;QACE,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,YAAY;QACtB,SAAS,EAAE,+BAA+B;QAC1C,SAAS,EAAE,CAAC;KACb,EACD,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,OAAe,EACf,MAAc,EACd,YAAgC,EAChC,KAAc;IAEd,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SAC5D,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,OAAO,EACP,YAAY,EACZ,GAAG,MAAM,IAAI,SAAS,IAAI,IAAI,OAAO,CACtC,CAAC;IAEF,MAAM,cAAc,GAAG;QACrB,MAAM;QACN,eAAe,EAAE,YAAY,CAAC,EAAE;QAChC,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QACpD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;IAEF,MAAM,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAS7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wCAAwC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,8BAA8B,EAAE,MAAM,wDAAwD,CAAC;AAGxG,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAavE,2EAA2E;AAC3E,MAAM,OAAO,GAAuC;IAClD,WAAW,EAAE,eAAe;IAC5B,UAAU,EAAE,cAAc;IAC1B,UAAU,EAAE,cAAc;IAC1B,QAAQ,EAAE,YAAY;CACvB,CAAC;AAuBF;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAQ,EAAE,KAAe;IACjD,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,OAAO,CAAC,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,OAAO,GAAG,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,MAAc,EAAE,KAAa;IAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,CAAC,IAAI,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,sBAAsB,CAAC,GAA4B;IAC1D,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,wBAAwB,KAAK,QAAQ;QAC7D,CAAC,CAAC,GAAG,CAAC,wBAAwB;QAC9B,CAAC,CAAC,EAAE,CAAC;IACP,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,OAAO,8BAA8B,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,sBAAsB,CAC7B,MAAc,EACd,KAAa,EACb,YAAoB;IAEpB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO;IACnC,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,IAAI,IAAI,KAAK,YAAY;YAAE,SAAS;QACpC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,WAAW,GAAI,OAAO,CAAC,OAAqC;YAC/D,GAAG,CAAC,OAAyB,CAAC;QAEjC,MAAM,OAAO,GAAG,WAAW,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,iBAAiB,GAAG,CAAC,IAAI,CAAC,wBAAwB,IAAI,wBAAwB,CAAC,CACnF,OAAO,EACP,IAAI,CACL,CAAC;QAEF,qCAAqC;QACrC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE/D,iCAAiC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,yBAAyB,OAAO,CAAC,MAAM,0BAA0B,CAClE,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,yDAAyD;YACzD,gEAAgE;YAChE,+DAA+D;YAC/D,mEAAmE;YACnE,yDAAyD;YACzD,iEAAiE;YACjE,iEAAiE;YACjE,MAAM,IAAI,KAAK,CACb,qCAAqC,OAAO,CAAC,QAAQ,GAAG,CACzD,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,eAAe,EACf,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CACpC,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAEjD,+DAA+D;QAC/D,iEAAiE;QACjE,kEAAkE;QAClE,6DAA6D;QAC7D,8DAA8D;QAC9D,wCAAwC;QACxC,MAAM,SAAS,GAA4B,EAAE,CAAC;QAC9C,MAAM,WAAW,GAA8B,EAAE,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YAClC,6DAA6D;YAC7D,+DAA+D;YAC/D,+DAA+D;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,8BAA8B,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAEzE,0BAA0B;YAC1B,+DAA+D;YAC/D,kEAAkE;YAClE,uEAAuE;YACvE,gEAAgE;YAChE,gDAAgD;YAChD,MAAM,WAAW,GAAG,MAAM,KAAK;iBAC5B,KAAK,EAAE;iBACP,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC;iBACxB,OAAO,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,KAAK,CAAC;gBAC3C,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,OAAO,EAAE;gBAC5D,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,aAAa,GAA0B,WAAW,CAAC,MAAM,GAAG,CAAC;gBACjE,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;oBACvB,CAAC,CAAC,oBAAoB;oBACtB,CAAC,CAAC,SAAS,CAAC;YAEhB,IAAI,aAAa,EAAE,CAAC;gBAClB,8DAA8D;gBAC9D,yDAAyD;gBACzD,6DAA6D;gBAC7D,4DAA4D;gBAC5D,mCAAmC;gBACnC,MAAM,WAAW,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAE,CAAC;gBACzD,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,WAAW,CAAC,CAAC;gBAEhE,IAAI,mBAAmB,IAAI,mBAAmB,KAAK,WAAW,EAAE,CAAC;oBAC/D,8CAA8C;oBAC9C,SAAS;gBACX,CAAC;gBAED,6DAA6D;gBAC7D,6DAA6D;gBAC7D,qDAAqD;gBACrD,0DAA0D;gBAC1D,iDAAiD;YACnD,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACnD,IAAI,eAAe,EAAE,CAAC;oBACpB,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;oBACtD,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,QAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,eAAe,CAChD,YAAY,CAAC,OAAO,CACrB,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,IAAI,SAAmB,CAAC;YACxB,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,WAAW,CAC3B,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,iBAAiB,EACtB;oBACE,QAAQ,EAAE,YAAY;oBACtB,SAAS,EAAE,wBAAwB;oBACnC,WAAW,EAAE,CAAC;iBACf,EACD,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CACzD,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,YAAY;gBACZ,KAAK;gBACL,WAAW;gBACX,QAAQ;gBACR,SAAS;gBACT,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,+BAA+B;QACzC,CAAC;QAED,8DAA8D;QAC9D,0DAA0D;QAC1D,+DAA+D;QAC/D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEjE,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC;YACjE,mEAAmE;YACnE,kEAAkE;YAClE,+DAA+D;YAC/D,0DAA0D;YAC1D,MAAM,WAAW,GAAc,IAAI,CAAC,YAAY,IAAI,mBAAmB,CAAC;YACxE,MAAM,OAAO,GAAG;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,eAAe,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;gBACrC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAC9B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS;gBACvC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,wBAAwB,EAAE,IAAI,CAAC,WAAW;gBAC1C,YAAY,EAAE,WAAW;gBACzB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;gBAClC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;gBACtC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,GAAG,CAAC,oBAAoB;oBACtB,CAAC,CAAC,EAAE,qBAAqB,EAAE,oBAAoB,EAAE;oBACjD,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;YACF,MAAM,WAAW,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,KAAK,OAAO,CAAC;YACtD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAC/C,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,kEAAkE;YAClE,mEAAmE;YACnE,iEAAiE;YACjE,gEAAgE;YAChE,iEAAiE;YACjE,mDAAmD;YACnD,MAAM,eAAe,GAAc,IAAI,CAAC,YAAY,IAAI,mBAAmB,CAAC;YAC5E,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,KAAK,oBAAoB;gBAC5D,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK,CAAC;YAEV,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YACnD,CAAC;YAED,MAAM,GAAG,GAAoB;gBAC3B,EAAE,EAAE,IAAI,CAAC,KAAK;gBACd,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAClD,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACtD,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;gBAC9C,wBAAwB,EAAE,IAAI,CAAC,WAAW;aAC3C,CAAC;YAEF,MAAM,gBAAgB,CACpB,IAAI,CAAC,OAAO,EACZ;gBACE,KAAK,EAAE,eAAe;gBACtB,QAAQ,EAAE,YAAY;gBACtB,SAAS,EAAE,IAAI,CAAC,YAAY;oBAC1B,CAAC,CAAC,sBAAsB;oBACxB,CAAC,CAAC,qBAAqB;gBACzB,SAAS,EAAE,CAAC;aACb,EACD,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAC7B,CAAC;YAEF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,8DAA8D;gBAC9D,2DAA2D;gBAC3D,8DAA8D;gBAC9D,oDAAoD;gBACpD,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;gBAC9C,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,yDAAyD;QACzD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,CAClB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,OAAO,EACP,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAC/B,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,OAAO,CACb,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAAsB,EACtB,IAAuB;IAEvB,IAAI,OAAO,CAAC,4BAA4B,KAAK,WAAW,EAAE,CAAC;QACzD,OAAO,IAAI,yBAAyB,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,iBAAiB,CAAC;AAChC,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,KAAa,EACb,OAAsB,EACtB,MAA+E,EAC/E,iBAAoC,EACpC,OAAe;IAEf,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CACP,CAAC;IAE7B,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC;IACnC,MAAM,eAAe,GACnB,SAAS,KAAK,oBAAoB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAClF,MAAM,WAAW,GACf,eAAe,KAAK,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IAEtE,MAAM,SAAS,GAAG,MAAM,WAAW,CACjC,OAAO,EACP,iBAAiB,EACjB;QACE,QAAQ,EAAE,YAAY;QACtB,SAAS,EAAE,kBAAkB;QAC7B,WAAW,EAAE,CAAC;KACf,EACD,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,CAClD,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,wBAAwB,KAAK,QAAQ;WAC7D,GAAG,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC;QAC5C,CAAC,CAAC,GAAG,CAAC,wBAAwB;QAC9B,CAAC,CAAC,OAAO;YACP,CAAC,CAAC,8BAA8B,CAAC,OAAO,CAAC;YACzC,CAAC,CAAC,EAAE,CAAC;IAET,MAAM,GAAG,GAAoB;QAC3B,EAAE,EAAE,KAAK;QACT,OAAO;QACP,OAAO;QACP,SAAS;QACT,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,MAAM;QAC3D,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,OAAO;QAC5D,OAAO,EAAG,GAAG,CAAC,OAAsC,IAAI,QAAQ;QAChE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,MAAmB,IAAI,EAAE,CAAC;QACtD,SAAS,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,SAAsB,IAAI,EAAE,CAAC;QAC5D,WAAW,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,WAAwB,IAAI,EAAE,CAAC;QAChE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9C,wBAAwB,EAAE,WAAW;KACtC,CAAC;IAEF,MAAM,gBAAgB,CACpB,OAAO,EACP;QACE,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,YAAY;QACtB,SAAS,EAAE,+BAA+B;QAC1C,SAAS,EAAE,CAAC;KACb,EACD,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,OAAe,EACf,MAAc,EACd,YAAgC,EAChC,KAAc;IAEd,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SAC5D,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,OAAO,EACP,YAAY,EACZ,GAAG,MAAM,IAAI,SAAS,IAAI,IAAI,OAAO,CACtC,CAAC;IAEF,MAAM,cAAc,GAAG;QACrB,MAAM;QACN,eAAe,EAAE,YAAY,CAAC,EAAE;QAChC,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QACpD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;IAEF,MAAM,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/src/jobs/handlers/reconcile.d.ts

@@ -12,6 +12,17 @@ export interface ReconcileDeps {
     /** Ollama embedding provider for local-mode sources (Slice 19). */
     localEmbeddingProvider?: EmbeddingProvider;
 }
+/**
+ * Optional job-payload args. When `dedupe_conversations` is true the
+ * reconcile handler runs only the dedupe pass (skipping the normal
+ * structured-doc / conversation / digest reconcile work). `apply`
+ * gates the mutation — without it the handler reports the plan but
+ * deletes nothing.
+ */
+export interface ReconcileJobArgs {
+    dedupe_conversations?: boolean;
+    apply?: boolean;
+}
 export interface ReconcileReport {
     files_checked: number;
     missing_reindexed: number;

package/dist/src/jobs/handlers/reconcile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/reconcile.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAElE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAiB1D,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,mEAAmE;IACnE,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAUD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CA4CtE"}
+{"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/reconcile.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAElE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAmB1D,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,mEAAmE;IACnE,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAUD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CA8DtE"}

package/dist/src/jobs/handlers/reconcile.js

@@ -29,6 +29,8 @@ import { recordIndexWrite } from "../../observability/index-events.js";
 import { Indexer, fileId } from "../../ingestion/indexer.js";
 import { isSupported } from "../../ingestion/extractor.js";
 import { getTable } from "../../storage/tables.js";
+import { conversationContentFingerprint } from "../../ingestion/inbox-core/conversation-fingerprint.js";
+import { dedupeConversations } from "./dedupe-conversations.js";
 /** All conversation source directories to reconcile. */
 const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
 /** Compute the union of ignored roots across all configured sources. */
@@ -38,6 +40,10 @@ function computeIgnoredPaths(deps) {
 }
 export function createReconcileHandler(deps) {
     return async (job) => {
+        // Read the job's input args before we overwrite `payload` with the report.
+        const args = job.payload ?? {};
+        const dedupeMode = args.dedupe_conversations === true;
+        const apply = args.apply === true;
         // --- 1. Pause all live indexers ---
         for (const w of deps.watchers)
             w.pause();
@@ -50,6 +56,18 @@ export function createReconcileHandler(deps) {
             engine_owned_purged: 0,
         };
         try {
+            if (dedupeMode) {
+                // Dedupe-conversations repair pass. Skips the normal structured-doc
+                // / conversation / digest reconciliation — this is a focused
+                // one-shot cleanup for pre-stable-id legacy duplicates.
+                const dedupeReport = await dedupeConversations({
+                    apply,
+                    dataDir: deps.dataDir,
+                    reportsDir: path.resolve(deps.dataDir, "..", "reports"),
+                });
+                job.payload = dedupeReport;
+                return;
+            }
             // --- 2. Reconcile structured_docs ---
             await reconcileStructuredDocs(deps, report);
             // --- 3. Reconcile conversations ---
@@ -399,6 +417,12 @@ async function reindexConversation(embedding, id, rawPath, dataDir) {
         operation: "reindex_conversation",
         input_count: 1,
     }, () => embedding.embed(content || summary));
+    const fingerprint = typeof raw.conversation_fingerprint === "string"
+        && raw.conversation_fingerprint.length > 0
+        ? raw.conversation_fingerprint
+        : content
+            ? conversationContentFingerprint(content)
+            : "";
     const row = {
         id,
         content,
@@ -413,6 +437,7 @@ async function reindexConversation(embedding, id, rawPath, dataDir) {
         key_outputs: JSON.stringify(raw.key_outputs ?? []),
         indexed_at: new Date().toISOString(),
         idempotency_key: raw.idempotency_key ?? "",
+        conversation_fingerprint: fingerprint,
     };
     await recordIndexWrite(dataDir, {
         table: "conversations_hot",