onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/jobs/handlers/dedupe-conversations.js

@@ -0,0 +1,371 @@
+/**
+ * Dedupe-conversations repair pass — opt-in tool.
+ *
+ * Closes the residual gap from the 2026-05-13 stable-row-id upgrade:
+ * rows written before that slice carry a legacy row id derived from
+ * `(conversationId, jobRootId)` instead of the new
+ * `stableConversationRowId(source, conversationId)`. A re-import after
+ * the upgrade lands the new stable-id row alongside the surviving legacy
+ * row instead of collapsing onto it, leaving two rows for the same
+ * `(source, conversation_id)`.
+ *
+ * Strategy: scan `conversations_hot` + `conversations_cold`, match each
+ * row to its raw artifact (by row id embedded in the filename), read
+ * `(source, conversation_id)` from the raw, and group rows by that
+ * canonical key. For groups with more than one member, pick a keeper
+ * and delete the rest (plus the matching raw files).
+ *
+ * Decision rule:
+ *  1. If any group member's row id matches
+ *     `stableConversationRowId(source, conversation_id)`, that member
+ *     wins ("stable-id is canonical post-upgrade"). Ties between
+ *     multiple stable members (degenerate — same row id would be one
+ *     row) fall through to the same secondary tie-breakers.
+ *  2. Otherwise (all members are legacy), prefer the newest
+ *     `indexed_at`, then prefer the `conversations_cold` member
+ *     (compaction promotes cold to canonical), then prefer larger
+ *     `content_length` (avoid silently shrinking the indexed snapshot),
+ *     then the lexicographically smaller row id (deterministic).
+ *
+ * Safety guard on the stable rule: a stable-id row may have been
+ * imported from an older export, so it can legitimately be older or
+ * shorter than a surviving legacy row from a more recent import. If
+ * any legacy member of the group is newer OR has more content than
+ * the chosen stable winner, the group is classified `suspicious` and
+ * `--apply` never mutates it. The operator triages by hand.
+ *
+ * Dry-run by default. Only `--apply` mutates rows or deletes raw files.
+ *
+ * CLI capture sources (`claude_code`, `codex_cli`, `gemini_cli`) opt out
+ * of `(source, conversation_id)` dedupe: they have no provider-side
+ * conversation id and their row id is the canonical identifier. The
+ * planner skips them entirely — they are neither grouped nor counted in
+ * `rows_without_dedupe_key`.
+ *
+ * For web-source rows whose raw file is missing, OR exists but lacks
+ * `conversation_id`, the planner reports them in `rows_without_dedupe_key`
+ * for operator triage. Truly orphan rows (no raw on disk) are also
+ * handled by the normal reconcile pass — that's where any deletion lives.
+ */
+import { stableConversationRowId } from "../../ingestion/inbox-core/conversation-key.js";
+/**
+ * CLI capture sources. These hit `POST /save` and have no provider-side
+ * conversation id — the row id (job chain root) IS the canonical
+ * identifier, so they cannot participate in stable-vs-legacy dedupe.
+ * The planner skips them entirely: they are neither grouped nor flagged
+ * in `rows_without_dedupe_key`.
+ */
+const CLI_CONVERSATION_SOURCES = new Set([
+    "claude_code",
+    "codex_cli",
+    "gemini_cli",
+]);
+/**
+ * Pure planner. Given a snapshot of rows + raw files, produces the
+ * dedupe plan with no I/O. Caller decides whether to apply.
+ */
+export function planDedupe(input) {
+    // Group raw files by the row id in the filename. A row id can have
+    // multiple raws (refresh-on-grow before the prune-prior-raw slice).
+    const rawsByRowId = new Map();
+    for (const raw of input.raw_files) {
+        const arr = rawsByRowId.get(raw.row_id_from_filename) ?? [];
+        arr.push(raw);
+        rawsByRowId.set(raw.row_id_from_filename, arr);
+    }
+    // Derive (source, conversation_id) per row id from its latest raw
+    // (lexicographically greatest filename). All raws for the same row
+    // id share the same provider conversation id, so this is just
+    // picking the most recent metadata snapshot.
+    const sourceAndConvByRowId = new Map();
+    for (const [rowId, raws] of rawsByRowId) {
+        const latest = raws.reduce((a, b) => (a.filename > b.filename ? a : b));
+        sourceAndConvByRowId.set(rowId, {
+            source: latest.source,
+            conversation_id: latest.conversation_id,
+        });
+    }
+    // Group rows by (source, conversation_id). Nested map (source ->
+    // conversation_id -> members) avoids string-key delimiter ambiguity
+    // since provider conversation ids can in principle contain any
+    // character.
+    const groups = new Map();
+    const rowsWithoutDedupeKey = [];
+    for (const row of input.rows) {
+        // CLI sources opt out of (source, conversation_id) dedupe: their
+        // row id IS the canonical key, so they cannot have stable-vs-legacy
+        // duplicates and should not be flagged when their raw lacks
+        // `conversation_id` (that's the expected shape).
+        if (CLI_CONVERSATION_SOURCES.has(row.source))
+            continue;
+        const lookup = sourceAndConvByRowId.get(row.row_id);
+        if (!lookup) {
+            rowsWithoutDedupeKey.push({
+                row_id: row.row_id,
+                table: row.table,
+                source: row.source,
+            });
+            continue;
+        }
+        const stableId = stableConversationRowId(lookup.source, lookup.conversation_id);
+        const member = {
+            row_id: row.row_id,
+            table: row.table,
+            indexed_at: row.indexed_at,
+            content_length: row.content_length,
+            is_stable_id: row.row_id === stableId,
+            raw_files: (rawsByRowId.get(row.row_id) ?? []).map((r) => r.path),
+        };
+        let bySource = groups.get(lookup.source);
+        if (!bySource) {
+            bySource = new Map();
+            groups.set(lookup.source, bySource);
+        }
+        const arr = bySource.get(lookup.conversation_id) ?? [];
+        arr.push(member);
+        bySource.set(lookup.conversation_id, arr);
+    }
+    // Build group plans for actual duplicates (size > 1).
+    const actionable = [];
+    const suspicious = [];
+    for (const [source, bySource] of groups) {
+        for (const [conversationId, members] of bySource) {
+            if (members.length < 2)
+                continue;
+            const stableId = stableConversationRowId(source, conversationId);
+            const decision = pickKeeper(members);
+            const deleted = members.filter((m) => m !== decision.kept);
+            const reasons = detectSuspicion(decision.kept, deleted);
+            const plan = {
+                source,
+                conversation_id: conversationId,
+                stable_id: stableId,
+                members,
+                kept_member: decision.kept,
+                deleted_members: deleted,
+                raw_files_to_delete: deleted.flatMap((m) => m.raw_files),
+                decision_reason: decision.reason,
+                classification: reasons.length > 0 ? "suspicious" : "actionable",
+                suspicious_reasons: reasons,
+            };
+            if (reasons.length > 0)
+                suspicious.push(plan);
+            else
+                actionable.push(plan);
+        }
+    }
+    // Deterministic ordering within each bucket: by source, then conversation_id.
+    const orderGroups = (a, b) => {
+        if (a.source !== b.source)
+            return a.source < b.source ? -1 : 1;
+        return a.conversation_id < b.conversation_id ? -1 : 1;
+    };
+    actionable.sort(orderGroups);
+    suspicious.sort(orderGroups);
+    return {
+        actionable_groups: actionable,
+        suspicious_groups: suspicious,
+        rows_without_dedupe_key: rowsWithoutDedupeKey,
+    };
+}
+/**
+ * A stable winner is only safe to delete legacy members when it is
+ * at least as new AND at least as long as every loser. Returns the
+ * list of suspicion reasons; empty list means the group is actionable.
+ * Legacy-only groups (where the keeper is itself legacy) are always
+ * actionable — the keeper rule already picked the safest legacy
+ * member (newest, then cold-preferred, then longer content).
+ */
+function detectSuspicion(kept, deleted) {
+    if (!kept.is_stable_id)
+        return [];
+    const reasons = new Set();
+    for (const loser of deleted) {
+        if (loser.indexed_at > kept.indexed_at)
+            reasons.add("legacy_member_is_newer");
+        if (loser.content_length > kept.content_length)
+            reasons.add("legacy_member_has_more_content");
+    }
+    return [...reasons];
+}
+function pickKeeper(members) {
+    const stable = members.filter((m) => m.is_stable_id);
+    if (stable.length > 0) {
+        const kept = stable.reduce(secondaryTieBreak);
+        return { kept, reason: "stable-id row is canonical post-upgrade" };
+    }
+    const kept = members.reduce(legacyPick);
+    return {
+        kept,
+        reason: "no stable-id row in group; kept newest legacy (cold-preferred, then longer content)",
+    };
+}
+/** Newest indexed_at wins; then prefer cold; then longer content; then lex-smaller id. */
+function legacyPick(a, b) {
+    if (a.indexed_at !== b.indexed_at)
+        return a.indexed_at > b.indexed_at ? a : b;
+    return secondaryTieBreak(a, b);
+}
+/** Prefer cold; then longer content; then lex-smaller row id. */
+function secondaryTieBreak(a, b) {
+    if (a.table !== b.table) {
+        return a.table === "conversations_cold" ? a : b;
+    }
+    if (a.content_length !== b.content_length) {
+        return a.content_length > b.content_length ? a : b;
+    }
+    return a.row_id < b.row_id ? a : b;
+}
+// ---------------------------------------------------------------------------
+// Side-effecting wrapper: reads tables + raw dirs, plans, optionally applies.
+// ---------------------------------------------------------------------------
+import fs from "node:fs";
+import path from "node:path";
+import { getTable } from "../../storage/tables.js";
+import { CONVERSATION_SOURCES } from "../../providers/types.js";
+const ALL_CONV_SOURCES = [...CONVERSATION_SOURCES, "inbox"];
+/**
+ * Scan `conversations_hot` + `conversations_cold`, plan the dedupe,
+ * optionally apply. Caller (the reconcile handler) wraps this in the
+ * watcher pause/resume protocol so live writes can't race.
+ */
+export async function dedupeConversations(opts) {
+    const hotTable = getTable("conversations_hot");
+    const coldTable = getTable("conversations_cold");
+    const hotRows = (await hotTable.query().toArray());
+    const coldRows = (await coldTable.query().toArray());
+    const rows = [
+        ...hotRows.map((r) => rowView(r, "conversations_hot")),
+        ...coldRows.map((r) => rowView(r, "conversations_cold")),
+    ];
+    const errors = [];
+    const rawFiles = [];
+    // Total raw JSON files visited on disk, regardless of whether the
+    // planner could derive a `(source, conversation_id)` key from them.
+    // CLI raws (no `conversation_id`) count here even though they don't
+    // land in `rawFiles`, so the metric matches its name.
+    let rawFilesScanned = 0;
+    for (const source of ALL_CONV_SOURCES) {
+        const rawDir = path.join(opts.dataDir, "raw", "conversations", source);
+        if (!fs.existsSync(rawDir))
+            continue;
+        for (const filename of fs.readdirSync(rawDir)) {
+            if (!filename.endsWith(".json"))
+                continue;
+            const rowIdFromFilename = extractIdFromFilename(filename);
+            if (!rowIdFromFilename)
+                continue;
+            const fullPath = path.join(rawDir, filename);
+            rawFilesScanned++;
+            try {
+                const raw = JSON.parse(fs.readFileSync(fullPath, "utf-8"));
+                const rawSource = typeof raw.source === "string" ? raw.source : source;
+                const conversationId = typeof raw.conversation_id === "string" ? raw.conversation_id : "";
+                if (!conversationId) {
+                    // Pre-stable-id raws and CLI captures don't carry
+                    // conversation_id. Without it we can't group, so skip silently
+                    // — but the raw was still scanned (counted above).
+                    continue;
+                }
+                rawFiles.push({
+                    path: fullPath,
+                    filename,
+                    row_id_from_filename: rowIdFromFilename,
+                    source: rawSource,
+                    conversation_id: conversationId,
+                });
+            }
+            catch (err) {
+                errors.push(`read raw failed: ${fullPath}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    const plan = planDedupe({ rows, raw_files: rawFiles });
+    const rowsToDelete = plan.actionable_groups.reduce((n, g) => n + g.deleted_members.length, 0);
+    const rawFilesToDelete = plan.actionable_groups.reduce((n, g) => n + g.raw_files_to_delete.length, 0);
+    let rowsDeleted = 0;
+    let rawFilesDeleted = 0;
+    if (opts.apply) {
+        // Suspicious groups are deliberately NOT touched, even with --apply.
+        for (const group of plan.actionable_groups) {
+            for (const member of group.deleted_members) {
+                const table = member.table === "conversations_cold"
+                    ? coldTable
+                    : hotTable;
+                try {
+                    await table.delete(`id = '${member.row_id}'`);
+                    rowsDeleted++;
+                }
+                catch (err) {
+                    errors.push(`delete row failed: ${member.table}/${member.row_id}: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            for (const rawPath of group.raw_files_to_delete) {
+                try {
+                    fs.unlinkSync(rawPath);
+                    rawFilesDeleted++;
+                }
+                catch (err) {
+                    if (err && typeof err === "object" && "code" in err
+                        && err.code === "ENOENT") {
+                        // Already gone — count as deleted, not an error.
+                        rawFilesDeleted++;
+                    }
+                    else {
+                        errors.push(`delete raw failed: ${rawPath}: ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                }
+            }
+        }
+    }
+    const report = {
+        apply: opts.apply,
+        rows_scanned: rows.length,
+        raw_files_scanned: rawFilesScanned,
+        duplicate_groups_found: plan.actionable_groups.length + plan.suspicious_groups.length,
+        rows_to_delete: rowsToDelete,
+        raw_files_to_delete: rawFilesToDelete,
+        rows_deleted: rowsDeleted,
+        raw_files_deleted: rawFilesDeleted,
+        rows_without_dedupe_key: plan.rows_without_dedupe_key.length,
+        suspicious_groups_skipped: plan.suspicious_groups.length,
+        actionable_groups: plan.actionable_groups,
+        suspicious_groups: plan.suspicious_groups,
+        errors,
+    };
+    if (opts.reportsDir) {
+        try {
+            fs.mkdirSync(opts.reportsDir, { recursive: true });
+            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+            const suffix = opts.apply ? "apply" : "dry-run";
+            const reportPath = path.join(opts.reportsDir, `dedupe-conversations-${stamp}-${suffix}.json`);
+            // Set report_path BEFORE serializing so the persisted file is
+            // self-identifying — `report_path` on disk equals its own location.
+            report.report_path = reportPath;
+            fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+        }
+        catch (err) {
+            report.errors.push(`write report failed: ${opts.reportsDir}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return report;
+}
+function rowView(row, table) {
+    return {
+        row_id: row.id,
+        table,
+        indexed_at: row.indexed_at,
+        content_length: (row.content ?? "").length,
+        source: row.source,
+    };
+}
+/**
+ * Filename format: `{ISO-timestamp}_{rowId}.json`. ISO timestamps in
+ * the writer use `:` and `.` replaced by `-`, so the first `_`
+ * is the unambiguous separator.
+ */
+function extractIdFromFilename(filename) {
+    const match = filename.match(/^[^_]+_(.+)\.json$/);
+    return match?.[1];
+}
+//# sourceMappingURL=dedupe-conversations.js.map

package/dist/src/jobs/handlers/dedupe-conversations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedupe-conversations.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/dedupe-conversations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AAKzF;;;;;;GAMG;AACH,MAAM,wBAAwB,GAAoC,IAAI,GAAG,CAAC;IACxE,aAAa;IACb,WAAW;IACX,YAAY;CACb,CAAC,CAAC;AA0FH;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAAsB;IAC/C,mEAAmE;IACnE,oEAAoE;IACpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;QAC5D,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACd,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED,kEAAkE;IAClE,mEAAmE;IACnE,8DAA8D;IAC9D,6CAA6C;IAC7C,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAGjC,CAAC;IACJ,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,oBAAoB,CAAC,GAAG,CAAC,KAAK,EAAE;YAC9B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,eAAe,EAAE,MAAM,CAAC,eAAe;SACxC,CAAC,CAAC;IACL,CAAC;IAED,iEAAiE;IACjE,oEAAoE;IACpE,+DAA+D;IAC/D,aAAa;IACb,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuC,CAAC;IAC9D,MAAM,oBAAoB,GAIpB,EAAE,CAAC;IAET,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,iEAAiE;QACjE,oEAAoE;QACpE,4DAA4D;QAC5D,iDAAiD;QACjD,IAAI,wBAAwB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QACvD,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,oBAAoB,CAAC,IAAI,CAAC;gBACxB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,uBAAuB,CACtC,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,eAAe,CACvB,CAAC;QACF,MAAM,MAAM,GAAiB;YAC3B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,YAAY,EAAE,GAAG,CAAC,MAAM,KAAK,QAAQ;YACrC,SAAS,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SAClE,CAAC;QACF,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjB,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,sDAAsD;IACtD,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YACjC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YACjE,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,IAAI,GAAoB;gBAC5B,MAAM;gBACN,eAAe,EAAE,cAAc;gBAC/B,SAAS,EAAE,QAAQ;gBACnB,OAAO;gBACP,WAAW,EAAE,QAAQ,CAAC,IAAI;gBAC1B,eAAe,EAAE,OAAO;gBACxB,mBAAmB,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBACxD,eAAe,EAAE,QAAQ,CAAC,MAAM;gBAChC,cAAc,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY;gBAChE,kBAAkB,EAAE,OAAO;aAC5B,CAAC;YACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBACzC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,MAAM,WAAW,GAAG,CAAC,CAAkB,EAAE,CAAkB,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC;IACF,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7B,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAE7B,OAAO;QACL,iBAAiB,EAAE,UAAU;QAC7B,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,oBAAoB;KAC9C,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CACtB,IAAkB,EAClB,OAAuB;IAEvB,IAAI,CAAC,IAAI,CAAC,YAAY;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC9E,IAAI,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc;YAC5C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,UAAU,CACjB,OAAuB;IAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,OAAO;QACL,IAAI;QACJ,MAAM,EACJ,qFAAqF;KACxF,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,SAAS,UAAU,CAAC,CAAe,EAAE,CAAe;IAClD,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,OAAO,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,iEAAiE;AACjE,SAAS,iBAAiB,CAAC,CAAe,EAAE,CAAe;IACzD,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC,KAAK,KAAK,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,cAAc,KAAK,CAAC,CAAC,cAAc,EAAE,CAAC;QAC1C,OAAO,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAE9E,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAEhE,MAAM,gBAAgB,GAAG,CAAC,GAAG,oBAAoB,EAAE,OAAO,CAAU,CAAC;AA6CrE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAA6B;IAE7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAEjD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAsB,CAAC;IACxE,MAAM,QAAQ,GAAG,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAsB,CAAC;IAE1E,MAAM,IAAI,GAAoB;QAC5B,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACtD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;KACzD,CAAC;IAEF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,kEAAkE;IAClE,oEAAoE;IACpE,oEAAoE;IACpE,sDAAsD;IACtD,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,KAAK,MAAM,QAAQ,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAC1C,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YAC1D,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC7C,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAGxD,CAAC;gBACF,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBACvE,MAAM,cAAc,GAClB,OAAO,GAAG,CAAC,eAAe,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,kDAAkD;oBAClD,+DAA+D;oBAC/D,mDAAmD;oBACnD,SAAS;gBACX,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,QAAQ;oBACd,QAAQ;oBACR,oBAAoB,EAAE,iBAAiB;oBACvC,MAAM,EAAE,SAAS;oBACjB,eAAe,EAAE,cAAc;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CACT,oBAAoB,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACpF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEvD,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC,MAAM,EACtC,CAAC,CACF,CAAC;IACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CACpD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,mBAAmB,CAAC,MAAM,EAC1C,CAAC,CACF,CAAC;IAEF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,qEAAqE;QACrE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,KAAK,oBAAoB;oBACjD,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;oBAC9C,WAAW,EAAE,CAAC;gBAChB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,sBAAsB,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;oBACvB,eAAe,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IACE,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG;2BAC3C,GAAwB,CAAC,IAAI,KAAK,QAAQ,EAC9C,CAAC;wBACD,iDAAiD;wBACjD,eAAe,EAAE,CAAC;oBACpB,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrF,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAA8B;QACxC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,YAAY,EAAE,IAAI,CAAC,MAAM;QACzB,iBAAiB,EAAE,eAAe;QAClC,sBAAsB,EACpB,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAC/D,cAAc,EAAE,YAAY;QAC5B,mBAAmB,EAAE,gBAAgB;QACrC,YAAY,EAAE,WAAW;QACzB,iBAAiB,EAAE,eAAe;QAClC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB,CAAC,MAAM;QAC5D,yBAAyB,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;QACxD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,MAAM;KACP,CAAC;IAEF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;YAChD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAC1B,IAAI,CAAC,UAAU,EACf,wBAAwB,KAAK,IAAI,MAAM,OAAO,CAC/C,CAAC;YACF,8DAA8D;YAC9D,oEAAoE;YACpE,MAAM,CAAC,WAAW,GAAG,UAAU,CAAC;YAChC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,wBAAwB,IAAI,CAAC,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,GAAoB,EAAE,KAAwB;IAC7D,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,EAAE;QACd,KAAK;QACL,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,cAAc,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM;QAC1C,MAAM,EAAE,GAAG,CAAC,MAAM;KACnB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC"}

package/dist/src/jobs/handlers/ingest.d.ts

@@ -0,0 +1,13 @@
+import type { JobHandler } from "../types.js";
+import type { JobQueue } from "../queue.js";
+import type { EmbeddingProvider, MetadataExtractor } from "../../providers/types.js";
+import type { IngestPayload } from "../../server/routes/ingest.js";
+export interface IngestHandlerDeps {
+    embeddingProvider: EmbeddingProvider;
+    metadataExtractor: MetadataExtractor;
+    dataDir: string;
+    queue: JobQueue;
+    metadataExtractorFactory?: (payload: IngestPayload, deps: IngestHandlerDeps) => MetadataExtractor;
+}
+export declare function createIngestHandler(deps: IngestHandlerDeps): JobHandler;
+//# sourceMappingURL=ingest.d.ts.map

package/dist/src/jobs/handlers/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAsDA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EACV,iBAAiB,EACjB,iBAAiB,EAGlB,MAAM,0BAA0B,CAAC;AAoBlC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAcnE,MAAM,WAAW,iBAAiB;IAChC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;IAChB,wBAAwB,CAAC,EAAE,CACzB,OAAO,EAAE,aAAa,EACtB,IAAI,EAAE,iBAAiB,KACpB,iBAAiB,CAAC;CACxB;AAuID,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,UAAU,CA+VvE"}