onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/jobs/handlers/ingest.js

@@ -0,0 +1,557 @@
+/**
+ * Ingest job handler — Slice 13.
+ *
+ * Processes a POST /ingest payload in two phases:
+ *
+ * Phase 1 — Process (no side effects):
+ *   1. Unpack archive securely (Slice 12 archive.ts)
+ *   2. Parse conversation export via source-specific parser
+ *   3. For each parsed conversation: extract metadata + embed
+ *   4. If ANY conversation fails: quarantine that one, throw (no persist)
+ *
+ * Phase 2 — Persist (only if all conversations passed Phase 1):
+ *   5. Write raw files (with full metadata — rebuild-safe)
+ *   6. Insert all rows into conversations_hot (or refresh into the
+ *      table where an older snapshot already lives)
+ *
+ * This two-phase design ensures atomicity: either all conversations in
+ * an upload are indexed, or none are. No partial ingest.
+ *
+ * Replay safety:
+ * - Row ID is deterministic on (source, provider conversation ID), so a
+ *   re-imported monthly export collapses to the same row across jobs.
+ * - Raw files are written before table insert (crash safety).
+ * - Existing rows in conversations_hot OR conversations_cold are checked
+ *   on replay (compacted-then-re-imported conversations don't re-enter hot).
+ *
+ * Refresh-on-grow:
+ * - For each existing row, the handler compares the incoming canonical
+ *   `conversationContentFingerprint(content)` against a fingerprint
+ *   derived from the existing row's own `content` column. The row, not
+ *   the raw artifact, is the source of truth for "what is currently
+ *   indexed"; using raw here would create a drift bug if Phase 2 crashes
+ *   between writing the new raw and replacing the row. If fingerprints
+ *   match, the conversation is unchanged and we skip. If they differ,
+ *   the conversation has grown and we refresh: write a new raw artifact
+ *   (which records the target table so crash recovery can route the
+ *   restored row back to the right place), delete the old row, insert
+ *   the new row in the same table where it lived (hot stays hot, cold
+ *   stays cold — we do not silently re-promote). After a successful
+ *   row replacement, prior raw artifacts for that row id are pruned
+ *   best-effort so rebuild does not insert duplicates.
+ *
+ * Persisted fingerprint:
+ * - `conversation_fingerprint` is written both into the raw artifact and
+ *   into the LanceDB row. Refresh-on-grow prefers the row column when it
+ *   is non-empty (avoids re-deriving from `content` on every replay), and
+ *   falls back to deriving from row `content` for legacy rows written
+ *   before the column existed. The migration in `initTables` adds the
+ *   column to existing tables filled with empty strings, so those rows
+ *   continue to go through the derivation fallback until next refresh.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { getTable } from "../../storage/tables.js";
+import { atomicWrite } from "../../storage/atomic.js";
+import { extractArchive } from "../../security/archive.js";
+import { parseChatGPTWeb } from "../../ingestion/parsers/chatgpt-web.js";
+import { parseClaudeWeb } from "../../ingestion/parsers/claude-web.js";
+import { parseGeminiWeb } from "../../ingestion/parsers/gemini-web.js";
+import { parseGrokWeb } from "../../ingestion/parsers/grok-web.js";
+import { stableConversationRowId } from "../../ingestion/inbox-core/conversation-key.js";
+import { conversationContentFingerprint } from "../../ingestion/inbox-core/conversation-fingerprint.js";
+import { eqFilter } from "../../storage/filter.js";
+import { chunkConversation, chunkingEnabled } from "../../ingestion/chunk-text.js";
+import { conversationChunkColumns, convChunkId, chunkSetComplete, } from "../../ingestion/chunk-meta.js";
+import { CodexCliMetadataExtractor } from "../../providers/codex-cli-metadata-extraction.js";
+import { BasicMetadataExtractor, RIFT_BASIC_METADATA_PROVIDER, } from "../../providers/basic-metadata-extraction.js";
+import { RIFT_NONE_EMBEDDING_PROVIDER, isPlaceholderEmbeddingProvider, } from "../../providers/placeholder-embed.js";
+import { recordEmbed } from "../../observability/embedding-events.js";
+import { withoutUserMarkers } from "../../capture/user-statements.js";
+import { recordIndexWrite } from "../../observability/index-events.js";
+/** Map of source -> parser. Only fixture-backed sources are registered. */
+const PARSERS = {
+    chatgpt_web: parseChatGPTWeb,
+    claude_web: parseClaudeWeb,
+    gemini_web: parseGeminiWeb,
+    grok_web: parseGrokWeb,
+};
+/**
+ * Walk the retry_of chain to the root job (same logic as save handler).
+ */
+function resolveChainRoot(job, queue) {
+    let current = job;
+    let depth = 0;
+    while (current.retry_of && depth < 50) {
+        const parent = queue.get(current.retry_of);
+        if (!parent)
+            break;
+        current = parent;
+        depth++;
+    }
+    return current;
+}
+/**
+ * Find the latest raw file for `rowId` under `rawDir`, if any.
+ *
+ * Raw filenames are `<timestamp>_<rowId>.json`. After a refresh, the
+ * same row id can have multiple files (one per version); we pick the
+ * lexicographically greatest filename, which matches the ISO-8601
+ * timestamp ordering used at write time.
+ */
+function findRawFile(rawDir, rowId) {
+    if (!fs.existsSync(rawDir))
+        return undefined;
+    const suffix = `_${rowId}.json`;
+    const matches = fs.readdirSync(rawDir).filter((f) => f.endsWith(suffix));
+    if (matches.length === 0)
+        return undefined;
+    matches.sort();
+    return path.join(rawDir, matches[matches.length - 1]);
+}
+/**
+ * Read (or derive) the canonical content fingerprint of an existing
+ * indexed row.
+ *
+ * Preference order:
+ *  1. The row's persisted `conversation_fingerprint` column (when
+ *     non-empty) — this is the source of truth written at last ingest.
+ *  2. Derive from the row's own `content` — used for legacy rows
+ *     written before the column existed (post-migration the column is
+ *     present but empty for those rows).
+ *
+ * Using the row content (rather than the raw artifact on disk) as the
+ * derivation fallback avoids a raw-vs-row drift bug: if Phase 2 crashes
+ * between writing the new raw and replacing the row, the raw on disk is
+ * ahead of the table, and a subsequent re-import comparing against raw
+ * would mistakenly decide "unchanged" while the row is still stale.
+ *
+ * Returns `undefined` if neither source yields a fingerprint; callers
+ * treat that as "cannot prove unchanged" and refresh.
+ */
+function existingRowFingerprint(row) {
+    const stored = typeof row.conversation_fingerprint === "string"
+        ? row.conversation_fingerprint
+        : "";
+    if (stored)
+        return stored;
+    const content = typeof row.content === "string" ? row.content : undefined;
+    if (!content)
+        return undefined;
+    return conversationContentFingerprint(content);
+}
+/**
+ * Best-effort prune of stale raw artifacts for `rowId` under `rawDir`,
+ * keeping only `keepFileName`. Called after a refresh has successfully
+ * replaced the row in its table, so the new raw is the only one rebuild
+ * needs to re-derive the row.
+ *
+ * Safe-by-design: any pre-existing raw still on disk has older content
+ * than the row now holds, so deleting it loses nothing the table cannot
+ * already reconstruct. If a prune fails (filesystem error, race with
+ * another process), we swallow the error rather than fail the ingest —
+ * the rebuild duplicate caused by an extra raw is a much cheaper bug
+ * than a failed ingest after the row is already correct.
+ */
+function pruneStaleRawArtifacts(rawDir, rowId, keepFileName) {
+    if (!fs.existsSync(rawDir))
+        return;
+    const suffix = `_${rowId}.json`;
+    for (const name of fs.readdirSync(rawDir)) {
+        if (!name.endsWith(suffix))
+            continue;
+        if (name === keepFileName)
+            continue;
+        try {
+            fs.unlinkSync(path.join(rawDir, name));
+        }
+        catch {
+            // best-effort
+        }
+    }
+}
+export function createIngestHandler(deps) {
+    return async (job) => {
+        const rootJob = resolveChainRoot(job, deps.queue);
+        const rootPayload = rootJob.payload ??
+            job.payload;
+        const payload = rootPayload;
+        if (!payload?.source || !payload.fileData) {
+            throw new Error("Invalid ingest job payload: missing source or fileData");
+        }
+        const metadataExtractor = (deps.metadataExtractorFactory ?? resolveMetadataExtractor)(payload, deps);
+        // Provenance markers stamped onto every row written by this job.
+        // `metadata_provider`: "rift-basic" when BasicMetadataExtractor produced
+        // the (deterministic, AI-free) metadata, else the enriching provider's
+        // name (or "" when unknown). `embedding_provider`: "rift-none" when the
+        // vector is a no-key placeholder, else "" (a valid embedding). Both are
+        // constant for the lifetime of one ingest job.
+        const metadataProviderLabel = metadataExtractor.getProvenance?.()?.provider ?? "";
+        const embeddingProviderLabel = isPlaceholderEmbeddingProvider(deps.embeddingProvider)
+            ? RIFT_NONE_EMBEDDING_PROVIDER
+            : "";
+        // Lazily created only if an AI extractor throws mid-import and we need the
+        // no-AI fallback (see Phase 1 below). One instance is reused for the job.
+        let basicFallback = null;
+        // --- 1. Unpack archive securely ---
+        const fileBuffer = Buffer.from(payload.fileData, "base64");
+        const { files } = extractArchive(fileBuffer, payload.filename);
+        // --- 2. Parse conversations ---
+        const parser = PARSERS[payload.source];
+        if (!parser) {
+            throw new Error(`No parser for source "${payload.source}": deferred (no fixture)`);
+        }
+        const parsed = parser(files);
+        job.progress = { phase: "reading", completed: 0, total: parsed.length };
+        if (parsed.length === 0) {
+            // A user-visible inbox drop that produced zero parseable
+            // conversations is an export the friend will want to know about
+            // and re-drop — not a silent no-op. Surfacing this as a failed
+            // job lets /status/friend show `last_error_reason: "empty_export"`
+            // and the existing Next: line route the user to re-drop.
+            // The phrase "No conversations" is what classifyJobError maps to
+            // the `empty_export` class; do not rephrase without updating it.
+            throw new Error(`No conversations found in export "${payload.filename}"`);
+        }
+        const dates = parsed.map((conversation) => conversation.createdAt)
+            .filter((date) => Number.isFinite(Date.parse(date))).sort();
+        const importSummary = {
+            conversations: parsed.length,
+            first_date: dates[0] ?? null,
+            last_date: dates.at(-1) ?? null,
+            titles: [...new Set([...parsed].sort((a, b) => b.createdAt.localeCompare(a.createdAt))
+                    .map((conversation) => conversation.title.trim()).filter(Boolean))].slice(0, 4),
+        };
+        const rawDir = path.join(deps.dataDir, "raw", "conversations", payload.rawSource ?? payload.source);
+        const table = getTable("conversations_hot");
+        const coldTable = getTable("conversations_cold");
+        // --- Phase 1: Process all conversations (no side effects) ---
+        // Extract metadata + embed for every conversation. If any fails,
+        // quarantine that one and throw — nothing has been persisted yet.
+        // Conversations with existing raw files (crash recovery) are
+        // collected but NOT inserted yet — they wait for Phase 2 so a
+        // mixed batch doesn't partially ingest.
+        const processed = [];
+        const recoverable = [];
+        for (const [conversationIndex, conversation] of parsed.entries()) {
+            job.progress.completed = conversationIndex;
+            // Row id is keyed on (source, provider conversation id) so a
+            // re-imported monthly export collapses to the same row instead
+            // of duplicating every conversation under a fresh job-root id.
+            const rowId = stableConversationRowId(payload.source, conversation.id);
+            const fingerprint = conversationContentFingerprint(conversation.content);
+            // Replay / refresh guard.
+            // Check both hot AND cold — a conversation compacted into cold
+            // is exactly the kind of "old" conversation a monthly full export
+            // re-includes; without the cold check we'd duplicate it back into hot.
+            // For each existing row, compare content fingerprints to decide
+            // between skip (unchanged) and refresh (grown).
+            // A chunked conversation is N rows sharing parent_id = rowId; an
+            // unchunked one is a single row id = rowId. Find the whole set either way.
+            const setFilter = `(${eqFilter("id", rowId, { validateAsRowId: true })}) OR (${eqFilter("parent_id", rowId, { validateAsRowId: true })})`;
+            const existingHot = await table.query().where(setFilter).toArray();
+            const existingCold = existingHot.length === 0
+                ? await coldTable.query().where(setFilter).toArray()
+                : [];
+            const existingTable = existingHot.length > 0
+                ? "conversations_hot"
+                : existingCold.length > 0
+                    ? "conversations_cold"
+                    : undefined;
+            if (existingTable) {
+                // Row is already indexed. Decide skip vs refresh by comparing the
+                // incoming content fingerprint against the set's stored fingerprint
+                // (every chunk carries the same whole-conversation fingerprint). Using
+                // the row (not the raw artifact on disk) sidesteps drift between raw
+                // and row during the Phase 2 crash window.
+                const existingRows = existingHot.length > 0 ? existingHot : existingCold;
+                const existingFingerprint = existingRowFingerprint(existingRows[0]);
+                if (existingFingerprint &&
+                    existingFingerprint === fingerprint &&
+                    chunkSetComplete(existingRows)) {
+                    // Unchanged AND the chunk set is complete — preserve the snapshot.
+                    continue;
+                }
+                // Grown, unprovable, or an incomplete set (interrupted write) → refresh.
+                // Either the conversation has grown (fingerprint differs) or
+                // we cannot prove it's unchanged (row content is empty / not
+                // a string). In both cases, fall through to refresh:
+                // re-extract metadata + re-embed, then in Phase 2 replace
+                // the row in the table where it currently lives.
+            }
+            else {
+                // Crash recovery: raw file exists but row missing → defer to Phase 2.
+                const existingRawPath = findRawFile(rawDir, rowId);
+                if (existingRawPath) {
+                    recoverable.push({ rawPath: existingRawPath, rowId });
+                    continue;
+                }
+            }
+            // Per-item provider/provenance, resolved AFTER extraction so providers
+            // that record run-specific details (e.g. Codex's cli_tag, set only on a
+            // successful run) report them. Overridden on fallback below.
+            let metadata;
+            let itemMetadataProvider;
+            let itemExtractionProvenance;
+            try {
+                metadata = await metadataExtractor.extractMetadata(conversation.content);
+                itemMetadataProvider = metadataProviderLabel;
+                itemExtractionProvenance = metadataExtractor.getProvenance?.();
+            }
+            catch (err) {
+                // Locked AI-worker policy (PLAN.MD §1): fast import must not require an
+                // AI worker. An installed-but-unauthenticated Codex — or any transient
+                // AI extractor failure — must NOT fail the import. Fall back to
+                // deterministic basic metadata and mark the row `rift-basic` so a later
+                // enrichment pass backfills exactly it. If the primary IS already basic,
+                // a throw is a real bug (BasicMetadataExtractor never throws), so keep
+                // the quarantine-and-fail behavior rather than masking it.
+                if (metadataProviderLabel === RIFT_BASIC_METADATA_PROVIDER) {
+                    await quarantine(deps.dataDir, payload.source, conversation, err);
+                    throw err;
+                }
+                basicFallback ??= new BasicMetadataExtractor();
+                metadata = await basicFallback.extractMetadata(conversation.content);
+                itemMetadataProvider = RIFT_BASIC_METADATA_PROVIDER;
+                itemExtractionProvenance = basicFallback.getProvenance();
+            }
+            // Chunk (flag-gated) and embed each chunk. chunkConversation
+            // short-circuits small conversations (and the flag-off case) to a single
+            // chunk, so unchunked behavior is byte-identical.
+            const chunkContents = chunkingEnabled()
+                ? chunkConversation(conversation.content)
+                : [conversation.content];
+            const chunkEmbeddings = [];
+            try {
+                for (const chunk of chunkContents) {
+                    chunkEmbeddings.push(await recordEmbed(deps.dataDir, deps.embeddingProvider, {
+                        pipeline: "ingest_job",
+                        operation: "conversation_embedding",
+                        input_count: 1,
+                    }, () => deps.embeddingProvider.embed(chunk)));
+                }
+            }
+            catch (err) {
+                await quarantine(deps.dataDir, payload.source, conversation, err);
+                throw err;
+            }
+            processed.push({
+                conversation,
+                rowId,
+                fingerprint,
+                metadata,
+                chunkContents,
+                chunkEmbeddings,
+                metadataProvider: itemMetadataProvider,
+                extractionProvenance: itemExtractionProvenance,
+                ...(existingTable ? { refreshTable: existingTable } : {}),
+            });
+        }
+        if (processed.length === 0 && recoverable.length === 0) {
+            job.import_summary = importSummary;
+            return; // All already indexed (replay)
+        }
+        // --- Phase 2: Persist (all conversations passed Phase 1) ---
+        job.progress = { phase: "indexing", completed: parsed.length - processed.length - recoverable.length, total: parsed.length };
+        // Write raw files first (crash safety), then insert rows.
+        // Recoverable items already have raw files — only insert rows.
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const newRawFileNames = new Map();
+        for (const item of processed) {
+            // Per-item provenance: a conversation that fell back to basic metadata
+            // carries the basic extractor's provenance, not the job-level provider's.
+            const extractionProvenance = item.extractionProvenance;
+            // `target_table` lets crash recovery route the restored row to the
+            // table the slice originally targeted. Without it, a cold refresh
+            // interrupted between delete-old and add-new would be restored
+            // into `conversations_hot` on replay (silent re-promote).
+            const targetTable = item.refreshTable ?? "conversations_hot";
+            const rawData = {
+                source: payload.source,
+                conversation_id: item.conversation.id,
+                title: item.conversation.title,
+                created_at: item.conversation.createdAt,
+                content: item.conversation.content,
+                conversation_fingerprint: item.fingerprint,
+                target_table: targetTable,
+                summary: item.conversation.title,
+                domain: item.metadata.domain,
+                intent: item.metadata.intent,
+                quality: item.metadata.quality,
+                topics: item.metadata.topics,
+                decisions: withoutUserMarkers(item.metadata.decisions),
+                key_outputs: item.metadata.key_outputs,
+                metadata_provider: item.metadataProvider,
+                embedding_provider: embeddingProviderLabel,
+                ingested_at: new Date().toISOString(),
+                ...(extractionProvenance
+                    ? { extraction_provenance: extractionProvenance }
+                    : {}),
+            };
+            const rawFileName = `${timestamp}_${item.rowId}.json`;
+            newRawFileNames.set(item.rowId, rawFileName);
+            const rawPath = path.join(rawDir, rawFileName);
+            await atomicWrite(rawPath, JSON.stringify(rawData, null, 2));
+        }
+        for (const item of processed) {
+            // For a refresh, delete the old row from whichever table it lives
+            // in BEFORE inserting the new one — otherwise LanceDB ends up with
+            // two rows sharing the same id. Raw artifact for the new version
+            // is already on disk above, so if the delete-then-add window is
+            // interrupted, the next replay finds rowId missing + raw present
+            // and reindexes from raw via the recoverable path.
+            const targetTableName = item.refreshTable ?? "conversations_hot";
+            const targetTable = item.refreshTable === "conversations_cold"
+                ? coldTable
+                : table;
+            if (item.refreshTable) {
+                // Delete the WHOLE prior set (single row or chunk set) for this
+                // conversation before inserting the new one — otherwise LanceDB ends
+                // up with stale chunks. The raw artifact for the new version is already
+                // on disk above, so if the delete-then-add window is interrupted, the
+                // next replay finds the set missing + raw present and reindexes from
+                // raw via the recoverable path.
+                await targetTable.delete(eqFilter("id", item.rowId, { validateAsRowId: true }));
+                await targetTable.delete(eqFilter("parent_id", item.rowId, { validateAsRowId: true }));
+            }
+            const count = item.chunkContents.length;
+            const indexedAt = new Date().toISOString();
+            const rows = item.chunkContents.map((content, i) => ({
+                id: count > 1 ? convChunkId(item.rowId, i) : item.rowId,
+                content,
+                summary: item.conversation.title,
+                embedding: item.chunkEmbeddings[i],
+                source: payload.source,
+                domain: item.metadata.domain,
+                intent: item.metadata.intent,
+                quality: item.metadata.quality,
+                topics: JSON.stringify(item.metadata.topics),
+                decisions: JSON.stringify(withoutUserMarkers(item.metadata.decisions)),
+                key_outputs: JSON.stringify(item.metadata.key_outputs),
+                indexed_at: indexedAt,
+                idempotency_key: payload.idempotency_key ?? "",
+                conversation_fingerprint: item.fingerprint,
+                metadata_provider: item.metadataProvider,
+                embedding_provider: embeddingProviderLabel,
+                ...conversationChunkColumns(item.rowId, i, count),
+            }));
+            await recordIndexWrite(deps.dataDir, {
+                table: targetTableName,
+                pipeline: "ingest_job",
+                operation: item.refreshTable
+                    ? "conversation_refresh"
+                    : "conversation_upsert",
+                row_count: rows.length,
+            }, () => targetTable.add(rows));
+            job.progress.completed += 1;
+            if (item.refreshTable) {
+                // Once the new row is in place, the prior raw artifact(s) for
+                // this rowId only carry the older content. Prune them so a
+                // future rebuild does not insert duplicate rows (one per raw)
+                // for the same conversation. Best-effort by design.
+                const keep = newRawFileNames.get(item.rowId);
+                pruneStaleRawArtifacts(rawDir, item.rowId, keep);
+            }
+        }
+        // Recover rows from existing raw files (crash recovery).
+        for (const item of recoverable) {
+            await reindexFromRaw(item.rawPath, item.rowId, payload, { hot: table, cold: coldTable }, deps.embeddingProvider, deps.dataDir);
+            job.progress.completed += 1;
+        }
+        job.import_summary = importSummary;
+        job.progress.completed = parsed.length;
+    };
+}
+function resolveMetadataExtractor(payload, deps) {
+    if (payload.extraction_provider_override === "codex-cli") {
+        return new CodexCliMetadataExtractor();
+    }
+    return deps.metadataExtractor;
+}
+/**
+ * Re-index a conversation from its raw file (crash recovery).
+ * Raw files written by Phase 2 include full metadata, so we can
+ * rebuild the row without calling the extractor again.
+ *
+ * Routing: the raw artifact's `target_table` decides hot vs cold so
+ * a cold refresh interrupted between delete-old and add-new is
+ * restored back into cold, not silently re-promoted to hot. Legacy
+ * artifacts (pre-2026-05-14) have no `target_table` field and default
+ * to hot — same behavior as before this slice.
+ */
+async function reindexFromRaw(rawPath, rowId, payload, tables, embeddingProvider, dataDir) {
+    const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+    const content = raw.content ?? "";
+    const summary = raw.summary ?? "";
+    const rawTarget = raw.target_table;
+    const targetTableName = rawTarget === "conversations_cold" ? "conversations_cold" : "conversations_hot";
+    const targetTable = targetTableName === "conversations_cold" ? tables.cold : tables.hot;
+    const embedding = await recordEmbed(dataDir, embeddingProvider, {
+        pipeline: "ingest_job",
+        operation: "reindex_from_raw",
+        input_count: 1,
+    }, () => embeddingProvider.embed(content || summary));
+    const fingerprint = typeof raw.conversation_fingerprint === "string"
+        && raw.conversation_fingerprint.length > 0
+        ? raw.conversation_fingerprint
+        : content
+            ? conversationContentFingerprint(content)
+            : "";
+    const row = {
+        id: rowId,
+        content,
+        summary,
+        embedding,
+        source: payload.source,
+        domain: raw.domain ?? "tech",
+        intent: raw.intent ?? "build",
+        quality: raw.quality ?? "medium",
+        topics: JSON.stringify(raw.topics ?? []),
+        decisions: JSON.stringify(raw.decisions ?? []),
+        key_outputs: JSON.stringify(raw.key_outputs ?? []),
+        indexed_at: new Date().toISOString(),
+        idempotency_key: payload.idempotency_key ?? "",
+        conversation_fingerprint: fingerprint,
+        // Metadata is restored from raw, so its provider marker is preserved as
+        // written. The embedding is recomputed here, so its marker reflects the
+        // provider that just ran (placeholder → "rift-none", real → "").
+        metadata_provider: typeof raw.metadata_provider === "string" ? raw.metadata_provider : "",
+        embedding_provider: isPlaceholderEmbeddingProvider(embeddingProvider)
+            ? RIFT_NONE_EMBEDDING_PROVIDER
+            : "",
+        // PR1: crash recovery reconstructs ONE whole row from the raw artifact
+        // (re-chunking on recovery is the deferred historical-backfill path).
+        parent_id: "",
+        chunk_index: "",
+        chunk_count: "",
+    };
+    await recordIndexWrite(dataDir, {
+        table: targetTableName,
+        pipeline: "ingest_job",
+        operation: "conversation_reindex_from_raw",
+        row_count: 1,
+    }, () => targetTable.add([row]));
+}
+/**
+ * Quarantine a failed conversation to data/quarantine/.
+ */
+async function quarantine(dataDir, source, conversation, error) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const hash = crypto
+        .createHash("sha256")
+        .update(conversation.id + conversation.content.slice(0, 500))
+        .digest("hex")
+        .slice(0, 12);
+    const quarantinePath = path.join(dataDir, "quarantine", `${source}_${timestamp}_${hash}.json`);
+    const quarantineData = {
+        source,
+        conversation_id: conversation.id,
+        title: conversation.title,
+        content_preview: conversation.content.slice(0, 1000),
+        error: error instanceof Error ? error.message : String(error),
+        quarantined_at: new Date().toISOString(),
+    };
+    await atomicWrite(quarantinePath, JSON.stringify(quarantineData, null, 2));
+}
+//# sourceMappingURL=ingest.js.map

package/dist/src/jobs/handlers/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAU7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wCAAwC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gDAAgD,CAAC;AACzF,OAAO,EAAE,8BAA8B,EAAE,MAAM,wDAAwD,CAAC;AACxG,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACnF,OAAO,EACL,wBAAwB,EACxB,WAAW,EACX,gBAAgB,GACjB,MAAM,+BAA+B,CAAC;AAGvC,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAC7F,OAAO,EACL,sBAAsB,EACtB,4BAA4B,GAC7B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,4BAA4B,EAC5B,8BAA8B,GAC/B,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAavE,2EAA2E;AAC3E,MAAM,OAAO,GAAuC;IAClD,WAAW,EAAE,eAAe;IAC5B,UAAU,EAAE,cAAc;IAC1B,UAAU,EAAE,cAAc;IAC1B,QAAQ,EAAE,YAAY;CACvB,CAAC;AAkCF;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAQ,EAAE,KAAe;IACjD,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,OAAO,CAAC,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,OAAO,GAAG,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,MAAc,EAAE,KAAa;IAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,CAAC,IAAI,EAAE,CAAC;IACf,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,sBAAsB,CAAC,GAA4B;IAC1D,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,wBAAwB,KAAK,QAAQ;QAC7D,CAAC,CAAC,GAAG,CAAC,wBAAwB;QAC9B,CAAC,CAAC,EAAE,CAAC;IACP,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,OAAO,GAAG,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,OAAO,8BAA8B,CAAC,OAAO,CAAC,CAAC;AACjD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,sBAAsB,CAC7B,MAAc,EACd,KAAa,EACb,YAAoB;IAEpB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO;IACnC,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,IAAI,IAAI,KAAK,YAAY;YAAE,SAAS;QACpC,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,WAAW,GAAI,OAAO,CAAC,OAAqC;YAC/D,GAAG,CAAC,OAAyB,CAAC;QAEjC,MAAM,OAAO,GAAG,WAAW,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,iBAAiB,GAAG,CAAC,IAAI,CAAC,wBAAwB,IAAI,wBAAwB,CAAC,CACnF,OAAO,EACP,IAAI,CACL,CAAC;QAEF,iEAAiE;QACjE,yEAAyE;QACzE,uEAAuE;QACvE,wEAAwE;QACxE,wEAAwE;QACxE,+CAA+C;QAC/C,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,aAAa,EAAE,EAAE,EAAE,QAAQ,IAAI,EAAE,CAAC;QAClF,MAAM,sBAAsB,GAAG,8BAA8B,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACnF,CAAC,CAAC,4BAA4B;YAC9B,CAAC,CAAC,EAAE,CAAC;QACP,2EAA2E;QAC3E,0EAA0E;QAC1E,IAAI,aAAa,GAAkC,IAAI,CAAC;QAExD,qCAAqC;QACrC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE/D,iCAAiC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,yBAAyB,OAAO,CAAC,MAAM,0BAA0B,CAClE,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,GAAG,CAAC,QAAQ,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QACxE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,yDAAyD;YACzD,gEAAgE;YAChE,+DAA+D;YAC/D,mEAAmE;YACnE,yDAAyD;YACzD,iEAAiE;YACjE,iEAAiE;YACjE,MAAM,IAAI,KAAK,CACb,qCAAqC,OAAO,CAAC,QAAQ,GAAG,CACzD,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC;aAC/D,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,aAAa,GAAG;YACpB,aAAa,EAAE,MAAM,CAAC,MAAM;YAC5B,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI;YAC5B,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI;YAC/B,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;qBACnF,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SAClF,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,eAAe,EACf,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CACpC,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAEjD,+DAA+D;QAC/D,iEAAiE;QACjE,kEAAkE;QAClE,6DAA6D;QAC7D,8DAA8D;QAC9D,wCAAwC;QACxC,MAAM,SAAS,GAA4B,EAAE,CAAC;QAC9C,MAAM,WAAW,GAA8B,EAAE,CAAC;QAElD,KAAK,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACjE,GAAG,CAAC,QAAQ,CAAC,SAAS,GAAG,iBAAiB,CAAC;YAC3C,6DAA6D;YAC7D,+DAA+D;YAC/D,+DAA+D;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;YACvE,MAAM,WAAW,GAAG,8BAA8B,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAEzE,0BAA0B;YAC1B,+DAA+D;YAC/D,kEAAkE;YAClE,uEAAuE;YACvE,gEAAgE;YAChE,gDAAgD;YAChD,iEAAiE;YACjE,2EAA2E;YAC3E,MAAM,SAAS,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,SAAS,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC;YAC1I,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,KAAK,CAAC;gBAC3C,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE;gBACpD,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,aAAa,GAA0B,WAAW,CAAC,MAAM,GAAG,CAAC;gBACjE,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;oBACvB,CAAC,CAAC,oBAAoB;oBACtB,CAAC,CAAC,SAAS,CAAC;YAEhB,IAAI,aAAa,EAAE,CAAC;gBAClB,kEAAkE;gBAClE,oEAAoE;gBACpE,uEAAuE;gBACvE,qEAAqE;gBACrE,2CAA2C;gBAC3C,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC;gBACzE,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC;gBAErE,IACE,mBAAmB;oBACnB,mBAAmB,KAAK,WAAW;oBACnC,gBAAgB,CAAC,YAAY,CAAC,EAC9B,CAAC;oBACD,mEAAmE;oBACnE,SAAS;gBACX,CAAC;gBACD,yEAAyE;gBAEzE,6DAA6D;gBAC7D,6DAA6D;gBAC7D,qDAAqD;gBACrD,0DAA0D;gBAC1D,iDAAiD;YACnD,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACnD,IAAI,eAAe,EAAE,CAAC;oBACpB,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;oBACtD,SAAS;gBACX,CAAC;YACH,CAAC;YAED,uEAAuE;YACvE,wEAAwE;YACxE,6DAA6D;YAC7D,IAAI,QAA8B,CAAC;YACnC,IAAI,oBAA4B,CAAC;YACjC,IAAI,wBAAkE,CAAC;YACvE,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,eAAe,CAChD,YAAY,CAAC,OAAO,CACrB,CAAC;gBACF,oBAAoB,GAAG,qBAAqB,CAAC;gBAC7C,wBAAwB,GAAG,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC;YACjE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wEAAwE;gBACxE,uEAAuE;gBACvE,gEAAgE;gBAChE,wEAAwE;gBACxE,yEAAyE;gBACzE,uEAAuE;gBACvE,2DAA2D;gBAC3D,IAAI,qBAAqB,KAAK,4BAA4B,EAAE,CAAC;oBAC3D,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;oBAClE,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,aAAa,KAAK,IAAI,sBAAsB,EAAE,CAAC;gBAC/C,QAAQ,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACrE,oBAAoB,GAAG,4BAA4B,CAAC;gBACpD,wBAAwB,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;YAC3D,CAAC;YAED,6DAA6D;YAC7D,yEAAyE;YACzE,kDAAkD;YAClD,MAAM,aAAa,GAAG,eAAe,EAAE;gBACrC,CAAC,CAAC,iBAAiB,CAAC,YAAY,CAAC,OAAO,CAAC;gBACzC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAC3B,MAAM,eAAe,GAAe,EAAE,CAAC;YACvC,IAAI,CAAC;gBACH,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAClC,eAAe,CAAC,IAAI,CAClB,MAAM,WAAW,CACf,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,iBAAiB,EACtB;wBACE,QAAQ,EAAE,YAAY;wBACtB,SAAS,EAAE,wBAAwB;wBACnC,WAAW,EAAE,CAAC;qBACf,EACD,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAC1C,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,YAAY;gBACZ,KAAK;gBACL,WAAW;gBACX,QAAQ;gBACR,aAAa;gBACb,eAAe;gBACf,gBAAgB,EAAE,oBAAoB;gBACtC,oBAAoB,EAAE,wBAAwB;gBAC9C,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,cAAc,GAAG,aAAa,CAAC;YACnC,OAAO,CAAC,+BAA+B;QACzC,CAAC;QAED,8DAA8D;QAC9D,GAAG,CAAC,QAAQ,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC7H,0DAA0D;QAC1D,+DAA+D;QAC/D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEjE,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,uEAAuE;YACvE,0EAA0E;YAC1E,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC;YACvD,mEAAmE;YACnE,kEAAkE;YAClE,+DAA+D;YAC/D,0DAA0D;YAC1D,MAAM,WAAW,GAAc,IAAI,CAAC,YAAY,IAAI,mBAAmB,CAAC;YACxE,MAAM,OAAO,GAAG;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,eAAe,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;gBACrC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAC9B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS;gBACvC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,wBAAwB,EAAE,IAAI,CAAC,WAAW;gBAC1C,YAAY,EAAE,WAAW;gBACzB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,SAAS,EAAE,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACtD,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;gBACtC,iBAAiB,EAAE,IAAI,CAAC,gBAAgB;gBACxC,kBAAkB,EAAE,sBAAsB;gBAC1C,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,GAAG,CAAC,oBAAoB;oBACtB,CAAC,CAAC,EAAE,qBAAqB,EAAE,oBAAoB,EAAE;oBACjD,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;YACF,MAAM,WAAW,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,KAAK,OAAO,CAAC;YACtD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAC/C,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,kEAAkE;YAClE,mEAAmE;YACnE,iEAAiE;YACjE,gEAAgE;YAChE,iEAAiE;YACjE,mDAAmD;YACnD,MAAM,eAAe,GAAc,IAAI,CAAC,YAAY,IAAI,mBAAmB,CAAC;YAC5E,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,KAAK,oBAAoB;gBAC5D,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK,CAAC;YAEV,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,gEAAgE;gBAChE,qEAAqE;gBACrE,wEAAwE;gBACxE,sEAAsE;gBACtE,qEAAqE;gBACrE,gCAAgC;gBAChC,MAAM,WAAW,CAAC,MAAM,CACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CACtD,CAAC;gBACF,MAAM,WAAW,CAAC,MAAM,CACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAC7D,CAAC;YACJ,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;YACxC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAsB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtE,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;gBACvD,OAAO;gBACP,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAE;gBACnC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACtE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACtD,UAAU,EAAE,SAAS;gBACrB,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;gBAC9C,wBAAwB,EAAE,IAAI,CAAC,WAAW;gBAC1C,iBAAiB,EAAE,IAAI,CAAC,gBAAgB;gBACxC,kBAAkB,EAAE,sBAAsB;gBAC1C,GAAG,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC;aAClD,CAAC,CAAC,CAAC;YAEJ,MAAM,gBAAgB,CACpB,IAAI,CAAC,OAAO,EACZ;gBACE,KAAK,EAAE,eAAe;gBACtB,QAAQ,EAAE,YAAY;gBACtB,SAAS,EAAE,IAAI,CAAC,YAAY;oBAC1B,CAAC,CAAC,sBAAsB;oBACxB,CAAC,CAAC,qBAAqB;gBACzB,SAAS,EAAE,IAAI,CAAC,MAAM;aACvB,EACD,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAC5B,CAAC;YAEF,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;YAC5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,8DAA8D;gBAC9D,2DAA2D;gBAC3D,8DAA8D;gBAC9D,oDAAoD;gBACpD,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;gBAC9C,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,yDAAyD;QACzD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,CAClB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,OAAO,EACP,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EAC/B,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,OAAO,CACb,CAAC;YACF,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,GAAG,CAAC,cAAc,GAAG,aAAa,CAAC;QACnC,GAAG,CAAC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;IACzC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAAsB,EACtB,IAAuB;IAEvB,IAAI,OAAO,CAAC,4BAA4B,KAAK,WAAW,EAAE,CAAC;QACzD,OAAO,IAAI,yBAAyB,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,iBAAiB,CAAC;AAChC,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,KAAa,EACb,OAAsB,EACtB,MAA+E,EAC/E,iBAAoC,EACpC,OAAe;IAEf,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CACP,CAAC;IAE7B,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC;IACnC,MAAM,eAAe,GACnB,SAAS,KAAK,oBAAoB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAClF,MAAM,WAAW,GACf,eAAe,KAAK,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IAEtE,MAAM,SAAS,GAAG,MAAM,WAAW,CACjC,OAAO,EACP,iBAAiB,EACjB;QACE,QAAQ,EAAE,YAAY;QACtB,SAAS,EAAE,kBAAkB;QAC7B,WAAW,EAAE,CAAC;KACf,EACD,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,CAClD,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,wBAAwB,KAAK,QAAQ;WAC7D,GAAG,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC;QAC5C,CAAC,CAAC,GAAG,CAAC,wBAAwB;QAC9B,CAAC,CAAC,OAAO;YACP,CAAC,CAAC,8BAA8B,CAAC,OAAO,CAAC;YACzC,CAAC,CAAC,EAAE,CAAC;IAET,MAAM,GAAG,GAAoB;QAC3B,EAAE,EAAE,KAAK;QACT,OAAO;QACP,OAAO;QACP,SAAS;QACT,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,MAAM;QAC3D,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,OAAO;QAC5D,OAAO,EAAG,GAAG,CAAC,OAAsC,IAAI,QAAQ;QAChE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,MAAmB,IAAI,EAAE,CAAC;QACtD,SAAS,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,SAAsB,IAAI,EAAE,CAAC;QAC5D,WAAW,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,WAAwB,IAAI,EAAE,CAAC;QAChE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9C,wBAAwB,EAAE,WAAW;QACrC,wEAAwE;QACxE,wEAAwE;QACxE,iEAAiE;QACjE,iBAAiB,EAAE,OAAO,GAAG,CAAC,iBAAiB,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE;QACzF,kBAAkB,EAAE,8BAA8B,CAAC,iBAAiB,CAAC;YACnE,CAAC,CAAC,4BAA4B;YAC9B,CAAC,CAAC,EAAE;QACN,uEAAuE;QACvE,sEAAsE;QACtE,SAAS,EAAE,EAAE;QACb,WAAW,EAAE,EAAE;QACf,WAAW,EAAE,EAAE;KAChB,CAAC;IAEF,MAAM,gBAAgB,CACpB,OAAO,EACP;QACE,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,YAAY;QACtB,SAAS,EAAE,+BAA+B;QAC1C,SAAS,EAAE,CAAC;KACb,EACD,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,OAAe,EACf,MAAc,EACd,YAAgC,EAChC,KAAc;IAEd,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SAC5D,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,OAAO,EACP,YAAY,EACZ,GAAG,MAAM,IAAI,SAAS,IAAI,IAAI,OAAO,CACtC,CAAC;IAEF,MAAM,cAAc,GAAG;QACrB,MAAM;QACN,eAAe,EAAE,YAAY,CAAC,EAAE;QAChC,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QACpD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;IAEF,MAAM,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/src/jobs/handlers/optimize.d.ts

@@ -0,0 +1,11 @@
+/**
+ * `optimize` job: LanceDB compaction, full-text index build/refresh and
+ * old-version cleanup (see src/storage/maintenance.ts).
+ *
+ * Exclusive: the queue never runs it next to another writer job, and the
+ * parent pauses watchers + scanner and refreshes its table handles after the
+ * worker exits. No embeddings, no network.
+ */
+import type { JobHandler } from "../types.js";
+export declare function createOptimizeHandler(): JobHandler;
+//# sourceMappingURL=optimize.d.ts.map

package/dist/src/jobs/handlers/optimize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/optimize.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,wBAAgB,qBAAqB,IAAI,UAAU,CAWlD"}

package/dist/src/jobs/handlers/optimize.js

@@ -0,0 +1,12 @@
+import { runTableMaintenance } from "../../storage/maintenance.js";
+export function createOptimizeHandler() {
+    return async () => {
+        const results = await runTableMaintenance();
+        for (const r of results) {
+            process.stderr.write(`optimize: ${r.table} fragments ${r.fragments_before}→${r.fragments_after}` +
+                (r.indexes_created.length ? `, fts built on ${r.indexes_created.join(", ")}` : "") +
+                ` (${r.ms}ms)\n`);
+        }
+    };
+}
+//# sourceMappingURL=optimize.js.map

package/dist/src/jobs/handlers/optimize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/optimize.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAEnE,MAAM,UAAU,qBAAqB;IACnC,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,OAAO,GAAG,MAAM,mBAAmB,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,aAAa,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,gBAAgB,IAAI,CAAC,CAAC,eAAe,EAAE;gBACzE,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,KAAK,CAAC,CAAC,EAAE,OAAO,CACnB,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}