@getrift/rift 0.0.0 → 0.1.0-beta.0

package/dist/src/jobs/handlers/ingest.js

@@ -0,0 +1,255 @@
+/**
+ * Ingest job handler — Slice 13.
+ *
+ * Processes a POST /ingest payload in two phases:
+ *
+ * Phase 1 — Process (no side effects):
+ *   1. Unpack archive securely (Slice 12 archive.ts)
+ *   2. Parse conversation export via source-specific parser
+ *   3. For each parsed conversation: extract metadata + embed
+ *   4. If ANY conversation fails: quarantine that one, throw (no persist)
+ *
+ * Phase 2 — Persist (only if all conversations passed Phase 1):
+ *   5. Write raw files (with full metadata — rebuild-safe)
+ *   6. Insert all rows into conversations_hot
+ *
+ * This two-phase design ensures atomicity: either all conversations in
+ * an upload are indexed, or none are. No partial ingest.
+ *
+ * Replay safety:
+ * - Row ID is derived from source conversation ID + job root ID
+ * - Raw files are written before table insert (crash safety)
+ * - Existing rows are skipped on replay
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { getTable } from "../../storage/tables.js";
+import { atomicWrite } from "../../storage/atomic.js";
+import { extractArchive } from "../../security/archive.js";
+import { parseChatGPTWeb } from "../../ingestion/parsers/chatgpt-web.js";
+import { parseClaudeWeb } from "../../ingestion/parsers/claude-web.js";
+import { parseGeminiWeb } from "../../ingestion/parsers/gemini-web.js";
+import { parseGrokWeb } from "../../ingestion/parsers/grok-web.js";
+import { CodexCliMetadataExtractor } from "../../providers/codex-cli-metadata-extraction.js";
+/** Map of source -> parser. Only fixture-backed sources are registered. */
+const PARSERS = {
+    chatgpt_web: parseChatGPTWeb,
+    claude_web: parseClaudeWeb,
+    gemini_web: parseGeminiWeb,
+    grok_web: parseGrokWeb,
+};
+/**
+ * Generate a stable row ID from the source conversation ID and job root.
+ * This ensures replay safety: retries converge on the same row ID.
+ */
+function stableRowId(conversationId, jobRootId) {
+    return crypto
+        .createHash("sha256")
+        .update(`${jobRootId}:${conversationId}`)
+        .digest("hex")
+        .slice(0, 32);
+}
+/**
+ * Walk the retry_of chain to the root job (same logic as save handler).
+ */
+function resolveChainRoot(job, queue) {
+    let current = job;
+    let depth = 0;
+    while (current.retry_of && depth < 50) {
+        const parent = queue.get(current.retry_of);
+        if (!parent)
+            break;
+        current = parent;
+        depth++;
+    }
+    return current;
+}
+/**
+ * Find the raw file for `rowId` under `rawDir`, if it exists.
+ */
+function findRawFile(rawDir, rowId) {
+    if (!fs.existsSync(rawDir))
+        return undefined;
+    const suffix = `_${rowId}.json`;
+    const match = fs.readdirSync(rawDir).find((f) => f.endsWith(suffix));
+    return match ? path.join(rawDir, match) : undefined;
+}
+export function createIngestHandler(deps) {
+    return async (job) => {
+        const rootJob = resolveChainRoot(job, deps.queue);
+        const rootPayload = rootJob.payload ??
+            job.payload;
+        const jobRootId = rootJob.id;
+        const payload = rootPayload;
+        if (!payload?.source || !payload.fileData) {
+            throw new Error("Invalid ingest job payload: missing source or fileData");
+        }
+        const metadataExtractor = (deps.metadataExtractorFactory ?? resolveMetadataExtractor)(payload, deps);
+        // --- 1. Unpack archive securely ---
+        const fileBuffer = Buffer.from(payload.fileData, "base64");
+        const { files } = extractArchive(fileBuffer, payload.filename);
+        // --- 2. Parse conversations ---
+        const parser = PARSERS[payload.source];
+        if (!parser) {
+            throw new Error(`No parser for source "${payload.source}": deferred (no fixture)`);
+        }
+        const parsed = parser(files);
+        if (parsed.length === 0) {
+            // Nothing to ingest — not an error, just empty export
+            return;
+        }
+        const rawDir = path.join(deps.dataDir, "raw", "conversations", payload.rawSource ?? payload.source);
+        const table = getTable("conversations_hot");
+        // --- Phase 1: Process all conversations (no side effects) ---
+        // Extract metadata + embed for every conversation. If any fails,
+        // quarantine that one and throw — nothing has been persisted yet.
+        // Conversations with existing raw files (crash recovery) are
+        // collected but NOT inserted yet — they wait for Phase 2 so a
+        // mixed batch doesn't partially ingest.
+        const processed = [];
+        const recoverable = [];
+        for (const conversation of parsed) {
+            const rowId = stableRowId(conversation.id, jobRootId);
+            // Replay guard: skip already-indexed conversations
+            const existing = await table
+                .query()
+                .where(`id = '${rowId}'`)
+                .toArray();
+            if (existing.length > 0) {
+                continue;
+            }
+            // Crash recovery: raw file exists but row missing → defer to Phase 2.
+            const existingRawPath = findRawFile(rawDir, rowId);
+            if (existingRawPath) {
+                recoverable.push({ rawPath: existingRawPath, rowId });
+                continue;
+            }
+            let metadata;
+            try {
+                metadata = await metadataExtractor.extractMetadata(conversation.content);
+            }
+            catch (err) {
+                await quarantine(deps.dataDir, payload.source, conversation, err);
+                throw err;
+            }
+            let embedding;
+            try {
+                embedding = await deps.embeddingProvider.embed(conversation.content);
+            }
+            catch (err) {
+                await quarantine(deps.dataDir, payload.source, conversation, err);
+                throw err;
+            }
+            processed.push({ conversation, rowId, metadata, embedding });
+        }
+        if (processed.length === 0 && recoverable.length === 0) {
+            return; // All already indexed (replay)
+        }
+        // --- Phase 2: Persist (all conversations passed Phase 1) ---
+        // Write raw files first (crash safety), then insert rows.
+        // Recoverable items already have raw files — only insert rows.
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        for (const item of processed) {
+            const extractionProvenance = metadataExtractor.getProvenance?.();
+            const rawData = {
+                source: payload.source,
+                conversation_id: item.conversation.id,
+                title: item.conversation.title,
+                created_at: item.conversation.createdAt,
+                content: item.conversation.content,
+                summary: item.conversation.title,
+                domain: item.metadata.domain,
+                intent: item.metadata.intent,
+                quality: item.metadata.quality,
+                topics: item.metadata.topics,
+                decisions: item.metadata.decisions,
+                key_outputs: item.metadata.key_outputs,
+                ingested_at: new Date().toISOString(),
+                ...(extractionProvenance
+                    ? { extraction_provenance: extractionProvenance }
+                    : {}),
+            };
+            const rawFileName = `${timestamp}_${item.rowId}.json`;
+            const rawPath = path.join(rawDir, rawFileName);
+            await atomicWrite(rawPath, JSON.stringify(rawData, null, 2));
+        }
+        for (const item of processed) {
+            const row = {
+                id: item.rowId,
+                content: item.conversation.content,
+                summary: item.conversation.title,
+                embedding: item.embedding,
+                source: payload.source,
+                domain: item.metadata.domain,
+                intent: item.metadata.intent,
+                quality: item.metadata.quality,
+                topics: JSON.stringify(item.metadata.topics),
+                decisions: JSON.stringify(item.metadata.decisions),
+                key_outputs: JSON.stringify(item.metadata.key_outputs),
+                indexed_at: new Date().toISOString(),
+                idempotency_key: payload.idempotency_key ?? "",
+            };
+            await table.add([row]);
+        }
+        // Recover rows from existing raw files (crash recovery).
+        for (const item of recoverable) {
+            await reindexFromRaw(item.rawPath, item.rowId, payload, table, deps.embeddingProvider);
+        }
+    };
+}
+function resolveMetadataExtractor(payload, deps) {
+    if (payload.extraction_provider_override === "codex-cli") {
+        return new CodexCliMetadataExtractor();
+    }
+    return deps.metadataExtractor;
+}
+/**
+ * Re-index a conversation from its raw file (crash recovery).
+ * Raw files written by Phase 2 include full metadata, so we can
+ * rebuild the row without calling the extractor again.
+ */
+async function reindexFromRaw(rawPath, rowId, payload, table, embeddingProvider) {
+    const raw = JSON.parse(fs.readFileSync(rawPath, "utf-8"));
+    const content = raw.content ?? "";
+    const summary = raw.summary ?? "";
+    const embedding = await embeddingProvider.embed(content || summary);
+    const row = {
+        id: rowId,
+        content,
+        summary,
+        embedding,
+        source: payload.source,
+        domain: raw.domain ?? "tech",
+        intent: raw.intent ?? "build",
+        quality: raw.quality ?? "medium",
+        topics: JSON.stringify(raw.topics ?? []),
+        decisions: JSON.stringify(raw.decisions ?? []),
+        key_outputs: JSON.stringify(raw.key_outputs ?? []),
+        indexed_at: new Date().toISOString(),
+        idempotency_key: payload.idempotency_key ?? "",
+    };
+    await table.add([row]);
+}
+/**
+ * Quarantine a failed conversation to data/quarantine/.
+ */
+async function quarantine(dataDir, source, conversation, error) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const hash = crypto
+        .createHash("sha256")
+        .update(conversation.id + conversation.content.slice(0, 500))
+        .digest("hex")
+        .slice(0, 12);
+    const quarantinePath = path.join(dataDir, "quarantine", `${source}_${timestamp}_${hash}.json`);
+    const quarantineData = {
+        source,
+        conversation_id: conversation.id,
+        title: conversation.title,
+        content_preview: conversation.content.slice(0, 1000),
+        error: error instanceof Error ? error.message : String(error),
+        quarantined_at: new Date().toISOString(),
+    };
+    await atomicWrite(quarantinePath, JSON.stringify(quarantineData, null, 2));
+}
+//# sourceMappingURL=ingest.js.map

package/dist/src/jobs/handlers/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../../src/jobs/handlers/ingest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAS7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wCAAwC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,uCAAuC,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAGnE,OAAO,EAAE,yBAAyB,EAAE,MAAM,kDAAkD,CAAC;AAa7F,2EAA2E;AAC3E,MAAM,OAAO,GAAuC;IAClD,WAAW,EAAE,eAAe;IAC5B,UAAU,EAAE,cAAc;IAC1B,UAAU,EAAE,cAAc;IAC1B,QAAQ,EAAE,YAAY;CACvB,CAAC;AAgBF;;;GAGG;AACH,SAAS,WAAW,CAAC,cAAsB,EAAE,SAAiB;IAC5D,OAAO,MAAM;SACV,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,GAAG,SAAS,IAAI,cAAc,EAAE,CAAC;SACxC,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAQ,EAAE,KAAe;IACjD,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,OAAO,CAAC,QAAQ,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,OAAO,GAAG,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,MAAc,EAAE,KAAa;IAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;IAChC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,OAAO,KAAK,EAAE,GAAQ,EAAiB,EAAE;QACvC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,WAAW,GAAI,OAAO,CAAC,OAAqC;YAC/D,GAAG,CAAC,OAAyB,CAAC;QACjC,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;QAE7B,MAAM,OAAO,GAAG,WAAW,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,iBAAiB,GAAG,CAAC,IAAI,CAAC,wBAAwB,IAAI,wBAAwB,CAAC,CACnF,OAAO,EACP,IAAI,CACL,CAAC;QAEF,qCAAqC;QACrC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE/D,iCAAiC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,yBAAyB,OAAO,CAAC,MAAM,0BAA0B,CAClE,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,sDAAsD;YACtD,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CACtB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,eAAe,EACf,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CACpC,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QAE5C,+DAA+D;QAC/D,iEAAiE;QACjE,kEAAkE;QAClE,6DAA6D;QAC7D,8DAA8D;QAC9D,wCAAwC;QACxC,MAAM,SAAS,GAA4B,EAAE,CAAC;QAC9C,MAAM,WAAW,GAA8B,EAAE,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAEtD,mDAAmD;YACnD,MAAM,QAAQ,GAAG,MAAM,KAAK;iBACzB,KAAK,EAAE;iBACP,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC;iBACxB,OAAO,EAAE,CAAC;YACb,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,SAAS;YACX,CAAC;YAED,sEAAsE;YACtE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACnD,IAAI,eAAe,EAAE,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;gBACtD,SAAS;YACX,CAAC;YAED,IAAI,QAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,eAAe,CAChD,YAAY,CAAC,OAAO,CACrB,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,IAAI,SAAmB,CAAC;YACxB,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YACvE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;gBAClE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,SAAS,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,+BAA+B;QACzC,CAAC;QAED,8DAA8D;QAC9D,0DAA0D;QAC1D,+DAA+D;QAC/D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEjE,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC;YACjE,MAAM,OAAO,GAAG;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,eAAe,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;gBACrC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAC9B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS;gBACvC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;gBAClC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;gBACtC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,GAAG,CAAC,oBAAoB;oBACtB,CAAC,CAAC,EAAE,qBAAqB,EAAE,oBAAoB,EAAE;oBACjD,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;YACF,MAAM,WAAW,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,KAAK,OAAO,CAAC;YACtD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAC/C,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAoB;gBAC3B,EAAE,EAAE,IAAI,CAAC,KAAK;gBACd,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO;gBAClC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK;gBAChC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAC5B,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAC9B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC5C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAClD,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACtD,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;aAC/C,CAAC;YAEF,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,yDAAyD;QACzD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,CAClB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,OAAO,EACP,KAAK,EACL,IAAI,CAAC,iBAAiB,CACvB,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAAsB,EACtB,IAAuB;IAEvB,IAAI,OAAO,CAAC,4BAA4B,KAAK,WAAW,EAAE,CAAC;QACzD,OAAO,IAAI,yBAAyB,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,iBAAiB,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,KAAa,EACb,OAAsB,EACtB,KAAkC,EAClC,iBAAoC;IAEpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CACP,CAAC;IAE7B,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC;IAEpE,MAAM,GAAG,GAAoB;QAC3B,EAAE,EAAE,KAAK;QACT,OAAO;QACP,OAAO;QACP,SAAS;QACT,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,MAAM;QAC3D,MAAM,EAAG,GAAG,CAAC,MAAoC,IAAI,OAAO;QAC5D,OAAO,EAAG,GAAG,CAAC,OAAsC,IAAI,QAAQ;QAChE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,MAAmB,IAAI,EAAE,CAAC;QACtD,SAAS,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,SAAsB,IAAI,EAAE,CAAC;QAC5D,WAAW,EAAE,IAAI,CAAC,SAAS,CAAE,GAAG,CAAC,WAAwB,IAAI,EAAE,CAAC;QAChE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,EAAE;KAC/C,CAAC;IAEF,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,OAAe,EACf,MAAc,EACd,YAAgC,EAChC,KAAc;IAEd,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SAC5D,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEhB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,OAAO,EACP,YAAY,EACZ,GAAG,MAAM,IAAI,SAAS,IAAI,IAAI,OAAO,CACtC,CAAC;IAEF,MAAM,cAAc,GAAG;QACrB,MAAM;QACN,eAAe,EAAE,YAAY,CAAC,EAAE;QAChC,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QACpD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACzC,CAAC;IAEF,MAAM,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/src/jobs/handlers/reconcile.d.ts

@@ -0,0 +1,29 @@
+import type { JobHandler } from "../types.js";
+import type { Config } from "../../config/schema.js";
+import type { EmbeddingProvider } from "../../providers/types.js";
+import type { Watcher } from "../../ingestion/watcher.js";
+import type { Scanner } from "../../ingestion/scanner.js";
+export interface ReconcileDeps {
+    config: Config;
+    embeddingProvider: EmbeddingProvider;
+    dataDir: string;
+    watchers: Watcher[];
+    scanner: Scanner | null;
+    /** Ollama embedding provider for local-mode sources (Slice 19). */
+    localEmbeddingProvider?: EmbeddingProvider;
+}
+export interface ReconcileReport {
+    files_checked: number;
+    missing_reindexed: number;
+    orphans_removed: number;
+    /**
+     * Rows deleted from `structured_docs` / `structured_docs_local` because
+     * their `source_path` lies under an engine-owned root per
+     * `buildIgnoredPaths` (pre-Scanner-hardening pollution). Tracked
+     * separately from `orphans_removed` so the retroactive-purge impact
+     * stays visible in reconcile reports.
+     */
+    engine_owned_purged: number;
+}
+export declare function createReconcileHandler(deps: ReconcileDeps): JobHandler;
+//# sourceMappingURL=reconcile.d.ts.map

package/dist/src/jobs/handlers/reconcile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../../../src/jobs/handlers/reconcile.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAO,UAAU,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAElE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAe1D,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,mEAAmE;IACnE,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAUD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CA4CtE"}