@getrift/rift 0.1.0-beta.2 → 0.1.0-beta.21

package/dist/src/ingestion/inbox-core/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,GAC1B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACL,gCAAgC,EAChC,2BAA2B,EAC3B,cAAc,GACf,MAAM,kBAAkB,CAAC"}

package/dist/src/ingestion/inbox-core/index.js

@@ -0,0 +1,20 @@
+/**
+ * inbox-core — pure file-detection primitives for an inbox watcher.
+ *
+ * Carved as a seam so the same logic can power both Rift's
+ * inbox-watcher.ts (which hands off to the job queue + ingest route)
+ * and a future standalone "AI Chat Backup Inbox" shell (which hands
+ * off to a local archive folder, with optional Rift indexing).
+ *
+ * Invariant: this module has zero Rift internals — no Fastify, no
+ * LanceDB, no Voyage, no JobQueue, no config loader, no parser
+ * registry. Only `node:` stdlib. The adapter wires it up.
+ */
+export { inboxIdempotencyKey } from "./idempotency.js";
+export { stableConversationRowId } from "./conversation-key.js";
+export { conversationContentFingerprint } from "./conversation-fingerprint.js";
+export { detectSourceFromSubdir } from "./source-detection.js";
+export { SUPPORTED_INBOX_EXTENSIONS, isSupportedInboxExtension, } from "./extensions.js";
+export { sniffInboxSource } from "./source-sniffer.js";
+export { readZipCentralDirectoryFilenames, sniffProviderFromZipEntries, sniffZipBuffer, } from "./zip-sniffer.js";
+//# sourceMappingURL=index.js.map

package/dist/src/ingestion/inbox-core/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAC1B,yBAAyB,GAC1B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAwB,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACL,gCAAgC,EAChC,2BAA2B,EAC3B,cAAc,GACf,MAAM,kBAAkB,CAAC"}

package/dist/src/ingestion/inbox-core/source-detection.d.ts

@@ -0,0 +1,2 @@
+export declare function detectSourceFromSubdir(filePath: string, inboxDir: string, knownSources: readonly string[]): string | null;
+//# sourceMappingURL=source-detection.d.ts.map

package/dist/src/ingestion/inbox-core/source-detection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-detection.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/source-detection.ts"],"names":[],"mappings":"AAaA,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,SAAS,MAAM,EAAE,GAC9B,MAAM,GAAG,IAAI,CAWf"}

package/dist/src/ingestion/inbox-core/source-detection.js

@@ -0,0 +1,23 @@
+/**
+ * Subdirectory-based source detection for inbox drops.
+ *
+ * If `data/inbox/<source>/file.json` exists and `<source>` is in
+ * `knownSources`, the source is `<source>`. Otherwise returns null
+ * and the caller decides on a default.
+ *
+ * Pure: no filesystem access, no Rift coupling. `knownSources` is
+ * passed in so the standalone shell can supply its own provider list
+ * without dragging in `parsers/types.ts`.
+ */
+import path from "node:path";
+export function detectSourceFromSubdir(filePath, inboxDir, knownSources) {
+    const relative = path.relative(inboxDir, filePath);
+    const firstSegment = relative.split(path.sep)[0];
+    if (firstSegment &&
+        firstSegment !== path.basename(filePath) &&
+        knownSources.includes(firstSegment)) {
+        return firstSegment;
+    }
+    return null;
+}
+//# sourceMappingURL=source-detection.js.map

package/dist/src/ingestion/inbox-core/source-detection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-detection.js","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/source-detection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,UAAU,sBAAsB,CACpC,QAAgB,EAChB,QAAgB,EAChB,YAA+B;IAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,IACE,YAAY;QACZ,YAAY,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACxC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EACnC,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/ingestion/inbox-core/source-sniffer.d.ts

@@ -0,0 +1,11 @@
+export type SniffedProvider = "claude_web" | "grok_web" | "gemini_web";
+/**
+ * Attempt to identify a non-ChatGPT provider for a file dropped in
+ * the inbox root. Returns null if no confident match — caller should
+ * fall back to its default (chatgpt_web in Rift's adapter).
+ *
+ * `data` may be undefined when the caller has only the filename
+ * (e.g., before reading the file). Filename-only heuristics still run.
+ */
+export declare function sniffInboxSource(filename: string, data?: Buffer, peekBytes?: number): SniffedProvider | null;
+//# sourceMappingURL=source-sniffer.d.ts.map

package/dist/src/ingestion/inbox-core/source-sniffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-sniffer.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/source-sniffer.ts"],"names":[],"mappings":"AAuBA,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,UAAU,GAAG,YAAY,CAAC;AAIvE;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,EACb,SAAS,GAAE,MAA2B,GACrC,eAAe,GAAG,IAAI,CAuCxB"}

package/dist/src/ingestion/inbox-core/source-sniffer.js

@@ -0,0 +1,69 @@
+/**
+ * Conservative content/filename sniffer for inbox drops.
+ *
+ * Purpose: when a file lands in the inbox root (no subdirectory hint),
+ * identify non-ChatGPT providers so they reach the right parser
+ * instead of silently failing under the ChatGPT default.
+ *
+ * Design: identify ONLY non-default providers (claude_web, grok_web,
+ * gemini_web). ChatGPT shapes are not matched here — the adapter's
+ * default fallback already handles them. Returning null means "no
+ * confident match; let the caller decide."
+ *
+ * Heuristics are cheap and ordered most-specific first:
+ *   1. Filename pattern (no I/O)
+ *   2. JSON content head substring match (bounded peek)
+ *
+ * Pure: no filesystem access, no Rift internals. The buffer is passed
+ * in by the caller — Rift's adapter reads with `fs`, a standalone
+ * shell would read however it likes.
+ */
+import path from "node:path";
+import { sniffZipBuffer } from "./zip-sniffer.js";
+const DEFAULT_PEEK_BYTES = 64 * 1024;
+/**
+ * Attempt to identify a non-ChatGPT provider for a file dropped in
+ * the inbox root. Returns null if no confident match — caller should
+ * fall back to its default (chatgpt_web in Rift's adapter).
+ *
+ * `data` may be undefined when the caller has only the filename
+ * (e.g., before reading the file). Filename-only heuristics still run.
+ */
+export function sniffInboxSource(filename, data, peekBytes = DEFAULT_PEEK_BYTES) {
+    const fnameLower = path.basename(filename).toLowerCase();
+    const ext = path.extname(fnameLower);
+    // --- Filename hints (cheap, no I/O) ---
+    if (fnameLower.startsWith("prod-grok-backend"))
+        return "grok_web";
+    if (fnameLower.includes("claude"))
+        return "claude_web";
+    if (fnameLower.includes("takeout"))
+        return "gemini_web";
+    // --- Content hints ---
+    if (!data || data.length === 0)
+        return null;
+    // ZIPs: walk the central directory and match on marker filenames.
+    // We never decompress here — the parser layer handles extraction.
+    if (ext === ".zip") {
+        return sniffZipBuffer(data);
+    }
+    if (ext !== ".json")
+        return null;
+    const head = data.subarray(0, Math.min(peekBytes, data.length)).toString("utf-8");
+    // Claude exports always carry chat_messages arrays on every conversation.
+    if (head.includes('"chat_messages"'))
+        return "claude_web";
+    // Gemini activity cards carry timestamp_text or assistant_turns.
+    if (head.includes('"timestamp_text"') || head.includes('"assistant_turns"')) {
+        return "gemini_web";
+    }
+    // Grok exports nest responses[] inside each conversation. The
+    // combined presence of both top-level keys is what distinguishes
+    // them from the (rare) ChatGPT JSON that happens to mention
+    // "conversations" or "responses" in isolation.
+    if (head.includes('"conversations"') && head.includes('"responses"')) {
+        return "grok_web";
+    }
+    return null;
+}
+//# sourceMappingURL=source-sniffer.js.map

package/dist/src/ingestion/inbox-core/source-sniffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-sniffer.js","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/source-sniffer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIlD,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AAErC;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAAgB,EAChB,IAAa,EACb,YAAoB,kBAAkB;IAEtC,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAErC,yCAAyC;IACzC,IAAI,UAAU,CAAC,UAAU,CAAC,mBAAmB,CAAC;QAAE,OAAO,UAAU,CAAC;IAClE,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,YAAY,CAAC;IAExD,wBAAwB;IACxB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE5C,kEAAkE;IAClE,kEAAkE;IAClE,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAElF,0EAA0E;IAC1E,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAAE,OAAO,YAAY,CAAC;IAE1D,iEAAiE;IACjE,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC5E,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,8DAA8D;IAC9D,iEAAiE;IACjE,4DAA4D;IAC5D,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACrE,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/ingestion/inbox-core/zip-sniffer.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Minimal ZIP central-directory reader for inbox source-sniffing.
+ *
+ * We do NOT extract or decompress. We only walk the central directory
+ * to learn the names of the entries inside the archive, so we can
+ * decide which provider an inbox-dropped ZIP belongs to.
+ *
+ * Why hand-rolled (no adm-zip): inbox-core must stay free of Rift
+ * internals AND of npm dependencies — only `node:` stdlib. The
+ * adapter (inbox-watcher) and a future standalone shell both compose
+ * this against their own filesystem.
+ *
+ * Scope: the regular (non-ZIP64) End-Of-Central-Directory record is
+ * enough for personal-use chat exports (Claude / ChatGPT / Gemini /
+ * Grok exports are all well under 4 GB and have far fewer than 65535
+ * entries). Archives that look like ZIP64, are truncated, or have a
+ * corrupt EOCD return null; the caller falls back to filename hints
+ * or the chatgpt_web default.
+ *
+ * ZIP layout we rely on:
+ *   - End of Central Directory (EOCD): 22 bytes + optional comment,
+ *     near the end of the file, signature 0x06054b50.
+ *   - Central Directory file headers: signature 0x02014b50, 46 bytes
+ *     fixed + variable filename / extra / comment fields.
+ */
+import type { SniffedProvider } from "./source-sniffer.js";
+/**
+ * Read entry filenames from a ZIP's central directory.
+ *
+ * Returns null when:
+ *   - buf is too small to contain a ZIP,
+ *   - the EOCD signature is not found in the trailing 64 KiB,
+ *   - the central directory pointers are out of range (truncated /
+ *     corrupted archive),
+ *   - the archive declares ZIP64 marker values (we don't follow the
+ *     ZIP64 locator — out of scope for inbox sniffing),
+ *   - any individual CD header signature mismatches.
+ *
+ * The list is in central-directory order, which is the order zip
+ * tools usually preserve. Filenames are decoded as UTF-8; ZIP entries
+ * default to CP437 if bit 11 of the general purpose flag is not set,
+ * but modern tools (Claude, Google Takeout, ChatGPT) all write UTF-8
+ * filenames for the marker files we look at, so we don't branch.
+ */
+export declare function readZipCentralDirectoryFilenames(buf: Buffer, options?: {
+    maxEntries?: number;
+}): string[] | null;
+/**
+ * Identify a non-ChatGPT provider from the entry list of a ZIP's
+ * central directory. ChatGPT-shaped zips intentionally return null so
+ * the adapter's default (`chatgpt_web`) handles them.
+ *
+ * Markers:
+ *   - Any entry path starts with `Takeout/` → gemini_web.
+ *     Google Takeout ZIPs use this exact top-level prefix.
+ *   - A root-level `projects.json` together with `conversations.json`
+ *     and NO `chat.html` / `shared_conversations.json` → claude_web.
+ *     ChatGPT exports contain conversations.json too but ship chat.html
+ *     and shared_conversations.json at root and have never shipped a
+ *     projects.json; Claude exports always ship projects.json at root.
+ *   - Grok ZIPs are caught upstream by filename hint
+ *     (`prod-grok-backend*.zip`), so we don't look for them here.
+ */
+export declare function sniffProviderFromZipEntries(entries: ReadonlyArray<string>): SniffedProvider | null;
+/**
+ * Convenience wrapper: read the CD and classify in one call. Returns
+ * null on a non-ZIP buffer, a corrupted CD, or an unclassified shape.
+ */
+export declare function sniffZipBuffer(buf: Buffer): SniffedProvider | null;
+//# sourceMappingURL=zip-sniffer.d.ts.map

package/dist/src/ingestion/inbox-core/zip-sniffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-sniffer.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/zip-sniffer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAiB3D;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,gCAAgC,CAC9C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAChC,MAAM,EAAE,GAAG,IAAI,CAmEjB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,GAC7B,eAAe,GAAG,IAAI,CAuCxB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAIlE"}

package/dist/src/ingestion/inbox-core/zip-sniffer.js

@@ -0,0 +1,161 @@
+const EOCD_SIGNATURE = 0x06054b50;
+const CD_HEADER_SIGNATURE = 0x02014b50;
+const EOCD_MIN_SIZE = 22;
+const EOCD_MAX_COMMENT = 0xffff;
+const EOCD_MAX_SCAN = EOCD_MIN_SIZE + EOCD_MAX_COMMENT;
+const CD_HEADER_FIXED_SIZE = 46;
+/**
+ * Cap how much of the central directory we read. Real chat exports
+ * are well under this; the cap guards against pathological archives
+ * (gzip-bomb-shaped CDs, untrusted inbox drops) without ever needing
+ * to extract data.
+ */
+const DEFAULT_MAX_ENTRIES = 4096;
+/**
+ * Read entry filenames from a ZIP's central directory.
+ *
+ * Returns null when:
+ *   - buf is too small to contain a ZIP,
+ *   - the EOCD signature is not found in the trailing 64 KiB,
+ *   - the central directory pointers are out of range (truncated /
+ *     corrupted archive),
+ *   - the archive declares ZIP64 marker values (we don't follow the
+ *     ZIP64 locator — out of scope for inbox sniffing),
+ *   - any individual CD header signature mismatches.
+ *
+ * The list is in central-directory order, which is the order zip
+ * tools usually preserve. Filenames are decoded as UTF-8; ZIP entries
+ * default to CP437 if bit 11 of the general purpose flag is not set,
+ * but modern tools (Claude, Google Takeout, ChatGPT) all write UTF-8
+ * filenames for the marker files we look at, so we don't branch.
+ */
+export function readZipCentralDirectoryFilenames(buf, options) {
+    if (!Buffer.isBuffer(buf) || buf.length < EOCD_MIN_SIZE)
+        return null;
+    const maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
+    // --- Locate EOCD by scanning backwards from the end for the
+    //     signature. The EOCD lives within the last 22 + 65535 bytes.
+    const scanStart = Math.max(0, buf.length - EOCD_MAX_SCAN);
+    let eocdOffset = -1;
+    for (let i = buf.length - EOCD_MIN_SIZE; i >= scanStart; i--) {
+        if (buf.readUInt32LE(i) === EOCD_SIGNATURE) {
+            eocdOffset = i;
+            break;
+        }
+    }
+    if (eocdOffset < 0)
+        return null;
+    // EOCD fields (little-endian):
+    //   off 4:  disk number (uint16)
+    //   off 6:  disk where CD starts (uint16)
+    //   off 8:  CD entries on this disk (uint16)
+    //   off 10: total CD entries (uint16)
+    //   off 12: CD size in bytes (uint32)
+    //   off 16: CD offset from archive start (uint32)
+    //   off 20: comment length (uint16)
+    const totalEntries = buf.readUInt16LE(eocdOffset + 10);
+    const cdSize = buf.readUInt32LE(eocdOffset + 12);
+    const cdOffset = buf.readUInt32LE(eocdOffset + 16);
+    // ZIP64 sentinel values — bail out rather than misread.
+    if (totalEntries === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) {
+        return null;
+    }
+    if (cdOffset + cdSize > buf.length)
+        return null;
+    if (cdOffset >= buf.length)
+        return null;
+    if (totalEntries === 0)
+        return [];
+    // --- Walk CD entries.
+    const names = [];
+    let cursor = cdOffset;
+    const cdEnd = cdOffset + cdSize;
+    const cap = Math.min(totalEntries, maxEntries);
+    for (let i = 0; i < cap; i++) {
+        if (cursor + CD_HEADER_FIXED_SIZE > cdEnd)
+            return null;
+        if (buf.readUInt32LE(cursor) !== CD_HEADER_SIGNATURE)
+            return null;
+        // CD header field offsets relative to cursor:
+        //   off 28: filename length (uint16)
+        //   off 30: extra field length (uint16)
+        //   off 32: file comment length (uint16)
+        const filenameLen = buf.readUInt16LE(cursor + 28);
+        const extraLen = buf.readUInt16LE(cursor + 30);
+        const commentLen = buf.readUInt16LE(cursor + 32);
+        const filenameStart = cursor + CD_HEADER_FIXED_SIZE;
+        const filenameEnd = filenameStart + filenameLen;
+        if (filenameEnd > cdEnd)
+            return null;
+        const filename = buf.toString("utf-8", filenameStart, filenameEnd);
+        names.push(filename);
+        cursor = filenameEnd + extraLen + commentLen;
+    }
+    return names;
+}
+/**
+ * Identify a non-ChatGPT provider from the entry list of a ZIP's
+ * central directory. ChatGPT-shaped zips intentionally return null so
+ * the adapter's default (`chatgpt_web`) handles them.
+ *
+ * Markers:
+ *   - Any entry path starts with `Takeout/` → gemini_web.
+ *     Google Takeout ZIPs use this exact top-level prefix.
+ *   - A root-level `projects.json` together with `conversations.json`
+ *     and NO `chat.html` / `shared_conversations.json` → claude_web.
+ *     ChatGPT exports contain conversations.json too but ship chat.html
+ *     and shared_conversations.json at root and have never shipped a
+ *     projects.json; Claude exports always ship projects.json at root.
+ *   - Grok ZIPs are caught upstream by filename hint
+ *     (`prod-grok-backend*.zip`), so we don't look for them here.
+ */
+export function sniffProviderFromZipEntries(entries) {
+    if (entries.length === 0)
+        return null;
+    let hasTakeoutPrefix = false;
+    let hasRootConversations = false;
+    let hasRootProjects = false;
+    let hasChatHtml = false;
+    let hasSharedConversations = false;
+    for (const raw of entries) {
+        const path = raw.replace(/^\.\//, "");
+        if (path.startsWith("Takeout/")) {
+            hasTakeoutPrefix = true;
+            // Keep scanning — a Takeout ZIP can legitimately co-ship root
+            // files, but the prefix is enough to commit to gemini_web.
+            // Short-circuit safe: nothing else will override.
+            break;
+        }
+        if (!path.includes("/")) {
+            const lower = path.toLowerCase();
+            if (lower === "conversations.json")
+                hasRootConversations = true;
+            else if (lower === "projects.json")
+                hasRootProjects = true;
+            else if (lower === "chat.html")
+                hasChatHtml = true;
+            else if (lower === "shared_conversations.json")
+                hasSharedConversations = true;
+        }
+    }
+    if (hasTakeoutPrefix)
+        return "gemini_web";
+    if (hasRootProjects &&
+        hasRootConversations &&
+        !hasChatHtml &&
+        !hasSharedConversations) {
+        return "claude_web";
+    }
+    return null;
+}
+/**
+ * Convenience wrapper: read the CD and classify in one call. Returns
+ * null on a non-ZIP buffer, a corrupted CD, or an unclassified shape.
+ */
+export function sniffZipBuffer(buf) {
+    const entries = readZipCentralDirectoryFilenames(buf);
+    if (!entries)
+        return null;
+    return sniffProviderFromZipEntries(entries);
+}
+//# sourceMappingURL=zip-sniffer.js.map

package/dist/src/ingestion/inbox-core/zip-sniffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-sniffer.js","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/zip-sniffer.ts"],"names":[],"mappings":"AA2BA,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,mBAAmB,GAAG,UAAU,CAAC;AACvC,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,aAAa,GAAG,aAAa,GAAG,gBAAgB,CAAC;AACvD,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,gCAAgC,CAC9C,GAAW,EACX,OAAiC;IAEjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,aAAa;QAAE,OAAO,IAAI,CAAC;IAErE,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,mBAAmB,CAAC;IAE9D,6DAA6D;IAC7D,kEAAkE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;IAC1D,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7D,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,cAAc,EAAE,CAAC;YAC3C,UAAU,GAAG,CAAC,CAAC;YACf,MAAM;QACR,CAAC;IACH,CAAC;IACD,IAAI,UAAU,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAEhC,+BAA+B;IAC/B,iCAAiC;IACjC,0CAA0C;IAC1C,6CAA6C;IAC7C,sCAAsC;IACtC,sCAAsC;IACtC,kDAAkD;IAClD,oCAAoC;IACpC,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAEnD,wDAAwD;IACxD,IAAI,YAAY,KAAK,MAAM,IAAI,MAAM,KAAK,UAAU,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAChF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,GAAG,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,QAAQ,IAAI,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,YAAY,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,uBAAuB;IACvB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,MAAM,GAAG,oBAAoB,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC;QACvD,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,mBAAmB;YAAE,OAAO,IAAI,CAAC;QAElE,8CAA8C;QAC9C,qCAAqC;QACrC,wCAAwC;QACxC,yCAAyC;QACzC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,MAAM,GAAG,oBAAoB,CAAC;QACpD,MAAM,WAAW,GAAG,aAAa,GAAG,WAAW,CAAC;QAChD,IAAI,WAAW,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC;QAErC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QACnE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErB,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,UAAU,CAAC;IAC/C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,2BAA2B,CACzC,OAA8B;IAE9B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,oBAAoB,GAAG,KAAK,CAAC;IACjC,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,sBAAsB,GAAG,KAAK,CAAC;IAEnC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAChC,gBAAgB,GAAG,IAAI,CAAC;YACxB,8DAA8D;YAC9D,2DAA2D;YAC3D,kDAAkD;YAClD,MAAM;QACR,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,IAAI,KAAK,KAAK,oBAAoB;gBAAE,oBAAoB,GAAG,IAAI,CAAC;iBAC3D,IAAI,KAAK,KAAK,eAAe;gBAAE,eAAe,GAAG,IAAI,CAAC;iBACtD,IAAI,KAAK,KAAK,WAAW;gBAAE,WAAW,GAAG,IAAI,CAAC;iBAC9C,IAAI,KAAK,KAAK,2BAA2B;gBAAE,sBAAsB,GAAG,IAAI,CAAC;QAChF,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB;QAAE,OAAO,YAAY,CAAC;IAE1C,IACE,eAAe;QACf,oBAAoB;QACpB,CAAC,WAAW;QACZ,CAAC,sBAAsB,EACvB,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,OAAO,GAAG,gCAAgC,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,OAAO,2BAA2B,CAAC,OAAO,CAAC,CAAC;AAC9C,CAAC"}

package/dist/src/ingestion/inbox-watcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"inbox-watcher.d.ts","sourceRoot":"","sources":["../../../src/ingestion/inbox-watcher.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAWvC,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;CACjB;AAyCD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAsCrE"}
+{"version":3,"file":"inbox-watcher.d.ts","sourceRoot":"","sources":["../../../src/ingestion/inbox-watcher.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAcvC,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CA2DrE"}

package/dist/src/ingestion/inbox-watcher.js

@@ -1,5 +1,5 @@
 /**
- * Inbox watcher — Slice 19.
+ * Inbox watcher — Slice 19, refactored to compose `inbox-core`.
  *
  * Watches data/inbox/ for dropped files and creates ingest jobs.
  * Reuses the existing secure ingest pipeline (archive security,
@@ -10,53 +10,18 @@
  *   or defaults to "chatgpt_web" for files in inbox root.
  * - Raw files land in data/raw/conversations/inbox/ via rawSource override.
  * - Idempotency key prevents duplicate processing on restart.
+ *
+ * This file is the Rift adapter — it owns `fs`, the JobQueue handoff,
+ * and Rift's parser registry. The pure detection primitives live in
+ * `inbox-core/` so a future standalone shell can reuse them.
  */
-import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 import { Watcher } from "./watcher.js";
 import { SUPPORTED_INGEST_SOURCES } from "./parsers/types.js";
-// Web conversation exports are JSON or archives of JSON. No parser for
-// raw markdown exists — markdown dropped here would fail at the parser
-// stage, so reject it up front.
-const SUPPORTED_EXTENSIONS = new Set([".json", ".zip"]);
+import { detectSourceFromSubdir, inboxIdempotencyKey, isSupportedInboxExtension, sniffInboxSource, } from "./inbox-core/index.js";
+import { writeSkipQuarantine } from "./skip-quarantine.js";
 const DEFAULT_SOURCE = "chatgpt_web";
-/**
- * Detect the parser source from a file path relative to the inbox directory.
- * Files in subdirectories use the subdirectory name as source
- * (e.g., data/inbox/chatgpt_web/export.zip → chatgpt_web).
- * Files directly in inbox/ default to chatgpt_web.
- */
-function detectSource(filePath, inboxDir) {
-    const relative = path.relative(inboxDir, filePath);
-    const firstSegment = relative.split(path.sep)[0];
-    if (firstSegment &&
-        firstSegment !== path.basename(filePath) &&
-        SUPPORTED_INGEST_SOURCES.includes(firstSegment)) {
-        return firstSegment;
-    }
-    return DEFAULT_SOURCE;
-}
-/**
- * Generate a stable idempotency key for an inbox file.
- * Includes path + mtime so re-dropping the same filename with new
- * content gets a fresh key, while restart doesn't re-process.
- */
-function inboxIdempotencyKey(filePath) {
-    let mtimeMs = 0;
-    try {
-        mtimeMs = fs.statSync(filePath).mtimeMs;
-    }
-    catch {
-        // File may have been moved; use 0
-    }
-    const hash = crypto
-        .createHash("sha256")
-        .update(`${filePath}:${mtimeMs}`)
-        .digest("hex")
-        .slice(0, 16);
-    return `inbox:${hash}`;
-}
 /**
  * Create an inbox watcher. Returns a Watcher instance that can be
  * started, paused, and stopped like any other watcher.
@@ -65,24 +30,43 @@ export function createInboxWatcher(opts) {
     const inboxDir = path.join(opts.dataDir, "inbox");
     fs.mkdirSync(inboxDir, { recursive: true });
     const handler = async (event) => {
-        // Only process new files (not changes or deletions).
-        if (event.type !== "add")
+        // Ignore deletions; otherwise process both `add` and `change` so a
+        // user who overwrites `export.zip` in place picks up the new
+        // content. The idempotency key includes mtime, so an unchanged
+        // `change` event collapses at the queue layer.
+        if (event.type === "unlink")
             return;
-        // Check file extension.
-        const ext = path.extname(event.path).toLowerCase();
-        if (!SUPPORTED_EXTENSIONS.has(ext))
+        if (!isSupportedInboxExtension(event.path)) {
+            // Markdown / images / other types are not parseable as web
+            // exports. Record a friendly skip so the user can see why their
+            // drop didn't ingest, instead of it disappearing silently.
+            await writeSkipQuarantine(opts.dataDir, {
+                reason: "inbox_unsupported_extension",
+                source_path: event.path,
+                metadata: { extension: path.extname(event.path).toLowerCase() },
+            }).catch(() => {
+                // Quarantine write failure must not break the watcher loop.
+            });
             return;
-        // Read file contents.
+        }
         let fileData;
+        let mtimeMs;
         try {
             fileData = fs.readFileSync(event.path);
+            mtimeMs = fs.statSync(event.path).mtimeMs;
         }
         catch {
             // File may have been removed between detection and read.
             return;
         }
-        const source = detectSource(event.path, inboxDir);
-        const idempotencyKey = inboxIdempotencyKey(event.path);
+        // Source resolution order:
+        //   1. Subdirectory hint (data/inbox/<provider>/file.json) — strongest.
+        //   2. Content/filename sniff — identifies non-ChatGPT providers.
+        //   3. DEFAULT_SOURCE — historical default for unidentified drops.
+        const source = detectSourceFromSubdir(event.path, inboxDir, SUPPORTED_INGEST_SOURCES) ??
+            sniffInboxSource(path.basename(event.path), fileData) ??
+            DEFAULT_SOURCE;
+        const idempotencyKey = inboxIdempotencyKey(event.path, mtimeMs);
         const payload = {
             source: source,
             filename: path.basename(event.path),

package/dist/src/ingestion/inbox-watcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"inbox-watcher.js","sourceRoot":"","sources":["../../../src/ingestion/inbox-watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,uEAAuE;AACvE,uEAAuE;AACvE,gCAAgC;AAChC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;AACxD,MAAM,cAAc,GAAG,aAAa,CAAC;AAOrC;;;;;GAKG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,QAAgB;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,IACE,YAAY;QACZ,YAAY,KAAK,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACvC,wBAA8C,CAAC,QAAQ,CAAC,YAAY,CAAC,EACtE,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;IACD,MAAM,IAAI,GAAG,MAAM;SAChB,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC;SAChC,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChB,OAAO,SAAS,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAyB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,KAAK,EAAE,KAAgB,EAAiB,EAAE;QACxD,qDAAqD;QACrD,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;YAAE,OAAO;QAEjC,wBAAwB;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACnD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAE3C,sBAAsB;QACtB,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;YACzD,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvD,MAAM,OAAO,GAAkB;YAC7B,MAAM,EAAE,MAAiC;YACzC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;YACnC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACrC,SAAS,EAAE,OAAO;SACnB,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE;YAChC,eAAe,EAAE,cAAc;YAC/B,OAAO;SACR,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"inbox-watcher.js","sourceRoot":"","sources":["../../../src/ingestion/inbox-watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EACL,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,cAAc,GAAG,aAAa,CAAC;AAOrC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAyB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,KAAK,EAAE,KAAgB,EAAiB,EAAE;QACxD,mEAAmE;QACnE,6DAA6D;QAC7D,+DAA+D;QAC/D,+CAA+C;QAC/C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO;QAEpC,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,2DAA2D;YAC3D,gEAAgE;YAChE,2DAA2D;YAC3D,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE;gBACtC,MAAM,EAAE,6BAA6B;gBACrC,WAAW,EAAE,KAAK,CAAC,IAAI;gBACvB,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;aAChE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;gBACZ,4DAA4D;YAC9D,CAAC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,QAAgB,CAAC;QACrB,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,OAAO,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;YACzD,OAAO;QACT,CAAC;QAED,2BAA2B;QAC3B,wEAAwE;QACxE,kEAAkE;QAClE,mEAAmE;QACnE,MAAM,MAAM,GACV,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,wBAAwB,CAAC;YACtE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC;YACrD,cAAc,CAAC;QACjB,MAAM,cAAc,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEhE,MAAM,OAAO,GAAkB;YAC7B,MAAM,EAAE,MAAiC;YACzC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;YACnC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACrC,SAAS,EAAE,OAAO;SACnB,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE;YAChC,eAAe,EAAE,cAAc;YAC/B,OAAO;SACR,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC"}

package/dist/src/ingestion/indexer.d.ts

@@ -10,6 +10,13 @@ export interface IndexerConfig {
     allowedRoots: readonly string[];
     /** Target table for document storage. Defaults to "structured_docs". */
     tableName?: "structured_docs" | "structured_docs_local";
+    /**
+     * Data directory root. Used to write skip-quarantine records when a file
+     * cannot be embedded (e.g. empty extracted content). Required so every
+     * production code path — watch, scheduled scan, reconcile — produces a
+     * visible explanation when a file is dropped, instead of silent stderr.
+     */
+    dataDir: string;
 }
 /**
  * Deterministic row ID from the canonical source path.

package/dist/src/ingestion/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../../src/ingestion/indexer.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMxF,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,wEAAwE;IACxE,SAAS,CAAC,EAAE,iBAAiB,GAAG,uBAAuB,CAAC;CACzD;AAED;;;GAGG;AACH,wBAAgB,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,aAAa;IAK/D;;;;OAIG;IACG,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;YAcpC,YAAY;YAwBZ,YAAY;CAK3B"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../../src/ingestion/indexer.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AASxF,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,wEAAwE;IACxE,SAAS,CAAC,EAAE,iBAAiB,GAAG,uBAAuB,CAAC;IACxD;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,aAAa;IAK/D;;;;OAIG;IACG,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;YAcpC,YAAY;YAiEZ,YAAY;CAK3B"}

package/dist/src/ingestion/indexer.js

@@ -9,6 +9,9 @@ import crypto from "node:crypto";
 import { getTable } from "../storage/tables.js";
 import { extract, isSupported } from "./extractor.js";
 import { validatePath, validateUnlinkPath } from "../security/paths.js";
+import { writeSkipQuarantine } from "./skip-quarantine.js";
+import { recordEmbed } from "../observability/embedding-events.js";
+import { recordIndexWrite } from "../observability/index-events.js";
 /**
  * Deterministic row ID from the canonical source path.
  * Same file always gets the same ID, enabling upsert via delete+add.
@@ -41,7 +44,28 @@ export class Indexer {
     }
     async handleUpsert(filePath) {
         const doc = await extract(filePath);
-        const embeddingVec = await this.embedding.embed(doc.content);
+        // Empty/blank extracted content cannot be embedded — Voyage rejects empty
+        // strings with HTTP 400, and indexing an empty row produces nothing
+        // searchable anyway. Quarantine the skip so it's visible (not stderr-only)
+        // and remove any stale row from a prior good extraction of the same path.
+        if (isBlank(doc.content)) {
+            const id = fileId(filePath);
+            const table = getTable(this.config.tableName ?? "structured_docs");
+            await table.delete(`id = '${id}'`);
+            await writeSkipQuarantine(this.config.dataDir, {
+                reason: "empty_extracted_content",
+                source_path: filePath,
+                metadata: doc.metadata,
+            });
+            return;
+        }
+        const embeddingVec = await recordEmbed(this.config.dataDir, this.embedding, {
+            pipeline: this.config.sourceType === "filesystem_watched"
+                ? "watcher"
+                : "scheduled_scan",
+            operation: "document_embedding",
+            input_count: 1,
+        }, () => this.embedding.embed(doc.content));
         const id = fileId(filePath);
         const table = getTable(this.config.tableName ?? "structured_docs");
         // Upsert: delete existing row (if any), then add new one.
@@ -57,7 +81,14 @@ export class Indexer {
             indexed_at: new Date().toISOString(),
             metadata: JSON.stringify(doc.metadata),
         };
-        await table.add([row]);
+        await recordIndexWrite(this.config.dataDir, {
+            table: this.config.tableName ?? "structured_docs",
+            pipeline: this.config.sourceType === "filesystem_watched"
+                ? "watcher"
+                : "scheduled_scan",
+            operation: "structured_doc_upsert",
+            row_count: 1,
+        }, () => table.add([row]));
     }
     async handleDelete(filePath) {
         const id = fileId(filePath);
@@ -65,4 +96,7 @@ export class Indexer {
         await table.delete(`id = '${id}'`);
     }
 }
+function isBlank(s) {
+    return s.trim().length === 0;
+}
 //# sourceMappingURL=indexer.js.map