onadit 0.0.1 → 1.0.0

package/dist/src/ingestion/inbox-core/zip-sniffer.js

@@ -0,0 +1,161 @@
+const EOCD_SIGNATURE = 0x06054b50;
+const CD_HEADER_SIGNATURE = 0x02014b50;
+const EOCD_MIN_SIZE = 22;
+const EOCD_MAX_COMMENT = 0xffff;
+const EOCD_MAX_SCAN = EOCD_MIN_SIZE + EOCD_MAX_COMMENT;
+const CD_HEADER_FIXED_SIZE = 46;
+/**
+ * Cap how much of the central directory we read. Real chat exports
+ * are well under this; the cap guards against pathological archives
+ * (gzip-bomb-shaped CDs, untrusted inbox drops) without ever needing
+ * to extract data.
+ */
+const DEFAULT_MAX_ENTRIES = 4096;
+/**
+ * Read entry filenames from a ZIP's central directory.
+ *
+ * Returns null when:
+ *   - buf is too small to contain a ZIP,
+ *   - the EOCD signature is not found in the trailing 64 KiB,
+ *   - the central directory pointers are out of range (truncated /
+ *     corrupted archive),
+ *   - the archive declares ZIP64 marker values (we don't follow the
+ *     ZIP64 locator — out of scope for inbox sniffing),
+ *   - any individual CD header signature mismatches.
+ *
+ * The list is in central-directory order, which is the order zip
+ * tools usually preserve. Filenames are decoded as UTF-8; ZIP entries
+ * default to CP437 if bit 11 of the general purpose flag is not set,
+ * but modern tools (Claude, Google Takeout, ChatGPT) all write UTF-8
+ * filenames for the marker files we look at, so we don't branch.
+ */
+export function readZipCentralDirectoryFilenames(buf, options) {
+    if (!Buffer.isBuffer(buf) || buf.length < EOCD_MIN_SIZE)
+        return null;
+    const maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
+    // --- Locate EOCD by scanning backwards from the end for the
+    //     signature. The EOCD lives within the last 22 + 65535 bytes.
+    const scanStart = Math.max(0, buf.length - EOCD_MAX_SCAN);
+    let eocdOffset = -1;
+    for (let i = buf.length - EOCD_MIN_SIZE; i >= scanStart; i--) {
+        if (buf.readUInt32LE(i) === EOCD_SIGNATURE) {
+            eocdOffset = i;
+            break;
+        }
+    }
+    if (eocdOffset < 0)
+        return null;
+    // EOCD fields (little-endian):
+    //   off 4:  disk number (uint16)
+    //   off 6:  disk where CD starts (uint16)
+    //   off 8:  CD entries on this disk (uint16)
+    //   off 10: total CD entries (uint16)
+    //   off 12: CD size in bytes (uint32)
+    //   off 16: CD offset from archive start (uint32)
+    //   off 20: comment length (uint16)
+    const totalEntries = buf.readUInt16LE(eocdOffset + 10);
+    const cdSize = buf.readUInt32LE(eocdOffset + 12);
+    const cdOffset = buf.readUInt32LE(eocdOffset + 16);
+    // ZIP64 sentinel values — bail out rather than misread.
+    if (totalEntries === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) {
+        return null;
+    }
+    if (cdOffset + cdSize > buf.length)
+        return null;
+    if (cdOffset >= buf.length)
+        return null;
+    if (totalEntries === 0)
+        return [];
+    // --- Walk CD entries.
+    const names = [];
+    let cursor = cdOffset;
+    const cdEnd = cdOffset + cdSize;
+    const cap = Math.min(totalEntries, maxEntries);
+    for (let i = 0; i < cap; i++) {
+        if (cursor + CD_HEADER_FIXED_SIZE > cdEnd)
+            return null;
+        if (buf.readUInt32LE(cursor) !== CD_HEADER_SIGNATURE)
+            return null;
+        // CD header field offsets relative to cursor:
+        //   off 28: filename length (uint16)
+        //   off 30: extra field length (uint16)
+        //   off 32: file comment length (uint16)
+        const filenameLen = buf.readUInt16LE(cursor + 28);
+        const extraLen = buf.readUInt16LE(cursor + 30);
+        const commentLen = buf.readUInt16LE(cursor + 32);
+        const filenameStart = cursor + CD_HEADER_FIXED_SIZE;
+        const filenameEnd = filenameStart + filenameLen;
+        if (filenameEnd > cdEnd)
+            return null;
+        const filename = buf.toString("utf-8", filenameStart, filenameEnd);
+        names.push(filename);
+        cursor = filenameEnd + extraLen + commentLen;
+    }
+    return names;
+}
+/**
+ * Identify a non-ChatGPT provider from the entry list of a ZIP's
+ * central directory. ChatGPT-shaped zips intentionally return null so
+ * the adapter's default (`chatgpt_web`) handles them.
+ *
+ * Markers:
+ *   - Any entry path starts with `Takeout/` → gemini_web.
+ *     Google Takeout ZIPs use this exact top-level prefix.
+ *   - A root-level `projects.json` together with `conversations.json`
+ *     and NO `chat.html` / `shared_conversations.json` → claude_web.
+ *     ChatGPT exports contain conversations.json too but ship chat.html
+ *     and shared_conversations.json at root and have never shipped a
+ *     projects.json; Claude exports always ship projects.json at root.
+ *   - Grok ZIPs are caught upstream by filename hint
+ *     (`prod-grok-backend*.zip`), so we don't look for them here.
+ */
+export function sniffProviderFromZipEntries(entries) {
+    if (entries.length === 0)
+        return null;
+    let hasTakeoutPrefix = false;
+    let hasRootConversations = false;
+    let hasRootProjects = false;
+    let hasChatHtml = false;
+    let hasSharedConversations = false;
+    for (const raw of entries) {
+        const path = raw.replace(/^\.\//, "");
+        if (path.startsWith("Takeout/")) {
+            hasTakeoutPrefix = true;
+            // Keep scanning — a Takeout ZIP can legitimately co-ship root
+            // files, but the prefix is enough to commit to gemini_web.
+            // Short-circuit safe: nothing else will override.
+            break;
+        }
+        if (!path.includes("/")) {
+            const lower = path.toLowerCase();
+            if (lower === "conversations.json")
+                hasRootConversations = true;
+            else if (lower === "projects.json")
+                hasRootProjects = true;
+            else if (lower === "chat.html")
+                hasChatHtml = true;
+            else if (lower === "shared_conversations.json")
+                hasSharedConversations = true;
+        }
+    }
+    if (hasTakeoutPrefix)
+        return "gemini_web";
+    if (hasRootProjects &&
+        hasRootConversations &&
+        !hasChatHtml &&
+        !hasSharedConversations) {
+        return "claude_web";
+    }
+    return null;
+}
+/**
+ * Convenience wrapper: read the CD and classify in one call. Returns
+ * null on a non-ZIP buffer, a corrupted CD, or an unclassified shape.
+ */
+export function sniffZipBuffer(buf) {
+    const entries = readZipCentralDirectoryFilenames(buf);
+    if (!entries)
+        return null;
+    return sniffProviderFromZipEntries(entries);
+}
+//# sourceMappingURL=zip-sniffer.js.map

package/dist/src/ingestion/inbox-core/zip-sniffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-sniffer.js","sourceRoot":"","sources":["../../../../src/ingestion/inbox-core/zip-sniffer.ts"],"names":[],"mappings":"AA2BA,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,mBAAmB,GAAG,UAAU,CAAC;AACvC,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,aAAa,GAAG,aAAa,GAAG,gBAAgB,CAAC;AACvD,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,gCAAgC,CAC9C,GAAW,EACX,OAAiC;IAEjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,aAAa;QAAE,OAAO,IAAI,CAAC;IAErE,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,mBAAmB,CAAC;IAE9D,6DAA6D;IAC7D,kEAAkE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;IAC1D,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7D,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,cAAc,EAAE,CAAC;YAC3C,UAAU,GAAG,CAAC,CAAC;YACf,MAAM;QACR,CAAC;IACH,CAAC;IACD,IAAI,UAAU,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAEhC,+BAA+B;IAC/B,iCAAiC;IACjC,0CAA0C;IAC1C,6CAA6C;IAC7C,sCAAsC;IACtC,sCAAsC;IACtC,kDAAkD;IAClD,oCAAoC;IACpC,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAEnD,wDAAwD;IACxD,IAAI,YAAY,KAAK,MAAM,IAAI,MAAM,KAAK,UAAU,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAChF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,GAAG,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,QAAQ,IAAI,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,YAAY,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,uBAAuB;IACvB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,MAAM,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,MAAM,GAAG,oBAAoB,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC;QACvD,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,mBAAmB;YAAE,OAAO,IAAI,CAAC;QAElE,8CAA8C;QAC9C,qCAAqC;QACrC,wCAAwC;QACxC,yCAAyC;QACzC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,MAAM,GAAG,oBAAoB,CAAC;QACpD,MAAM,WAAW,GAAG,aAAa,GAAG,WAAW,CAAC;QAChD,IAAI,WAAW,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC;QAErC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QACnE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErB,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,UAAU,CAAC;IAC/C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,2BAA2B,CACzC,OAA8B;IAE9B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,oBAAoB,GAAG,KAAK,CAAC;IACjC,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,sBAAsB,GAAG,KAAK,CAAC;IAEnC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAChC,gBAAgB,GAAG,IAAI,CAAC;YACxB,8DAA8D;YAC9D,2DAA2D;YAC3D,kDAAkD;YAClD,MAAM;QACR,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,IAAI,KAAK,KAAK,oBAAoB;gBAAE,oBAAoB,GAAG,IAAI,CAAC;iBAC3D,IAAI,KAAK,KAAK,eAAe;gBAAE,eAAe,GAAG,IAAI,CAAC;iBACtD,IAAI,KAAK,KAAK,WAAW;gBAAE,WAAW,GAAG,IAAI,CAAC;iBAC9C,IAAI,KAAK,KAAK,2BAA2B;gBAAE,sBAAsB,GAAG,IAAI,CAAC;QAChF,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB;QAAE,OAAO,YAAY,CAAC;IAE1C,IACE,eAAe;QACf,oBAAoB;QACpB,CAAC,WAAW;QACZ,CAAC,sBAAsB,EACvB,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,OAAO,GAAG,gCAAgC,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,OAAO,2BAA2B,CAAC,OAAO,CAAC,CAAC;AAC9C,CAAC"}

package/dist/src/ingestion/inbox-watcher.d.ts

@@ -0,0 +1,12 @@
+import type { JobQueue } from "../jobs/queue.js";
+import { Watcher } from "./watcher.js";
+export interface InboxWatcherOptions {
+    dataDir: string;
+    queue: JobQueue;
+}
+/**
+ * Create an inbox watcher. Returns a Watcher instance that can be
+ * started, paused, and stopped like any other watcher.
+ */
+export declare function createInboxWatcher(opts: InboxWatcherOptions): Watcher;
+//# sourceMappingURL=inbox-watcher.d.ts.map

package/dist/src/ingestion/inbox-watcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inbox-watcher.d.ts","sourceRoot":"","sources":["../../../src/ingestion/inbox-watcher.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAcvC,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,QAAQ,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CA2DrE"}

package/dist/src/ingestion/inbox-watcher.js

@@ -0,0 +1,83 @@
+/**
+ * Inbox watcher — Slice 19, refactored to compose `inbox-core`.
+ *
+ * Watches data/inbox/ for dropped files and creates ingest jobs.
+ * Reuses the existing secure ingest pipeline (archive security,
+ * parser validation, extraction, quarantine). Does NOT implement
+ * a second parser path.
+ *
+ * - Source detection: subdirectory name (e.g., data/inbox/chatgpt_web/file.zip)
+ *   or defaults to "chatgpt_web" for files in inbox root.
+ * - Raw files land in data/raw/conversations/inbox/ via rawSource override.
+ * - Idempotency key prevents duplicate processing on restart.
+ *
+ * This file is the Onadit adapter — it owns `fs`, the JobQueue handoff,
+ * and Onadit's parser registry. The pure detection primitives live in
+ * `inbox-core/` so a future standalone shell can reuse them.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { Watcher } from "./watcher.js";
+import { SUPPORTED_INGEST_SOURCES } from "./parsers/types.js";
+import { detectSourceFromSubdir, inboxIdempotencyKey, isSupportedInboxExtension, sniffInboxSource, } from "./inbox-core/index.js";
+import { writeSkipQuarantine } from "./skip-quarantine.js";
+const DEFAULT_SOURCE = "chatgpt_web";
+/**
+ * Create an inbox watcher. Returns a Watcher instance that can be
+ * started, paused, and stopped like any other watcher.
+ */
+export function createInboxWatcher(opts) {
+    const inboxDir = path.join(opts.dataDir, "inbox");
+    fs.mkdirSync(inboxDir, { recursive: true });
+    const handler = async (event) => {
+        // Ignore deletions; otherwise process both `add` and `change` so a
+        // user who overwrites `export.zip` in place picks up the new
+        // content. The idempotency key includes mtime, so an unchanged
+        // `change` event collapses at the queue layer.
+        if (event.type === "unlink")
+            return;
+        if (!isSupportedInboxExtension(event.path)) {
+            // Markdown / images / other types are not parseable as web
+            // exports. Record a friendly skip so the user can see why their
+            // drop didn't ingest, instead of it disappearing silently.
+            await writeSkipQuarantine(opts.dataDir, {
+                reason: "inbox_unsupported_extension",
+                source_path: event.path,
+                metadata: { extension: path.extname(event.path).toLowerCase() },
+            }).catch(() => {
+                // Quarantine write failure must not break the watcher loop.
+            });
+            return;
+        }
+        let fileData;
+        let mtimeMs;
+        try {
+            fileData = fs.readFileSync(event.path);
+            mtimeMs = fs.statSync(event.path).mtimeMs;
+        }
+        catch {
+            // File may have been removed between detection and read.
+            return;
+        }
+        // Source resolution order:
+        //   1. Subdirectory hint (data/inbox/<provider>/file.json) — strongest.
+        //   2. Content/filename sniff — identifies non-ChatGPT providers.
+        //   3. DEFAULT_SOURCE — historical default for unidentified drops.
+        const source = detectSourceFromSubdir(event.path, inboxDir, SUPPORTED_INGEST_SOURCES) ??
+            sniffInboxSource(path.basename(event.path), fileData) ??
+            DEFAULT_SOURCE;
+        const idempotencyKey = inboxIdempotencyKey(event.path, mtimeMs);
+        const payload = {
+            source: source,
+            filename: path.basename(event.path),
+            fileData: fileData.toString("base64"),
+            rawSource: "inbox",
+        };
+        await opts.queue.create("ingest", {
+            idempotency_key: idempotencyKey,
+            payload,
+        });
+    };
+    return new Watcher([inboxDir], handler);
+}
+//# sourceMappingURL=inbox-watcher.js.map

package/dist/src/ingestion/inbox-watcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inbox-watcher.js","sourceRoot":"","sources":["../../../src/ingestion/inbox-watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EACL,sBAAsB,EACtB,mBAAmB,EACnB,yBAAyB,EACzB,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,cAAc,GAAG,aAAa,CAAC;AAOrC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAyB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,KAAK,EAAE,KAAgB,EAAiB,EAAE;QACxD,mEAAmE;QACnE,6DAA6D;QAC7D,+DAA+D;QAC/D,+CAA+C;QAC/C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO;QAEpC,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,2DAA2D;YAC3D,gEAAgE;YAChE,2DAA2D;YAC3D,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE;gBACtC,MAAM,EAAE,6BAA6B;gBACrC,WAAW,EAAE,KAAK,CAAC,IAAI;gBACvB,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE;aAChE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;gBACZ,4DAA4D;YAC9D,CAAC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,QAAgB,CAAC;QACrB,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,OAAO,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;YACzD,OAAO;QACT,CAAC;QAED,2BAA2B;QAC3B,wEAAwE;QACxE,kEAAkE;QAClE,mEAAmE;QACnE,MAAM,MAAM,GACV,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,wBAAwB,CAAC;YACtE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC;YACrD,cAAc,CAAC;QACjB,MAAM,cAAc,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEhE,MAAM,OAAO,GAAkB;YAC7B,MAAM,EAAE,MAAiC;YACzC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;YACnC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACrC,SAAS,EAAE,OAAO;SACnB,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE;YAChC,eAAe,EAAE,cAAc;YAC/B,OAAO;SACR,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,IAAI,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC"}

package/dist/src/ingestion/indexer.d.ts

@@ -0,0 +1,39 @@
+import type { EmbeddingProvider, SourceType, SourceScope } from "../providers/types.js";
+export interface FileEvent {
+    type: "add" | "change" | "unlink";
+    path: string;
+}
+export interface IndexerConfig {
+    sourceType: SourceType;
+    sourceScope: SourceScope;
+    clientName: string;
+    allowedRoots: readonly string[];
+    /** Target table for document storage. Defaults to "structured_docs". */
+    tableName?: "structured_docs" | "structured_docs_local";
+    /**
+     * Data directory root. Used to write skip-quarantine records when a file
+     * cannot be embedded (e.g. empty extracted content). Required so every
+     * production code path — watch, scheduled scan, reconcile — produces a
+     * visible explanation when a file is dropped, instead of silent stderr.
+     */
+    dataDir: string;
+}
+/**
+ * Deterministic row ID from the canonical source path.
+ * Same file always gets the same ID, enabling upsert via delete+add.
+ */
+export declare function fileId(sourcePath: string): string;
+export declare class Indexer {
+    private readonly embedding;
+    private readonly config;
+    constructor(embedding: EmbeddingProvider, config: IndexerConfig);
+    /**
+     * Handle a file system event. Validates the path, extracts content,
+     * embeds, and upserts to structured_docs. For unlink events, removes
+     * the row. Errors are thrown to the caller (watcher catches and logs).
+     */
+    handleEvent(event: FileEvent): Promise<void>;
+    private handleUpsert;
+    private handleDelete;
+}
+//# sourceMappingURL=indexer.d.ts.map

package/dist/src/ingestion/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../../src/ingestion/indexer.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAYxF,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,wEAAwE;IACxE,SAAS,CAAC,EAAE,iBAAiB,GAAG,uBAAuB,CAAC;IACxD;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,aAAa;IAK/D;;;;OAIG;IACG,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;YAcpC,YAAY;YAuEZ,YAAY;CAK3B"}

package/dist/src/ingestion/indexer.js

@@ -0,0 +1,111 @@
+/**
+ * Indexer: file event -> extract -> embed -> upsert structured_docs.
+ *
+ * Each Indexer instance is bound to a single source directory config,
+ * providing the correct source_type, source_scope, and client_name
+ * for every row it writes.
+ */
+import crypto from "node:crypto";
+import { getTable } from "../storage/tables.js";
+import { extract, isSupported } from "./extractor.js";
+import { validatePath, validateUnlinkPath } from "../security/paths.js";
+import { writeSkipQuarantine } from "./skip-quarantine.js";
+import { recordEmbed } from "../observability/embedding-events.js";
+import { recordIndexWrite } from "../observability/index-events.js";
+import { chunkText, chunkingEnabled } from "./chunk-text.js";
+import { docChunkId, docChunkMetadata } from "./chunk-meta.js";
+import { eqFilter } from "../storage/filter.js";
+/**
+ * Deterministic row ID from the canonical source path.
+ * Same file always gets the same ID, enabling upsert via delete+add.
+ */
+export function fileId(sourcePath) {
+    return crypto.createHash("sha256").update(sourcePath).digest("hex").slice(0, 32);
+}
+export class Indexer {
+    embedding;
+    config;
+    constructor(embedding, config) {
+        this.embedding = embedding;
+        this.config = config;
+    }
+    /**
+     * Handle a file system event. Validates the path, extracts content,
+     * embeds, and upserts to structured_docs. For unlink events, removes
+     * the row. Errors are thrown to the caller (watcher catches and logs).
+     */
+    async handleEvent(event) {
+        if (event.type === "unlink") {
+            const canonical = validateUnlinkPath(event.path, this.config.allowedRoots);
+            await this.handleDelete(canonical);
+            return;
+        }
+        const canonical = validatePath(event.path, this.config.allowedRoots);
+        if (!isSupported(canonical))
+            return;
+        await this.handleUpsert(canonical);
+    }
+    async handleUpsert(filePath) {
+        const doc = await extract(filePath);
+        // Empty/blank extracted content cannot be embedded — Voyage rejects empty
+        // strings with HTTP 400, and indexing an empty row produces nothing
+        // searchable anyway. Quarantine the skip so it's visible (not stderr-only)
+        // and remove any stale row from a prior good extraction of the same path.
+        const table = getTable(this.config.tableName ?? "structured_docs");
+        const sourcePathFilter = eqFilter("source_path", filePath);
+        if (isBlank(doc.content)) {
+            // Remove the whole prior set (single row or chunk set) for this path.
+            await table.delete(sourcePathFilter);
+            await writeSkipQuarantine(this.config.dataDir, {
+                reason: "empty_extracted_content",
+                source_path: filePath,
+                metadata: doc.metadata,
+            });
+            return;
+        }
+        const pipeline = this.config.sourceType === "filesystem_watched" ? "watcher" : "scheduled_scan";
+        const baseId = fileId(filePath);
+        // Chunk (flag-gated). chunkText short-circuits small docs to a single
+        // chunk, so a below-threshold doc — and every doc with the flag off —
+        // produces exactly one unmarked row, byte-identical to the prior behavior.
+        const chunks = chunkingEnabled() ? chunkText(doc.content) : [doc.content];
+        const count = chunks.length;
+        // Data-safety: embed the FULL new set BEFORE the destructive delete, so a
+        // failure leaves the old set intact (filesystem is the source of truth).
+        const indexedAt = new Date().toISOString();
+        const rows = [];
+        for (let i = 0; i < count; i++) {
+            const chunk = chunks[i];
+            const embeddingVec = await recordEmbed(this.config.dataDir, this.embedding, { pipeline, operation: "document_embedding", input_count: 1 }, () => this.embedding.embed(chunk));
+            rows.push({
+                id: count > 1 ? docChunkId(baseId, i) : baseId,
+                source_path: filePath,
+                content: chunk,
+                embedding: embeddingVec,
+                source_type: this.config.sourceType,
+                source_scope: this.config.sourceScope,
+                client_name: this.config.clientName,
+                indexed_at: indexedAt,
+                metadata: JSON.stringify(docChunkMetadata(doc.metadata, i, count)),
+            });
+        }
+        // Upsert by source_path: removes any prior chunk set AND any legacy
+        // single row for this file before writing the new set (no dup/stale chunks).
+        await table.delete(sourcePathFilter);
+        await recordIndexWrite(this.config.dataDir, {
+            table: this.config.tableName ?? "structured_docs",
+            pipeline,
+            operation: "structured_doc_upsert",
+            row_count: rows.length,
+        }, () => table.add(rows));
+    }
+    async handleDelete(filePath) {
+        const table = getTable(this.config.tableName ?? "structured_docs");
+        // Delete by source_path so all chunks of the file are removed together.
+        await table.delete(eqFilter("source_path", filePath));
+    }
+}
+function isBlank(s) {
+    return s.trim().length === 0;
+}
+//# sourceMappingURL=indexer.js.map

package/dist/src/ingestion/indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../../src/ingestion/indexer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAuBhD;;;GAGG;AACH,MAAM,UAAU,MAAM,CAAC,UAAkB;IACvC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,OAAO,OAAO;IACD,SAAS,CAAoB;IAC7B,MAAM,CAAgB;IAEvC,YAAY,SAA4B,EAAE,MAAqB;QAC7D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,KAAgB;QAChC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3E,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAErE,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;YAAE,OAAO;QAEpC,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IACrC,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAgB;QACzC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEpC,0EAA0E;QAC1E,oEAAoE;QACpE,2EAA2E;QAC3E,0EAA0E;QAC1E,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC,CAAC;QACnE,MAAM,gBAAgB,GAAG,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAE3D,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,sEAAsE;YACtE,MAAM,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACrC,MAAM,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;gBAC7C,MAAM,EAAE,yBAAyB;gBACjC,WAAW,EAAE,QAAQ;gBACrB,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GACZ,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEhC,sEAAsE;QACtE,sEAAsE;QACtE,2EAA2E;QAC3E,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QAE5B,0EAA0E;QAC1E,yEAAyE;QACzE,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAuB,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACzB,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,IAAI,CAAC,SAAS,EACd,EAAE,QAAQ,EAAE,SAAS,EAAE,oBAAoB,EAAE,WAAW,EAAE,CAAC,EAAE,EAC7D,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAClC,CAAC;YACF,IAAI,CAAC,IAAI,CAAC;gBACR,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM;gBAC9C,WAAW,EAAE,QAAQ;gBACrB,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,YAAY;gBACvB,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;gBACnC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACrC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;gBACnC,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;aACnE,CAAC,CAAC;QACL,CAAC;QAED,oEAAoE;QACpE,6EAA6E;QAC7E,MAAM,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACrC,MAAM,gBAAgB,CACpB,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB;YACE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB;YACjD,QAAQ;YACR,SAAS,EAAE,uBAAuB;YAClC,SAAS,EAAE,IAAI,CAAC,MAAM;SACvB,EACD,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CACtB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAgB;QACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,iBAAiB,CAAC,CAAC;QACnE,wEAAwE;QACxE,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC;IACxD,CAAC;CACF;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;AAC/B,CAAC"}

package/dist/src/ingestion/metadata-extraction.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Structured metadata extraction via Anthropic API — Slice 13.
+ *
+ * Implements the MetadataExtractor interface (Slice 4 types).
+ * Calls the Anthropic Messages API with the fast_model to extract:
+ *   domain, intent, quality, topics, decisions, key_outputs
+ *
+ * Output is Zod-validated. Structural invalidity (missing fields, wrong
+ * types, empty `topics`) throws so the caller can quarantine the item.
+ * Enum drift on `domain` / `intent` / `quality` is normalized to a safe
+ * default and logged to stderr via `normalizedEnum` — visible but
+ * non-fatal.
+ */
+import { z } from "zod";
+import type { MetadataExtractor, ConversationMetadata, RetryConfig } from "../providers/types.js";
+export declare const ConversationMetadataSchema: z.ZodObject<{
+    domain: z.ZodEffects<z.ZodUnknown, "business" | "tech" | "personal" | "travel" | "health" | "finance" | "creative", unknown>;
+    intent: z.ZodEffects<z.ZodUnknown, "research" | "decision" | "brainstorm" | "build" | "learn" | "troubleshoot", unknown>;
+    quality: z.ZodEffects<z.ZodUnknown, "high" | "medium" | "low", unknown>;
+    topics: z.ZodArray<z.ZodString, "many">;
+    decisions: z.ZodArray<z.ZodString, "many">;
+    key_outputs: z.ZodArray<z.ZodString, "many">;
+}, "strip", z.ZodTypeAny, {
+    domain: "business" | "tech" | "personal" | "travel" | "health" | "finance" | "creative";
+    intent: "research" | "decision" | "brainstorm" | "build" | "learn" | "troubleshoot";
+    quality: "high" | "medium" | "low";
+    topics: string[];
+    decisions: string[];
+    key_outputs: string[];
+}, {
+    topics: string[];
+    decisions: string[];
+    key_outputs: string[];
+    domain?: unknown;
+    intent?: unknown;
+    quality?: unknown;
+}>;
+export interface AnthropicExtractorOptions {
+    apiKey: string;
+    model: string;
+    retry?: Partial<RetryConfig>;
+}
+export declare class AnthropicMetadataExtractor implements MetadataExtractor {
+    readonly retry: RetryConfig;
+    private readonly apiKey;
+    private readonly model;
+    constructor(opts: AnthropicExtractorOptions);
+    extractMetadata(text: string): Promise<ConversationMetadata>;
+    private callApi;
+}
+/**
+ * Parse raw LLM JSON output and validate against the metadata schema.
+ * Throws on invalid output (caller should quarantine).
+ */
+export declare function parseAndValidate(raw: string): ConversationMetadata;
+//# sourceMappingURL=metadata-extraction.d.ts.map

package/dist/src/ingestion/metadata-extraction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata-extraction.d.ts","sourceRoot":"","sources":["../../../src/ingestion/metadata-extraction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAiC/B,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;EAOrC,CAAC;AAoCH,MAAM,WAAW,yBAAyB;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9B;AAED,qBAAa,0BAA2B,YAAW,iBAAiB;IAClE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;gBAEnB,IAAI,EAAE,yBAAyB;IAMrC,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;YASpD,OAAO;CA2DtB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,oBAAoB,CA0BlE"}

package/dist/src/ingestion/metadata-extraction.js

@@ -0,0 +1,151 @@
+/**
+ * Structured metadata extraction via Anthropic API — Slice 13.
+ *
+ * Implements the MetadataExtractor interface (Slice 4 types).
+ * Calls the Anthropic Messages API with the fast_model to extract:
+ *   domain, intent, quality, topics, decisions, key_outputs
+ *
+ * Output is Zod-validated. Structural invalidity (missing fields, wrong
+ * types, empty `topics`) throws so the caller can quarantine the item.
+ * Enum drift on `domain` / `intent` / `quality` is normalized to a safe
+ * default and logged to stderr via `normalizedEnum` — visible but
+ * non-fatal.
+ */
+import { z } from "zod";
+import { DOMAINS, INTENTS, QUALITY_LEVELS, DEFAULT_RETRY_CONFIG, } from "../providers/types.js";
+// --- Zod schema for extraction output validation ---
+/**
+ * Normalize an LLM-emitted enum value: accept it if it matches the allowed
+ * set, otherwise remap to `fallback` and warn on stderr. Keeps ingest
+ * tolerant (we don't fail the whole record on a stray value) but makes
+ * extraction drift observable instead of silent — which is what
+ * `.catch(default)` was masking.
+ */
+function normalizedEnum(values, fallback, field) {
+    return z.unknown().transform((raw) => {
+        if (typeof raw === "string" && values.includes(raw)) {
+            return raw;
+        }
+        process.stderr.write(`metadata-extraction: ${field}=${JSON.stringify(raw)} not in [${values.join("|")}], remapped to "${fallback}"\n`);
+        return fallback;
+    });
+}
+export const ConversationMetadataSchema = z.object({
+    domain: normalizedEnum(DOMAINS, "tech", "domain"),
+    intent: normalizedEnum(INTENTS, "research", "intent"),
+    quality: normalizedEnum(QUALITY_LEVELS, "medium", "quality"),
+    topics: z.array(z.string().min(1)).min(1),
+    decisions: z.array(z.string()),
+    key_outputs: z.array(z.string()),
+});
+// --- Anthropic API types ---
+const ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages";
+// --- Extraction prompt ---
+const EXTRACTION_PROMPT = `Analyze this conversation and extract structured metadata. Return ONLY valid JSON matching this exact schema:
+
+{
+  "domain": one of: "business", "tech", "personal", "travel", "health", "finance", "creative",
+  "intent": one of: "research", "decision", "brainstorm", "build", "learn", "troubleshoot",
+  "quality": one of: "high", "medium", "low" (based on depth, specificity, and usefulness of the exchange),
+  "topics": ["array", "of", "key", "topics"] (3-8 concise topic strings),
+  "decisions": ["array of decisions made"] (empty if none),
+  "key_outputs": ["array of key outputs produced"] (empty if none)
+}
+
+Rules:
+- Return ONLY the JSON object, no markdown fences, no explanation
+- topics must have at least 1 entry
+- decisions and key_outputs can be empty arrays
+- quality "high" = substantive, detailed exchange; "medium" = useful but shallow; "low" = minimal value`;
+export class AnthropicMetadataExtractor {
+    retry;
+    apiKey;
+    model;
+    constructor(opts) {
+        this.apiKey = opts.apiKey;
+        this.model = opts.model;
+        this.retry = { ...DEFAULT_RETRY_CONFIG, ...opts.retry };
+    }
+    async extractMetadata(text) {
+        const messages = [
+            { role: "user", content: `${EXTRACTION_PROMPT}\n\n---\n\n${text}` },
+        ];
+        const rawJson = await this.callApi(messages);
+        return parseAndValidate(rawJson);
+    }
+    async callApi(messages) {
+        let lastError;
+        for (let attempt = 0; attempt <= this.retry.maxRetries; attempt++) {
+            if (attempt > 0) {
+                const delay = Math.min(this.retry.baseDelayMs * 2 ** (attempt - 1), this.retry.maxDelayMs);
+                await sleep(delay);
+            }
+            try {
+                const res = await fetch(ANTHROPIC_API_URL, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        "x-api-key": this.apiKey,
+                        "anthropic-version": "2023-06-01",
+                    },
+                    body: JSON.stringify({
+                        model: this.model,
+                        max_tokens: 1024,
+                        messages,
+                    }),
+                });
+                if (res.status === 429 || res.status >= 500) {
+                    lastError = new Error(`Anthropic API ${res.status}: ${await res.text()}`);
+                    continue;
+                }
+                if (!res.ok) {
+                    throw new Error(`Anthropic API ${res.status}: ${await res.text()}`);
+                }
+                const body = (await res.json());
+                const textBlock = body.content.find((c) => c.type === "text");
+                if (!textBlock?.text) {
+                    throw new Error("Anthropic API returned no text content");
+                }
+                return textBlock.text;
+            }
+            catch (err) {
+                if (err instanceof TypeError) {
+                    // Network error — transient, retry
+                    lastError = err;
+                    continue;
+                }
+                throw err;
+            }
+        }
+        throw lastError;
+    }
+}
+/**
+ * Parse raw LLM JSON output and validate against the metadata schema.
+ * Throws on invalid output (caller should quarantine).
+ */
+export function parseAndValidate(raw) {
+    // Strip markdown fences if the LLM wraps the JSON
+    let cleaned = raw.trim();
+    if (cleaned.startsWith("```")) {
+        cleaned = cleaned
+            .replace(/^```(?:json)?\s*\n?/, "")
+            .replace(/\n?```\s*$/, "");
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(cleaned);
+    }
+    catch {
+        throw new Error(`Extraction output is not valid JSON: ${cleaned.slice(0, 200)}`);
+    }
+    const result = ConversationMetadataSchema.safeParse(parsed);
+    if (!result.success) {
+        throw new Error(`Extraction output failed validation: ${result.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`);
+    }
+    return result.data;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=metadata-extraction.js.map