@getrift/rift 0.0.0 → 0.1.0-beta.0

package/dist/src/ingestion/parsers/gemini-web.js

@@ -0,0 +1,170 @@
+const MONTHS = new Map([
+    ["jan", 0],
+    ["janv", 0],
+    ["january", 0],
+    ["feb", 1],
+    ["fev", 1],
+    ["fevr", 1],
+    ["fevrier", 1],
+    ["february", 1],
+    ["mar", 2],
+    ["mars", 2],
+    ["march", 2],
+    ["apr", 3],
+    ["avr", 3],
+    ["april", 3],
+    ["may", 4],
+    ["mai", 4],
+    ["jun", 5],
+    ["juin", 5],
+    ["june", 5],
+    ["jul", 6],
+    ["juil", 6],
+    ["july", 6],
+    ["aug", 7],
+    ["aout", 7],
+    ["august", 7],
+    ["sep", 8],
+    ["sept", 8],
+    ["september", 8],
+    ["oct", 9],
+    ["october", 9],
+    ["nov", 10],
+    ["november", 10],
+    ["dec", 11],
+    ["decembre", 11],
+    ["december", 11],
+]);
+const ACTIVITY_TIMESTAMP_PATTERN = /\b(\d{1,2})\s+([A-Za-zÀ-ÿ.]+)\s+(\d{4}),\s+(\d{2}):(\d{2}):(\d{2})\s+(CEST|CET|UTC)\b/g;
+function normalizeMonthToken(value) {
+    return value
+        .normalize("NFD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .replace(/\./g, "")
+        .toLowerCase();
+}
+function toIsoIfValid(date) {
+    return Number.isNaN(date.getTime()) ? null : date.toISOString();
+}
+function parseActivityTimestamp(value) {
+    const match = value.trim().match(/^(\d{1,2})\s+([A-Za-zÀ-ÿ.]+)\s+(\d{4}),\s+(\d{2}):(\d{2}):(\d{2})\s+(CEST|CET|UTC)$/);
+    if (!match)
+        return null;
+    const dayText = match[1];
+    const monthText = match[2];
+    const yearText = match[3];
+    const hourText = match[4];
+    const minuteText = match[5];
+    const secondText = match[6];
+    const zone = match[7];
+    if (!dayText ||
+        !monthText ||
+        !yearText ||
+        !hourText ||
+        !minuteText ||
+        !secondText ||
+        !zone) {
+        return null;
+    }
+    const month = MONTHS.get(normalizeMonthToken(monthText));
+    if (month === undefined)
+        return null;
+    const offset = zone === "CEST" ? "+02:00" : zone === "CET" ? "+01:00" : "Z";
+    const isoLike = `${yearText}-${String(month + 1).padStart(2, "0")}-${dayText.padStart(2, "0")}T${hourText}:${minuteText}:${secondText}${offset}`;
+    return toIsoIfValid(new Date(isoLike));
+}
+function extractTimestampFromContent(content) {
+    let match = null;
+    let last = null;
+    ACTIVITY_TIMESTAMP_PATTERN.lastIndex = 0;
+    while ((match = ACTIVITY_TIMESTAMP_PATTERN.exec(content)) !== null) {
+        last = match[0];
+    }
+    return last ? parseActivityTimestamp(last) : null;
+}
+function normalizeCreatedAt(card) {
+    if (typeof card.create_time === "number" && Number.isFinite(card.create_time)) {
+        const ms = card.create_time > 1_000_000_000_000
+            ? card.create_time
+            : card.create_time * 1000;
+        const fromNumber = toIsoIfValid(new Date(ms));
+        if (fromNumber)
+            return fromNumber;
+    }
+    if (typeof card.create_time === "string" && card.create_time.trim()) {
+        const fromString = parseActivityTimestamp(card.create_time) ??
+            toIsoIfValid(new Date(card.create_time));
+        if (fromString)
+            return fromString;
+    }
+    if (typeof card.timestamp_text === "string" && card.timestamp_text.trim()) {
+        const fromTimestampText = parseActivityTimestamp(card.timestamp_text);
+        if (fromTimestampText)
+            return fromTimestampText;
+    }
+    if (typeof card.content === "string" && card.content.trim()) {
+        const fromContent = extractTimestampFromContent(card.content);
+        if (fromContent)
+            return fromContent;
+    }
+    return "";
+}
+function isGeminiActivityCard(entry) {
+    if (!entry || typeof entry !== "object")
+        return false;
+    const card = entry;
+    return (typeof card.conversation_id === "string" &&
+        card.conversation_id.length > 0 &&
+        card.source === "gemini_web" &&
+        card.operative_unit === "gemini_activity_card" &&
+        card.original_export_format === "google_takeout_html");
+}
+/**
+ * Parse the repo's staged Gemini activity-card JSON format into the
+ * shared ParsedConversation shape used by ingest/backfill.
+ */
+export const parseGeminiWeb = (files) => {
+    const results = [];
+    for (const file of files) {
+        if (!file.path.toLowerCase().endsWith(".json"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(file.data.toString("utf-8"));
+        }
+        catch {
+            continue;
+        }
+        if (!Array.isArray(parsed))
+            continue;
+        for (const entry of parsed) {
+            if (!isGeminiActivityCard(entry))
+                continue;
+            if ((entry.assistant_turns ?? 0) < 1)
+                continue;
+            const id = entry.conversation_id;
+            if (typeof id !== "string" || id.length === 0)
+                continue;
+            const content = typeof entry.content === "string" && entry.content.trim()
+                ? entry.content.trim()
+                : typeof entry.preview === "string" && entry.preview.trim()
+                    ? entry.preview.trim()
+                    : null;
+            if (!content)
+                continue;
+            const title = typeof entry.title === "string" && entry.title.trim()
+                ? entry.title.trim()
+                : typeof entry.prompt === "string" && entry.prompt.trim()
+                    ? entry.prompt.trim()
+                    : "Untitled Gemini activity";
+            results.push({
+                id,
+                title,
+                createdAt: normalizeCreatedAt(entry),
+                content,
+            });
+        }
+    }
+    return results;
+};
+//# sourceMappingURL=gemini-web.js.map

package/dist/src/ingestion/parsers/gemini-web.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini-web.js","sourceRoot":"","sources":["../../../../src/ingestion/parsers/gemini-web.ts"],"names":[],"mappings":"AAyBA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAiB;IACrC,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,UAAU,EAAE,CAAC,CAAC;IACf,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,OAAO,EAAE,CAAC,CAAC;IACZ,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,OAAO,EAAE,CAAC,CAAC;IACZ,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,MAAM,EAAE,CAAC,CAAC;IACX,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,KAAK,EAAE,CAAC,CAAC;IACV,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,KAAK,EAAE,EAAE,CAAC;IACX,CAAC,UAAU,EAAE,EAAE,CAAC;IAChB,CAAC,KAAK,EAAE,EAAE,CAAC;IACX,CAAC,UAAU,EAAE,EAAE,CAAC;IAChB,CAAC,UAAU,EAAE,EAAE,CAAC;CACjB,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAC9B,wFAAwF,CAAC;AAE3F,SAAS,mBAAmB,CAAC,KAAa;IACxC,OAAO,KAAK;SACT,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,IAAU;IAC9B,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAC9B,qFAAqF,CACtF,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,IACE,CAAC,OAAO;QACR,CAAC,SAAS;QACV,CAAC,QAAQ;QACT,CAAC,QAAQ;QACT,CAAC,UAAU;QACX,CAAC,UAAU;QACX,CAAC,IAAI,EACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC;IACzD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,QAAQ,IAAI,UAAU,IAAI,UAAU,GAAG,MAAM,EAAE,CAAC;IACjJ,OAAO,YAAY,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,2BAA2B,CAAC,OAAe;IAClD,IAAI,KAAK,GAA2B,IAAI,CAAC;IACzC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,0BAA0B,CAAC,SAAS,GAAG,CAAC,CAAC;IACzC,OAAO,CAAC,KAAK,GAAG,0BAA0B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAwB;IAClD,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9E,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,GAAG,iBAAiB;YAC7C,CAAC,CAAC,IAAI,CAAC,WAAW;YAClB,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;IACpC,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACpE,MAAM,UAAU,GAAG,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC;YACzD,YAAY,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3C,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;IACpC,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,cAAc,KAAK,QAAQ,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1E,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtE,IAAI,iBAAiB;YAAE,OAAO,iBAAiB,CAAC;IAClD,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC5D,MAAM,WAAW,GAAG,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,IAAI,WAAW;YAAE,OAAO,WAAW,CAAC;IACtC,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,IAAI,GAAG,KAA2B,CAAC;IACzC,OAAO,CACL,OAAO,IAAI,CAAC,eAAe,KAAK,QAAQ;QACxC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAC/B,IAAI,CAAC,MAAM,KAAK,YAAY;QAC5B,IAAI,CAAC,cAAc,KAAK,sBAAsB;QAC9C,IAAI,CAAC,sBAAsB,KAAK,qBAAqB,CACtD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAuB,CAAC,KAAK,EAAE,EAAE;IAC1D,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAS;QAEzD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAS;QAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC;gBAAE,SAAS;YAC3C,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC/C,MAAM,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC;YACjC,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAExD,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;gBACvE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;gBACtB,CAAC,CAAC,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;oBACzD,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;oBACtB,CAAC,CAAC,IAAI,CAAC;YACX,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,MAAM,KAAK,GAAG,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;gBACjE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE;gBACpB,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;oBACvD,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;oBACrB,CAAC,CAAC,0BAA0B,CAAC;YAEjC,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE;gBACF,KAAK;gBACL,SAAS,EAAE,kBAAkB,CAAC,KAAK,CAAC;gBACpC,OAAO;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}

package/dist/src/ingestion/parsers/grok-web.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Grok web export parser.
+ *
+ * Parses `prod-grok-backend.json` from Grok's account-export bundle.
+ * Shape (verified April 2026, 347 conversations, 1869 messages):
+ *   {
+ *     conversations: [
+ *       {
+ *         conversation: { id, title, create_time, ... },
+ *         responses: [
+ *           { response: { _id, message, sender, create_time,
+ *                         parent_response_id, model, ... },
+ *             share_link }
+ *         ]
+ *       }
+ *     ],
+ *     projects, tasks, media_posts
+ *   }
+ *
+ * Messages are in temporal order within `responses[]` (0 / 1869
+ * out-of-order in the corpus). Branching exists via
+ * `parent_response_id` but affects only 13 / 347 conversations; v1
+ * walks `responses[]` linearly as a deliberate accuracy tradeoff —
+ * matches what the Claude web parser does.
+ *
+ * Not captured in v1 (deferred): `web_search_results`, `xpost_ids`,
+ * `agent_thinking_traces`, `steps[]`, per-response `model`,
+ * workspace personalities.
+ */
+import type { ConversationParser } from "./types.js";
+/**
+ * Parse Grok web export files into normalized conversations.
+ *
+ * Accepts `prod-grok-backend.json` (the only meaningful payload in
+ * the account-export zip). Empty conversations and those without a
+ * non-human turn are skipped; the staging gate already filters these
+ * out, this is belt-and-braces.
+ */
+export declare const parseGrokWeb: ConversationParser;
+//# sourceMappingURL=grok-web.d.ts.map

package/dist/src/ingestion/parsers/grok-web.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grok-web.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/parsers/grok-web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,KAAK,EAAsB,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAwCzE;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,EAAE,kBA+C1B,CAAC"}

package/dist/src/ingestion/parsers/grok-web.js

@@ -0,0 +1,67 @@
+function isHumanSender(sender) {
+    return typeof sender === "string" && sender.toLowerCase() === "human";
+}
+function isoOrNow(value) {
+    if (typeof value === "string" && value.trim()) {
+        const d = new Date(value);
+        if (!Number.isNaN(d.getTime()))
+            return d.toISOString();
+    }
+    return new Date().toISOString();
+}
+/**
+ * Parse Grok web export files into normalized conversations.
+ *
+ * Accepts `prod-grok-backend.json` (the only meaningful payload in
+ * the account-export zip). Empty conversations and those without a
+ * non-human turn are skipped; the staging gate already filters these
+ * out, this is belt-and-braces.
+ */
+export const parseGrokWeb = (files) => {
+    const results = [];
+    for (const file of files) {
+        if (!file.path.toLowerCase().endsWith(".json"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(file.data.toString("utf-8"));
+        }
+        catch {
+            continue;
+        }
+        const root = parsed;
+        const conversations = root?.conversations;
+        if (!Array.isArray(conversations))
+            continue;
+        for (const wrapper of conversations) {
+            const meta = wrapper?.conversation;
+            const responses = wrapper?.responses;
+            if (!meta?.id || !Array.isArray(responses))
+                continue;
+            const lines = [];
+            let sawAssistant = false;
+            for (const resp of responses) {
+                const r = resp?.response;
+                if (!r)
+                    continue;
+                const text = typeof r.message === "string" ? r.message.trim() : "";
+                if (!text)
+                    continue;
+                const prefix = isHumanSender(r.sender) ? "User" : "Assistant";
+                if (prefix === "Assistant")
+                    sawAssistant = true;
+                lines.push(`${prefix}: ${text}`);
+            }
+            if (lines.length === 0 || !sawAssistant)
+                continue;
+            results.push({
+                id: meta.id,
+                title: meta.title?.trim() || "Untitled",
+                createdAt: isoOrNow(meta.create_time),
+                content: lines.join("\n\n"),
+            });
+        }
+    }
+    return results;
+};
+//# sourceMappingURL=grok-web.js.map

package/dist/src/ingestion/parsers/grok-web.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grok-web.js","sourceRoot":"","sources":["../../../../src/ingestion/parsers/grok-web.ts"],"names":[],"mappings":"AAyDA,SAAS,aAAa,CAAC,MAA0B;IAC/C,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAyB;IACzC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;IACzD,CAAC;IACD,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,YAAY,GAAuB,CAAC,KAAK,EAAE,EAAE;IACxD,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAS;QAEzD,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,MAA2B,CAAC;QACzC,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;YAAE,SAAS;QAE5C,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,OAAO,EAAE,YAAY,CAAC;YACnC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;YACrC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;gBAAE,SAAS;YAErD,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC;gBACzB,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnE,IAAI,CAAC,IAAI;oBAAE,SAAS;gBAEpB,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;gBAC9D,IAAI,MAAM,KAAK,WAAW;oBAAE,YAAY,GAAG,IAAI,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY;gBAAE,SAAS;YAElD,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,UAAU;gBACvC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC;gBACrC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}

package/dist/src/ingestion/parsers/types.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Shared types for web conversation parsers — Slice 13.
+ *
+ * Each source-specific parser returns ParsedConversation[].
+ * Metadata (domain, intent, quality, etc.) is added later by
+ * the MetadataExtractor, not by the parser.
+ */
+export interface ParsedConversation {
+    /** Conversation UUID from the source export. */
+    id: string;
+    /** Conversation title. */
+    title: string;
+    /** ISO 8601 creation timestamp. */
+    createdAt: string;
+    /** Formatted conversation text (all messages, role-prefixed). */
+    content: string;
+}
+/**
+ * A parser takes extracted files (from archive.ts) and returns
+ * normalized conversation entries.
+ */
+export type ConversationParser = (files: Array<{
+    path: string;
+    data: Buffer;
+}>) => ParsedConversation[];
+/** Sources with implemented parsers (backed by real fixtures). */
+export declare const SUPPORTED_INGEST_SOURCES: readonly ["chatgpt_web", "claude_web", "gemini_web", "grok_web"];
+export type SupportedIngestSource = (typeof SUPPORTED_INGEST_SOURCES)[number];
+/** All web sources that POST /ingest accepts in the enum (some deferred). */
+export declare const INGEST_SOURCES: readonly ["chatgpt_web", "claude_web", "grok_web", "gemini_web"];
+export type IngestSource = (typeof INGEST_SOURCES)[number];
+/** Check if a source has an implemented parser. */
+export declare function isSourceSupported(source: string): source is SupportedIngestSource;
+//# sourceMappingURL=types.d.ts.map

package/dist/src/ingestion/parsers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/parsers/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,kBAAkB;IACjC,gDAAgD;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KACzC,kBAAkB,EAAE,CAAC;AAE1B,kEAAkE;AAClE,eAAO,MAAM,wBAAwB,kEAK3B,CAAC;AACX,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9E,6EAA6E;AAC7E,eAAO,MAAM,cAAc,kEAKjB,CAAC;AACX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D,mDAAmD;AACnD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,qBAAqB,CAEjF"}

package/dist/src/ingestion/parsers/types.js

@@ -0,0 +1,26 @@
+/**
+ * Shared types for web conversation parsers — Slice 13.
+ *
+ * Each source-specific parser returns ParsedConversation[].
+ * Metadata (domain, intent, quality, etc.) is added later by
+ * the MetadataExtractor, not by the parser.
+ */
+/** Sources with implemented parsers (backed by real fixtures). */
+export const SUPPORTED_INGEST_SOURCES = [
+    "chatgpt_web",
+    "claude_web",
+    "gemini_web",
+    "grok_web",
+];
+/** All web sources that POST /ingest accepts in the enum (some deferred). */
+export const INGEST_SOURCES = [
+    "chatgpt_web",
+    "claude_web",
+    "grok_web",
+    "gemini_web",
+];
+/** Check if a source has an implemented parser. */
+export function isSourceSupported(source) {
+    return SUPPORTED_INGEST_SOURCES.includes(source);
+}
+//# sourceMappingURL=types.js.map

package/dist/src/ingestion/parsers/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/ingestion/parsers/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAqBH,kEAAkE;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,UAAU;CACF,CAAC;AAGX,6EAA6E;AAC7E,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,aAAa;IACb,YAAY;IACZ,UAAU;IACV,YAAY;CACJ,CAAC;AAGX,mDAAmD;AACnD,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,OAAQ,wBAA8C,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC1E,CAAC"}

package/dist/src/ingestion/scanner.d.ts

@@ -0,0 +1,48 @@
+import { type FileEvent } from "./indexer.js";
+export interface ScanResult {
+    events: FileEvent[];
+    filesChecked: number;
+}
+export interface ScanOptions {
+    ignoredPaths?: readonly string[];
+}
+export interface WalkOptions {
+    ignoredPaths?: readonly string[];
+}
+export interface ScanState {
+    last_scan_at: string | null;
+    next_scan_at: string | null;
+}
+export declare class Scanner {
+    private paused;
+    private readonly statePath;
+    private state;
+    constructor(dataDir: string);
+    /**
+     * Scan the given directories for new or modified files.
+     * Returns FileEvents for files that need indexing.
+     * Returns empty result if paused.
+     */
+    scan(dirs: readonly string[], opts?: ScanOptions): Promise<ScanResult>;
+    pause(): void;
+    resume(): void;
+    get isPaused(): boolean;
+    get scanState(): ScanState;
+    setNextScan(nextScanAt: string): void;
+    /**
+     * Whether a scan is due based on the configured interval.
+     * True if we have never scanned or if `last_scan_at + interval` is in the past.
+     */
+    isDue(intervalSeconds: number): boolean;
+    private loadState;
+    private saveState;
+}
+/**
+ * Recursively walk a directory, returning all file paths.
+ * Honors the same ignored-path policy as the live Watcher via
+ * `isWatcherPathIgnored`: always skips .git / node_modules / .DS_Store,
+ * plus any caller-supplied roots (e.g. engine data_dir, dist, logs).
+ * Does not follow symlinks.
+ */
+export declare function walkDirectory(dir: string, opts?: WalkOptions): string[];
+//# sourceMappingURL=scanner.d.ts.map

package/dist/src/ingestion/scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scanner.d.ts","sourceRoot":"","sources":["../../../src/ingestion/scanner.ts"],"names":[],"mappings":"AAYA,OAAO,EAAU,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAMtD,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,KAAK,CAAyD;gBAE1D,OAAO,EAAE,MAAM;IAK3B;;;;OAIG;IACG,IAAI,CACR,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,IAAI,GAAE,WAAgB,GACrB,OAAO,CAAC,UAAU,CAAC;IAyCtB,KAAK,IAAI,IAAI;IAIb,MAAM,IAAI,IAAI;IAId,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAKrC;;;OAGG;IACH,KAAK,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO;IAMvC,OAAO,CAAC,SAAS;YASH,SAAS;CAGxB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,MAAM,EAAE,CAgB3E"}

package/dist/src/ingestion/scanner.js

@@ -0,0 +1,131 @@
+/**
+ * Scheduled scanner for mode: "scheduled_scan" directories.
+ *
+ * Walks directories, compares file mtimes against indexed state
+ * in structured_docs, and returns FileEvents for new/modified files.
+ * Reuses the Indexer pipeline from Slice 5.
+ *
+ * Exposes pause()/resume() for Live-Indexer / Repair-Job Coordination.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { isSupported } from "./extractor.js";
+import { fileId } from "./indexer.js";
+import { getTable } from "../storage/tables.js";
+import { atomicWrite } from "../storage/atomic.js";
+import { isWatcherPathIgnored } from "./watcher.js";
+export class Scanner {
+    paused = false;
+    statePath;
+    state = { last_scan_at: null, next_scan_at: null };
+    constructor(dataDir) {
+        this.statePath = path.join(dataDir, "scan-state.json");
+        this.loadState();
+    }
+    /**
+     * Scan the given directories for new or modified files.
+     * Returns FileEvents for files that need indexing.
+     * Returns empty result if paused.
+     */
+    async scan(dirs, opts = {}) {
+        if (this.paused)
+            return { events: [], filesChecked: 0 };
+        let filesChecked = 0;
+        const events = [];
+        const table = getTable("structured_docs");
+        const walkOpts = opts.ignoredPaths
+            ? { ignoredPaths: opts.ignoredPaths }
+            : {};
+        for (const dir of dirs) {
+            const files = walkDirectory(dir, walkOpts);
+            for (const filePath of files) {
+                if (!isSupported(filePath))
+                    continue;
+                filesChecked++;
+                const id = fileId(filePath);
+                const existing = (await table
+                    .query()
+                    .where(`id = '${id}'`)
+                    .toArray());
+                if (existing.length === 0) {
+                    events.push({ type: "add", path: filePath });
+                }
+                else {
+                    const stats = fs.statSync(filePath);
+                    const indexedAt = new Date(existing[0].indexed_at);
+                    if (stats.mtime > indexedAt) {
+                        events.push({ type: "change", path: filePath });
+                    }
+                }
+            }
+        }
+        this.state.last_scan_at = new Date().toISOString();
+        await this.saveState();
+        return { events, filesChecked };
+    }
+    pause() {
+        this.paused = true;
+    }
+    resume() {
+        this.paused = false;
+    }
+    get isPaused() {
+        return this.paused;
+    }
+    get scanState() {
+        return { ...this.state };
+    }
+    setNextScan(nextScanAt) {
+        this.state.next_scan_at = nextScanAt;
+        this.saveState().catch(() => { });
+    }
+    /**
+     * Whether a scan is due based on the configured interval.
+     * True if we have never scanned or if `last_scan_at + interval` is in the past.
+     */
+    isDue(intervalSeconds) {
+        if (!this.state.last_scan_at)
+            return true;
+        const lastScan = new Date(this.state.last_scan_at).getTime();
+        return Date.now() >= lastScan + intervalSeconds * 1000;
+    }
+    loadState() {
+        try {
+            const raw = fs.readFileSync(this.statePath, "utf-8");
+            this.state = JSON.parse(raw);
+        }
+        catch {
+            // No state file yet — use defaults
+        }
+    }
+    async saveState() {
+        await atomicWrite(this.statePath, JSON.stringify(this.state, null, 2));
+    }
+}
+/**
+ * Recursively walk a directory, returning all file paths.
+ * Honors the same ignored-path policy as the live Watcher via
+ * `isWatcherPathIgnored`: always skips .git / node_modules / .DS_Store,
+ * plus any caller-supplied roots (e.g. engine data_dir, dist, logs).
+ * Does not follow symlinks.
+ */
+export function walkDirectory(dir, opts = {}) {
+    const results = [];
+    if (!fs.existsSync(dir))
+        return results;
+    const ignoredPaths = opts.ignoredPaths ?? [];
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (isWatcherPathIgnored(fullPath, ignoredPaths))
+            continue;
+        if (entry.isDirectory() && !entry.isSymbolicLink()) {
+            results.push(...walkDirectory(fullPath, opts));
+        }
+        else if (entry.isFile()) {
+            results.push(fullPath);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=scanner.js.map

package/dist/src/ingestion/scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scanner.js","sourceRoot":"","sources":["../../../src/ingestion/scanner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAkB,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAoBpD,MAAM,OAAO,OAAO;IACV,MAAM,GAAG,KAAK,CAAC;IACN,SAAS,CAAS;IAC3B,KAAK,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAEtE,YAAY,OAAe;QACzB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CACR,IAAuB,EACvB,OAAoB,EAAE;QAEtB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAgB,IAAI,CAAC,YAAY;YAC7C,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC;QAEP,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC3C,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBACrC,YAAY,EAAE,CAAC;gBAEf,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC5B,MAAM,QAAQ,GAAG,CAAC,MAAM,KAAK;qBAC1B,KAAK,EAAE;qBACP,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC;qBACrB,OAAO,EAAE,CAAuB,CAAC;gBAEpC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBACpC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,CAAC;oBACpD,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;wBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;oBAClD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEvB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAClC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,SAAS;QACX,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED,WAAW,CAAC,UAAkB;QAC5B,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,UAAU,CAAC;QACrC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAuB;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;QAC7D,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,GAAG,eAAe,GAAG,IAAI,CAAC;IACzD,CAAC;IAEO,SAAS;QACf,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAc,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,mCAAmC;QACrC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAoB,EAAE;IAC/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACxC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;IAE7C,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,oBAAoB,CAAC,QAAQ,EAAE,YAAY,CAAC;YAAE,SAAS;QAC3D,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/ingestion/staging.d.ts

@@ -0,0 +1,109 @@
+import { type IngestSource } from "./parsers/types.js";
+/** Files smaller than this are classified as `too_small`. */
+export declare const MIN_USEFUL_BYTES = 512;
+/** Per-source extension allowlist for preflight classification. */
+export declare const SOURCE_EXTENSIONS: Record<IngestSource, readonly string[]>;
+/**
+ * Junk markers — see module header for meanings. Stable strings; persisted
+ * into preflight.json and inspected by tests.
+ */
+export type JunkMarker = "empty" | "too_small" | "non_utf8" | "duplicate_hash" | "no_assistant_turns" | "unsupported_format";
+export interface ManifestFile {
+    name: string;
+    path: string;
+    size_bytes: number;
+}
+export interface Manifest {
+    batch_name: string;
+    created_at: string;
+    source: IngestSource;
+    path: string;
+    files: ManifestFile[];
+}
+export interface PreflightFile {
+    name: string;
+    path: string;
+    size_bytes: number;
+    extension: string;
+    supported: boolean;
+    utf8_ok: boolean;
+    content_hash: string | null;
+    duplicate_hash: boolean;
+    junk_markers: JunkMarker[];
+    notes: string[];
+}
+export interface SizeDistribution {
+    min: number;
+    max: number;
+    median: number;
+    total: number;
+}
+export interface Preflight {
+    batch_name: string;
+    source: IngestSource;
+    generated_at: string;
+    total_files: number;
+    supported_files: number;
+    unsupported_files: number;
+    size_distribution: SizeDistribution;
+    duplicate_candidates: number;
+    parseability: {
+        parseable: number;
+        unparseable: number;
+    };
+    junk_candidates: number;
+    files: PreflightFile[];
+}
+export interface StagingArtifacts {
+    manifest: Manifest;
+    preflight: Preflight;
+}
+/**
+ * Cheap-sample size for utf-8 / zip-magic checks on files that are too
+ * large to fully validate. Files at or below this size are read in full
+ * by the same code path (sample === whole file).
+ */
+export declare const PARSE_SAMPLE_BYTES: number;
+/**
+ * Largest file we will fully read into memory for structural JSON
+ * validation. Above this cap, JSON files are flagged as
+ * `unvalidated_too_large` instead of being trusted on a sniff. The cap
+ * comfortably accommodates real ChatGPT/Claude exports while protecting
+ * against pathological inputs.
+ */
+export declare const FULL_VALIDATION_CAP_BYTES: number;
+/**
+ * List every file in a batch folder, excluding our own manifest/preflight
+ * artifacts. Returns absolute paths, sorted by name for deterministic output.
+ */
+export declare function listBatchFiles(batchDir: string): string[];
+/**
+ * Build a manifest for a staged batch. Does not touch file contents.
+ */
+export declare function buildManifest(params: {
+    batchDir: string;
+    source: IngestSource;
+    now?: Date;
+}): Manifest;
+/**
+ * Run preflight classification on every file listed in the manifest.
+ * No network, no model calls. Reads up to PARSE_SAMPLE_BYTES per file.
+ */
+export declare function runPreflight(params: {
+    manifest: Manifest;
+    now?: Date;
+}): Preflight;
+/**
+ * Persist manifest and preflight as `manifest.json` and `preflight.json`
+ * inside the batch directory. Overwrites any previous artifacts.
+ */
+export declare function writeStagingArtifacts(batchDir: string, artifacts: StagingArtifacts): void;
+/**
+ * Convenience wrapper: build manifest, run preflight, persist both.
+ */
+export declare function stageBatch(params: {
+    batchDir: string;
+    source: IngestSource;
+    now?: Date;
+}): StagingArtifacts;
+//# sourceMappingURL=staging.d.ts.map