@flowselections/mailbox-orders 1.0.9 → 1.0.11

package/dist-lib/lib/mailbox-ingest.server.js

@@ -0,0 +1,928 @@
+// Server-only ingestpijplijn voor de Mailbox-bestellingen module.
+// Deze module wordt aangeroepen door:
+//   • pg_cron via het publieke endpoint /api/public/mailbox-orders/process-events
+//   • de client Realtime-subscriber via server-fn `syncMailboxNow`
+//   • de "Sync nu" knop via server-fn `syncMailboxNow`
+//
+// Alle triggers doorlopen exact dezelfde pipeline: één source of truth.
+// GEEN imports van "@/integrations/supabase/client.server" op module-scope —
+// die worden per aanroep dynamisch geladen (client-reachable modules mogen
+// server-only helpers niet direct binnenhalen).
+import { emailBodyText } from "./mailbox-body";
+import { detectLoadCarrierInText, isLoadCarrierAllowed, loadCarrierLabel, normalizeAllowedLoadCarriers, } from "./load-carriers";
+/**
+ * Ladingdragers komen live uit de Logistics Engine — nooit cachen, nooit een
+ * eigen lijst bijhouden.
+ */
+async function fetchAllowedLoadCarriers(admin) {
+    const { data } = await admin.rpc("logistics_allowed_load_carriers", {
+        p_strategy_id: undefined,
+    });
+    return normalizeAllowedLoadCarriers(data);
+}
+/**
+ * Bepaalt de ladingdrager voor een bericht: herkenning uit de mail wint,
+ * anders de standaard van de Engine. Staat de herkende drager uit, dan wordt
+ * het bericht op handmatig controleren gezet.
+ */
+async function resolveLoadCarrier(admin, messageId, aiValue) {
+    const list = await fetchAllowedLoadCarriers(admin);
+    const { data: msg } = await admin
+        .from("mailbox_messages")
+        .select("subject, body_text, body_html")
+        .eq("id", messageId)
+        .maybeSingle();
+    const text = [
+        msg?.subject ?? "",
+        emailBodyText(msg?.body_text, msg?.body_html),
+    ].join("\n");
+    const fromAi = typeof aiValue === "string" && list.carriers.some((c) => c.code === aiValue) ? aiValue : null;
+    const detected = fromAi ?? detectLoadCarrierInText(list, text);
+    if (detected && !isLoadCarrierAllowed(list, detected)) {
+        return { code: null, blocked: true, detectedLabel: loadCarrierLabel(list, detected) };
+    }
+    return { code: detected ?? list.default, blocked: false, detectedLabel: null };
+}
+function pickStr(obj, ...keys) {
+    for (const k of keys) {
+        const v = k.split(".").reduce((a, p) => (a == null ? a : a[p]), obj);
+        if (typeof v === "string" && v.length > 0)
+            return v;
+    }
+    return null;
+}
+/** Haalt het kale e-mailadres uit "Naam <adres@dom.nl>" of een los adres. */
+function normalizeEmail(raw) {
+    const s = (raw ?? "").trim();
+    const m = s.match(/<([^>]+)>/);
+    return (m ? m[1] : s).trim().toLowerCase();
+}
+async function matchCustomerByEmail(admin, fromEmail) {
+    const email = normalizeEmail(fromEmail);
+    if (!email)
+        return { customerId: null, source: null, warning: null, candidates: [] };
+    const { data: directHits } = await admin
+        .from("customers")
+        .select("id, company_name")
+        .ilike("email", email)
+        .limit(5);
+    const direct = Array.isArray(directHits) ? directHits : [];
+    if (direct.length === 1) {
+        return {
+            customerId: direct[0].id,
+            source: "customers",
+            warning: null,
+            candidates: direct.map((c) => ({ id: c.id, name: c.company_name })),
+        };
+    }
+    if (direct.length > 1) {
+        return {
+            customerId: null,
+            source: "customers",
+            warning: `Meerdere klanten met e-mailadres ${email} — kies handmatig.`,
+            candidates: direct.map((c) => ({ id: c.id, name: c.company_name })),
+        };
+    }
+    const { data: contactHits } = await admin
+        .from("customer_contacts")
+        .select("customer_id")
+        .ilike("email", email)
+        .limit(5);
+    const contacts = Array.isArray(contactHits) ? contactHits : [];
+    const uniqueCustomerIds = Array.from(new Set(contacts.map((c) => c.customer_id).filter(Boolean)));
+    if (uniqueCustomerIds.length === 0) {
+        return { customerId: null, source: null, warning: null, candidates: [] };
+    }
+    const { data: custRows } = await admin
+        .from("customers")
+        .select("id, company_name")
+        .in("id", uniqueCustomerIds);
+    const cands = (Array.isArray(custRows) ? custRows : []).map((c) => ({
+        id: c.id,
+        name: c.company_name,
+    }));
+    if (uniqueCustomerIds.length === 1) {
+        return {
+            customerId: uniqueCustomerIds[0],
+            source: "customer_contacts",
+            warning: null,
+            candidates: cands,
+        };
+    }
+    return {
+        customerId: null,
+        source: "customer_contacts",
+        warning: `Meerdere klanten gevonden via contactpersoon met e-mailadres ${email} — kies handmatig.`,
+        candidates: cands,
+    };
+}
+// Standaard veldspecificatie voor de AI wanneer er (nog) geen rijen staan in
+// mailbox_order_template_fields. Afhaallocatie/tuin en vervoerder staan hier
+// bewust NIET in: dat zijn logistieke keuzes, geen maildata.
+const DEFAULT_AI_FIELDS = [
+    {
+        key: "delivery_date",
+        label: "Gewenste leverdatum",
+        field_type: "date",
+        ai_hint: "Alleen invullen als de mail een datum of week noemt. Formaat YYYY-MM-DD.",
+    },
+    {
+        key: "reference",
+        label: "Referentie klant",
+        field_type: "text",
+        ai_hint: "Order-, referentie- of PO-nummer van de klant.",
+    },
+    {
+        key: "notes",
+        label: "Opmerking",
+        field_type: "text",
+        ai_hint: "Korte samenvatting van bijzondere wensen uit de mail.",
+    },
+    {
+        key: "delivery_location",
+        label: "Afleveradres",
+        field_type: "delivery_location",
+        ai_hint: "Kies uitsluitend een id uit de meegegeven afleverlocaties van deze klant.",
+    },
+];
+/** Centrale AI-instellingen van deze module (aanpasbaar via Instellingen). */
+export async function loadMailboxAiSettings(admin) {
+    const { data } = await admin
+        .from("mailbox_ai_settings")
+        .select("system_prompt, required_fields")
+        .limit(1)
+        .maybeSingle();
+    const req = Array.isArray(data?.required_fields) ? data.required_fields : [];
+    return {
+        systemPrompt: data?.system_prompt ?? null,
+        requiredFields: req.length > 0 ? req : ["customer", "delivery_date", "delivery_location", "lines"],
+    };
+}
+async function buildAiContext(admin, customerId) {
+    const [{ data: products }, { data: locations }, { data: tplFields }, { data: plantRows }, settings] = await Promise.all([
+        admin.from("products").select("id, product").order("product").range(0, 999),
+        customerId
+            ? admin
+                .from("customer_delivery_locations")
+                .select("id, name, city")
+                .eq("customer_id", customerId)
+                .limit(100)
+            : Promise.resolve({ data: [] }),
+        admin.from("mailbox_order_template_fields").select("*").order("sort_order", { ascending: true }),
+        admin.from("products").select("product_type").not("product_type", "is", null).range(0, 999),
+        loadMailboxAiSettings(admin),
+    ]);
+    const catalog = (Array.isArray(products) ? products : []).map((p) => ({
+        id: p.id,
+        name: p.product ?? "",
+    }));
+    const plantTypes = Array.from(new Set((Array.isArray(plantRows) ? plantRows : [])
+        .map((r) => (typeof r?.product_type === "string" ? r.product_type.trim() : ""))
+        .filter(Boolean))).sort();
+    const deliveryLocations = (Array.isArray(locations) ? locations : []).map((l) => ({
+        id: l.id,
+        name: [l.name, l.city].filter(Boolean).join(" — ") || "Locatie",
+    }));
+    const rows = (Array.isArray(tplFields) ? tplFields : []).filter((f) => f.ai_enabled !== false);
+    const templateFields = rows.length > 0
+        ? rows.map((f) => ({
+            key: f.key,
+            label: f.label ?? f.key,
+            field_type: f.field_type ?? "text",
+            ai_hint: f.ai_hint ?? null,
+            allowed: f.field_type === "product"
+                ? catalog
+                : f.field_type === "delivery_location"
+                    ? deliveryLocations
+                    : f.field_type === "plant_type"
+                        ? plantTypes.map((n) => ({ id: n, name: n }))
+                        : undefined,
+        }))
+        : DEFAULT_AI_FIELDS.map((f) => ({
+            ...f,
+            allowed: f.field_type === "delivery_location" ? deliveryLocations : undefined,
+        }));
+    return { catalog, deliveryLocations, templateFields, plantTypes, settings };
+}
+/** Voert de AI-parse uit voor één bericht. Gooit nooit — fouten komen terug in `error`. */
+async function aiParseMessage(admin, messageId, customerId) {
+    try {
+        const { data: msg } = await admin
+            .from("mailbox_messages")
+            .select("subject, body_text, body_html")
+            .eq("id", messageId)
+            .maybeSingle();
+        const bodyText = emailBodyText(msg?.body_text, msg?.body_html);
+        if (!bodyText.trim())
+            throw new Error("Bericht bevat geen tekst om te lezen.");
+        const ctx = await buildAiContext(admin, customerId);
+        const { parseOrderEmail } = await import("./mailbox-ai.server");
+        const res = await parseOrderEmail(admin, {
+            bodyText,
+            subject: msg?.subject ?? "",
+            templateFields: ctx.templateFields,
+            productCatalog: ctx.catalog,
+            deliveryLocations: ctx.deliveryLocations,
+            plantTypes: ctx.plantTypes,
+            requiredFields: ctx.settings.requiredFields,
+            systemPrompt: ctx.settings.systemPrompt,
+        });
+        return { parsed: res.parsed, meta: res.meta, error: null, transient: false, requiredFields: ctx.settings.requiredFields };
+    }
+    catch (e) {
+        const msg = String(e?.message ?? e);
+        const { isTransientAiError } = await import("./mailbox-ai.server");
+        const transient = e?.name === "AiUnavailableError" ? Boolean(e.transient) : isTransientAiError(msg);
+        return { parsed: null, meta: null, error: msg.slice(0, 500), transient, requiredFields: [] };
+    }
+}
+const FIELD_LABELS = {
+    customer: "Klant",
+    delivery_date: "Bezorgdatum",
+    delivery_location: "Afleveradres",
+    pickup_location: "Afhaallocatie / Tuin",
+    plant_type: "Plantsoort",
+    lines: "Bestelregels",
+    reference: "Referentie klant",
+    notes: "Opmerking",
+};
+/**
+ * Combineert de zelfcontrole van de AI met een harde servercheck op de
+ * verplichte velden. Ontbrekende of onzekere gegevens worden expliciet
+ * benoemd in plaats van stilzwijgend leeg gelaten.
+ */
+export function computeValidation(parsed, requiredFields, extra) {
+    const label = (k) => FIELD_LABELS[k] ?? k;
+    const missing = new Set(parsed.validation?.missing ?? []);
+    const uncertain = new Set(parsed.validation?.uncertain ?? []);
+    for (const key of requiredFields) {
+        if (key === "lines") {
+            if (parsed.lines.length === 0)
+                missing.add(label("lines"));
+            continue;
+        }
+        if (key === "customer") {
+            if (!extra.customerId && !parsed.fields?.customer?.value)
+                missing.add(label("customer"));
+            continue;
+        }
+        const f = parsed.fields?.[key];
+        const empty = f?.value === null || f?.value === undefined || f?.value === "";
+        if (empty)
+            missing.add(label(key));
+        else if (f && (f.needs_review || f.confidence < 0.8))
+            uncertain.add(label(key));
+    }
+    for (const l of parsed.lines) {
+        if (!l.product_id)
+            uncertain.add(`Artikel niet gekoppeld: ${l.raw_product_text || "onbekend"}`);
+        if (l.quantity === null || l.quantity <= 0)
+            missing.add(`Aantal ontbreekt: ${l.raw_product_text || "onbekend"}`);
+        if (!l.plant_type)
+            uncertain.add(`Plantsoort onbekend: ${l.raw_product_text || "onbekend"}`);
+    }
+    const m = Array.from(missing).slice(0, 50);
+    const u = Array.from(uncertain).slice(0, 50);
+    return {
+        missing: m,
+        uncertain: u,
+        notes: parsed.validation?.notes ?? null,
+        ok: m.length === 0 && u.length === 0,
+    };
+}
+/** Maximaal aantal achtergrondpogingen voordat we het voorstel handmatig laten. */
+export const AI_MAX_BACKGROUND_RETRIES = 6;
+function aiPayloadBlock(attempt, fingerprint, prevAi) {
+    if (attempt.parsed) {
+        return {
+            ok: true,
+            provider: attempt.meta?.providerSlug,
+            model: attempt.meta?.modelName,
+            duration_ms: attempt.meta?.durationMs,
+            fingerprint: attempt.meta?.fingerprint ?? fingerprint,
+            parsed_at: new Date().toISOString(),
+            retry_count: 0,
+            retry_pending: false,
+        };
+    }
+    const prevCount = Number(prevAi?.retry_count ?? 0);
+    const retryCount = Number.isFinite(prevCount) && prevCount > 0 ? prevCount + 1 : 1;
+    const transient = attempt.transient === true;
+    return {
+        ok: false,
+        error: attempt.error,
+        transient,
+        retry_count: retryCount,
+        retry_pending: transient && retryCount < AI_MAX_BACKGROUND_RETRIES,
+        last_attempt_at: new Date().toISOString(),
+        fingerprint,
+        parsed_at: new Date().toISOString(),
+    };
+}
+/**
+ * Vult het afleveradres aan met de hoofdafleverlocatie (is_primary) van de klant
+ * wanneer de mail/AI géén geldige locatie heeft herkend. Herkent de AI wél een
+ * locatie van deze klant, dan blijft die staan.
+ */
+export async function resolveDeliveryLocation(admin, customerId, aiValue) {
+    if (!customerId)
+        return null;
+    const { data: rows } = await admin
+        .from("customer_delivery_locations")
+        .select("id, is_primary")
+        .eq("customer_id", customerId)
+        .order("is_primary", { ascending: false })
+        .limit(100);
+    const locations = Array.isArray(rows) ? rows : [];
+    if (locations.length === 0)
+        return null;
+    if (typeof aiValue === "string" && aiValue) {
+        const hit = locations.find((l) => String(l.id) === aiValue);
+        if (hit)
+            return { id: String(hit.id), primary: false };
+    }
+    const primary = locations.find((l) => l.is_primary === true) ?? locations[0];
+    return primary ? { id: String(primary.id), primary: true } : null;
+}
+/**
+ * Afhaallocatie (eigen tuin/warehouse): AI-herkenning wint, anders de hoofdlocatie
+ * (Floriday warehouse met isDefault = true).
+ */
+export async function resolvePickupLocation(admin, aiValue) {
+    const { data: rows } = await admin
+        .from("floriday_warehouses_cache")
+        .select("floriday_id, data")
+        .eq("is_deleted", false);
+    const own = (Array.isArray(rows) ? rows : []).filter((r) => {
+        const d = r?.data ?? {};
+        return d?.hasAccess === true && d?.isExternal === false;
+    });
+    if (own.length === 0)
+        return null;
+    if (typeof aiValue === "string" && aiValue) {
+        const hit = own.find((r) => String(r.floriday_id) === aiValue);
+        if (hit)
+            return { id: String(hit.floriday_id), primary: false };
+    }
+    const main = own.find((r) => r?.data?.isDefault === true) ?? own[0];
+    return main ? { id: String(main.floriday_id), primary: true } : null;
+}
+/** Schrijft velden + regels van een AI-parse weg (na het opschonen van oude rijen). */
+async function writeParsedDetails(admin, proposalId, parsed) {
+    const fieldRows = Object.entries(parsed.fields).map(([key, v]) => ({
+        proposal_id: proposalId,
+        field_key: key,
+        value: v.value === undefined ? null : v.value,
+        ai_confidence: v.confidence,
+        ai_filled: v.value !== null && v.value !== undefined,
+        needs_review: v.needs_review,
+    }));
+    if (fieldRows.length > 0) {
+        await admin.from("mailbox_proposal_field_values").insert(fieldRows);
+    }
+    if (parsed.lines.length > 0) {
+        await admin.from("mailbox_proposal_lines").insert(parsed.lines.map((l, idx) => ({
+            proposal_id: proposalId,
+            product_id: l.product_id,
+            plant_type: l.plant_type,
+            raw_product_text: l.raw_product_text,
+            quantity: l.quantity,
+            unit: l.unit,
+            match_confidence: l.match_confidence,
+            sort_order: idx,
+        })));
+    }
+}
+/**
+ * Maakt exact één voorstel per bericht. Probeert eerst de AI-parse via de
+ * centrale AI-Authenticatie module; lukt dat niet, dan valt hij terug op een
+ * leeg voorstel met de reden erbij. Ingest mag nooit stukvallen op AI.
+ */
+export async function createStubProposal(admin, messageId, fromEmail) {
+    const { data: existing } = await admin
+        .from("mailbox_order_proposals")
+        .select("id")
+        .eq("message_id", messageId)
+        .maybeSingle();
+    if (existing?.id)
+        return;
+    // Val terug op het afzenderadres van het bericht zelf wanneer de event-payload
+    // geen adres meegaf — anders wordt de klant nooit automatisch herkend.
+    let senderEmail = normalizeEmail(fromEmail);
+    if (!senderEmail) {
+        const { data: msg } = await admin
+            .from("mailbox_messages")
+            .select("from_email")
+            .eq("id", messageId)
+            .maybeSingle();
+        senderEmail = normalizeEmail(msg?.from_email ?? null);
+    }
+    const matchResult = await matchCustomerByEmail(admin, senderEmail);
+    const basePayload = {
+        auto_matched_customer: matchResult.customerId !== null,
+        auto_match_source: matchResult.source,
+        auto_match_email: senderEmail || null,
+        auto_match_warning: matchResult.warning,
+        auto_match_candidates: matchResult.candidates,
+    };
+    // --- AI-parse (best effort) ---------------------------------------------
+    const { currentAiFingerprint } = await import("./mailbox-ai.server");
+    const fingerprint = await currentAiFingerprint(admin).catch(() => null);
+    const attempt = await aiParseMessage(admin, messageId, matchResult.customerId);
+    const parsed = attempt.parsed;
+    const fieldValue = (key) => {
+        const v = parsed?.fields?.[key]?.value;
+        return v === undefined ? null : v;
+    };
+    const deliveryDate = typeof fieldValue("delivery_date") === "string" ? fieldValue("delivery_date") : null;
+    const notesValue = typeof fieldValue("notes") === "string" ? fieldValue("notes") : null;
+    // Afleveradres: AI-herkenning wint, anders de hoofdafleverlocatie van de klant.
+    const loc = await resolveDeliveryLocation(admin, matchResult.customerId, fieldValue("delivery_location"));
+    if (loc && parsed) {
+        parsed.fields = {
+            ...parsed.fields,
+            delivery_location: loc.primary
+                ? { value: loc.id, confidence: 0.5, needs_review: true }
+                : (parsed.fields?.delivery_location ?? { value: loc.id, confidence: 0.9, needs_review: false }),
+        };
+    }
+    // Afhaallocatie: AI-herkenning wint, anders de hoofd-afhaallocatie.
+    const pick = await resolvePickupLocation(admin, fieldValue("pickup_location"));
+    if (pick && parsed) {
+        parsed.fields = {
+            ...parsed.fields,
+            pickup_location: pick.primary
+                ? { value: pick.id, confidence: 0.5, needs_review: true }
+                : (parsed.fields?.pickup_location ?? { value: pick.id, confidence: 0.9, needs_review: false }),
+        };
+    }
+    // Ladingdrager: alleen wat de Engine toestaat.
+    const lc = await resolveLoadCarrier(admin, messageId, fieldValue("load_carrier"));
+    if (lc.code && parsed) {
+        parsed.fields = {
+            ...parsed.fields,
+            load_carrier: { value: lc.code, confidence: 0.6, needs_review: false },
+        };
+    }
+    const blockedNote = lc.blocked
+        ? `Handmatig controleren - ladingdrager uitgeschakeld: "${lc.detectedLabel}" staat uit in de Rule Engine.`
+        : null;
+    const validation = parsed
+        ? computeValidation(parsed, attempt.requiredFields, { customerId: matchResult.customerId })
+        : null;
+    const { data: proposal, error } = await admin
+        .from("mailbox_order_proposals")
+        .insert({
+        message_id: messageId,
+        matched_customer_id: matchResult.customerId,
+        confidence: parsed ? parsed.overall_confidence : matchResult.customerId ? 1 : 0,
+        delivery_date: deliveryDate,
+        notes: blockedNote ? [blockedNote, notesValue].filter(Boolean).join("\n") : notesValue,
+        status: "pending",
+        ai_validation: validation,
+        parsed_payload: {
+            ...basePayload,
+            stub: parsed === null,
+            reason: parsed === null ? "ai_unavailable" : null,
+            load_carrier: lc.code,
+            load_carrier_blocked: lc.blocked,
+            load_carrier_detected: lc.detectedLabel,
+            ai: aiPayloadBlock(attempt, fingerprint),
+            fields: parsed?.fields ?? null,
+            validation,
+        },
+    })
+        .select("id")
+        .single();
+    // 23505 = unique-violation op message_id: een concurrent trigger heeft 'm
+    // net ingevoegd. Idempotent — negeren. Alle andere fouten laten we bubbelen.
+    if (error) {
+        if (error.code === "23505")
+            return;
+        throw new Error(error.message);
+    }
+    const proposalId = proposal.id;
+    if (parsed)
+        await writeParsedDetails(admin, proposalId, parsed);
+    else if (loc) {
+        await admin.from("mailbox_proposal_field_values").insert({
+            proposal_id: proposalId,
+            field_key: "delivery_location",
+            value: loc.id,
+            ai_confidence: 0.5,
+            ai_filled: false,
+            needs_review: true,
+        });
+    }
+    if (!parsed && lc.code) {
+        await admin.from("mailbox_proposal_field_values").upsert({
+            proposal_id: proposalId,
+            field_key: "load_carrier",
+            value: lc.code,
+            ai_confidence: 0.6,
+            ai_filled: false,
+            needs_review: false,
+        }, { onConflict: "proposal_id,field_key" });
+    }
+    if (lc.blocked) {
+        await admin
+            .from("mailbox_messages")
+            .update({ status: "manual_review_load_carrier_disabled" })
+            .eq("id", messageId);
+    }
+}
+/**
+ * Analyseert een bestaand (nog niet goedgekeurd) voorstel opnieuw met de
+ * huidige AI-agent. De vorige AI-analyse wordt vervangen.
+ */
+export async function reparseProposal(admin, proposalId) {
+    const { data: prop } = await admin
+        .from("mailbox_order_proposals")
+        .select("id, message_id, status, matched_customer_id, parsed_payload")
+        .eq("id", proposalId)
+        .maybeSingle();
+    if (!prop?.id)
+        return { ok: false, error: "Voorstel niet gevonden." };
+    if (prop.status !== "pending") {
+        return { ok: false, error: "Voorstel is al verwerkt — niet opnieuw geanalyseerd." };
+    }
+    // Klantherkenning opnieuw proberen op basis van het afzenderadres.
+    let matchedCustomerId = prop.matched_customer_id ?? null;
+    const { data: msgRow } = await admin
+        .from("mailbox_messages")
+        .select("from_email")
+        .eq("id", prop.message_id)
+        .maybeSingle();
+    const senderEmail = normalizeEmail(msgRow?.from_email ?? null);
+    const rematch = matchedCustomerId ? null : await matchCustomerByEmail(admin, senderEmail);
+    if (rematch?.customerId)
+        matchedCustomerId = rematch.customerId;
+    const { currentAiFingerprint } = await import("./mailbox-ai.server");
+    const fingerprint = await currentAiFingerprint(admin).catch(() => null);
+    const attempt = await aiParseMessage(admin, prop.message_id, matchedCustomerId);
+    const prevBase = (prop.parsed_payload && typeof prop.parsed_payload === "object" ? prop.parsed_payload : {});
+    const base = rematch
+        ? {
+            ...prevBase,
+            auto_matched_customer: rematch.customerId !== null,
+            auto_match_source: rematch.source,
+            auto_match_email: senderEmail || null,
+            auto_match_warning: rematch.warning,
+            auto_match_candidates: rematch.candidates,
+        }
+        : prevBase;
+    if (!attempt.parsed) {
+        // Mislukte hernieuwde analyse: bestaande gegevens laten staan, alleen de
+        // AI-status bijwerken zodat we niet in een lus blijven proberen.
+        await admin
+            .from("mailbox_order_proposals")
+            .update({ parsed_payload: { ...base, ai: aiPayloadBlock(attempt, fingerprint, prevBase?.ai) } })
+            .eq("id", proposalId);
+        return { ok: false, error: attempt.error };
+    }
+    const parsed = attempt.parsed;
+    const fv = (key) => {
+        const v = parsed.fields?.[key]?.value;
+        return v === undefined ? null : v;
+    };
+    // Afleveradres: AI-herkenning wint, anders de hoofdafleverlocatie van de klant.
+    const rloc = await resolveDeliveryLocation(admin, matchedCustomerId, fv("delivery_location"));
+    if (rloc) {
+        parsed.fields = {
+            ...parsed.fields,
+            delivery_location: rloc.primary
+                ? { value: rloc.id, confidence: 0.5, needs_review: true }
+                : (parsed.fields?.delivery_location ?? { value: rloc.id, confidence: 0.9, needs_review: false }),
+        };
+    }
+    // Afhaallocatie: AI-herkenning wint, anders de hoofd-afhaallocatie.
+    const rpick = await resolvePickupLocation(admin, fv("pickup_location"));
+    if (rpick) {
+        parsed.fields = {
+            ...parsed.fields,
+            pickup_location: rpick.primary
+                ? { value: rpick.id, confidence: 0.5, needs_review: true }
+                : (parsed.fields?.pickup_location ?? { value: rpick.id, confidence: 0.9, needs_review: false }),
+        };
+    }
+    // Ladingdrager: alleen wat de Engine toestaat.
+    const rlc = await resolveLoadCarrier(admin, prop.message_id, fv("load_carrier"));
+    if (rlc.code) {
+        parsed.fields = {
+            ...parsed.fields,
+            load_carrier: { value: rlc.code, confidence: 0.6, needs_review: false },
+        };
+    }
+    else if (parsed.fields) {
+        delete parsed.fields.load_carrier;
+    }
+    const rBlockedNote = rlc.blocked
+        ? `Handmatig controleren - ladingdrager uitgeschakeld: "${rlc.detectedLabel}" staat uit in de Rule Engine.`
+        : null;
+    await admin.from("mailbox_proposal_lines").delete().eq("proposal_id", proposalId);
+    await admin.from("mailbox_proposal_field_values").delete().eq("proposal_id", proposalId);
+    const rvalidation = computeValidation(parsed, attempt.requiredFields, {
+        customerId: matchedCustomerId,
+    });
+    const notesFromAi = typeof fv("notes") === "string" ? fv("notes") : null;
+    const { error: updErr } = await admin
+        .from("mailbox_order_proposals")
+        .update({
+        matched_customer_id: matchedCustomerId,
+        confidence: parsed.overall_confidence,
+        delivery_date: typeof fv("delivery_date") === "string" ? fv("delivery_date") : null,
+        notes: rBlockedNote ? [rBlockedNote, notesFromAi].filter(Boolean).join("\n") : notesFromAi,
+        ai_validation: rvalidation,
+        parsed_payload: {
+            ...base,
+            stub: false,
+            reason: null,
+            reparsed: true,
+            load_carrier: rlc.code,
+            load_carrier_blocked: rlc.blocked,
+            load_carrier_detected: rlc.detectedLabel,
+            ai: aiPayloadBlock(attempt, fingerprint),
+            fields: parsed.fields,
+            validation: rvalidation,
+        },
+    })
+        .eq("id", proposalId);
+    if (updErr)
+        return { ok: false, error: updErr.message };
+    if (rlc.blocked) {
+        await admin
+            .from("mailbox_messages")
+            .update({ status: "manual_review_load_carrier_disabled" })
+            .eq("id", prop.message_id);
+    }
+    await writeParsedDetails(admin, proposalId, parsed);
+    return { ok: true, error: null };
+}
+/**
+ * Detecteert een gewijzigde AI-agent: alle openstaande voorstellen waarvan de
+ * opgeslagen vingerafdruk afwijkt van de huidige worden opnieuw geanalyseerd.
+ */
+export async function reparseChangedAgentProposals(admin, limit = 25) {
+    const { currentAiFingerprint } = await import("./mailbox-ai.server");
+    const fingerprint = await currentAiFingerprint(admin).catch(() => null);
+    if (!fingerprint)
+        return { checked: 0, reparsed: 0, failed: 0, fingerprint: null };
+    const { data: rows } = await admin
+        .from("mailbox_order_proposals")
+        .select("id, parsed_payload")
+        .eq("status", "pending")
+        .order("created_at", { ascending: false })
+        .limit(200);
+    const list = Array.isArray(rows) ? rows : [];
+    let reparsed = 0;
+    let failed = 0;
+    let checked = 0;
+    for (const r of list) {
+        const stored = r.parsed_payload?.ai?.fingerprint ?? null;
+        if (stored === fingerprint)
+            continue;
+        checked++;
+        if (reparsed + failed >= limit)
+            break;
+        const res = await reparseProposal(admin, r.id);
+        if (res.ok)
+            reparsed++;
+        else
+            failed++;
+    }
+    return { checked, reparsed, failed, fingerprint };
+}
+/**
+ * Achtergrondherstel: voorstellen waarvan de AI-analyse door een tijdelijke
+ * storing (bijv. model overbelast) mislukte, worden bij een volgende run
+ * automatisch opnieuw geanalyseerd — tot AI_MAX_BACKGROUND_RETRIES pogingen.
+ * Blijft strikt idempotent: alleen 'pending' voorstellen worden aangeraakt.
+ */
+export async function retryFailedAiProposals(admin, limit = 10) {
+    const { data: rows } = await admin
+        .from("mailbox_order_proposals")
+        .select("id, parsed_payload")
+        .eq("status", "pending")
+        .order("created_at", { ascending: false })
+        .limit(200);
+    const list = Array.isArray(rows) ? rows : [];
+    let retried = 0;
+    let recovered = 0;
+    let stillFailing = 0;
+    for (const r of list) {
+        const ai = r.parsed_payload?.ai ?? null;
+        if (!ai || ai.ok !== false)
+            continue;
+        if (ai.retry_pending !== true)
+            continue;
+        if (Number(ai.retry_count ?? 0) >= AI_MAX_BACKGROUND_RETRIES)
+            continue;
+        if (retried >= limit)
+            break;
+        retried++;
+        try {
+            const res = await reparseProposal(admin, r.id);
+            if (res.ok)
+                recovered++;
+            else
+                stillFailing++;
+        }
+        catch {
+            stillFailing++;
+        }
+    }
+    return { retried, recovered, stillFailing };
+}
+async function ingestSingleEvent(admin, connectionId, ev) {
+    const p = ev.payload ?? {};
+    const externalId = ev.external_id ?? pickStr(p, "message_id", "messageId", "id") ?? ev.id;
+    // Idempotent: bestaat er al een message voor deze external_id op deze connectie?
+    if (externalId) {
+        const { data: existing } = await admin
+            .from("mailbox_messages")
+            .select("id")
+            .eq("connection_id", connectionId)
+            .eq("external_id", externalId)
+            .maybeSingle();
+        if (existing?.id) {
+            // Zorg dat er in elk geval een voorstel is (herstel oude berichten zonder stub).
+            await createStubProposal(admin, existing.id, pickStr(p, "from_email", "fromEmail", "from.address"));
+            return { inserted: false, messageId: existing.id };
+        }
+    }
+    const fromEmail = pickStr(p, "from_email", "fromEmail", "from.address", "from.email", "sender.address") ?? "";
+    const fromName = pickStr(p, "from_name", "fromName", "from.name", "sender.name");
+    const subject = pickStr(p, "subject", "title") ?? "";
+    const bodyText = pickStr(p, "body_text", "bodyText", "text", "body.text", "snippet");
+    const bodyHtml = pickStr(p, "body_html", "bodyHtml", "html", "body.html");
+    const receivedAt = pickStr(p, "received_at", "receivedAt", "date", "internalDate") ??
+        ev.received_at ??
+        new Date().toISOString();
+    const { data: inserted, error: insErr } = await admin
+        .from("mailbox_messages")
+        .insert({
+        external_id: externalId,
+        message_id: pickStr(p, "message_id", "messageId") ?? externalId,
+        connection_id: connectionId,
+        from_email: fromEmail,
+        from_name: fromName,
+        subject,
+        body_text: bodyText,
+        body_html: bodyHtml,
+        received_at: receivedAt,
+        status: "new",
+    })
+        .select("id")
+        .single();
+    if (insErr) {
+        // Race met cron/realtime — de andere trigger heeft 'm net ingevoegd.
+        if (insErr.code === "23505") {
+            const { data: dup } = await admin
+                .from("mailbox_messages")
+                .select("id")
+                .eq("connection_id", connectionId)
+                .eq("external_id", externalId)
+                .maybeSingle();
+            if (dup?.id)
+                await createStubProposal(admin, dup.id, fromEmail);
+            return { inserted: false, messageId: dup?.id ?? null };
+        }
+        throw new Error(insErr.message);
+    }
+    await createStubProposal(admin, inserted.id, fromEmail);
+    return { inserted: true, messageId: inserted.id };
+}
+// De volledige pipeline. Alle triggers roepen dit aan.
+export async function runIngest(trigger) {
+    const { supabaseAdmin } = await import("../integrations/supabase/client.server");
+    // 1. Actieve orders/receive connectie ophalen (service_role → geen bearer nodig).
+    const { data: active, error: activeErr } = await supabaseAdmin.rpc("email_auth_for_purpose", {
+        _purpose: "orders",
+        _direction: "receive",
+    });
+    if (activeErr)
+        throw new Error(`email_auth_for_purpose: ${activeErr.message}`);
+    const row = Array.isArray(active) ? active[0] : null;
+    const connectionId = row?.id ?? null;
+    if (!connectionId) {
+        return {
+            ok: true,
+            runId: null,
+            processed: 0,
+            inserted: 0,
+            errors: 0,
+            errorSample: null,
+            connectionId: null,
+            skipped: "no_active_connection",
+        };
+    }
+    // 2. Open run-record voor observability.
+    const { data: runRow } = await supabaseAdmin
+        .from("mailbox_ingest_runs")
+        .insert({ trigger, connection_id: connectionId })
+        .select("id")
+        .single();
+    const runId = runRow?.id ?? null;
+    let processed = 0;
+    let inserted = 0;
+    let errors = 0;
+    let errorSample = null;
+    const processedEventIds = [];
+    try {
+        // 3. Onverwerkte events voor deze connectie ophalen (max 200 per run).
+        const { data: events, error: evErr } = await supabaseAdmin
+            .from("email_auth_events")
+            .select("id, connection_id, external_id, payload, received_at")
+            .eq("connection_id", connectionId)
+            .is("consumed_at", null)
+            .order("received_at", { ascending: true })
+            .limit(200);
+        if (evErr)
+            throw new Error(`select email_auth_events: ${evErr.message}`);
+        for (const ev of (events ?? [])) {
+            processed++;
+            try {
+                const res = await ingestSingleEvent(supabaseAdmin, connectionId, {
+                    id: ev.id,
+                    external_id: ev.external_id ?? null,
+                    payload: ev.payload ?? {},
+                    received_at: ev.received_at ?? null,
+                });
+                if (res.inserted)
+                    inserted++;
+                processedEventIds.push(ev.id);
+            }
+            catch (e) {
+                errors++;
+                if (!errorSample)
+                    errorSample = String(e?.message ?? e).slice(0, 500);
+                // We laten dit event onverwerkt zodat de volgende run 'm opnieuw probeert.
+            }
+        }
+        // 4. Markeer succesvolle events als consumed via directe update (service_role).
+        if (processedEventIds.length > 0) {
+            const { error: markErr } = await supabaseAdmin
+                .from("email_auth_events")
+                .update({ consumed_at: new Date().toISOString() })
+                .in("id", processedEventIds);
+            if (markErr) {
+                errors++;
+                if (!errorSample)
+                    errorSample = `mark consumed: ${markErr.message}`.slice(0, 500);
+            }
+        }
+        // 5. Backfill: bestaande messages zonder proposal krijgen alsnog een stub.
+        const { data: orphans } = await supabaseAdmin
+            .from("mailbox_messages")
+            .select("id, from_email, mailbox_order_proposals(id)")
+            .eq("connection_id", connectionId)
+            .order("received_at", { ascending: false })
+            .limit(200);
+        for (const m of (orphans ?? [])) {
+            const props = Array.isArray(m.mailbox_order_proposals) ? m.mailbox_order_proposals : [];
+            if (props.length > 0)
+                continue;
+            try {
+                await createStubProposal(supabaseAdmin, m.id, m.from_email ?? null);
+            }
+            catch (e) {
+                errors++;
+                if (!errorSample)
+                    errorSample = `orphan-stub: ${String(e?.message ?? e)}`.slice(0, 500);
+            }
+        }
+        // 6. Gewijzigde AI-agent? Openstaande voorstellen opnieuw analyseren.
+        try {
+            await reparseChangedAgentProposals(supabaseAdmin);
+        }
+        catch (e) {
+            errors++;
+            if (!errorSample)
+                errorSample = `reparse: ${String(e?.message ?? e)}`.slice(0, 500);
+        }
+        // 7. Herstelronde: mislukte AI-analyses door tijdelijke storingen opnieuw proberen.
+        try {
+            await retryFailedAiProposals(supabaseAdmin);
+        }
+        catch (e) {
+            errors++;
+            if (!errorSample)
+                errorSample = `reparse: ${String(e?.message ?? e)}`.slice(0, 500);
+        }
+    }
+    finally {
+        if (runId) {
+            await supabaseAdmin
+                .from("mailbox_ingest_runs")
+                .update({
+                finished_at: new Date().toISOString(),
+                processed,
+                inserted,
+                errors,
+                error_sample: errorSample,
+            })
+                .eq("id", runId);
+        }
+    }
+    return { ok: true, runId, processed, inserted, errors, errorSample, connectionId };
+}