@iann29/rastro 0.9.0 → 0.10.0

package/dist/component/metaCapi.js

@@ -0,0 +1,754 @@
+import { v } from "convex/values";
+import { internal } from "./_generated/api.js";
+import { env, internalAction, internalMutation, } from "./_generated/server.js";
+import { DAY_MS } from "./constants.js";
+import { metaGraph, MetaGraphError, } from "./metaGraph.js";
+import { decryptSecret } from "./secrets.js";
+import { metaConversionEventNameValidator, metaUserDataValidator, } from "./validators.js";
+/** Meta refuses events older than this; the sender gives up first. */
+export const MAX_EVENT_AGE_MS = 7 * DAY_MS;
+export const MAX_SEND_ATTEMPTS = 3;
+/** The wait before the second and the third attempt. */
+export const RETRY_DELAYS_MS = [60_000, 10 * 60_000];
+/**
+ * A pending row whose attempt was due this long ago was lost (an action
+ * outlives neither this nor the longest retry wait): the sweep sends it again.
+ */
+export const STALL_MS = 15 * 60_000;
+const SWEEP_BATCH = 25;
+const MAX_PHONES = 10;
+const MAX_LEAD_CONVERSIONS = 20;
+const SHA256_HEX = /^[a-f0-9]{64}$/;
+export async function sha256Hex(value) {
+    const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value));
+    return [...new Uint8Array(digest)]
+        .map((byte) => byte.toString(16).padStart(2, "0"))
+        .join("");
+}
+/** SHA-256 of the empty string: a hash of nothing, which matches nobody. */
+const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+const digestOf = (value) => {
+    if (typeof value !== "string")
+        return null;
+    const hash = value.trim().toLowerCase();
+    return SHA256_HEX.test(hash) ? hash : null;
+};
+function ignored() {
+    // Never the value itself: it may be a plain phone number.
+    console.warn("[rastro] meta ignored: expected { ph: SHA-256 hex digests, externalId?: a SHA-256 hex digest }");
+    return null;
+}
+/**
+ * The host's hashes, checked, or null when there is nothing to send with.
+ * Never throws and never echoes what it got: anything but SHA-256 hex digests
+ * in the documented shape (a plain phone number, a wrong type, more than
+ * MAX_PHONES phones) drops the whole `meta`, so the lead's own write goes on
+ * without the Conversions API. The hash of an empty string is left out.
+ */
+export function parseMetaUserData(value) {
+    if (value === undefined || value === null)
+        return null;
+    if (typeof value !== "object" || Array.isArray(value))
+        return ignored();
+    const fields = value;
+    const phones = fields.ph ?? [];
+    if (!Array.isArray(phones) || phones.length > MAX_PHONES)
+        return ignored();
+    const ph = [];
+    for (const item of phones) {
+        const hash = digestOf(item);
+        if (hash === null)
+            return ignored();
+        if (hash !== EMPTY_SHA256 && !ph.includes(hash))
+            ph.push(hash);
+    }
+    let externalId;
+    if (fields.externalId !== undefined) {
+        const hash = digestOf(fields.externalId);
+        if (hash === null)
+            return ignored();
+        if (hash !== EMPTY_SHA256)
+            externalId = hash;
+    }
+    if (ph.length === 0 && externalId === undefined)
+        return null;
+    return { ph, ...(externalId ? { externalId } : {}) };
+}
+// ---------------------------------------------------------------------------
+// The pixel and its token.
+/**
+ * The site's pixel when it may send: enabled, and its token not refused (its
+ * own, or its connection's, which must still list the pixel: a pixel marked
+ * `PIXEL_NOT_GRANTED` queues nothing). `meta.loadPixelToken` resolves the
+ * token itself, by the same rules.
+ */
+async function sendingPixel(ctx, siteId) {
+    const pixel = await ctx.db
+        .query("sitePixels")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique();
+    if (!pixel?.enabled)
+        return null;
+    if (pixel.source === "connection") {
+        const connection = pixel.connectionId
+            ? await ctx.db.get("metaConnections", pixel.connectionId)
+            : null;
+        if (connection?.status !== "active")
+            return null;
+        return connection.pixels?.some((row) => row.pixelId === pixel.pixelId)
+            ? pixel
+            : null;
+    }
+    return pixel.invalidSince === undefined ? pixel : null;
+}
+/** Meta refused the pixel's own token: the pixel stops until set again. */
+async function refusePixelToken(ctx, pixel, message) {
+    await ctx.db.patch("sitePixels", pixel._id, {
+        invalidSince: Date.now(),
+        lastError: message.slice(0, 300),
+    });
+}
+// ---------------------------------------------------------------------------
+// Counts.
+/** The count documents per site, so concurrent writes rarely meet. */
+export const COUNT_SHARDS = 8;
+/** An event's count shard: fixed by its id, so every write of it agrees. */
+const shardOf = (eventId) => Number.parseInt(eventId.slice(0, 8), 16) % COUNT_SHARDS;
+/**
+ * Moves an event's counts; returns the pending count its shard had before,
+ * which tells `enqueue` whether the sweep may be idle.
+ */
+async function bumpCounts(ctx, row, delta) {
+    const shard = shardOf(row.eventId);
+    const doc = await ctx.db
+        .query("metaConversionCounts")
+        .withIndex("by_siteId_and_shard", (range) => range.eq("siteId", row.siteId).eq("shard", shard))
+        .unique();
+    const next = {
+        pending: (doc?.pending ?? 0) + (delta.pending ?? 0),
+        sent: (doc?.sent ?? 0) + (delta.sent ?? 0),
+        failed: (doc?.failed ?? 0) + (delta.failed ?? 0),
+    };
+    if (doc)
+        await ctx.db.patch("metaConversionCounts", doc._id, next);
+    else {
+        await ctx.db.insert("metaConversionCounts", {
+            siteId: row.siteId,
+            shard,
+            ...next,
+        });
+    }
+    return doc?.pending ?? 0;
+}
+/** The days `last30d` covers: today (UTC) and the 29 before. */
+export const SENT_WINDOW_DAYS = 30;
+const utcDay = (time) => Math.floor(time / DAY_MS) * DAY_MS;
+/** Moves the sent count of an event's UTC day, in its shard. */
+async function bumpDaily(ctx, row, sentAt, delta) {
+    const shard = shardOf(row.eventId);
+    const dayStart = utcDay(sentAt);
+    const doc = await ctx.db
+        .query("metaConversionDaily")
+        .withIndex("by_siteId_and_dayStart_and_shard", (range) => range
+        .eq("siteId", row.siteId)
+        .eq("dayStart", dayStart)
+        .eq("shard", shard))
+        .unique();
+    if (doc) {
+        await ctx.db.patch("metaConversionDaily", doc._id, {
+            [row.eventName]: Math.max(doc[row.eventName] + delta, 0),
+        });
+    }
+    else if (delta > 0) {
+        await ctx.db.insert("metaConversionDaily", {
+            siteId: row.siteId,
+            dayStart,
+            shard,
+            Purchase: 0,
+            QualifiedLead: 0,
+            Lead: 0,
+            Schedule: 0,
+            [row.eventName]: 1,
+        });
+    }
+}
+/**
+ * The site's queue by status and, given the reader's `now`, what Meta
+ * accepted in the last 30 UTC days by event and today. Without `now` (a
+ * query never reads the clock) those two are null.
+ */
+export async function conversionCounts(ctx, siteId, now) {
+    const shards = await ctx.db
+        .query("metaConversionCounts")
+        .withIndex("by_siteId_and_shard", (range) => range.eq("siteId", siteId))
+        .take(COUNT_SHARDS);
+    const total = { pending: 0, sent: 0, failed: 0 };
+    for (const shard of shards) {
+        total.pending += shard.pending;
+        total.sent += shard.sent;
+        total.failed += shard.failed;
+    }
+    if (now === undefined)
+        return { ...total, last30d: null, sentToday: null };
+    const today = utcDay(now);
+    const days = await ctx.db
+        .query("metaConversionDaily")
+        .withIndex("by_siteId_and_dayStart_and_shard", (range) => range
+        .eq("siteId", siteId)
+        .gte("dayStart", today - (SENT_WINDOW_DAYS - 1) * DAY_MS)
+        .lte("dayStart", today))
+        .take(SENT_WINDOW_DAYS * COUNT_SHARDS);
+    const last30d = {
+        Purchase: 0,
+        QualifiedLead: 0,
+        Lead: 0,
+        Schedule: 0,
+    };
+    let sentToday = 0;
+    for (const day of days) {
+        last30d.Purchase += day.Purchase;
+        last30d.QualifiedLead += day.QualifiedLead;
+        last30d.Lead += day.Lead;
+        last30d.Schedule += day.Schedule;
+        if (day.dayStart === today) {
+            sentToday += day.Purchase + day.QualifiedLead + day.Lead + day.Schedule;
+        }
+    }
+    return { ...total, last30d, sentToday };
+}
+// ---------------------------------------------------------------------------
+// Enqueueing.
+async function enqueue(ctx, args) {
+    const userData = parseMetaUserData(args.meta);
+    if (!userData)
+        return false;
+    if (!(await sendingPixel(ctx, args.siteId)))
+        return false;
+    const now = Date.now();
+    const eventId = await sha256Hex(`${args.leadId}:${args.eventName}`);
+    const fields = {
+        eventTime: args.eventTime,
+        ...(args.value ?? {}),
+        userData,
+        status: "pending",
+        attempts: 0,
+        nextAttemptAt: now,
+    };
+    const seen = await ctx.db
+        .query("metaConversionOutbox")
+        .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", eventId))
+        .unique();
+    // Queued or delivered: once per lead per event. A failed one is not
+    // delivered, so the host's next call with hashes opens it again.
+    if (seen && seen.status !== "failed")
+        return false;
+    let outboxId;
+    let pendingBefore;
+    if (seen) {
+        outboxId = seen._id;
+        await ctx.db.patch("metaConversionOutbox", seen._id, {
+            ...fields,
+            attemptStartedAt: undefined,
+            lastError: undefined,
+            fbtraceId: undefined,
+        });
+        pendingBefore = await bumpCounts(ctx, seen, { failed: -1, pending: 1 });
+    }
+    else {
+        outboxId = await ctx.db.insert("metaConversionOutbox", {
+            siteId: args.siteId,
+            leadId: args.leadId,
+            eventId,
+            eventName: args.eventName,
+            ...fields,
+            createdAt: now,
+        });
+        pendingBefore = await bumpCounts(ctx, { siteId: args.siteId, eventId }, { pending: 1 });
+    }
+    await ctx.scheduler.runAfter(0, internal.metaCapi.send, { outboxId });
+    // A shard with pending rows already has the sweep running for them (it
+    // stops only when none is pending), so only an idle shard reads its state.
+    if (pendingBefore === 0)
+        await kickSweep(ctx, now + STALL_MS + 1_000);
+    return true;
+}
+/**
+ * Queues the event for a lead that entered a column the site's event map
+ * points at. Once per lead per event: a lead that comes back to the column,
+ * or reached the event another way, is not sent again. Without `meta` or an
+ * active pixel nothing is queued. Returns whether a row was queued.
+ */
+export async function enqueueColumnEvent(ctx, args) {
+    return await enqueue(ctx, {
+        siteId: args.siteId,
+        leadId: args.leadId,
+        eventName: args.eventName,
+        eventTime: args.at,
+        meta: args.meta,
+    });
+}
+/** `Lead` for a lead just recorded, when the pixel sends new leads. */
+export async function enqueueNewLead(ctx, args) {
+    if (args.meta === undefined)
+        return false;
+    const pixel = await sendingPixel(ctx, args.siteId);
+    if (!pixel?.sendLeadEvent)
+        return false;
+    return await enqueue(ctx, { ...args, eventName: "Lead" });
+}
+/** `Purchase` for a sale the host recorded, with its value. */
+export async function enqueuePurchase(ctx, args) {
+    // A sale without a value tells Meta nothing to optimize for.
+    if (args.valueCents <= 0)
+        return false;
+    return await enqueue(ctx, {
+        siteId: args.siteId,
+        leadId: args.leadId,
+        eventName: "Purchase",
+        eventTime: args.eventTime,
+        meta: args.meta,
+        value: { valueCents: args.valueCents, currency: args.currency },
+    });
+}
+/** Forgets a removed lead's conversions, hashes and history alike. */
+export async function removeLeadConversions(ctx, lead) {
+    const rows = await ctx.db
+        .query("metaConversionOutbox")
+        .withIndex("by_leadId", (range) => range.eq("leadId", lead._id))
+        .take(MAX_LEAD_CONVERSIONS);
+    for (const row of rows) {
+        await ctx.db.delete("metaConversionOutbox", row._id);
+        await bumpCounts(ctx, row, { [row.status]: -1 });
+        if (row.status === "sent" && row.sentAt !== undefined) {
+            await bumpDaily(ctx, row, row.sentAt, -1);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Reads.
+/**
+ * The site's conversions, newest first, with the lead each one reports. The
+ * cursor is the last row read: its `createdAt` (`before`) and, since one call
+ * can queue several events in the same millisecond, its `eventId`
+ * (`beforeEventId`): the page goes on right after that row. With `before`
+ * alone, the page starts strictly before that time.
+ */
+export async function listConversions(ctx, args) {
+    const { before, beforeEventId } = args;
+    const query = ctx.db
+        .query("metaConversionOutbox")
+        .withIndex("by_siteId_and_createdAt", (range) => before === undefined
+        ? range.eq("siteId", args.siteId)
+        : beforeEventId === undefined
+            ? range.eq("siteId", args.siteId).lt("createdAt", before)
+            : range.eq("siteId", args.siteId).lte("createdAt", before))
+        .order("desc");
+    const rows = [];
+    // Rows tied with the cursor's time come in a fixed order (the index ends
+    // with the creation time): skip them up to the cursor row itself.
+    let skipping = before !== undefined && beforeEventId !== undefined;
+    for await (const row of query) {
+        if (skipping) {
+            if (row.createdAt === before) {
+                if (row.eventId === beforeEventId)
+                    skipping = false;
+                continue;
+            }
+            skipping = false;
+        }
+        rows.push(row);
+        if (rows.length >= args.limit)
+            break;
+    }
+    const out = [];
+    for (const row of rows) {
+        const lead = await ctx.db.get("leads", row.leadId);
+        const adId = lead?.ad?.adId ?? null;
+        const ad = adId === null
+            ? null
+            : await ctx.db
+                .query("adCatalog")
+                .withIndex("by_siteId_and_adId", (range) => range.eq("siteId", row.siteId).eq("adId", adId))
+                .unique();
+        out.push({
+            eventId: row.eventId,
+            eventName: row.eventName,
+            leadId: row.leadId,
+            leadLabel: lead?.label ?? null,
+            adId,
+            adName: ad?.adName ?? null,
+            valueCents: row.valueCents ?? null,
+            status: row.status,
+            createdAt: row.createdAt,
+            sentAt: row.sentAt ?? null,
+        });
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+// Sending.
+/** The Conversions API event for one outbox row. */
+export async function conversionEvent(row, userData) {
+    return {
+        event_name: row.eventName,
+        event_time: Math.floor(row.eventTime / 1000),
+        event_id: row.eventId,
+        action_source: "chat",
+        user_data: {
+            ...(userData.ph.length > 0 ? { ph: userData.ph } : {}),
+            ...(userData.externalId ? { external_id: [userData.externalId] } : {}),
+            // Every site Rastro serves today sells in Brazil.
+            country: [await sha256Hex("br")],
+        },
+        ...(row.eventName === "Purchase" &&
+            row.valueCents !== undefined &&
+            row.currency !== undefined
+            ? {
+                custom_data: {
+                    value: row.valueCents / 100,
+                    currency: row.currency,
+                },
+            }
+            : {}),
+    };
+}
+/**
+ * Takes one attempt at a pending row: stamps its lease (`attemptStartedAt`,
+ * so the sweep leaves a send that may still be in flight alone) and returns
+ * what to send, or why the row is dropped.
+ */
+export const claimSend = internalMutation({
+    args: { outboxId: v.id("metaConversionOutbox"), now: v.number() },
+    returns: v.union(v.null(), v.object({ kind: v.literal("drop"), error: v.string() }), v.object({
+        kind: v.literal("send"),
+        siteId: v.id("sites"),
+        attempts: v.number(),
+        testEventCode: v.union(v.string(), v.null()),
+        event: v.object({
+            eventId: v.string(),
+            eventName: metaConversionEventNameValidator,
+            eventTime: v.number(),
+            valueCents: v.optional(v.number()),
+            currency: v.optional(v.string()),
+        }),
+        userData: metaUserDataValidator,
+    })),
+    handler: async (ctx, args) => {
+        const row = await ctx.db.get("metaConversionOutbox", args.outboxId);
+        if (row?.status !== "pending")
+            return null;
+        await ctx.db.patch("metaConversionOutbox", row._id, {
+            attemptStartedAt: args.now,
+        });
+        if (!row.userData)
+            return { kind: "drop", error: "NO_USER_DATA" };
+        if (args.now - row.eventTime > MAX_EVENT_AGE_MS) {
+            return { kind: "drop", error: "EVENT_TOO_OLD" };
+        }
+        const pixel = await sendingPixel(ctx, row.siteId);
+        if (!pixel)
+            return { kind: "drop", error: "PIXEL_UNAVAILABLE" };
+        return {
+            kind: "send",
+            siteId: row.siteId,
+            attempts: row.attempts,
+            testEventCode: pixel.testEventCode ?? null,
+            event: {
+                eventId: row.eventId,
+                eventName: row.eventName,
+                eventTime: row.eventTime,
+                ...(row.valueCents !== undefined ? { valueCents: row.valueCents } : {}),
+                ...(row.currency !== undefined ? { currency: row.currency } : {}),
+            },
+            userData: row.userData,
+        };
+    },
+});
+/** Ends a row for good, sent or failed; either way its hashes go. */
+async function closeRow(ctx, row, status, fields = {}) {
+    const now = Date.now();
+    await ctx.db.patch("metaConversionOutbox", row._id, {
+        status,
+        attempts: row.attempts + 1,
+        userData: undefined,
+        lastError: fields.error?.slice(0, 300),
+        ...(fields.fbtraceId ? { fbtraceId: fields.fbtraceId.slice(0, 100) } : {}),
+        ...(status === "sent" ? { sentAt: now } : {}),
+    });
+    await bumpCounts(ctx, row, { pending: -1, [status]: 1 });
+    if (status === "sent")
+        await bumpDaily(ctx, row, now, 1);
+}
+/**
+ * Counts a failed attempt: the next one after the backoff, or, when it was
+ * the last, the row fails for good.
+ */
+async function failAttempt(ctx, row, error) {
+    if (row.attempts + 1 >= MAX_SEND_ATTEMPTS) {
+        await closeRow(ctx, row, "failed", { error });
+        return;
+    }
+    const attempts = row.attempts + 1;
+    const delay = RETRY_DELAYS_MS[Math.min(attempts, RETRY_DELAYS_MS.length) - 1];
+    await ctx.db.patch("metaConversionOutbox", row._id, {
+        attempts,
+        nextAttemptAt: Date.now() + delay,
+        lastError: error.slice(0, 300),
+    });
+    await ctx.scheduler.runAfter(delay, internal.metaCapi.send, {
+        outboxId: row._id,
+    });
+}
+/**
+ * Closes a row: sent, or failed for good. When Meta refused the token, the
+ * pixel that sent stops too.
+ */
+export const finishSend = internalMutation({
+    args: {
+        outboxId: v.id("metaConversionOutbox"),
+        status: v.union(v.literal("sent"), v.literal("failed")),
+        error: v.optional(v.string()),
+        fbtraceId: v.optional(v.string()),
+        /** The pixel whose own token Meta refused, and the token's IV. */
+        refusedPixel: v.optional(v.object({ pixelId: v.string(), tokenIv: v.string() })),
+    },
+    returns: v.null(),
+    handler: async (ctx, args) => {
+        const row = await ctx.db.get("metaConversionOutbox", args.outboxId);
+        if (row?.status !== "pending")
+            return null;
+        await closeRow(ctx, row, args.status, {
+            ...(args.error !== undefined ? { error: args.error } : {}),
+            ...(args.fbtraceId !== undefined ? { fbtraceId: args.fbtraceId } : {}),
+        });
+        if (args.refusedPixel) {
+            const pixel = await sendingPixel(ctx, row.siteId);
+            // Only the token that was refused: one stored meanwhile keeps sending.
+            if (pixel?.pixelId === args.refusedPixel.pixelId &&
+                pixel.source !== "connection" &&
+                pixel.tokenIv === args.refusedPixel.tokenIv) {
+                await refusePixelToken(ctx, pixel, args.error ?? "token refused");
+            }
+        }
+        return null;
+    },
+});
+/** Records a failed attempt: retried after the backoff, or failed for good. */
+export const retrySend = internalMutation({
+    args: { outboxId: v.id("metaConversionOutbox"), error: v.string() },
+    returns: v.null(),
+    handler: async (ctx, args) => {
+        const row = await ctx.db.get("metaConversionOutbox", args.outboxId);
+        if (row?.status !== "pending")
+            return null;
+        await failAttempt(ctx, row, args.error);
+        return null;
+    },
+});
+const graph = () => metaGraph({ fake: env.RASTRO_META_FAKE === "true" });
+/**
+ * One attempt at one outbox row. Whatever goes wrong counts as an attempt:
+ * Meta's answers by their kind, anything else as `UNEXPECTED`.
+ */
+export const send = internalAction({
+    args: { outboxId: v.id("metaConversionOutbox") },
+    returns: v.null(),
+    handler: async (ctx, args) => {
+        try {
+            await attempt(ctx, args.outboxId);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            await ctx.runMutation(internal.metaCapi.retrySend, {
+                outboxId: args.outboxId,
+                error: `UNEXPECTED: ${message}`,
+            });
+        }
+        return null;
+    },
+});
+async function attempt(ctx, outboxId) {
+    const job = await ctx.runMutation(internal.metaCapi.claimSend, {
+        outboxId,
+        now: Date.now(),
+    });
+    if (!job)
+        return;
+    const close = (status, extra) => ctx.runMutation(internal.metaCapi.finishSend, {
+        outboxId,
+        status,
+        ...extra,
+    });
+    if (job.kind === "drop") {
+        await close("failed", { error: job.error });
+        return;
+    }
+    const sealed = await ctx.runQuery(internal.meta.loadPixelToken, {
+        siteId: job.siteId,
+    });
+    if (!sealed) {
+        await close("failed", { error: "PIXEL_UNAVAILABLE" });
+        return;
+    }
+    let token;
+    try {
+        token = await decryptSecret(env.RASTRO_SECRETS_KEY, {
+            cipher: sealed.tokenCipher,
+            iv: sealed.tokenIv,
+        });
+    }
+    catch {
+        await close("failed", { error: "SECRETS_KEY_MISSING" });
+        return;
+    }
+    let result;
+    try {
+        result = await graph().conversions(token, sealed.pixelId, {
+            data: [await conversionEvent(job.event, job.userData)],
+            ...(job.testEventCode ? { test_event_code: job.testEventCode } : {}),
+        });
+    }
+    catch (error) {
+        // Anything but Meta's own answer is the caller's UNEXPECTED.
+        if (!(error instanceof MetaGraphError))
+            throw error;
+        if (error.kind === "token") {
+            // Whoever owns the refused token stops: the pixel, or its connection
+            // (and with it every pixel and ad account it fed).
+            switch (sealed.source) {
+                case "token":
+                    await close("failed", {
+                        error: error.message,
+                        refusedPixel: {
+                            pixelId: sealed.pixelId,
+                            tokenIv: sealed.tokenIv,
+                        },
+                    });
+                    break;
+                case "connection":
+                    await close("failed", { error: error.message });
+                    if (sealed.connectionId) {
+                        // Only if it is still the stored token (a reconnect wins).
+                        await ctx.runMutation(internal.meta.markConnectionInvalid, {
+                            connectionId: sealed.connectionId,
+                            tokenIv: sealed.tokenIv,
+                            message: error.message,
+                        });
+                    }
+                    break;
+            }
+        }
+        else if (error.kind === "permission") {
+            // A permission error does not heal by waiting.
+            await close("failed", { error: error.message });
+        }
+        else {
+            await ctx.runMutation(internal.metaCapi.retrySend, {
+                outboxId,
+                error: error.message,
+            });
+        }
+        return;
+    }
+    await close("sent", {
+        ...(result.fbtraceId ? { fbtraceId: result.fbtraceId } : {}),
+    });
+}
+// ---------------------------------------------------------------------------
+// The sweep: a send lost on the way (a crashed action, a failed write) leaves
+// its row pending. A chain re-sends such rows while any row is pending.
+/** Makes sure the sweep runs by `at`, starting the chain if it stopped. */
+async function kickSweep(ctx, at) {
+    const state = await ctx.db
+        .query("metaCapiSweep")
+        .withIndex("by_key", (range) => range.eq("key", "sweep"))
+        .unique();
+    if (state) {
+        const job = await ctx.db.system.get("_scheduled_functions", state.jobId);
+        const alive = job?.state.kind === "pending" || job?.state.kind === "inProgress";
+        if (alive && state.scheduledFor <= at)
+            return;
+        if (job?.state.kind === "pending")
+            await ctx.scheduler.cancel(state.jobId);
+    }
+    const jobId = await ctx.scheduler.runAt(at, internal.metaCapi.sweep, {});
+    if (state) {
+        await ctx.db.patch("metaCapiSweep", state._id, { scheduledFor: at, jobId });
+    }
+    else {
+        await ctx.db.insert("metaCapiSweep", {
+            key: "sweep",
+            scheduledFor: at,
+            jobId,
+        });
+    }
+}
+/**
+ * Re-sends the rows whose attempt was due more than STALL_MS ago, the lost
+ * attempt counted: past the last one, or past MAX_EVENT_AGE_MS, the row
+ * fails for good. Then runs again when the oldest pending row could stall,
+ * or stops.
+ */
+export const sweep = internalMutation({
+    args: {},
+    returns: v.null(),
+    handler: async (ctx) => {
+        const now = Date.now();
+        const stalled = await ctx.db
+            .query("metaConversionOutbox")
+            .withIndex("by_status_and_nextAttemptAt", (range) => range.eq("status", "pending").lt("nextAttemptAt", now - STALL_MS))
+            .take(SWEEP_BATCH);
+        let handled = 0;
+        for (const row of stalled) {
+            // A send that took its attempt less than STALL_MS ago may still be in
+            // flight: its lease holds.
+            if ((row.attemptStartedAt ?? 0) >= now - STALL_MS)
+                continue;
+            handled += 1;
+            if (now - row.eventTime > MAX_EVENT_AGE_MS) {
+                await closeRow(ctx, row, "failed", { error: "EVENT_TOO_OLD" });
+                continue;
+            }
+            await failAttempt(ctx, row, "UNEXPECTED: the send stalled");
+        }
+        const state = await ctx.db
+            .query("metaCapiSweep")
+            .withIndex("by_key", (range) => range.eq("key", "sweep"))
+            .unique();
+        let at = now;
+        // A full batch it acted on may have more behind it; otherwise wait for
+        // the oldest pending row to be able to stall.
+        if (stalled.length < SWEEP_BATCH || handled === 0) {
+            const oldest = await ctx.db
+                .query("metaConversionOutbox")
+                .withIndex("by_status_and_nextAttemptAt", (range) => range.eq("status", "pending"))
+                .first();
+            at = oldest
+                ? Math.max(oldest.nextAttemptAt, oldest.attemptStartedAt ?? 0) +
+                    STALL_MS +
+                    1_000
+                : null;
+        }
+        if (at === null) {
+            if (state)
+                await ctx.db.delete("metaCapiSweep", state._id);
+            return null;
+        }
+        const jobId = await ctx.scheduler.runAt(at, internal.metaCapi.sweep, {});
+        if (state) {
+            await ctx.db.patch("metaCapiSweep", state._id, {
+                scheduledFor: at,
+                jobId,
+            });
+        }
+        else {
+            await ctx.db.insert("metaCapiSweep", {
+                key: "sweep",
+                scheduledFor: at,
+                jobId,
+            });
+        }
+        return null;
+    },
+});
+//# sourceMappingURL=metaCapi.js.map