@iann29/rastro 0.9.0 → 0.10.1

package/src/component/metaCapi.ts

@@ -0,0 +1,921 @@
+import { v, type Infer } from "convex/values";
+import { internal } from "./_generated/api.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import {
+  env,
+  internalAction,
+  internalMutation,
+  type ActionCtx,
+  type MutationCtx,
+  type QueryCtx,
+} from "./_generated/server.js";
+import { DAY_MS } from "./constants.js";
+import {
+  metaGraph,
+  MetaGraphError,
+  type ConversionEvent,
+} from "./metaGraph.js";
+import { decryptSecret } from "./secrets.js";
+import {
+  metaConversionEventNameValidator,
+  metaUserDataValidator,
+  type metaConversionValidator,
+} from "./validators.js";
+
+/*
+ * The Conversions API sender. A lead event the host reports (a sale, a new
+ * lead, a lead entering a mapped column) becomes one row of
+ * `metaConversionOutbox`, at most once per lead per event, and is sent to the
+ * site's pixel at once. The host's hashes live on the row only until the
+ * final attempt; the row itself stays as the site's conversion history.
+ */
+
+export type MetaUserData = Infer<typeof metaUserDataValidator>;
+export type MetaConversionEventName = Infer<
+  typeof metaConversionEventNameValidator
+>;
+type MetaConversion = Infer<typeof metaConversionValidator>;
+type Status = "pending" | "sent" | "failed";
+
+/** Meta refuses events older than this; the sender gives up first. */
+export const MAX_EVENT_AGE_MS = 7 * DAY_MS;
+export const MAX_SEND_ATTEMPTS = 3;
+/** The wait before the second and the third attempt. */
+export const RETRY_DELAYS_MS = [60_000, 10 * 60_000] as const;
+/**
+ * A pending row whose attempt was due this long ago was lost (an action
+ * outlives neither this nor the longest retry wait): the sweep sends it again.
+ */
+export const STALL_MS = 15 * 60_000;
+const SWEEP_BATCH = 25;
+const MAX_PHONES = 10;
+const MAX_LEAD_CONVERSIONS = 20;
+const SHA256_HEX = /^[a-f0-9]{64}$/;
+
+export async function sha256Hex(value: string): Promise<string> {
+  const digest = await crypto.subtle.digest(
+    "SHA-256",
+    new TextEncoder().encode(value),
+  );
+  return [...new Uint8Array(digest)]
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+/** SHA-256 of the empty string: a hash of nothing, which matches nobody. */
+const EMPTY_SHA256 =
+  "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
+const digestOf = (value: unknown): string | null => {
+  if (typeof value !== "string") return null;
+  const hash = value.trim().toLowerCase();
+  return SHA256_HEX.test(hash) ? hash : null;
+};
+
+function ignored(): null {
+  // Never the value itself: it may be a plain phone number.
+  console.warn(
+    "[rastro] meta ignored: expected { ph: SHA-256 hex digests, externalId?: a SHA-256 hex digest }",
+  );
+  return null;
+}
+
+/**
+ * The host's hashes, checked, or null when there is nothing to send with.
+ * Never throws and never echoes what it got: anything but SHA-256 hex digests
+ * in the documented shape (a plain phone number, a wrong type, more than
+ * MAX_PHONES phones) drops the whole `meta`, so the lead's own write goes on
+ * without the Conversions API. The hash of an empty string is left out.
+ */
+export function parseMetaUserData(value: unknown): MetaUserData | null {
+  if (value === undefined || value === null) return null;
+  if (typeof value !== "object" || Array.isArray(value)) return ignored();
+  const fields = value as { ph?: unknown; externalId?: unknown };
+  const phones = fields.ph ?? [];
+  if (!Array.isArray(phones) || phones.length > MAX_PHONES) return ignored();
+  const ph: string[] = [];
+  for (const item of phones) {
+    const hash = digestOf(item);
+    if (hash === null) return ignored();
+    if (hash !== EMPTY_SHA256 && !ph.includes(hash)) ph.push(hash);
+  }
+  let externalId: string | undefined;
+  if (fields.externalId !== undefined) {
+    const hash = digestOf(fields.externalId);
+    if (hash === null) return ignored();
+    if (hash !== EMPTY_SHA256) externalId = hash;
+  }
+  if (ph.length === 0 && externalId === undefined) return null;
+  return { ph, ...(externalId ? { externalId } : {}) };
+}
+
+// ---------------------------------------------------------------------------
+// The pixel and its token.
+
+/**
+ * The site's pixel when it may send: enabled, and its token not refused (its
+ * own, or its connection's, which must still list the pixel: a pixel marked
+ * `PIXEL_NOT_GRANTED` queues nothing). `meta.loadPixelToken` resolves the
+ * token itself, by the same rules.
+ */
+async function sendingPixel(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+): Promise<Doc<"sitePixels"> | null> {
+  const pixel = await ctx.db
+    .query("sitePixels")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  if (!pixel?.enabled) return null;
+  if (pixel.source === "connection") {
+    const connection = pixel.connectionId
+      ? await ctx.db.get("metaConnections", pixel.connectionId)
+      : null;
+    if (connection?.status !== "active") return null;
+    return connection.pixels?.some((row) => row.pixelId === pixel.pixelId)
+      ? pixel
+      : null;
+  }
+  return pixel.invalidSince === undefined ? pixel : null;
+}
+
+/** Meta refused the pixel's own token: the pixel stops until set again. */
+async function refusePixelToken(
+  ctx: MutationCtx,
+  pixel: Doc<"sitePixels">,
+  message: string,
+) {
+  await ctx.db.patch("sitePixels", pixel._id, {
+    invalidSince: Date.now(),
+    lastError: message.slice(0, 300),
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Counts.
+
+/** The count documents per site, so concurrent writes rarely meet. */
+export const COUNT_SHARDS = 8;
+
+/** An event's count shard: fixed by its id, so every write of it agrees. */
+const shardOf = (eventId: string) =>
+  Number.parseInt(eventId.slice(0, 8), 16) % COUNT_SHARDS;
+
+/**
+ * Moves an event's counts; returns the pending count its shard had before,
+ * which tells `enqueue` whether the sweep may be idle.
+ */
+async function bumpCounts(
+  ctx: MutationCtx,
+  row: { siteId: Id<"sites">; eventId: string },
+  delta: Partial<Record<Status, number>>,
+): Promise<number> {
+  const shard = shardOf(row.eventId);
+  const doc = await ctx.db
+    .query("metaConversionCounts")
+    .withIndex("by_siteId_and_shard", (range) =>
+      range.eq("siteId", row.siteId).eq("shard", shard),
+    )
+    .unique();
+  const next = {
+    pending: (doc?.pending ?? 0) + (delta.pending ?? 0),
+    sent: (doc?.sent ?? 0) + (delta.sent ?? 0),
+    failed: (doc?.failed ?? 0) + (delta.failed ?? 0),
+  };
+  if (doc) await ctx.db.patch("metaConversionCounts", doc._id, next);
+  else {
+    await ctx.db.insert("metaConversionCounts", {
+      siteId: row.siteId,
+      shard,
+      ...next,
+    });
+  }
+  return doc?.pending ?? 0;
+}
+
+/** The days `last30d` covers: today (UTC) and the 29 before. */
+export const SENT_WINDOW_DAYS = 30;
+
+const utcDay = (time: number) => Math.floor(time / DAY_MS) * DAY_MS;
+
+/** Moves the sent count of an event's UTC day, in its shard. */
+async function bumpDaily(
+  ctx: MutationCtx,
+  row: {
+    siteId: Id<"sites">;
+    eventId: string;
+    eventName: MetaConversionEventName;
+  },
+  sentAt: number,
+  delta: 1 | -1,
+) {
+  const shard = shardOf(row.eventId);
+  const dayStart = utcDay(sentAt);
+  const doc = await ctx.db
+    .query("metaConversionDaily")
+    .withIndex("by_siteId_and_dayStart_and_shard", (range) =>
+      range
+        .eq("siteId", row.siteId)
+        .eq("dayStart", dayStart)
+        .eq("shard", shard),
+    )
+    .unique();
+  if (doc) {
+    await ctx.db.patch("metaConversionDaily", doc._id, {
+      [row.eventName]: Math.max(doc[row.eventName] + delta, 0),
+    });
+  } else if (delta > 0) {
+    await ctx.db.insert("metaConversionDaily", {
+      siteId: row.siteId,
+      dayStart,
+      shard,
+      Purchase: 0,
+      QualifiedLead: 0,
+      Lead: 0,
+      Schedule: 0,
+      [row.eventName]: 1,
+    });
+  }
+}
+
+type SentByEvent = Record<MetaConversionEventName, number>;
+
+/**
+ * The site's queue by status and, given the reader's `now`, what Meta
+ * accepted in the last 30 UTC days by event and today. Without `now` (a
+ * query never reads the clock) those two are null.
+ */
+export async function conversionCounts(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  now?: number,
+): Promise<
+  Record<Status, number> & {
+    last30d: SentByEvent | null;
+    sentToday: number | null;
+  }
+> {
+  const shards = await ctx.db
+    .query("metaConversionCounts")
+    .withIndex("by_siteId_and_shard", (range) => range.eq("siteId", siteId))
+    .take(COUNT_SHARDS);
+  const total = { pending: 0, sent: 0, failed: 0 };
+  for (const shard of shards) {
+    total.pending += shard.pending;
+    total.sent += shard.sent;
+    total.failed += shard.failed;
+  }
+  if (now === undefined) return { ...total, last30d: null, sentToday: null };
+  const today = utcDay(now);
+  const days = await ctx.db
+    .query("metaConversionDaily")
+    .withIndex("by_siteId_and_dayStart_and_shard", (range) =>
+      range
+        .eq("siteId", siteId)
+        .gte("dayStart", today - (SENT_WINDOW_DAYS - 1) * DAY_MS)
+        .lte("dayStart", today),
+    )
+    .take(SENT_WINDOW_DAYS * COUNT_SHARDS);
+  const last30d: SentByEvent = {
+    Purchase: 0,
+    QualifiedLead: 0,
+    Lead: 0,
+    Schedule: 0,
+  };
+  let sentToday = 0;
+  for (const day of days) {
+    last30d.Purchase += day.Purchase;
+    last30d.QualifiedLead += day.QualifiedLead;
+    last30d.Lead += day.Lead;
+    last30d.Schedule += day.Schedule;
+    if (day.dayStart === today) {
+      sentToday += day.Purchase + day.QualifiedLead + day.Lead + day.Schedule;
+    }
+  }
+  return { ...total, last30d, sentToday };
+}
+
+// ---------------------------------------------------------------------------
+// Enqueueing.
+
+async function enqueue(
+  ctx: MutationCtx,
+  args: {
+    siteId: Id<"sites">;
+    leadId: Id<"leads">;
+    eventName: MetaConversionEventName;
+    eventTime: number;
+    meta: unknown;
+    value?: { valueCents: number; currency: string };
+  },
+): Promise<boolean> {
+  const userData = parseMetaUserData(args.meta);
+  if (!userData) return false;
+  if (!(await sendingPixel(ctx, args.siteId))) return false;
+  const now = Date.now();
+  const eventId = await sha256Hex(`${args.leadId}:${args.eventName}`);
+  const fields = {
+    eventTime: args.eventTime,
+    ...(args.value ?? {}),
+    userData,
+    status: "pending" as const,
+    attempts: 0,
+    nextAttemptAt: now,
+  };
+  const seen = await ctx.db
+    .query("metaConversionOutbox")
+    .withIndex("by_siteId_and_eventId", (range) =>
+      range.eq("siteId", args.siteId).eq("eventId", eventId),
+    )
+    .unique();
+  // Queued or delivered: once per lead per event. A failed one is not
+  // delivered, so the host's next call with hashes opens it again.
+  if (seen && seen.status !== "failed") return false;
+  let outboxId: Id<"metaConversionOutbox">;
+  let pendingBefore: number;
+  if (seen) {
+    outboxId = seen._id;
+    await ctx.db.patch("metaConversionOutbox", seen._id, {
+      ...fields,
+      attemptStartedAt: undefined,
+      lastError: undefined,
+      fbtraceId: undefined,
+    });
+    pendingBefore = await bumpCounts(ctx, seen, { failed: -1, pending: 1 });
+  } else {
+    outboxId = await ctx.db.insert("metaConversionOutbox", {
+      siteId: args.siteId,
+      leadId: args.leadId,
+      eventId,
+      eventName: args.eventName,
+      ...fields,
+      createdAt: now,
+    });
+    pendingBefore = await bumpCounts(
+      ctx,
+      { siteId: args.siteId, eventId },
+      { pending: 1 },
+    );
+  }
+  await ctx.scheduler.runAfter(0, internal.metaCapi.send, { outboxId });
+  // A shard with pending rows already has the sweep running for them (it
+  // stops only when none is pending), so only an idle shard reads its state.
+  if (pendingBefore === 0) await kickSweep(ctx, now + STALL_MS + 1_000);
+  return true;
+}
+
+/**
+ * Queues the event for a lead that entered a column the site's event map
+ * points at. Once per lead per event: a lead that comes back to the column,
+ * or reached the event another way, is not sent again. Without `meta` or an
+ * active pixel nothing is queued. Returns whether a row was queued.
+ */
+export async function enqueueColumnEvent(
+  ctx: MutationCtx,
+  args: {
+    siteId: Id<"sites">;
+    leadId: Id<"leads">;
+    eventName: "Lead" | "QualifiedLead" | "Schedule";
+    /** When the lead entered the column. */
+    at: number;
+    /** The hashes the host passed with the move; nothing is sent without. */
+    meta?: MetaUserData;
+  },
+): Promise<boolean> {
+  return await enqueue(ctx, {
+    siteId: args.siteId,
+    leadId: args.leadId,
+    eventName: args.eventName,
+    eventTime: args.at,
+    meta: args.meta,
+  });
+}
+
+/** `Lead` for a lead just recorded, when the pixel sends new leads. */
+export async function enqueueNewLead(
+  ctx: MutationCtx,
+  args: {
+    siteId: Id<"sites">;
+    leadId: Id<"leads">;
+    eventTime: number;
+    meta: unknown;
+  },
+): Promise<boolean> {
+  if (args.meta === undefined) return false;
+  const pixel = await sendingPixel(ctx, args.siteId);
+  if (!pixel?.sendLeadEvent) return false;
+  return await enqueue(ctx, { ...args, eventName: "Lead" });
+}
+
+/** `Purchase` for a sale the host recorded, with its value. */
+export async function enqueuePurchase(
+  ctx: MutationCtx,
+  args: {
+    siteId: Id<"sites">;
+    leadId: Id<"leads">;
+    eventTime: number;
+    valueCents: number;
+    currency: string;
+    meta: unknown;
+  },
+): Promise<boolean> {
+  // A sale without a value tells Meta nothing to optimize for.
+  if (args.valueCents <= 0) return false;
+  return await enqueue(ctx, {
+    siteId: args.siteId,
+    leadId: args.leadId,
+    eventName: "Purchase",
+    eventTime: args.eventTime,
+    meta: args.meta,
+    value: { valueCents: args.valueCents, currency: args.currency },
+  });
+}
+
+/** Forgets a removed lead's conversions, hashes and history alike. */
+export async function removeLeadConversions(
+  ctx: MutationCtx,
+  lead: { _id: Id<"leads">; siteId: Id<"sites"> },
+) {
+  const rows = await ctx.db
+    .query("metaConversionOutbox")
+    .withIndex("by_leadId", (range) => range.eq("leadId", lead._id))
+    .take(MAX_LEAD_CONVERSIONS);
+  for (const row of rows) {
+    await ctx.db.delete("metaConversionOutbox", row._id);
+    await bumpCounts(ctx, row, { [row.status]: -1 });
+    if (row.status === "sent" && row.sentAt !== undefined) {
+      await bumpDaily(ctx, row, row.sentAt, -1);
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Reads.
+
+/**
+ * The site's conversions, newest first, with the lead each one reports. The
+ * cursor is the last row read: its `createdAt` (`before`) and, since one call
+ * can queue several events in the same millisecond, its `eventId`
+ * (`beforeEventId`): the page goes on right after that row. With `before`
+ * alone, the page starts strictly before that time.
+ */
+export async function listConversions(
+  ctx: QueryCtx,
+  args: {
+    siteId: Id<"sites">;
+    limit: number;
+    before?: number;
+    beforeEventId?: string;
+  },
+): Promise<MetaConversion[]> {
+  const { before, beforeEventId } = args;
+  const query = ctx.db
+    .query("metaConversionOutbox")
+    .withIndex("by_siteId_and_createdAt", (range) =>
+      before === undefined
+        ? range.eq("siteId", args.siteId)
+        : beforeEventId === undefined
+          ? range.eq("siteId", args.siteId).lt("createdAt", before)
+          : range.eq("siteId", args.siteId).lte("createdAt", before),
+    )
+    .order("desc");
+  const rows: Doc<"metaConversionOutbox">[] = [];
+  // Rows tied with the cursor's time come in a fixed order (the index ends
+  // with the creation time): skip them up to the cursor row itself.
+  let skipping = before !== undefined && beforeEventId !== undefined;
+  for await (const row of query) {
+    if (skipping) {
+      if (row.createdAt === before) {
+        if (row.eventId === beforeEventId) skipping = false;
+        continue;
+      }
+      skipping = false;
+    }
+    rows.push(row);
+    if (rows.length >= args.limit) break;
+  }
+  const out: MetaConversion[] = [];
+  for (const row of rows) {
+    const lead = await ctx.db.get("leads", row.leadId);
+    const adId = lead?.ad?.adId ?? null;
+    const ad =
+      adId === null
+        ? null
+        : await ctx.db
+            .query("adCatalog")
+            .withIndex("by_siteId_and_adId", (range) =>
+              range.eq("siteId", row.siteId).eq("adId", adId),
+            )
+            .unique();
+    out.push({
+      eventId: row.eventId,
+      eventName: row.eventName,
+      leadId: row.leadId,
+      leadLabel: lead?.label ?? null,
+      adId,
+      adName: ad?.adName ?? null,
+      valueCents: row.valueCents ?? null,
+      status: row.status,
+      createdAt: row.createdAt,
+      sentAt: row.sentAt ?? null,
+    });
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Sending.
+
+/** The Conversions API event for one outbox row. */
+export async function conversionEvent(
+  row: Pick<
+    Doc<"metaConversionOutbox">,
+    "eventId" | "eventName" | "eventTime" | "valueCents" | "currency"
+  >,
+  userData: MetaUserData,
+): Promise<ConversionEvent> {
+  return {
+    event_name: row.eventName,
+    event_time: Math.floor(row.eventTime / 1000),
+    event_id: row.eventId,
+    action_source: "chat",
+    user_data: {
+      ...(userData.ph.length > 0 ? { ph: userData.ph } : {}),
+      ...(userData.externalId ? { external_id: [userData.externalId] } : {}),
+      // Every site Rastro serves today sells in Brazil.
+      country: [await sha256Hex("br")],
+    },
+    ...(row.eventName === "Purchase" &&
+    row.valueCents !== undefined &&
+    row.currency !== undefined
+      ? {
+          custom_data: {
+            value: row.valueCents / 100,
+            currency: row.currency,
+          },
+        }
+      : {}),
+  };
+}
+
+/**
+ * Takes one attempt at a pending row: stamps its lease (`attemptStartedAt`,
+ * so the sweep leaves a send that may still be in flight alone) and returns
+ * what to send, or why the row is dropped.
+ */
+export const claimSend = internalMutation({
+  args: { outboxId: v.id("metaConversionOutbox"), now: v.number() },
+  returns: v.union(
+    v.null(),
+    v.object({ kind: v.literal("drop"), error: v.string() }),
+    v.object({
+      kind: v.literal("send"),
+      siteId: v.id("sites"),
+      attempts: v.number(),
+      testEventCode: v.union(v.string(), v.null()),
+      event: v.object({
+        eventId: v.string(),
+        eventName: metaConversionEventNameValidator,
+        eventTime: v.number(),
+        valueCents: v.optional(v.number()),
+        currency: v.optional(v.string()),
+      }),
+      userData: metaUserDataValidator,
+    }),
+  ),
+  handler: async (ctx, args) => {
+    const row = await ctx.db.get("metaConversionOutbox", args.outboxId);
+    if (row?.status !== "pending") return null;
+    await ctx.db.patch("metaConversionOutbox", row._id, {
+      attemptStartedAt: args.now,
+    });
+    if (!row.userData) return { kind: "drop" as const, error: "NO_USER_DATA" };
+    if (args.now - row.eventTime > MAX_EVENT_AGE_MS) {
+      return { kind: "drop" as const, error: "EVENT_TOO_OLD" };
+    }
+    const pixel = await sendingPixel(ctx, row.siteId);
+    if (!pixel) return { kind: "drop" as const, error: "PIXEL_UNAVAILABLE" };
+    return {
+      kind: "send" as const,
+      siteId: row.siteId,
+      attempts: row.attempts,
+      testEventCode: pixel.testEventCode ?? null,
+      event: {
+        eventId: row.eventId,
+        eventName: row.eventName,
+        eventTime: row.eventTime,
+        ...(row.valueCents !== undefined ? { valueCents: row.valueCents } : {}),
+        ...(row.currency !== undefined ? { currency: row.currency } : {}),
+      },
+      userData: row.userData,
+    };
+  },
+});
+
+/** Ends a row for good, sent or failed; either way its hashes go. */
+async function closeRow(
+  ctx: MutationCtx,
+  row: Doc<"metaConversionOutbox">,
+  status: "sent" | "failed",
+  fields: { error?: string; fbtraceId?: string } = {},
+) {
+  const now = Date.now();
+  await ctx.db.patch("metaConversionOutbox", row._id, {
+    status,
+    attempts: row.attempts + 1,
+    userData: undefined,
+    lastError: fields.error?.slice(0, 300),
+    ...(fields.fbtraceId ? { fbtraceId: fields.fbtraceId.slice(0, 100) } : {}),
+    ...(status === "sent" ? { sentAt: now } : {}),
+  });
+  await bumpCounts(ctx, row, { pending: -1, [status]: 1 });
+  if (status === "sent") await bumpDaily(ctx, row, now, 1);
+}
+
+/**
+ * Counts a failed attempt: the next one after the backoff, or, when it was
+ * the last, the row fails for good.
+ */
+async function failAttempt(
+  ctx: MutationCtx,
+  row: Doc<"metaConversionOutbox">,
+  error: string,
+) {
+  if (row.attempts + 1 >= MAX_SEND_ATTEMPTS) {
+    await closeRow(ctx, row, "failed", { error });
+    return;
+  }
+  const attempts = row.attempts + 1;
+  const delay =
+    RETRY_DELAYS_MS[Math.min(attempts, RETRY_DELAYS_MS.length) - 1]!;
+  await ctx.db.patch("metaConversionOutbox", row._id, {
+    attempts,
+    nextAttemptAt: Date.now() + delay,
+    lastError: error.slice(0, 300),
+  });
+  await ctx.scheduler.runAfter(delay, internal.metaCapi.send, {
+    outboxId: row._id,
+  });
+}
+
+/**
+ * Closes a row: sent, or failed for good. When Meta refused the token, the
+ * pixel that sent stops too.
+ */
+export const finishSend = internalMutation({
+  args: {
+    outboxId: v.id("metaConversionOutbox"),
+    status: v.union(v.literal("sent"), v.literal("failed")),
+    error: v.optional(v.string()),
+    fbtraceId: v.optional(v.string()),
+    /** The pixel whose own token Meta refused, and the token's IV. */
+    refusedPixel: v.optional(
+      v.object({ pixelId: v.string(), tokenIv: v.string() }),
+    ),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const row = await ctx.db.get("metaConversionOutbox", args.outboxId);
+    if (row?.status !== "pending") return null;
+    await closeRow(ctx, row, args.status, {
+      ...(args.error !== undefined ? { error: args.error } : {}),
+      ...(args.fbtraceId !== undefined ? { fbtraceId: args.fbtraceId } : {}),
+    });
+    if (args.refusedPixel) {
+      const pixel = await sendingPixel(ctx, row.siteId);
+      // Only the token that was refused: one stored meanwhile keeps sending.
+      if (
+        pixel?.pixelId === args.refusedPixel.pixelId &&
+        pixel.source !== "connection" &&
+        pixel.tokenIv === args.refusedPixel.tokenIv
+      ) {
+        await refusePixelToken(ctx, pixel, args.error ?? "token refused");
+      }
+    }
+    return null;
+  },
+});
+
+/** Records a failed attempt: retried after the backoff, or failed for good. */
+export const retrySend = internalMutation({
+  args: { outboxId: v.id("metaConversionOutbox"), error: v.string() },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const row = await ctx.db.get("metaConversionOutbox", args.outboxId);
+    if (row?.status !== "pending") return null;
+    await failAttempt(ctx, row, args.error);
+    return null;
+  },
+});
+
+const graph = () => metaGraph({ fake: env.RASTRO_META_FAKE === "true" });
+
+/**
+ * One attempt at one outbox row. Whatever goes wrong counts as an attempt:
+ * Meta's answers by their kind, anything else as `UNEXPECTED`.
+ */
+export const send = internalAction({
+  args: { outboxId: v.id("metaConversionOutbox") },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    try {
+      await attempt(ctx, args.outboxId);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      await ctx.runMutation(internal.metaCapi.retrySend, {
+        outboxId: args.outboxId,
+        error: `UNEXPECTED: ${message}`,
+      });
+    }
+    return null;
+  },
+});
+
+async function attempt(ctx: ActionCtx, outboxId: Id<"metaConversionOutbox">) {
+  const job = await ctx.runMutation(internal.metaCapi.claimSend, {
+    outboxId,
+    now: Date.now(),
+  });
+  if (!job) return;
+  const close = (
+    status: "sent" | "failed",
+    extra: {
+      error?: string;
+      fbtraceId?: string;
+      refusedPixel?: { pixelId: string; tokenIv: string };
+    },
+  ) =>
+    ctx.runMutation(internal.metaCapi.finishSend, {
+      outboxId,
+      status,
+      ...extra,
+    });
+  if (job.kind === "drop") {
+    await close("failed", { error: job.error });
+    return;
+  }
+  const sealed = await ctx.runQuery(internal.meta.loadPixelToken, {
+    siteId: job.siteId,
+  });
+  if (!sealed) {
+    await close("failed", { error: "PIXEL_UNAVAILABLE" });
+    return;
+  }
+  let token: string;
+  try {
+    token = await decryptSecret(env.RASTRO_SECRETS_KEY, {
+      cipher: sealed.tokenCipher,
+      iv: sealed.tokenIv,
+    });
+  } catch {
+    await close("failed", { error: "SECRETS_KEY_MISSING" });
+    return;
+  }
+  let result: { fbtraceId: string | null };
+  try {
+    result = await graph().conversions(token, sealed.pixelId, {
+      data: [await conversionEvent(job.event, job.userData)],
+      ...(job.testEventCode ? { test_event_code: job.testEventCode } : {}),
+    });
+  } catch (error) {
+    // Anything but Meta's own answer is the caller's UNEXPECTED.
+    if (!(error instanceof MetaGraphError)) throw error;
+    if (error.kind === "token") {
+      // Whoever owns the refused token stops: the pixel, or its connection
+      // (and with it every pixel and ad account it fed).
+      switch (sealed.source) {
+        case "token":
+          await close("failed", {
+            error: error.message,
+            refusedPixel: {
+              pixelId: sealed.pixelId,
+              tokenIv: sealed.tokenIv,
+            },
+          });
+          break;
+        case "connection":
+          await close("failed", { error: error.message });
+          if (sealed.connectionId) {
+            // Only if it is still the stored token (a reconnect wins).
+            await ctx.runMutation(internal.meta.markConnectionInvalid, {
+              connectionId: sealed.connectionId,
+              tokenIv: sealed.tokenIv,
+              message: error.message,
+            });
+          }
+          break;
+      }
+    } else if (error.kind === "permission") {
+      // A permission error does not heal by waiting.
+      await close("failed", { error: error.message });
+    } else {
+      await ctx.runMutation(internal.metaCapi.retrySend, {
+        outboxId,
+        error: error.message,
+      });
+    }
+    return;
+  }
+  await close("sent", {
+    ...(result.fbtraceId ? { fbtraceId: result.fbtraceId } : {}),
+  });
+}
+
+// ---------------------------------------------------------------------------
+// The sweep: a send lost on the way (a crashed action, a failed write) leaves
+// its row pending. A chain re-sends such rows while any row is pending.
+
+/** Makes sure the sweep runs by `at`, starting the chain if it stopped. */
+async function kickSweep(ctx: MutationCtx, at: number) {
+  const state = await ctx.db
+    .query("metaCapiSweep")
+    .withIndex("by_key", (range) => range.eq("key", "sweep"))
+    .unique();
+  if (state) {
+    const job = await ctx.db.system.get("_scheduled_functions", state.jobId);
+    const alive =
+      job?.state.kind === "pending" || job?.state.kind === "inProgress";
+    if (alive && state.scheduledFor <= at) return;
+    if (job?.state.kind === "pending") await ctx.scheduler.cancel(state.jobId);
+  }
+  const jobId = await ctx.scheduler.runAt(at, internal.metaCapi.sweep, {});
+  if (state) {
+    await ctx.db.patch("metaCapiSweep", state._id, { scheduledFor: at, jobId });
+  } else {
+    await ctx.db.insert("metaCapiSweep", {
+      key: "sweep",
+      scheduledFor: at,
+      jobId,
+    });
+  }
+}
+
+/**
+ * Re-sends the rows whose attempt was due more than STALL_MS ago, the lost
+ * attempt counted: past the last one, or past MAX_EVENT_AGE_MS, the row
+ * fails for good. Then runs again when the oldest pending row could stall,
+ * or stops.
+ */
+export const sweep = internalMutation({
+  args: {},
+  returns: v.null(),
+  handler: async (ctx) => {
+    const now = Date.now();
+    const stalled = await ctx.db
+      .query("metaConversionOutbox")
+      .withIndex("by_status_and_nextAttemptAt", (range) =>
+        range.eq("status", "pending").lt("nextAttemptAt", now - STALL_MS),
+      )
+      .take(SWEEP_BATCH);
+    let handled = 0;
+    for (const row of stalled) {
+      // A send that took its attempt less than STALL_MS ago may still be in
+      // flight: its lease holds.
+      if ((row.attemptStartedAt ?? 0) >= now - STALL_MS) continue;
+      handled += 1;
+      if (now - row.eventTime > MAX_EVENT_AGE_MS) {
+        await closeRow(ctx, row, "failed", { error: "EVENT_TOO_OLD" });
+        continue;
+      }
+      await failAttempt(ctx, row, "UNEXPECTED: the send stalled");
+    }
+    const state = await ctx.db
+      .query("metaCapiSweep")
+      .withIndex("by_key", (range) => range.eq("key", "sweep"))
+      .unique();
+    let at: number | null = now;
+    // A full batch it acted on may have more behind it; otherwise wait for
+    // the oldest pending row to be able to stall.
+    if (stalled.length < SWEEP_BATCH || handled === 0) {
+      const oldest = await ctx.db
+        .query("metaConversionOutbox")
+        .withIndex("by_status_and_nextAttemptAt", (range) =>
+          range.eq("status", "pending"),
+        )
+        .first();
+      at = oldest
+        ? Math.max(oldest.nextAttemptAt, oldest.attemptStartedAt ?? 0) +
+          STALL_MS +
+          1_000
+        : null;
+    }
+    if (at === null) {
+      if (state) await ctx.db.delete("metaCapiSweep", state._id);
+      return null;
+    }
+    const jobId = await ctx.scheduler.runAt(at, internal.metaCapi.sweep, {});
+    if (state) {
+      await ctx.db.patch("metaCapiSweep", state._id, {
+        scheduledFor: at,
+        jobId,
+      });
+    } else {
+      await ctx.db.insert("metaCapiSweep", {
+        key: "sweep",
+        scheduledFor: at,
+        jobId,
+      });
+    }
+    return null;
+  },
+});