@iann29/rastro 0.9.0 → 0.10.1

package/src/component/metaSync.ts

@@ -0,0 +1,1164 @@
+import { v, type Infer } from "convex/values";
+import { internal } from "./_generated/api.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import {
+  env,
+  internalAction,
+  internalMutation,
+  internalQuery,
+  type MutationCtx,
+} from "./_generated/server.js";
+import { DAY_MS } from "./constants.js";
+import { localDayStart, localOffsetMs } from "./localTime.js";
+import {
+  metaGraph,
+  MetaGraphError,
+  type GraphAd,
+  type GraphSpendRow,
+  type GraphUsage,
+  type MetaGraph,
+} from "./metaGraph.js";
+import { decryptSecret } from "./secrets.js";
+
+/*
+ * The Meta sync. Components have no crons, so one scheduler chain (`loop`)
+ * takes the linked ad accounts whose `nextSyncAt` has come, a few at a time,
+ * and hands each to `syncAccount` under a fresh `runId`. The action reads
+ * Meta and writes back through small mutations, each of which first checks
+ * that the link still exists and that the run still holds it: an account
+ * unlinked, or taken over by a newer run, takes no write from an old one.
+ * The chain stops when no linked account is waiting; a link, a new token or
+ * `syncNow` starts it again.
+ *
+ * Cadence per account, in the account's own timezone:
+ * - after linking, a 90-day backfill, one 30-day block per run, resumable;
+ * - then today and yesterday every 45 minutes;
+ * - the trailing 7 days and the full catalog once a day, from 04:00;
+ * - the trailing 28 days once a week (Meta settles spend within 28 days);
+ * - back after a pause of more than 27 days, the gap is backfilled first.
+ */
+
+export const SYNC_INTERVAL_MS = 45 * 60_000;
+/** `nextSyncAt` of an account the chain must not take: its token failed. */
+export const PAUSED_AT = Number.MAX_SAFE_INTEGER;
+/** A link to one connection at most; the chain pauses and resumes them all. */
+export const MAX_LINKS_PER_CONNECTION = 500;
+const LEASE_MS = 15 * 60_000;
+const RETRY_MS = 15 * 60_000;
+const MAX_RETRY_MS = 6 * 60 * 60_000;
+const PERMISSION_RETRY_MS = 6 * 60 * 60_000;
+const THROTTLE_BACKOFF_MS = 60 * 60_000;
+const BACKFILL_NEXT_BLOCK_MS = 1_000;
+/** Above this share of any Meta rate limit, the next sync waits longer. */
+export const USAGE_BACKOFF_PCT = 75;
+const LOOP_BATCH = 4;
+const LOOP_STAGGER_MS = 30_000;
+const BACKFILL_DAYS = 90;
+const BACKFILL_BLOCK_DAYS = 30;
+const GAP_DAYS = 27;
+const DAILY_RESYNC_DAYS = 7;
+const WEEKLY_RESYNC_DAYS = 28;
+const DAILY_RESYNC_HOUR = 4;
+const WEEK_MS = 7 * DAY_MS;
+const SPEND_CHUNK = 400;
+const CATALOG_CHUNK = 100;
+const PAGE = 500;
+const MISSING_NAMES_LOOKBACK_DAYS = 7;
+const MAX_MISSING_NAMES = 50;
+const MAX_THUMBNAILS_PER_RUN = 300;
+const THUMBNAIL_CONCURRENCY = 8;
+const MAX_ERROR = 300;
+
+// ---------------------------------------------------------------------------
+// Dates. Meta files spend under the account-local date; Rastro stores that
+// date's UTC midnight as `dayStart`.
+
+/** The account-local calendar date of an instant, `YYYY-MM-DD`. */
+export function localDate(timestamp: number, timeZone: string): string {
+  return new Date(timestamp + localOffsetMs(timestamp, timeZone))
+    .toISOString()
+    .slice(0, 10);
+}
+
+export function dateDayStart(date: string): number {
+  const [year, month, day] = date.split("-").map(Number);
+  return Date.UTC(year!, month! - 1, day!);
+}
+
+const shiftDate = (date: string, days: number) =>
+  new Date(dateDayStart(date) + days * DAY_MS).toISOString().slice(0, 10);
+const minDate = (left: string, right: string) => (left < right ? left : right);
+const maxDate = (left: string, right: string) => (left > right ? left : right);
+
+export type SyncRange = { since: string; until: string };
+export type SyncPlan =
+  | {
+      kind: "backfill";
+      range: SyncRange;
+      /** The date the next block starts from, or null when this is the last. */
+      nextFrom: string | null;
+      fullCatalog: boolean;
+    }
+  | {
+      kind: "live";
+      range: SyncRange;
+      fullCatalog: boolean;
+      daily: boolean;
+      weekly: boolean;
+    };
+
+/** What one run of an account reads from Meta, by its state and the time. */
+export function planSync(
+  account: Pick<
+    Doc<"siteAdAccounts">,
+    | "timezone"
+    | "lastSyncAt"
+    | "lastDailyResyncAt"
+    | "lastFullResyncAt"
+    | "backfillFrom"
+  >,
+  now: number,
+): SyncPlan {
+  const today = localDate(now, account.timezone);
+  const earliest = shiftDate(today, -(BACKFILL_DAYS - 1));
+  let from = account.backfillFrom;
+  // Never synced (whatever failed before): the backfill is still owed.
+  if (from === undefined && account.lastSyncAt === undefined) from = earliest;
+  // Back after a long pause: fill the gap before the live cadence resumes.
+  if (
+    from === undefined &&
+    account.lastSyncAt !== undefined &&
+    now - account.lastSyncAt > GAP_DAYS * DAY_MS
+  ) {
+    from = maxDate(
+      shiftDate(localDate(account.lastSyncAt, account.timezone), -1),
+      earliest,
+    );
+  }
+  if (from !== undefined) {
+    const since = minDate(maxDate(from, earliest), today);
+    const until = minDate(shiftDate(since, BACKFILL_BLOCK_DAYS - 1), today);
+    const last = until === today;
+    return {
+      kind: "backfill",
+      range: { since, until },
+      nextFrom: last ? null : shiftDate(until, 1),
+      fullCatalog: last,
+    };
+  }
+  const dailySlot =
+    localDayStart(now, account.timezone) + DAILY_RESYNC_HOUR * 3_600_000;
+  const daily =
+    now >= dailySlot && (account.lastDailyResyncAt ?? 0) < dailySlot;
+  const weekly =
+    daily && now - (account.lastFullResyncAt ?? 0) >= WEEK_MS - DAY_MS / 2;
+  const days = weekly ? WEEKLY_RESYNC_DAYS : daily ? DAILY_RESYNC_DAYS : 2;
+  return {
+    kind: "live",
+    range: { since: shiftDate(today, -(days - 1)), until: today },
+    fullCatalog: daily,
+    daily,
+    weekly,
+  };
+}
+
+/** The wait after `failures` failed runs in a row: 15 min doubling to 6 h. */
+export function retryDelay(failures: number): number {
+  return Math.min(RETRY_MS * 2 ** Math.max(failures - 1, 0), MAX_RETRY_MS);
+}
+
+// ---------------------------------------------------------------------------
+// The chain.
+
+/**
+ * Makes sure the chain runs by `at`: starts it, or pulls a later run earlier.
+ * Called by every write that gives the chain work.
+ */
+export async function kickSyncLoop(ctx: MutationCtx, at: number) {
+  const state = await ctx.db
+    .query("metaSyncLoop")
+    .withIndex("by_key", (range) => range.eq("key", "loop"))
+    .unique();
+  if (state) {
+    const job = await ctx.db.system.get("_scheduled_functions", state.jobId);
+    const alive =
+      job?.state.kind === "pending" || job?.state.kind === "inProgress";
+    if (alive && state.scheduledFor <= at) return;
+    if (job?.state.kind === "pending") await ctx.scheduler.cancel(state.jobId);
+  }
+  const jobId = await ctx.scheduler.runAt(at, internal.metaSync.loop, {});
+  if (state) {
+    await ctx.db.patch("metaSyncLoop", state._id, { scheduledFor: at, jobId });
+  } else {
+    await ctx.db.insert("metaSyncLoop", {
+      key: "loop",
+      scheduledFor: at,
+      jobId,
+    });
+  }
+}
+
+/** One link of the chain: hands the due accounts to fresh runs, moves on. */
+export const loop = internalMutation({
+  args: {},
+  returns: v.null(),
+  handler: async (ctx) => {
+    const now = Date.now();
+    const due = await ctx.db
+      .query("siteAdAccounts")
+      .withIndex("by_nextSyncAt", (range) => range.lte("nextSyncAt", now))
+      .take(LOOP_BATCH);
+    for (const account of due) {
+      const runId = crypto.randomUUID();
+      await ctx.db.patch("siteAdAccounts", account._id, {
+        runId,
+        nextSyncAt: now + LEASE_MS,
+        // A run that still held the account never finished: say so. Its
+        // writes are refused from here on, since the account has a new run.
+        ...(account.runId !== undefined
+          ? { lastError: "RUN_TIMEOUT: the previous sync did not finish" }
+          : {}),
+      });
+      await ctx.scheduler.runAfter(0, internal.metaSync.syncAccount, {
+        siteAdAccountId: account._id,
+        runId,
+      });
+    }
+    const state = await ctx.db
+      .query("metaSyncLoop")
+      .withIndex("by_key", (range) => range.eq("key", "loop"))
+      .unique();
+    const next = await ctx.db
+      .query("siteAdAccounts")
+      .withIndex("by_nextSyncAt", (range) => range.lt("nextSyncAt", PAUSED_AT))
+      .first();
+    if (!next) {
+      if (state) await ctx.db.delete("metaSyncLoop", state._id);
+      return null;
+    }
+    // More accounts due than one batch: spread them out a little.
+    const at = Math.max(
+      next.nextSyncAt,
+      now + (due.length ? LOOP_STAGGER_MS : 0),
+    );
+    const jobId = await ctx.scheduler.runAt(at, internal.metaSync.loop, {});
+    if (state) {
+      await ctx.db.patch("metaSyncLoop", state._id, {
+        scheduledFor: at,
+        jobId,
+      });
+    } else {
+      await ctx.db.insert("metaSyncLoop", {
+        key: "loop",
+        scheduledFor: at,
+        jobId,
+      });
+    }
+    return null;
+  },
+});
+
+// ---------------------------------------------------------------------------
+// One account.
+
+const runArgs = {
+  siteAdAccountId: v.id("siteAdAccounts"),
+  runId: v.string(),
+};
+
+/** The link, if it still exists and this run still holds it. */
+async function heldLink(
+  ctx: MutationCtx,
+  args: { siteAdAccountId: Id<"siteAdAccounts">; runId: string },
+) {
+  const link = await ctx.db.get("siteAdAccounts", args.siteAdAccountId);
+  return link && link.runId === args.runId ? link : null;
+}
+
+export const loadJob = internalQuery({
+  args: runArgs,
+  returns: v.union(
+    v.null(),
+    v.object({
+      siteId: v.id("sites"),
+      adAccountId: v.string(),
+      timezone: v.string(),
+      lastSyncAt: v.optional(v.number()),
+      lastDailyResyncAt: v.optional(v.number()),
+      lastFullResyncAt: v.optional(v.number()),
+      backfillFrom: v.optional(v.string()),
+      connectionActive: v.boolean(),
+      tokenCipher: v.string(),
+      tokenIv: v.string(),
+    }),
+  ),
+  handler: async (ctx, args) => {
+    const account = await ctx.db.get("siteAdAccounts", args.siteAdAccountId);
+    if (!account || account.runId !== args.runId) return null;
+    const connection = await ctx.db.get(
+      "metaConnections",
+      account.connectionId,
+    );
+    if (!connection) return null;
+    return {
+      siteId: account.siteId,
+      adAccountId: account.adAccountId,
+      timezone: account.timezone,
+      lastSyncAt: account.lastSyncAt,
+      lastDailyResyncAt: account.lastDailyResyncAt,
+      lastFullResyncAt: account.lastFullResyncAt,
+      backfillFrom: account.backfillFrom,
+      connectionActive: connection.status === "active",
+      tokenCipher: connection.tokenCipher,
+      tokenIv: connection.tokenIv,
+    };
+  },
+});
+
+type SpendFigures = { spendCents: number; impressions: number; clicks: number };
+type SpendKey = { siteId: Id<"sites">; adAccountId: string; dayStart: number };
+
+/** Adds `delta` to the site-day aggregate, dropping a row that reaches 0. */
+async function bumpSiteSpend(
+  ctx: MutationCtx,
+  key: SpendKey,
+  delta: SpendFigures,
+) {
+  if (!delta.spendCents && !delta.impressions && !delta.clicks) return;
+  const row = await ctx.db
+    .query("siteSpendDaily")
+    .withIndex("by_siteId_and_adAccountId_and_dayStart", (range) =>
+      range
+        .eq("siteId", key.siteId)
+        .eq("adAccountId", key.adAccountId)
+        .eq("dayStart", key.dayStart),
+    )
+    .unique();
+  const next = {
+    spendCents: (row?.spendCents ?? 0) + delta.spendCents,
+    impressions: (row?.impressions ?? 0) + delta.impressions,
+    clicks: (row?.clicks ?? 0) + delta.clicks,
+  };
+  if (row && !next.spendCents && !next.impressions && !next.clicks) {
+    await ctx.db.delete("siteSpendDaily", row._id);
+  } else if (row) {
+    await ctx.db.patch("siteSpendDaily", row._id, next);
+  } else {
+    await ctx.db.insert("siteSpendDaily", { ...key, ...next });
+  }
+}
+
+const figures = (row: SpendFigures): SpendFigures => ({
+  spendCents: row.spendCents,
+  impressions: row.impressions,
+  clicks: row.clicks,
+});
+const negate = (row: SpendFigures): SpendFigures => ({
+  spendCents: -row.spendCents,
+  impressions: -row.impressions,
+  clicks: -row.clicks,
+});
+
+const spendRowValidator = v.object({
+  adId: v.string(),
+  dayStart: v.number(),
+  spendCents: v.number(),
+  impressions: v.number(),
+  clicks: v.number(),
+  conversations: v.number(),
+});
+
+/**
+ * Upserts one chunk of spend rows, stamping them with the run's start, and
+ * keeps the site-day aggregate in step. `stop` when the run lost the link.
+ */
+export const saveSpend = internalMutation({
+  args: {
+    ...runArgs,
+    syncedAt: v.number(),
+    rows: v.array(spendRowValidator),
+  },
+  returns: v.object({ stop: v.boolean() }),
+  handler: async (ctx, args) => {
+    const link = await heldLink(ctx, args);
+    if (!link) return { stop: true };
+    for (const row of args.rows) {
+      const key = {
+        siteId: link.siteId,
+        adAccountId: link.adAccountId,
+        dayStart: row.dayStart,
+      };
+      const existing = await ctx.db
+        .query("adSpendDaily")
+        .withIndex("by_siteId_and_adId_and_dayStart", (range) =>
+          range
+            .eq("siteId", link.siteId)
+            .eq("adId", row.adId)
+            .eq("dayStart", row.dayStart),
+        )
+        .unique();
+      const fields = {
+        adAccountId: link.adAccountId,
+        spendCents: row.spendCents,
+        impressions: row.impressions,
+        clicks: row.clicks,
+        metaConversations: row.conversations,
+        // Never older than what a later run already wrote.
+        updatedAt: Math.max(existing?.updatedAt ?? 0, args.syncedAt),
+      };
+      if (existing) {
+        await bumpSiteSpend(
+          ctx,
+          { ...key, adAccountId: existing.adAccountId },
+          negate(existing),
+        );
+        await ctx.db.patch("adSpendDaily", existing._id, fields);
+      } else {
+        await ctx.db.insert("adSpendDaily", {
+          siteId: link.siteId,
+          adId: row.adId,
+          dayStart: row.dayStart,
+          ...fields,
+        });
+      }
+      await bumpSiteSpend(ctx, key, figures(row));
+    }
+    return { stop: false };
+  },
+});
+
+/**
+ * After a complete range is saved, drops the account's rows in it that the
+ * run did not touch: Meta leaves out an ad-day with nothing, so a day
+ * corrected to zero disappears from its answer. Pages until done.
+ */
+export const dropStaleSpend = internalMutation({
+  args: {
+    ...runArgs,
+    firstDay: v.number(),
+    lastDay: v.number(),
+    syncedAt: v.number(),
+    cursor: v.union(v.string(), v.null()),
+  },
+  returns: v.object({
+    stop: v.boolean(),
+    done: v.boolean(),
+    cursor: v.union(v.string(), v.null()),
+  }),
+  handler: async (ctx, args) => {
+    const link = await heldLink(ctx, args);
+    if (!link) return { stop: true, done: true, cursor: null };
+    const page = await ctx.db
+      .query("adSpendDaily")
+      .withIndex("by_siteId_and_adAccountId_and_dayStart", (range) =>
+        range
+          .eq("siteId", link.siteId)
+          .eq("adAccountId", link.adAccountId)
+          .gte("dayStart", args.firstDay)
+          .lte("dayStart", args.lastDay),
+      )
+      .paginate({ cursor: args.cursor, numItems: PAGE });
+    for (const row of page.page) {
+      if (row.updatedAt < args.syncedAt) {
+        await ctx.db.delete("adSpendDaily", row._id);
+        await bumpSiteSpend(
+          ctx,
+          {
+            siteId: row.siteId,
+            adAccountId: row.adAccountId,
+            dayStart: row.dayStart,
+          },
+          negate(row),
+        );
+      }
+    }
+    return { stop: false, done: page.isDone, cursor: page.continueCursor };
+  },
+});
+
+const catalogAdValidator = v.object({
+  adId: v.string(),
+  adName: v.optional(v.string()),
+  adsetId: v.optional(v.string()),
+  adsetName: v.optional(v.string()),
+  campaignId: v.optional(v.string()),
+  campaignName: v.optional(v.string()),
+  adStatus: v.optional(v.string()),
+  creativeType: v.optional(
+    v.union(
+      v.literal("video"),
+      v.literal("image"),
+      v.literal("carousel"),
+      v.literal("other"),
+    ),
+  ),
+  /** Meta's thumbnail, only to decide whether one must be copied. */
+  hasThumbnail: v.optional(v.boolean()),
+});
+
+/**
+ * Names, status and creative for the account's ads. A missing field keeps the
+ * stored value. Returns the ads that have a thumbnail at Meta but none in
+ * storage yet; `stop` when the run lost the link.
+ */
+export const upsertCatalog = internalMutation({
+  args: { ...runArgs, ads: v.array(catalogAdValidator) },
+  returns: v.object({
+    stop: v.boolean(),
+    needThumbnail: v.array(v.string()),
+  }),
+  handler: async (ctx, args) => {
+    const link = await heldLink(ctx, args);
+    if (!link) return { stop: true, needThumbnail: [] };
+    const now = Date.now();
+    const needThumbnail: string[] = [];
+    for (const { hasThumbnail, ...ad } of args.ads) {
+      const defined = Object.fromEntries(
+        Object.entries(ad).filter(([, value]) => value !== undefined),
+      );
+      const row = await ctx.db
+        .query("adCatalog")
+        .withIndex("by_siteId_and_adId", (range) =>
+          range.eq("siteId", link.siteId).eq("adId", ad.adId),
+        )
+        .unique();
+      if (row) {
+        await ctx.db.patch("adCatalog", row._id, {
+          ...defined,
+          adAccountId: link.adAccountId,
+          updatedAt: now,
+        });
+        if (hasThumbnail && !row.thumbnailStorageId) {
+          needThumbnail.push(ad.adId);
+        }
+      } else {
+        await ctx.db.insert("adCatalog", {
+          siteId: link.siteId,
+          ...defined,
+          adId: ad.adId,
+          adAccountId: link.adAccountId,
+          updatedAt: now,
+        });
+        if (hasThumbnail) needThumbnail.push(ad.adId);
+      }
+    }
+    return { stop: false, needThumbnail };
+  },
+});
+
+export const setThumbnail = internalMutation({
+  args: {
+    siteId: v.id("sites"),
+    adId: v.string(),
+    storageId: v.id("_storage"),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const row = await ctx.db
+      .query("adCatalog")
+      .withIndex("by_siteId_and_adId", (range) =>
+        range.eq("siteId", args.siteId).eq("adId", args.adId),
+      )
+      .unique();
+    if (!row) {
+      await ctx.storage.delete(args.storageId);
+      return null;
+    }
+    if (row.thumbnailStorageId) {
+      await ctx.storage.delete(row.thumbnailStorageId);
+    }
+    await ctx.db.patch("adCatalog", row._id, {
+      thumbnailStorageId: args.storageId,
+      updatedAt: Date.now(),
+    });
+    return null;
+  },
+});
+
+/**
+ * Ads that brought the site leads lately but have no name yet, to ask the
+ * account for. An ad the catalog already files under another account is not
+ * this account's to name.
+ */
+export const missingAdNames = internalQuery({
+  args: {
+    siteId: v.id("sites"),
+    adAccountId: v.string(),
+    since: v.number(),
+  },
+  returns: v.array(v.string()),
+  handler: async (ctx, args) => {
+    const rows = await ctx.db
+      .query("leadDailyRollups")
+      .withIndex("by_siteId_and_dimension_and_dayStart_and_key", (range) =>
+        range
+          .eq("siteId", args.siteId)
+          .eq("dimension", "ad")
+          .gte("dayStart", args.since),
+      )
+      .take(2_000);
+    const missing: string[] = [];
+    for (const adId of new Set(rows.map((row) => row.key))) {
+      const catalog = await ctx.db
+        .query("adCatalog")
+        .withIndex("by_siteId_and_adId", (range) =>
+          range.eq("siteId", args.siteId).eq("adId", adId),
+        )
+        .unique();
+      if (catalog?.adName) continue;
+      if (catalog?.adAccountId && catalog.adAccountId !== args.adAccountId) {
+        continue;
+      }
+      missing.push(adId);
+      if (missing.length >= MAX_MISSING_NAMES) break;
+    }
+    return missing;
+  },
+});
+
+const outcomeValidator = v.union(
+  v.object({
+    kind: v.literal("ok"),
+    usagePct: v.number(),
+    regainMs: v.union(v.number(), v.null()),
+    plan: v.union(
+      v.object({
+        kind: v.literal("backfill"),
+        nextFrom: v.union(v.string(), v.null()),
+      }),
+      v.object({
+        kind: v.literal("live"),
+        daily: v.boolean(),
+        weekly: v.boolean(),
+      }),
+    ),
+    /** Meta had more rows than the page cap for some day of the range. */
+    truncated: v.boolean(),
+  }),
+  v.object({
+    kind: v.union(
+      v.literal("token"),
+      v.literal("throttled"),
+      v.literal("permission"),
+      v.literal("other"),
+      v.literal("key"),
+    ),
+    message: v.string(),
+    regainMs: v.union(v.number(), v.null()),
+  }),
+);
+type Outcome = Infer<typeof outcomeValidator>;
+
+/** Records how a run ended and when the account syncs next. */
+export const finishSync = internalMutation({
+  args: {
+    ...runArgs,
+    /** Where the run wrote, to purge it if the link is gone. */
+    siteId: v.id("sites"),
+    adAccountId: v.string(),
+    startedAt: v.number(),
+    /** The token the run used: a failure only condemns that token. */
+    tokenIv: v.string(),
+    outcome: outcomeValidator,
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const account = await ctx.db.get("siteAdAccounts", args.siteAdAccountId);
+    if (!account) {
+      // Unlinked while the run wrote: take back whatever it wrote after the
+      // unlink's own purge ran.
+      await ctx.scheduler.runAfter(0, internal.metaSync.purgeSpend, {
+        siteId: args.siteId,
+        adAccountId: args.adAccountId,
+        cursor: null,
+      });
+      return null;
+    }
+    // A newer run holds the account: this one's ending is not the account's.
+    if (account.runId !== args.runId) return null;
+    const now = Date.now();
+    const { outcome } = args;
+    const requested = account.syncRequested === true;
+    const base = { runId: undefined, syncRequested: undefined };
+    if (outcome.kind === "ok") {
+      const busy = outcome.usagePct > USAGE_BACKOFF_PCT;
+      const wait = busy
+        ? Math.max(outcome.regainMs ?? 0, THROTTLE_BACKOFF_MS)
+        : 0;
+      const lastError = outcome.truncated
+        ? "META_TRUNCATED: Meta listed more rows than one call reads for a day; that day may be short"
+        : busy
+          ? "RATE_LIMITED: backing off"
+          : undefined;
+      let nextSyncAt: number;
+      if (outcome.plan.kind === "backfill" && outcome.plan.nextFrom !== null) {
+        // More blocks to read: the next one right away (unless Meta is busy).
+        nextSyncAt = now + Math.max(wait, BACKFILL_NEXT_BLOCK_MS);
+        await ctx.db.patch("siteAdAccounts", account._id, {
+          ...base,
+          backfillFrom: outcome.plan.nextFrom,
+          failures: 0,
+          lastError,
+          nextSyncAt,
+        });
+      } else {
+        nextSyncAt = requested ? now : now + (wait || SYNC_INTERVAL_MS);
+        const backfilled = outcome.plan.kind === "backfill";
+        const daily =
+          backfilled || (outcome.plan.kind === "live" && outcome.plan.daily);
+        const weekly =
+          backfilled || (outcome.plan.kind === "live" && outcome.plan.weekly);
+        await ctx.db.patch("siteAdAccounts", account._id, {
+          ...base,
+          syncState: "live",
+          backfillFrom: undefined,
+          failures: 0,
+          lastSyncAt: args.startedAt,
+          ...(daily ? { lastDailyResyncAt: args.startedAt } : {}),
+          ...(weekly ? { lastFullResyncAt: args.startedAt } : {}),
+          lastError,
+          nextSyncAt,
+        });
+      }
+      await kickSyncLoop(ctx, nextSyncAt);
+      return null;
+    }
+    const message = outcome.message.slice(0, MAX_ERROR);
+    if (outcome.kind === "token") {
+      const connection = await ctx.db.get(
+        "metaConnections",
+        account.connectionId,
+      );
+      if (connection && connection.tokenIv === args.tokenIv) {
+        await pauseConnection(ctx, connection._id, message, now);
+        return null;
+      }
+      // The host pasted a new token while this run used the old one: the
+      // new one gets its chance at once.
+      await ctx.db.patch("siteAdAccounts", account._id, {
+        ...base,
+        nextSyncAt: now,
+      });
+      await kickSyncLoop(ctx, now);
+      return null;
+    }
+    const failures = (account.failures ?? 0) + 1;
+    const delay =
+      outcome.kind === "throttled"
+        ? Math.max(outcome.regainMs ?? 0, THROTTLE_BACKOFF_MS)
+        : outcome.kind === "permission"
+          ? PERMISSION_RETRY_MS
+          : retryDelay(failures);
+    const code =
+      outcome.kind === "throttled"
+        ? "RATE_LIMITED"
+        : outcome.kind === "key"
+          ? "SECRETS_KEY_MISSING"
+          : "META_UNAVAILABLE";
+    const nextSyncAt = requested ? now : now + delay;
+    await ctx.db.patch("siteAdAccounts", account._id, {
+      ...base,
+      // A backfill stays owed (lastSyncAt unset, or backfillFrom kept), so a
+      // retry resumes it; a permission error is shown as an error.
+      syncState: outcome.kind === "permission" ? "error" : account.syncState,
+      failures,
+      lastError: `${code}: ${message}`.slice(0, MAX_ERROR),
+      nextSyncAt,
+    });
+    await kickSyncLoop(ctx, nextSyncAt);
+    return null;
+  },
+});
+
+/** A dead token stops every account it serves until a new one is pasted. */
+async function pauseConnection(
+  ctx: MutationCtx,
+  connectionId: Id<"metaConnections">,
+  message: string,
+  now: number,
+) {
+  const connection = await ctx.db.get("metaConnections", connectionId);
+  await ctx.db.patch("metaConnections", connectionId, {
+    status: "invalid",
+    invalidSince: connection?.invalidSince ?? now,
+    lastError: message,
+    lastCheckedAt: now,
+    updatedAt: now,
+  });
+  // Bounded: a connection links at most MAX_LINKS_PER_CONNECTION accounts.
+  const links = await ctx.db
+    .query("siteAdAccounts")
+    .withIndex("by_connectionId", (range) =>
+      range.eq("connectionId", connectionId),
+    )
+    .take(MAX_LINKS_PER_CONNECTION);
+  for (const link of links) {
+    await ctx.db.patch("siteAdAccounts", link._id, {
+      syncState: "error",
+      runId: undefined,
+      syncRequested: undefined,
+      lastError: `META_TOKEN_INVALID: ${message}`.slice(0, MAX_ERROR),
+      nextSyncAt: PAUSED_AT,
+    });
+  }
+}
+
+function catalogFromSpend(rows: GraphSpendRow[]) {
+  const byAd = new Map<string, GraphSpendRow>();
+  for (const row of rows) byAd.set(row.adId, row);
+  return [...byAd.values()].map((row) => ({
+    adId: row.adId,
+    ...(row.adName ? { adName: row.adName } : {}),
+    ...(row.adsetId ? { adsetId: row.adsetId } : {}),
+    ...(row.adsetName ? { adsetName: row.adsetName } : {}),
+    ...(row.campaignId ? { campaignId: row.campaignId } : {}),
+    ...(row.campaignName ? { campaignName: row.campaignName } : {}),
+  }));
+}
+
+function catalogFromAds(ads: GraphAd[]) {
+  return ads.map((ad) => ({
+    adId: ad.adId,
+    ...(ad.name ? { adName: ad.name } : {}),
+    ...(ad.adsetId ? { adsetId: ad.adsetId } : {}),
+    ...(ad.adsetName ? { adsetName: ad.adsetName } : {}),
+    ...(ad.campaignId ? { campaignId: ad.campaignId } : {}),
+    ...(ad.campaignName ? { campaignName: ad.campaignName } : {}),
+    ...(ad.effectiveStatus ? { adStatus: ad.effectiveStatus } : {}),
+    ...(ad.creativeType ? { creativeType: ad.creativeType } : {}),
+    hasThumbnail: ad.thumbnailUrl !== null,
+  }));
+}
+
+function chunks<T>(items: T[], size: number): T[][] {
+  const out: T[][] = [];
+  for (let index = 0; index < items.length; index += size) {
+    out.push(items.slice(index, index + size));
+  }
+  return out;
+}
+
+const dayCount = (range: SyncRange) =>
+  (dateDayStart(range.until) - dateDayStart(range.since)) / DAY_MS + 1;
+
+type RangePart = { range: SyncRange; rows: GraphSpendRow[]; complete: boolean };
+
+/**
+ * A range's spend, split in halves while Meta has more rows than one call
+ * reads, down to single days. A day still over the cap comes back partial
+ * (`complete: false`): its rows are kept, but nothing is inferred from what
+ * it lacks.
+ */
+async function readRange(
+  graph: MetaGraph,
+  token: string,
+  adAccountId: string,
+  range: SyncRange,
+  track: (usage: GraphUsage) => void,
+): Promise<RangePart[]> {
+  const result = await graph.insights(
+    token,
+    adAccountId,
+    range.since,
+    range.until,
+  );
+  track(result.usage);
+  if (result.complete || dayCount(range) === 1) {
+    return [{ range, rows: result.rows, complete: result.complete }];
+  }
+  const firstUntil = shiftDate(
+    range.since,
+    Math.floor(dayCount(range) / 2) - 1,
+  );
+  return [
+    ...(await readRange(
+      graph,
+      token,
+      adAccountId,
+      { since: range.since, until: firstUntil },
+      track,
+    )),
+    ...(await readRange(
+      graph,
+      token,
+      adAccountId,
+      { since: shiftDate(firstUntil, 1), until: range.until },
+      track,
+    )),
+  ];
+}
+
+/** Ends the run quietly: the link is gone or a newer run holds it. */
+class LostLink extends Error {}
+
+/** Reads one account from Meta and writes what it found, as run `runId`. */
+export const syncAccount = internalAction({
+  args: runArgs,
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const startedAt = Date.now();
+    const job = await ctx.runQuery(internal.metaSync.loadJob, args);
+    if (!job) return null;
+    const finish = (outcome: Outcome) =>
+      ctx.runMutation(internal.metaSync.finishSync, {
+        ...args,
+        siteId: job.siteId,
+        adAccountId: job.adAccountId,
+        startedAt,
+        tokenIv: job.tokenIv,
+        outcome,
+      });
+    let usage: GraphUsage = { pct: 0, regainMs: null };
+    const track = (next: GraphUsage) => {
+      usage = {
+        pct: Math.max(usage.pct, next.pct),
+        regainMs: Math.max(usage.regainMs ?? 0, next.regainMs ?? 0) || null,
+      };
+    };
+    try {
+      if (!job.connectionActive) {
+        await finish({
+          kind: "token",
+          message: "the connection's token was marked invalid",
+          regainMs: null,
+        });
+        return null;
+      }
+      let token: string;
+      try {
+        token = await decryptSecret(env.RASTRO_SECRETS_KEY, {
+          cipher: job.tokenCipher,
+          iv: job.tokenIv,
+        });
+      } catch {
+        await finish({
+          kind: "key",
+          message:
+            "RASTRO_SECRETS_KEY is missing or not the one the token was sealed with",
+          regainMs: null,
+        });
+        return null;
+      }
+      const graph = metaGraph({ fake: env.RASTRO_META_FAKE === "true" });
+      const plan = planSync(job, startedAt);
+
+      const parts = await readRange(
+        graph,
+        token,
+        job.adAccountId,
+        plan.range,
+        track,
+      );
+      const catalog = new Map<
+        string,
+        ReturnType<typeof catalogFromSpend>[number]
+      >();
+      for (const part of parts) {
+        const rows = part.rows.map((row) => ({
+          adId: row.adId,
+          dayStart: dateDayStart(row.date),
+          spendCents: row.spendCents,
+          impressions: row.impressions,
+          clicks: row.clicks,
+          conversations: row.conversations,
+        }));
+        for (const chunk of chunks(rows, SPEND_CHUNK)) {
+          const saved = await ctx.runMutation(internal.metaSync.saveSpend, {
+            ...args,
+            syncedAt: startedAt,
+            rows: chunk,
+          });
+          if (saved.stop) throw new LostLink();
+        }
+        // A partial day proves nothing about the rows it lacks.
+        if (part.complete) {
+          let cursor: string | null = null;
+          for (;;) {
+            const page: {
+              stop: boolean;
+              done: boolean;
+              cursor: string | null;
+            } = await ctx.runMutation(internal.metaSync.dropStaleSpend, {
+              ...args,
+              firstDay: dateDayStart(part.range.since),
+              lastDay: dateDayStart(part.range.until),
+              syncedAt: startedAt,
+              cursor,
+            });
+            if (page.stop) throw new LostLink();
+            if (page.done) break;
+            cursor = page.cursor;
+          }
+        }
+        for (const entry of catalogFromSpend(part.rows)) {
+          catalog.set(entry.adId, { ...catalog.get(entry.adId), ...entry });
+        }
+      }
+
+      let ads: GraphAd[] = [];
+      if (plan.fullCatalog) {
+        const listed = await graph.ads(token, job.adAccountId);
+        track(listed.usage);
+        ads = listed.ads;
+      } else {
+        const missing = (
+          await ctx.runQuery(internal.metaSync.missingAdNames, {
+            siteId: job.siteId,
+            adAccountId: job.adAccountId,
+            since:
+              dateDayStart(localDate(startedAt, job.timezone)) -
+              MISSING_NAMES_LOOKBACK_DAYS * DAY_MS,
+          })
+        ).filter((adId) => !catalog.get(adId)?.adName);
+        if (missing.length > 0) {
+          const listed = await graph.ads(token, job.adAccountId, missing);
+          track(listed.usage);
+          ads = listed.ads;
+        }
+      }
+      const listedIds = new Set(ads.map((ad) => ad.adId));
+      const entries = [
+        ...[...catalog.values()].filter((entry) => !listedIds.has(entry.adId)),
+        ...catalogFromAds(ads),
+      ];
+      const needThumbnail: string[] = [];
+      for (const chunk of chunks(entries, CATALOG_CHUNK)) {
+        const upserted = await ctx.runMutation(
+          internal.metaSync.upsertCatalog,
+          { ...args, ads: chunk },
+        );
+        if (upserted.stop) throw new LostLink();
+        needThumbnail.push(...upserted.needThumbnail);
+      }
+      const thumbnails = new Map(
+        ads.flatMap((ad) =>
+          ad.thumbnailUrl ? [[ad.adId, ad.thumbnailUrl] as const] : [],
+        ),
+      );
+      const queue = needThumbnail.slice(0, MAX_THUMBNAILS_PER_RUN);
+      for (const batch of chunks(queue, THUMBNAIL_CONCURRENCY)) {
+        await Promise.all(
+          batch.map(async (adId) => {
+            const url = thumbnails.get(adId);
+            const blob = url ? await graph.thumbnail(url) : null;
+            if (!blob) return;
+            const storageId: Id<"_storage"> = await ctx.storage.store(blob);
+            await ctx.runMutation(internal.metaSync.setThumbnail, {
+              siteId: job.siteId,
+              adId,
+              storageId,
+            });
+          }),
+        );
+      }
+      await finish({
+        kind: "ok",
+        usagePct: usage.pct,
+        regainMs: usage.regainMs,
+        plan:
+          plan.kind === "backfill"
+            ? { kind: "backfill", nextFrom: plan.nextFrom }
+            : { kind: "live", daily: plan.daily, weekly: plan.weekly },
+        truncated: parts.some((part) => !part.complete),
+      });
+    } catch (error) {
+      if (error instanceof LostLink) {
+        // finishSync sees the link gone (and purges) or a newer run holding
+        // it (and leaves the account to that run).
+        await finish({ kind: "other", message: "link lost", regainMs: null });
+      } else if (error instanceof MetaGraphError) {
+        await finish({
+          // `app` comes from the login exchanges only, never a sync call.
+          kind: error.kind === "app" ? "other" : error.kind,
+          message: error.message,
+          regainMs: error.regainMs ?? usage.regainMs,
+        });
+      } else {
+        await finish({
+          kind: "other",
+          message: error instanceof Error ? error.message : "sync failed",
+          regainMs: null,
+        });
+      }
+    }
+    return null;
+  },
+});
+
+/**
+ * Deletes a site's spend from one ad account after it is unlinked, a batch
+ * per run, then its site-day aggregate. Stops if the account is linked to the
+ * site again meanwhile (its backfill rewrites the rows).
+ */
+export const purgeSpend = internalMutation({
+  args: {
+    siteId: v.id("sites"),
+    adAccountId: v.string(),
+    cursor: v.union(v.string(), v.null()),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const relinked = await ctx.db
+      .query("siteAdAccounts")
+      .withIndex("by_siteId_and_adAccountId", (range) =>
+        range.eq("siteId", args.siteId).eq("adAccountId", args.adAccountId),
+      )
+      .unique();
+    if (relinked) return null;
+    const rows = await ctx.db
+      .query("adSpendDaily")
+      .withIndex("by_siteId_and_adAccountId_and_dayStart", (range) =>
+        range.eq("siteId", args.siteId).eq("adAccountId", args.adAccountId),
+      )
+      .take(PAGE);
+    for (const row of rows) await ctx.db.delete("adSpendDaily", row._id);
+    const sums =
+      rows.length < PAGE
+        ? await ctx.db
+            .query("siteSpendDaily")
+            .withIndex("by_siteId_and_adAccountId_and_dayStart", (range) =>
+              range
+                .eq("siteId", args.siteId)
+                .eq("adAccountId", args.adAccountId),
+            )
+            .take(PAGE)
+        : [];
+    for (const row of sums) await ctx.db.delete("siteSpendDaily", row._id);
+    if (rows.length === PAGE || sums.length === PAGE) {
+      await ctx.scheduler.runAfter(0, internal.metaSync.purgeSpend, {
+        ...args,
+        cursor: null,
+      });
+    }
+    return null;
+  },
+});
+
+/**
+ * How far the first backfill has read, from the cursor the chain keeps:
+ * whole 30-day blocks done of the 90 days' total. Null once the account has
+ * synced (a later gap fill runs under `live`). Reads no clock: the range is
+ * counted from the account-local date it was linked on.
+ */
+export function backfillProgress(
+  link: Pick<
+    Doc<"siteAdAccounts">,
+    "lastSyncAt" | "backfillFrom" | "createdAt" | "timezone"
+  >,
+): { done: number; total: number } | null {
+  if (link.lastSyncAt !== undefined) return null;
+  const total = Math.ceil(BACKFILL_DAYS / BACKFILL_BLOCK_DAYS);
+  if (link.backfillFrom === undefined) return { done: 0, total };
+  const start = shiftDate(
+    localDate(link.createdAt, link.timezone),
+    -(BACKFILL_DAYS - 1),
+  );
+  const days = Math.round(
+    (dateDayStart(link.backfillFrom) - dateDayStart(start)) / DAY_MS,
+  );
+  const done = Math.min(
+    Math.max(Math.floor(days / BACKFILL_BLOCK_DAYS), 0),
+    total - 1,
+  );
+  return { done, total };
+}