@agent-native/core 0.152.0 → 0.153.0

package/corpus/templates/analytics/server/jobs/analytics-bigquery-backfill.ts

@@ -5,20 +5,23 @@ import { runWithRequestContext } from "@agent-native/core/server";
 
 import {
   backfillFirstPartyAnalyticsBatch,
+  type FirstPartyAnalyticsBackfillCursor,
   type FirstPartyAnalyticsScope,
 } from "../lib/first-party-analytics-backend.js";
 
 const JOB_TABLE = "analytics_bigquery_backfill_jobs";
+const SHARD_TABLE = "analytics_bigquery_backfill_shards";
 const LEASE_MS = 5 * 60 * 1000;
 const DEDICATED_LEASE_MS = 24 * 60 * 60 * 1000;
 const ERROR_RETRY_MS = 5 * 60 * 1000;
 const DEFAULT_BATCH_SIZE = 250;
 const MAX_BATCH_SIZE = 750;
-const DEFAULT_MAX_BATCHES_PER_SWEEP = 2;
+const DEFAULT_MAX_BATCHES_PER_SWEEP = 4;
 const MAX_BATCHES_PER_SWEEP = 4;
+const DEFAULT_PARALLELISM = 8;
+const MAX_PARALLELISM = 8;
 const DEFAULT_MAX_ACTIVE_SESSIONS = 80;
 const DEFAULT_MAX_TOTAL_SESSIONS = 250;
-const BATCH_SIZE_ENV = "ANALYTICS_BIGQUERY_BACKFILL_BATCH_SIZE";
 
 type Query =
   | string
@@ -73,6 +76,26 @@ export interface BigQueryBackfillSweepResult {
   error?: string;
 }
 
+interface BigQueryBackfillShard {
+  id: string;
+  jobId: string;
+  orgId: string;
+  ownerEmail: string;
+  table: string;
+  startAt: string;
+  startId: string;
+  endAt: string;
+  endId: string;
+  endInclusive: boolean;
+  batchSize: number;
+  cursor: FirstPartyAnalyticsBackfillCursor | null;
+  status: BigQueryBackfillJobStatus;
+  copied: number;
+  leaseToken: string | null;
+  leaseExpiresAt: string | null;
+  nextRunAt: string;
+}
+
 function executor(): Executor {
   return getDbExec() as unknown as Executor;
 }
@@ -86,16 +109,6 @@ function boundedBatchSize(value: number | undefined): number {
   return Math.min(Math.max(Math.floor(value as number), 1), MAX_BATCH_SIZE);
 }
 
-function runtimeBatchSize(job: BigQueryBackfillJob): number {
-  const raw = process.env[BATCH_SIZE_ENV]?.trim();
-  if (!raw) return job.batchSize;
-  const parsed = Number.parseInt(raw, 10);
-  if (!Number.isFinite(parsed) || parsed <= 0) return job.batchSize;
-  // Keep the override bounded so operators can lower throughput without
-  // rewriting the cursor-bearing job row.
-  return boundedBatchSize(parsed);
-}
-
 function maxBatchesPerSweep(): number {
   const raw = process.env.ANALYTICS_BIGQUERY_BACKFILL_SWEEP_LIMIT?.trim();
   if (!raw) return DEFAULT_MAX_BATCHES_PER_SWEEP;
@@ -105,6 +118,15 @@ function maxBatchesPerSweep(): number {
     : DEFAULT_MAX_BATCHES_PER_SWEEP;
 }
 
+function parallelism(): number {
+  const raw = process.env.ANALYTICS_BIGQUERY_BACKFILL_PARALLELISM?.trim();
+  if (!raw) return DEFAULT_PARALLELISM;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0
+    ? Math.min(parsed, MAX_PARALLELISM)
+    : DEFAULT_PARALLELISM;
+}
+
 function positiveEnvNumber(name: string, fallback: number): number {
   const raw = process.env[name]?.trim();
   if (!raw) return fallback;
@@ -145,7 +167,7 @@ function rowToJob(row: Record<string, unknown>): BigQueryBackfillJob {
     );
   }
   return {
-    id: stringValue(row, "id"),
+    id: stringValue(row, "id", "shard_id"),
     orgId: stringValue(row, "org_id", "orgId"),
     ownerEmail: stringValue(row, "owner_email", "ownerEmail"),
     table: stringValue(row, "table_ref", "tableRef"),
@@ -166,6 +188,54 @@ function rowToJob(row: Record<string, unknown>): BigQueryBackfillJob {
   };
 }
 
+function cursorFromFields(
+  receivedAt: unknown,
+  id: unknown,
+): FirstPartyAnalyticsBackfillCursor | null {
+  return typeof receivedAt === "string" && receivedAt && typeof id === "string"
+    ? { receivedAt, id }
+    : null;
+}
+
+function booleanValue(row: Record<string, unknown>, key: string): boolean {
+  return row[key] === true || row[key] === 1 || row[key] === "1";
+}
+
+function rowToShard(row: Record<string, unknown>): BigQueryBackfillShard {
+  const status = stringValue(row, "status");
+  if (status !== "pending" && status !== "running" && status !== "completed") {
+    throw new Error(
+      `Unknown BigQuery backfill shard status: ${status || "empty"}`,
+    );
+  }
+  return {
+    id: stringValue(row, "shard_id", "id"),
+    jobId: stringValue(row, "job_id", "jobId"),
+    orgId: stringValue(row, "org_id", "orgId"),
+    ownerEmail: stringValue(row, "owner_email", "ownerEmail"),
+    table: stringValue(row, "table_ref", "tableRef"),
+    startAt: stringValue(row, "start_at", "startAt"),
+    startId: stringValue(row, "start_id", "startId"),
+    endAt: stringValue(row, "end_at", "endAt"),
+    endId: stringValue(row, "end_id", "endId"),
+    endInclusive: booleanValue(row, "end_inclusive"),
+    batchSize: boundedBatchSize(numberValue(row, "batch_size", "batchSize")),
+    cursor: cursorFromFields(
+      row.backfill_cursor_at ?? row.backfillCursorAt,
+      row.backfill_cursor_id ?? row.backfillCursorId,
+    ),
+    status: status as BigQueryBackfillJobStatus,
+    copied: numberValue(row, "copied_count", "copiedCount"),
+    leaseToken: nullableStringValue(row, "lease_token", "leaseToken"),
+    leaseExpiresAt: nullableStringValue(
+      row,
+      "lease_expires_at",
+      "leaseExpiresAt",
+    ),
+    nextRunAt: stringValue(row, "next_run_at", "nextRunAt"),
+  };
+}
+
 function rowFromResult(result: QueryResult): Record<string, unknown> | null {
   const row = result.rows?.[0];
   return row && typeof row === "object"
@@ -192,6 +262,423 @@ export async function getFirstPartyAnalyticsBigQueryBackfillJob(
   return row ? rowToJob(row) : null;
 }
 
+async function getNextFirstPartyAnalyticsBigQueryBackfillJob(
+  db: Executor,
+  now: string,
+): Promise<BigQueryBackfillJob | null> {
+  const result = await db.execute({
+    sql: `SELECT id, org_id, owner_email, table_ref, batch_size,
+                 backfill_cursor, status, copied_count, lease_token,
+                 lease_expires_at, next_run_at, last_error, completed_at,
+                 updated_at
+            FROM ${JOB_TABLE}
+           WHERE next_run_at <= ?
+             AND (
+               status = 'pending'
+               OR (status = 'running' AND lease_expires_at IS NOT NULL
+                   AND lease_expires_at <= ?)
+             )
+           ORDER BY updated_at ASC
+           LIMIT 1`,
+    args: [now, now],
+    timeoutMs: 3_000,
+    maxAttempts: 1,
+  });
+  const row = rowFromResult(result);
+  return row ? rowToJob(row) : null;
+}
+
+function parseJobCursor(
+  cursor: string | null,
+): FirstPartyAnalyticsBackfillCursor | null {
+  if (!cursor) return null;
+  try {
+    const parsed = JSON.parse(
+      cursor,
+    ) as Partial<FirstPartyAnalyticsBackfillCursor>;
+    if (
+      typeof parsed.receivedAt !== "string" ||
+      !parsed.receivedAt ||
+      typeof parsed.id !== "string"
+    ) {
+      throw new Error("cursor fields are missing");
+    }
+    return { receivedAt: parsed.receivedAt, id: parsed.id };
+  } catch (error) {
+    throw new Error(
+      `The durable BigQuery backfill cursor is invalid: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  }
+}
+
+function compareCursors(
+  left: FirstPartyAnalyticsBackfillCursor,
+  right: FirstPartyAnalyticsBackfillCursor,
+): number {
+  return (
+    left.receivedAt.localeCompare(right.receivedAt) ||
+    left.id.localeCompare(right.id)
+  );
+}
+
+function configuredLookbackDays(): number {
+  const raw = process.env.ANALYTICS_BIGQUERY_BACKFILL_LOOKBACK_DAYS?.trim();
+  const parsed = raw ? Number.parseInt(raw, 10) : 60;
+  return Number.isInteger(parsed) && parsed >= 30 && parsed <= 60 ? parsed : 60;
+}
+
+function utcDayStart(value: string): string {
+  const date = new Date(value);
+  date.setUTCHours(0, 0, 0, 0);
+  return date.toISOString();
+}
+
+function addUtcDays(value: string, days: number): string {
+  const date = new Date(value);
+  date.setUTCDate(date.getUTCDate() + days);
+  return date.toISOString();
+}
+
+function initialShardCursor(
+  job: BigQueryBackfillJob,
+  now: string,
+): FirstPartyAnalyticsBackfillCursor {
+  return (
+    parseJobCursor(job.cursor) ?? {
+      receivedAt: new Date(
+        Date.parse(now) - configuredLookbackDays() * 24 * 60 * 60 * 1000,
+      ).toISOString(),
+      id: "",
+    }
+  );
+}
+
+export function buildBackfillShardRanges(
+  job: BigQueryBackfillJob,
+  now: string,
+): Array<{
+  start: FirstPartyAnalyticsBackfillCursor;
+  end: FirstPartyAnalyticsBackfillCursor;
+  endInclusive: boolean;
+}> {
+  const start = initialShardCursor(job, now);
+  const currentDayEnd = addUtcDays(utcDayStart(now), 1);
+  const ranges: Array<{
+    start: FirstPartyAnalyticsBackfillCursor;
+    end: FirstPartyAnalyticsBackfillCursor;
+    endInclusive: boolean;
+  }> = [];
+  let rangeStart = start;
+  while (
+    compareCursors(rangeStart, {
+      receivedAt: currentDayEnd,
+      id: "",
+    }) < 0
+  ) {
+    const nextDay = addUtcDays(utcDayStart(rangeStart.receivedAt), 1);
+    const endAt = nextDay < currentDayEnd ? nextDay : currentDayEnd;
+    const end = { receivedAt: endAt, id: "" };
+    ranges.push({ start: rangeStart, end, endInclusive: false });
+    rangeStart = end;
+  }
+  return ranges;
+}
+
+async function finalizeCoordinatorIfComplete(
+  db: Executor,
+  job: BigQueryBackfillJob,
+): Promise<boolean> {
+  const remainingResult = await db.execute({
+    sql: `SELECT COUNT(*) AS remaining
+            FROM ${SHARD_TABLE}
+           WHERE job_id = ? AND status <> 'completed'`,
+    args: [job.id],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  const remaining = numberValue(
+    rowFromResult(remainingResult) ?? {},
+    "remaining",
+  );
+  if (remaining !== 0) return false;
+
+  const summaryResult = await db.execute({
+    sql: `SELECT COALESCE(SUM(copied_count), 0) AS copied_count
+            FROM ${SHARD_TABLE}
+           WHERE job_id = ?`,
+    args: [job.id],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  const endpointResult = await db.execute({
+    sql: `SELECT end_at, end_id
+            FROM ${SHARD_TABLE}
+           WHERE job_id = ?
+           ORDER BY end_at DESC, end_id DESC
+           LIMIT 1`,
+    args: [job.id],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  const summary = rowFromResult(summaryResult);
+  const endpoint = rowFromResult(endpointResult);
+  const endAt = stringValue(endpoint ?? {}, "end_at", "endAt");
+  const endId = stringValue(endpoint ?? {}, "end_id", "endId");
+  const cursor = endAt
+    ? JSON.stringify({ receivedAt: endAt, id: endId })
+    : job.cursor;
+  await db.execute({
+    sql: `UPDATE ${JOB_TABLE}
+             SET status = 'completed', backfill_cursor = ?,
+                 copied_count = copied_count + ?, lease_token = NULL, lease_expires_at = NULL,
+                 next_run_at = ?, last_error = NULL, completed_at = ?, updated_at = ?
+           WHERE id = ? AND status <> 'completed'`,
+    args: [
+      cursor,
+      numberValue(summary ?? {}, "copied_count", "copiedCount"),
+      new Date().toISOString(),
+      new Date().toISOString(),
+      new Date().toISOString(),
+      job.id,
+    ],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  return true;
+}
+
+async function ensureBackfillShards(
+  db: Executor,
+  job: BigQueryBackfillJob,
+): Promise<void> {
+  if (job.status === "completed") return;
+  const existing = await db.execute({
+    sql: `SELECT shard_id FROM ${SHARD_TABLE} WHERE job_id = ? LIMIT 1`,
+    args: [job.id],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  if (rowFromResult(existing)) return;
+
+  const now = new Date().toISOString();
+  for (const range of buildBackfillShardRanges(job, now)) {
+    const id = `${job.id}:${range.start.receivedAt}:${range.start.id}:${range.end.receivedAt}`;
+    await db.execute({
+      sql: `INSERT INTO ${SHARD_TABLE} (
+              shard_id, job_id, org_id, owner_email, table_ref,
+              start_at, start_id, end_at, end_id, end_inclusive,
+              batch_size, backfill_cursor_at, backfill_cursor_id,
+              status, copied_count, lease_token, lease_expires_at,
+              next_run_at, last_error, completed_at, updated_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL,
+                      'pending', 0, NULL, NULL, ?, NULL, NULL, ?)
+            ON CONFLICT (shard_id) DO NOTHING`,
+      args: [
+        id,
+        job.id,
+        job.orgId,
+        job.ownerEmail,
+        job.table,
+        range.start.receivedAt,
+        range.start.id,
+        range.end.receivedAt,
+        range.end.id,
+        range.endInclusive,
+        job.batchSize,
+        now,
+        now,
+      ],
+      timeoutMs: 5_000,
+      maxAttempts: 1,
+    });
+  }
+}
+
+async function claimNextShard(
+  db: Executor,
+  coordinatorId: string,
+  now: string,
+): Promise<BigQueryBackfillShard | null> {
+  if (!db.transaction) {
+    throw new Error("BigQuery backfill requires a database transaction");
+  }
+  const token = randomUUID();
+  const leaseExpiresAt = new Date(Date.now() + LEASE_MS).toISOString();
+  return db.transaction(async (tx) => {
+    const candidateResult = await tx.execute({
+      sql: `SELECT shard_id, job_id, org_id, owner_email, table_ref,
+                   start_at, start_id, end_at, end_id, end_inclusive,
+                   batch_size, backfill_cursor_at, backfill_cursor_id,
+                   status, copied_count, lease_token, lease_expires_at,
+                   next_run_at, updated_at
+             FROM ${SHARD_TABLE}
+             WHERE job_id = ?
+               AND next_run_at <= ?
+               AND (
+                 status = 'pending'
+                 OR (status = 'running' AND lease_expires_at IS NOT NULL
+                     AND lease_expires_at <= ?)
+               )
+             ORDER BY start_at DESC, start_id DESC, updated_at ASC
+             LIMIT 1`,
+      args: [coordinatorId, now, now],
+      timeoutMs: 3_000,
+      maxAttempts: 1,
+    });
+    const candidate = rowFromResult(candidateResult);
+    if (!candidate) return null;
+    const updated = await tx.execute({
+      sql: `UPDATE ${SHARD_TABLE}
+               SET status = 'running', lease_token = ?, lease_expires_at = ?,
+                   updated_at = ?
+             WHERE shard_id = ? AND job_id = ? AND next_run_at <= ?
+               AND (
+                 status = 'pending'
+                 OR (status = 'running' AND lease_expires_at IS NOT NULL
+                     AND lease_expires_at <= ?)
+               )`,
+      args: [
+        token,
+        leaseExpiresAt,
+        now,
+        stringValue(candidate, "shard_id", "id"),
+        coordinatorId,
+        now,
+        now,
+      ],
+      timeoutMs: 3_000,
+      maxAttempts: 1,
+    });
+    if (updated.rowsAffected !== 1) return null;
+    return rowToShard({
+      ...candidate,
+      status: "running",
+      lease_token: token,
+      lease_expires_at: leaseExpiresAt,
+      updated_at: now,
+    });
+  });
+}
+
+async function finishShard(
+  db: Executor,
+  shard: BigQueryBackfillShard,
+  result: { nextCursor: string | null; copied: number; complete: boolean },
+): Promise<void> {
+  let nextCursor: FirstPartyAnalyticsBackfillCursor | null = null;
+  if (result.nextCursor) {
+    try {
+      const parsed = JSON.parse(
+        result.nextCursor,
+      ) as Partial<FirstPartyAnalyticsBackfillCursor>;
+      if (
+        typeof parsed.receivedAt !== "string" ||
+        !parsed.receivedAt ||
+        typeof parsed.id !== "string"
+      ) {
+        throw new Error("cursor fields are missing");
+      }
+      nextCursor = { receivedAt: parsed.receivedAt, id: parsed.id };
+    } catch (error) {
+      throw new Error(
+        `BigQuery backfill returned an invalid shard cursor: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    }
+  }
+  const now = new Date().toISOString();
+  const updated = await db.execute({
+    sql: `UPDATE ${SHARD_TABLE}
+             SET status = ?, backfill_cursor_at = ?, backfill_cursor_id = ?,
+                 copied_count = copied_count + ?, lease_token = NULL,
+                 lease_expires_at = NULL, next_run_at = ?, last_error = NULL,
+                 completed_at = ?, updated_at = ?
+           WHERE shard_id = ? AND lease_token = ?`,
+    args: [
+      result.complete ? "completed" : "pending",
+      nextCursor?.receivedAt ?? null,
+      nextCursor?.id ?? null,
+      result.copied,
+      now,
+      result.complete ? now : null,
+      now,
+      shard.id,
+      shard.leaseToken,
+    ],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  if (updated.rowsAffected !== 1) {
+    throw new Error(
+      "BigQuery backfill shard lost its lease before saving progress",
+    );
+  }
+}
+
+async function scheduleShardRetry(
+  db: Executor,
+  shard: BigQueryBackfillShard,
+  error: string,
+): Promise<void> {
+  const retryAt = new Date(Date.now() + ERROR_RETRY_MS).toISOString();
+  await db.execute({
+    sql: `UPDATE ${SHARD_TABLE}
+             SET status = 'pending', lease_token = NULL, lease_expires_at = NULL,
+                 next_run_at = ?, last_error = ?, updated_at = ?
+           WHERE shard_id = ? AND lease_token = ?`,
+    args: [
+      retryAt,
+      error.slice(0, 1_000),
+      new Date().toISOString(),
+      shard.id,
+      shard.leaseToken,
+    ],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  await db.execute({
+    sql: `UPDATE ${JOB_TABLE}
+             SET last_error = ?, updated_at = ?
+           WHERE id = ? AND status <> 'completed'`,
+    args: [error.slice(0, 1_000), new Date().toISOString(), shard.jobId],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+}
+
+async function runClaimedShard(
+  db: Executor,
+  shard: BigQueryBackfillShard,
+): Promise<{ copied: number; complete: boolean }> {
+  const cursor = shard.cursor ? JSON.stringify(shard.cursor) : null;
+  const result = await runWithRequestContext(
+    { userEmail: shard.ownerEmail, orgId: shard.orgId },
+    () =>
+      backfillFirstPartyAnalyticsBatch(
+        { userEmail: shard.ownerEmail, orgId: shard.orgId },
+        cursor,
+        shard.batchSize,
+        shard.table,
+        {
+          rangeStart: {
+            receivedAt: shard.startAt,
+            id: shard.startId,
+          },
+          rangeEnd: {
+            receivedAt: shard.endAt,
+            id: shard.endId,
+          },
+          rangeEndInclusive: shard.endInclusive,
+        },
+      ),
+  );
+  await finishShard(db, shard, result);
+  return { copied: result.copied, complete: result.complete };
+}
+
 export async function acquireDedicatedFirstPartyAnalyticsBackfillLease(
   scope: FirstPartyAnalyticsScope,
   table: string,
@@ -200,6 +687,17 @@ export async function acquireDedicatedFirstPartyAnalyticsBackfillLease(
     throw new Error("Dedicated BigQuery backfill requires an organization");
   }
   const id = jobId(scope.orgId);
+  const sharded = await executor().execute({
+    sql: `SELECT shard_id FROM ${SHARD_TABLE} WHERE job_id = ? LIMIT 1`,
+    args: [id],
+    timeoutMs: 5_000,
+    maxAttempts: 1,
+  });
+  if (rowFromResult(sharded)) {
+    throw new Error(
+      "The sharded BigQuery backfill owns this migration; use the durable scheduled worker",
+    );
+  }
   const token = randomUUID();
   const now = new Date().toISOString();
   const leaseExpiresAt = new Date(
@@ -313,113 +811,6 @@ export async function queueFirstPartyAnalyticsBigQueryBackfill(
   return job;
 }
 
-async function claimNextJob(
-  db: Executor,
-  now: string,
-): Promise<BigQueryBackfillJob | null> {
-  if (!db.transaction) {
-    throw new Error("BigQuery backfill requires a database transaction");
-  }
-  const token = randomUUID();
-  const leaseExpiresAt = new Date(Date.now() + LEASE_MS).toISOString();
-  return db.transaction(async (tx) => {
-    const candidateResult = await tx.execute({
-      sql: `SELECT id, org_id, owner_email, table_ref, batch_size,
-                   backfill_cursor, status, copied_count, lease_token,
-                   lease_expires_at, next_run_at, last_error, completed_at,
-                   updated_at
-              FROM ${JOB_TABLE}
-             WHERE next_run_at <= ?
-               AND (
-                 status = 'pending'
-                 OR (status = 'running' AND lease_expires_at IS NOT NULL
-                     AND lease_expires_at <= ?)
-               )
-             ORDER BY updated_at ASC
-             LIMIT 1`,
-      args: [now, now],
-      timeoutMs: 3_000,
-      maxAttempts: 1,
-    });
-    const candidate = rowFromResult(candidateResult);
-    if (!candidate) return null;
-
-    const updated = await tx.execute({
-      sql: `UPDATE ${JOB_TABLE}
-               SET status = 'running', lease_token = ?, lease_expires_at = ?,
-                   updated_at = ?
-             WHERE id = ?
-               AND next_run_at <= ?
-               AND (
-                 status = 'pending'
-                 OR (status = 'running' AND lease_expires_at IS NOT NULL
-                     AND lease_expires_at <= ?)
-               )`,
-      args: [token, leaseExpiresAt, now, candidate.id, now, now],
-      timeoutMs: 3_000,
-      maxAttempts: 1,
-    });
-    if (updated.rowsAffected !== 1) return null;
-    return rowToJob({
-      ...candidate,
-      status: "running",
-      lease_token: token,
-      lease_expires_at: leaseExpiresAt,
-      updated_at: now,
-    });
-  });
-}
-
-async function finishJob(
-  db: Executor,
-  job: BigQueryBackfillJob,
-  result: { nextCursor: string | null; copied: number; complete: boolean },
-  now: string,
-): Promise<void> {
-  const completedAt = result.complete ? now : null;
-  const updated = await db.execute({
-    sql: `UPDATE ${JOB_TABLE}
-             SET status = ?, backfill_cursor = ?, copied_count = copied_count + ?,
-                 lease_token = NULL, lease_expires_at = NULL,
-                 next_run_at = ?, last_error = NULL, completed_at = ?,
-                 updated_at = ?
-           WHERE id = ? AND lease_token = ?`,
-    args: [
-      result.complete ? "completed" : "pending",
-      result.nextCursor,
-      result.copied,
-      now,
-      completedAt,
-      now,
-      job.id,
-      job.leaseToken,
-    ],
-    timeoutMs: 5_000,
-    maxAttempts: 1,
-  });
-  if (updated.rowsAffected !== 1) {
-    throw new Error("BigQuery backfill lost its lease before saving progress");
-  }
-}
-
-async function scheduleRetry(
-  db: Executor,
-  job: BigQueryBackfillJob,
-  error: string,
-  now: string,
-): Promise<void> {
-  const retryAt = new Date(Date.now() + ERROR_RETRY_MS).toISOString();
-  await db.execute({
-    sql: `UPDATE ${JOB_TABLE}
-             SET status = 'pending', lease_token = NULL, lease_expires_at = NULL,
-                 next_run_at = ?, last_error = ?, updated_at = ?
-           WHERE id = ? AND lease_token = ?`,
-    args: [retryAt, error.slice(0, 1_000), now, job.id, job.leaseToken],
-    timeoutMs: 5_000,
-    maxAttempts: 1,
-  });
-}
-
 async function pressureSnapshot(
   db: Executor,
 ): Promise<{ paused: boolean; reason?: string }> {
@@ -482,6 +873,16 @@ export async function runFirstPartyAnalyticsBigQueryBackfillOnce(): Promise<BigQ
     };
   }
 
+  const coordinator = await getNextFirstPartyAnalyticsBigQueryBackfillJob(
+    db,
+    new Date().toISOString(),
+  );
+  if (!coordinator) {
+    return { status: "idle", batches: 0, copied: 0, remaining: 0 };
+  }
+
+  await ensureBackfillShards(db, coordinator);
+
   let batches = 0;
   let copied = 0;
   for (let index = 0; index < maxBatchesPerSweep(); index += 1) {
@@ -496,47 +897,58 @@ export async function runFirstPartyAnalyticsBigQueryBackfillOnce(): Promise<BigQ
       };
     }
 
-    const job = await claimNextJob(db, new Date().toISOString());
-    if (!job) {
+    const claimed = (
+      await Promise.all(
+        Array.from({ length: parallelism() }, () =>
+          claimNextShard(db, coordinator.id, new Date().toISOString()),
+        ),
+      )
+    ).filter((shard): shard is BigQueryBackfillShard => shard !== null);
+    if (!claimed.length) {
+      const completed = await finalizeCoordinatorIfComplete(db, coordinator);
       return {
-        status: batches ? "progress" : "idle",
+        status: completed ? "completed" : batches ? "progress" : "idle",
         batches,
         copied,
-        remaining: batches ? 1 : 0,
+        remaining: completed ? 0 : 1,
       };
     }
 
-    try {
-      const batch = await runWithRequestContext(
-        { userEmail: job.ownerEmail, orgId: job.orgId },
-        () =>
-          backfillFirstPartyAnalyticsBatch(
-            { userEmail: job.ownerEmail, orgId: job.orgId },
-            job.cursor,
-            runtimeBatchSize(job),
-            job.table,
-          ),
-      );
-      const now = new Date().toISOString();
-      // The lease row is the progress source of truth; the backend setting is
-      // only the migration configuration and cutover marker.
-      await finishJob(db, job, batch, now);
-      batches += 1;
-      copied += batch.copied;
-      if (batch.complete) {
-        return { status: "completed", batches, copied, remaining: 0 };
+    const outcomes = await Promise.allSettled(
+      claimed.map((shard) => runClaimedShard(db, shard)),
+    );
+    let retryScheduled = false;
+    for (
+      let outcomeIndex = 0;
+      outcomeIndex < outcomes.length;
+      outcomeIndex += 1
+    ) {
+      const outcome = outcomes[outcomeIndex]!;
+      const shard = claimed[outcomeIndex]!;
+      if (outcome.status === "fulfilled") {
+        batches += 1;
+        copied += outcome.value.copied;
+      } else {
+        retryScheduled = true;
+        const message =
+          outcome.reason instanceof Error
+            ? outcome.reason.message
+            : String(outcome.reason);
+        await scheduleShardRetry(db, shard, message);
       }
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      await scheduleRetry(db, job, message, new Date().toISOString());
+    }
+    if (retryScheduled) {
       return {
         status: "retry-scheduled",
         batches,
         copied,
         remaining: 1,
-        error: message,
       };
     }
+
+    if (await finalizeCoordinatorIfComplete(db, coordinator)) {
+      return { status: "completed", batches, copied, remaining: 0 };
+    }
   }
 
   return { status: "progress", batches, copied, remaining: 1 };