@ultimat3/jobs 29.4.0 → 30.0.0

package/src/backfill-definition.ts

@@ -0,0 +1,184 @@
+// What `backfill()` takes and answers: the two source shapes (a local chain, a remote page
+// function), the scoped variant keyed per input, and the payload a queue row carries. Apart from
+// `backfill.ts` so the factory, the pass and the registry read one set of types.
+
+import type { Ctx, Environment } from '@ultimat3/core';
+import type { ReadBuilder } from '@ultimat3/entity';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import type { DurationInput } from './clock';
+import type { RetryPolicy } from './retry';
+import type { JobTenant } from './tenant';
+
+export interface BackfillBatch<Row, I = undefined> {
+  /** The page `page()` would have returned here: tenancy, soft delete, projection, preloads. */
+  readonly rows: readonly Row[];
+  readonly ctx: Ctx;
+  /**
+   * The run's cancellation composed with this batch's own ceiling — the same seam `ctx.signal` is
+   * elsewhere. Past it this step may no longer write, so hand it to whatever the body calls.
+   */
+  readonly signal: AbortSignal;
+  /** 0-based position in the pass, and the step name this batch is checkpointed under. */
+  readonly index: number;
+  /** The scoped pass's own input — `undefined` for a backfill declaring none. */
+  readonly input: I;
+}
+
+interface BackfillCommon<Row, I, P> {
+  /**
+   * REQUIRED, unlike a job's. A backfill's name is a durable key — the queue row, the step trace
+   * and the ledger of what has already been run all carry it — so it is never left to whichever
+   * export name a module happened to use.
+   */
+  readonly name: string;
+  /**
+   * REQUIRED, exactly as it is on `job()` — a backfill IS a job, so it declares the org its pass
+   * runs under rather than inheriting the worker's. A payload carries only `force`, so the two
+   * honest spellings are `tenant: () => '<org>'` for a sweep declared against one tenant, and
+   * `tenant: 'none'` for one that spans every tenant.
+   *
+   * `'none'` is where a backfill differs from a plain `job`, and the difference is forced by the
+   * shape of `source`: it hands back a LAZY chain, so every page's plan is built inside the
+   * iteration — after the declaring frame has closed — and there is nothing an author could wrap
+   * in `crossTenant(reason, fn)`. So `backfillPass` opens that scope itself, for a `'none'`
+   * declaration only (`backfill-scope.ts`). A declared tenant is handed its context untouched and
+   * every page is scoped to that org, exactly as a request would be.
+   */
+  readonly tenant: JobTenant<P>;
+  /**
+   * One page, in the batch's own durable step. Deliberately handed no `step`: a step name minted
+   * inside this body would have to be unique across the whole run, and the natural spelling
+   * (`step.run('rewrite', …)`) collides with itself on the second batch (`X_STEP_DUPLICATE`).
+   *
+   * At least once, like every other job body — an attempt cancelled between the last row and the
+   * checkpoint replays this page. Write through `upsertAll`, `updateWhere` or an idempotent
+   * statement; never `count + 1`.
+   */
+  handle(batch: BackfillBatch<Row, I>): Promise<void> | void;
+  /** Rows per statement and per step. Defaults to `DEFAULT_BACKFILL_BATCH`. */
+  readonly batch?: number;
+  /**
+   * Batches per second. Defaults to `DEFAULT_BACKFILL_RATE`, which is slow on purpose: this pass
+   * shares its pool with the requests the app is still serving. Fractions are a rate too —
+   * `rate: 0.5` is one batch every two seconds. To sweep faster raise it; there is no way to
+   * turn it off, because a backfill that saturates the pool has no correct value here.
+   */
+  readonly rate?: number;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  /** Per attempt, not per pass: a resumed attempt picks up at the last checkpoint. */
+  readonly timeout?: DurationInput;
+  /**
+   * The migration this sweep needs applied first — the id `x db gen` wrote, e.g.
+   * `20260814120000_add_publish_at`. Declared DATA, and checked by whoever can read `x_migrations`
+   * (`x db backfill`): this package holds no `@ultimat3/db` dependency, and growing one so a queue
+   * could read a migration ledger would put the migration engine on tier 3's import graph.
+   *
+   * Deliberately NOT a `dependsOn` graph over other backfills. The real dependency is almost
+   * always "after code that tolerates both shapes is serving", which the framework cannot observe,
+   * so a graph would encode an ordering it has no way to be right about.
+   */
+  readonly requires?: string;
+  /**
+   * The environments this sweep may run in. Omitted means EVERY one — never an implied
+   * "cleanups are production": a staging rehearsal is correct practice, so which deploys a sweep
+   * belongs to is the app's convention and this field is only the mechanism that carries it
+   * (axiom 8). A mismatch is `X_BACKFILL_ENVIRONMENT`, refused inside the pass as well as by the
+   * CLI, because a backfill enqueued by app code never passes through a command.
+   */
+  readonly environments?: readonly Environment[];
+  /**
+   * How many rows still match — the SAME predicate `source` selects on, counted rather than read.
+   * Optional for `pages` too: a remote may not know, and then the stall check is silent.
+   * Optional, and what it buys is that a dry run cannot lie and "did it converge" becomes
+   * arithmetic: a pass whose source is exhausted while this still answers above zero has two
+   * predicates that disagree, which is `X_BACKFILL_STALLED` and an authoring bug in any business.
+   */
+  count?(args: { readonly ctx: Ctx; readonly input: I }): Promise<number> | number;
+}
+
+/** One page of a remote source: its rows, and where the next page starts — `null` ends the pass. */
+export interface BackfillPage<Row> {
+  readonly rows: readonly Row[];
+  readonly next: string | null;
+}
+
+/**
+ * A remote source, one page per durable step. Only `{ cursor, rows: n }` is checkpointed — never
+ * the rows (`backfill-pass.ts`) — so a pass killed mid-page asks for THAT page again and no earlier
+ * one. `signal` is the run's cancellation composed with the step's ceiling.
+ */
+export type BackfillPages<Row, I> = (args: {
+  readonly ctx: Ctx;
+  readonly cursor: string | null;
+  readonly signal: AbortSignal;
+  readonly input: I;
+}) => Promise<BackfillPage<Row>>;
+
+interface FromSource<Row> {
+  /**
+   * The rows to visit, as a chain: `() => db.posts.where({ published: true })`, or one narrowed by
+   * the run's own context — `({ ctx }) => db.posts.where({ orgId: ctx.actor.orgId })`. Read once
+   * per attempt and never enqueued, so what a run visits cannot drift from what was declared.
+   * `orderBy` is optional — the driver's own total order ends in the primary key, which is what
+   * lets every batch resume from the last one's cursor.
+   */
+  source(args: { readonly ctx: Ctx }): ReadBuilder<Row>;
+  readonly pages?: never;
+}
+
+interface FromPages<Row, I> {
+  readonly source?: never;
+  /** A remote source: exactly one of `source` and `pages` (`X_BACKFILL_SOURCE_INVALID`). */
+  readonly pages: BackfillPages<Row, I>;
+}
+
+/** An unscoped sweep: one pass per name, over a local chain or a remote page function. */
+export type BackfillDefinition<Row> = BackfillCommon<Row, undefined, BackfillInput> &
+  (FromSource<Row> | FromPages<Row, undefined>) & {
+    readonly input?: never;
+    readonly scope?: never;
+  };
+
+/**
+ * One pass per SCOPE: `sync-invoices` per account. The ledger key and the live idempotency key are
+ * `<name>:<scope(input)>`, so two scopes keep two ledgers and never dedupe onto each other. An
+ * `input` with no `scope` is `X_BACKFILL_SOURCE_INVALID`.
+ */
+export type ScopedBackfillDefinition<Row, I> = BackfillCommon<Row, I, ScopedBackfillInput<I>> &
+  (FromSource<Row> | FromPages<Row, I>) & {
+    readonly input: StandardSchemaV1<unknown, I>;
+    scope(input: I): string;
+  };
+
+/** What one completed pass reports — bounded, so `x jobs show` can print it. */
+export interface BackfillReport {
+  readonly name: string;
+  /** Batches THIS pass handled, replayed ones included. `0` when it was skipped. */
+  readonly batches: number;
+  readonly rows: number;
+  /** True when `x_backfills` already held a completed pass under this name and `force` was not set. */
+  readonly skipped: boolean;
+  /** The completed pass the ledger answered with, when there was one. */
+  readonly previousRunId?: string | undefined;
+}
+
+/**
+ * A backfill's payload is its identity plus the one decision a queue row is allowed to carry.
+ * What to visit is declared, so nothing here can drift from the definition; `force` changes only
+ * whether a name the ledger already records as completed runs a SECOND pass, and never what that
+ * pass would do.
+ */
+export interface BackfillInput {
+  /**
+   * Run even though `x_backfills` holds a completed pass under this name. The rerun is a NEW
+   * ledger row — history is never overwritten, so what each pass swept stays readable.
+   */
+  readonly force?: boolean | undefined;
+}
+
+/** A scoped pass's payload: the force decision, and the input its scope is derived from. */
+export interface ScopedBackfillInput<I> {
+  readonly force?: boolean | undefined;
+  readonly input: I;
+}

package/src/backfill-errors.ts

@@ -1,6 +1,6 @@
-// The seven `X_BACKFILL_*` codes, apart from `errors.ts` for the reason `driver-pg-rows.ts` is
+// The eight `X_BACKFILL_*` codes, apart from `errors.ts` for the reason `driver-pg-rows.ts` is
 // apart from `driver-pg.ts`: one file, one job, and `errors.ts` was over the 500-line ceiling
-// `x verify`'s `filesize` step enforces. The registry stays there — `JOB_OWNED_ERROR_CODES`,
+// `x verify`'s `filesize` step enforces. The registry is `error-codes.ts` — `JOB_OWNED_ERROR_CODES`,
 // `JOB_ERROR_TITLES` and the single `registerErrorCodes()` call — because a package's codes are
 // declared in ONE place; only the classes that throw them live here, beside `backfill-ledger.ts`,
 // `backfill-pending.ts` and `backfill-registry.ts`.
@@ -128,3 +128,19 @@ export class BackfillUnknownError extends UltimateError {
     });
   }
 }
+
+/**
+ * A declaration whose rows cannot be read as written: both or neither of `source` and `pages`, or
+ * an `input` with no `scope` to key one pass per input. Refused at declaration, before the name is
+ * taken, so the corrected declaration still can be.
+ */
+export class BackfillSourceInvalidError extends UltimateError {
+  constructor(input: { backfill: string; problem: string; fix: string }) {
+    super({
+      code: 'X_BACKFILL_SOURCE_INVALID',
+      cause: `backfill "${input.backfill}" ${input.problem}`,
+      fix: input.fix,
+      meta: { backfill: input.backfill },
+    });
+  }
+}

package/src/backfill-pass.ts

@@ -14,18 +14,26 @@
 // a report an operator reads and the record that a completed name has already been swept.
 
 import { appVersion, assert, logger, renderThrowable, resolveEnvironment } from '@ultimat3/core';
-import type { BatchIterator } from '@ultimat3/entity';
-import type { BackfillDefinition, BackfillInput, BackfillReport } from './backfill';
+import { expectedQueryLoop, expectedSeqScan } from '@ultimat3/db';
+import type { BackfillInput, BackfillReport } from './backfill-definition';
 import { BackfillStalledError } from './backfill-errors';
 import { checkBackfillEnvironment } from './backfill-gate';
 import type { BackfillLedger, BackfillRun } from './backfill-ledger';
 import { decideBackfill } from './backfill-ledger';
 import type { Pacer } from './backfill-rate';
+import { pageReader } from './backfill-reader';
 import { withBackfillScope } from './backfill-scope';
+import type { AnyBackfill } from './backfill-source';
+import { ledgerNameFor } from './backfill-source';
 import { jobDriver } from './driver';
 import type { JobRunArgs } from './job';
 import { isStepSuspension } from './steps';
 
+/** Why a backfill's page read and its count read every row: the sweep is the job. */
+const BACKFILL_SCAN = 'a backfill visits every row once, in key order';
+/** Why the scale sweep is told a page's writes grow with the page: the rows ARE the work. */
+const BACKFILL_LOOP = 'a backfill writes every row it visits; a page of N rows is N rows of work';
+
 /** Every batch is a step, and this is the name it is checkpointed under. See `backfillPass()`. */
 const STEP_PREFIX = 'batch:';
 
@@ -36,15 +44,12 @@ interface Checkpoint {
   readonly rows: number;
 }
 
-/** One live iteration and the pull that advances it. Rebuilt when the checkpoints disagree. */
-interface Iteration<Row> {
-  readonly batches: BatchIterator<Row>;
-  readonly pull: AsyncIterator<readonly Row[]>;
-}
+/** What a run's payload carries: `force`, and the input a scoped pass is keyed by. */
+type PassInput = BackfillInput & { readonly input?: unknown };
 
 /** Everything about the backfill that is fixed at declaration, hashed, sized and paced once. */
 export interface BackfillPlan<Row> {
-  readonly definition: BackfillDefinition<Row>;
+  readonly definition: AnyBackfill<Row>;
   readonly size: number;
   readonly checksum: string;
   /** The declared `rate`, already an interval. Built at declaration for the reason `size` is. */
@@ -115,12 +120,14 @@ async function markFailed(
  * to close.
  */
 async function assertConverged<Row>(
-  definition: BackfillDefinition<Row>,
-  ctx: JobRunArgs<BackfillInput>['ctx'],
+  definition: AnyBackfill<Row>,
+  ctx: JobRunArgs<PassInput>['ctx'],
+  input: unknown,
   swept: number,
 ): Promise<void> {
-  if (definition.count === undefined) return;
-  const remaining = await definition.count({ ctx });
+  const count = definition.count;
+  if (count === undefined) return;
+  const remaining = await expectedSeqScan(BACKFILL_SCAN, () => count({ ctx, input }));
   // Parsed, not trusted: `count()` is app code feeding a framework decision, and both `NaN > 0`
   // and `-1 > 0` are FALSE — an unchecked bad number reads as "converged" and writes the completed
   // ledger row that stops the next deploy ever re-running this sweep. The one failure mode this
@@ -142,11 +149,14 @@ async function assertConverged<Row>(
  */
 export async function backfillPass<Row>(
   plan: BackfillPlan<Row>,
-  args: JobRunArgs<BackfillInput>,
+  args: JobRunArgs<PassInput>,
 ): Promise<BackfillReport> {
   const { definition, size, checksum, pace } = plan;
   const { step, runId } = args;
-  const name = definition.name;
+  const input = args.input.input;
+  // The LEDGER's name: the declaration's, or `<name>:<scope>` for a scoped pass — so two scopes
+  // keep two histories, and an unscoped pass writes exactly what it always wrote.
+  const name = ledgerNameFor(definition.name, definition.scope, args.input);
   // Before the ledger is even opened: a sweep this deploy may not run must leave no row saying it
   // started. Enforced here and not only in `x db backfill`, because app code that calls
   // `.enqueue()` directly never passes through a command — a rail only the CLI holds is a
@@ -157,7 +167,11 @@ export async function backfillPass<Row>(
   // nothing must not start failing over a variable it never reads.
   const declaredEnvironments = definition.environments;
   if (declaredEnvironments !== undefined && declaredEnvironments.length > 0) {
-    const mismatch = checkBackfillEnvironment(name, declaredEnvironments, resolveEnvironment());
+    const mismatch = checkBackfillEnvironment(
+      definition.name,
+      declaredEnvironments,
+      resolveEnvironment(),
+    );
     if (mismatch !== undefined) throw mismatch;
   }
   // Absent on a driver that ships no ledger, which runs the pass with no bookkeeping rather than
@@ -214,24 +228,7 @@ export async function backfillPass<Row>(
       let cursor: string | null = null;
       let rows = 0;
       let batches = 0;
-      let live: Iteration<Row> | undefined;
-
-      /**
-       * The iteration positioned at `cursor`, opened on the first batch this attempt actually runs so
-       * a resumed pass sends no statement for a page it already handled.
-       *
-       * `batches.cursor` IS where the next statement starts, so comparing it with the checkpoint's is
-       * the whole staleness test — and it is not academic: `retryFromStep` re-opens ONE step in the
-       * middle of a finished run, which moves the checkpoints somewhere this iteration is not. Rebuilt
-       * from the checkpoint rather than read from wherever the old one was parked.
-       */
-      const iterate = async (): Promise<Iteration<Row>> => {
-        if (live !== undefined && live.batches.cursor === cursor) return live;
-        await live?.batches.close();
-        const opened = definition.source({ ctx }).after(cursor).inBatches(size);
-        live = { batches: opened, pull: opened[Symbol.asyncIterator]() };
-        return live;
-      };
+      const reader = pageReader(definition, ctx, size, input);
 
       try {
         for (let index = 0; ; index += 1) {
@@ -252,16 +249,22 @@ export async function backfillPass<Row>(
               // The signal is the run's cancellation composed with this step's ceiling, so a cancelled
               // pass unwinds out of the wait instead of sitting in a timer nobody is waiting for.
               await pace.wait({ signal, step: stepName });
-              const iteration = await iterate();
-              const next = await iteration.pull.next();
-              if (next.done === true) return { cursor: null, rows: 0 };
+              // A backfill visits every row once, in key order: the scan is the point, and the query-plan
+              // sweep is told so rather than asked to index a column no read filters on.
+              const page = await expectedSeqScan(BACKFILL_SCAN, () => reader.read(cursor, signal));
+              // An exhausted chain, or a remote page with nothing on it: no body, no ledger write.
+              if (page.rows.length === 0) return { cursor: page.next, rows: 0 };
               if (!recording) {
                 recording = true;
                 await open();
               }
-              await definition.handle({ rows: next.value, ctx, signal, index });
+              // Its writes grow with the page by definition — every row visited is a row of work —
+              // so the scale sweep hears the reason instead of judging a loop it was asked to run.
+              await expectedQueryLoop(BACKFILL_LOOP, () =>
+                definition.handle({ rows: page.rows, ctx, signal, index, input }),
+              );
               swept = true;
-              return { cursor: iteration.batches.cursor, rows: next.value.length };
+              return { cursor: page.next, rows: page.rows.length };
             }),
             stepName,
           );
@@ -288,7 +291,7 @@ export async function backfillPass<Row>(
         // this converge", asked once, where the answer is finally decidable. Inside the `try` so the
         // ledger records the attempt as `failed`: a pass that left rows behind must not write the
         // completed row that stops the next deploy re-running it.
-        if (recording) await assertConverged(definition, ctx, rows);
+        if (recording) await assertConverged(definition, ctx, input, rows);
       } catch (error) {
         // Control flow, not a failure: a suspended run is parked and will be back on this step.
         if (!isStepSuspension(error) && recording) await markFailed(ledger, runId, rows);
@@ -296,7 +299,7 @@ export async function backfillPass<Row>(
       } finally {
         // Whatever the iteration holds belongs to this attempt, and an attempt that failed, was
         // cancelled or finished is done with it either way.
-        await live?.batches.close();
+        await reader.close();
       }
 
       if (recording) await ledger?.finish(runId, { status: 'completed', rows });

package/src/backfill-pending.ts

@@ -97,6 +97,36 @@ function stateOf(
   return newest.status === 'running' ? 'running' : 'failed';
 }
 
+/** One declaration judged against the ledger rows under one ledger name. */
+function judged(
+  declaration: BackfillDeclaration,
+  name: string,
+  under: readonly BackfillProgress[],
+  environment: Environment,
+): BackfillStateRow {
+  const newest = under[0];
+  const completed = under.find((run) => run.status === 'completed');
+  return {
+    name,
+    state: stateOf(declaration, under, completed, environment),
+    checksum: declaration.checksum,
+    ledgerChecksum: newest?.checksum ?? null,
+    changed: completed !== undefined && completed.checksum !== declaration.checksum,
+    requires: declaration.requires,
+    environments: declaration.environments,
+    lastRunId: newest?.runId ?? null,
+    rows: newest?.rows ?? null,
+  };
+}
+
+/**
+ * The ledger names a scoped declaration owns: `<name>:<scope>`. A scope is app data — an account,
+ * a vendor — that the framework cannot enumerate, so a scoped declaration reports one row per scope
+ * it has RUN and is never `pending` on its own: "which accounts should have synced" is the app's.
+ */
+const scopesOf = (name: string, ledgerNames: Iterable<string>): string[] =>
+  [...ledgerNames].filter((ledgerName) => ledgerName.startsWith(`${name}:`)).sort();
+
 export function pendingBackfills(input: {
   readonly declarations: readonly BackfillDeclaration[];
   /** Every `x_backfills` row, newest first — `inspectBackfills`' own order, unfiltered. */
@@ -104,24 +134,17 @@ export function pendingBackfills(input: {
   readonly environment: Environment;
 }): BackfillPendingReport {
   const byName = groupByName(input.runs);
-  const rows = input.declarations.map((declaration): BackfillStateRow => {
-    const under = byName.get(declaration.name) ?? [];
-    const newest = under[0];
-    const completed = under.find((run) => run.status === 'completed');
-    return {
-      name: declaration.name,
-      state: stateOf(declaration, under, completed, input.environment),
-      checksum: declaration.checksum,
-      ledgerChecksum: newest?.checksum ?? null,
-      changed: completed !== undefined && completed.checksum !== declaration.checksum,
-      requires: declaration.requires,
-      environments: declaration.environments,
-      lastRunId: newest?.runId ?? null,
-      rows: newest?.rows ?? null,
-    };
+  const owned = new Set<string>();
+  const rows = input.declarations.flatMap((declaration): BackfillStateRow[] => {
+    const names = declaration.scoped
+      ? scopesOf(declaration.name, byName.keys())
+      : [declaration.name];
+    for (const name of names) owned.add(name);
+    return names.map((name) =>
+      judged(declaration, name, byName.get(name) ?? [], input.environment),
+    );
   });
-  const declared = new Set(input.declarations.map((declaration) => declaration.name));
-  const orphaned = [...byName.keys()].filter((name) => !declared.has(name)).sort();
+  const orphaned = [...byName.keys()].filter((name) => !owned.has(name)).sort();
   return {
     environment: input.environment,
     rows,

package/src/backfill-reader.ts

@@ -0,0 +1,94 @@
+// One page of a backfill, from wherever its rows live: a local chain read through `inBatches()`, or
+// the app's remote page function. The pass (`backfill-pass.ts`) asks "the page at this cursor" and
+// checkpoints `{ cursor: next, rows: n }`; this file is the only place the two sources differ.
+
+import type { Ctx } from '@ultimat3/core';
+import { assert } from '@ultimat3/core';
+import type { BatchIterator } from '@ultimat3/entity';
+import type { BackfillPage } from './backfill-definition';
+import type { AnyBackfill } from './backfill-source';
+
+export interface PageReader<Row> {
+  /** The page starting at `cursor`. `next: null` with no rows is an exhausted source. */
+  read(cursor: string | null, signal: AbortSignal): Promise<BackfillPage<Row>>;
+  close(): Promise<void>;
+}
+
+/** One live iteration and the pull that advances it. Rebuilt when the checkpoints disagree. */
+interface Iteration<Row> {
+  readonly batches: BatchIterator<Row>;
+  readonly pull: AsyncIterator<readonly Row[]>;
+}
+
+/**
+ * A local chain. The iteration is opened on the first page an attempt actually reads, so a resumed
+ * pass sends no statement for a page it already handled, and positioned at `cursor`:
+ * `batches.cursor` IS where the next statement starts, so comparing it with the checkpoint's is the
+ * whole staleness test — `retryFromStep` re-opens ONE step mid-run, which moves the checkpoints
+ * somewhere a parked iteration is not.
+ */
+function sourceReader<Row>(
+  source: NonNullable<AnyBackfill<Row>['source']>,
+  ctx: Ctx,
+  size: number,
+): PageReader<Row> {
+  let live: Iteration<Row> | undefined;
+  return {
+    async read(cursor) {
+      if (live === undefined || live.batches.cursor !== cursor) {
+        await live?.batches.close();
+        const opened = source({ ctx }).after(cursor).inBatches(size);
+        live = { batches: opened, pull: opened[Symbol.asyncIterator]() };
+      }
+      const next = await live.pull.next();
+      if (next.done === true) return { rows: [], next: null };
+      return { rows: next.value, next: live.batches.cursor };
+    },
+    async close() {
+      await live?.batches.close();
+    },
+  };
+}
+
+/**
+ * The app's page function. Parsed, not trusted: an absent `next` is not `null`, and a pass that
+ * read it as "more" would ask the vendor for page `undefined` forever.
+ */
+function pagesReader<Row>(
+  definition: AnyBackfill<Row>,
+  pages: NonNullable<AnyBackfill<Row>['pages']>,
+  ctx: Ctx,
+  input: unknown,
+): PageReader<Row> {
+  return {
+    async read(cursor, signal) {
+      const page = await pages({ ctx, cursor, signal, input });
+      const next: unknown = page?.next;
+      assert(
+        Array.isArray(page?.rows) && (next === null || typeof next === 'string'),
+        `backfill "${definition.name}" pages() answered a value that is not { rows: [], next: string | null }`,
+        `pages: async ({ cursor }) => ({ rows: page.items, next: page.nextCursor ?? null }) on backfill({ name: "${definition.name}" })`,
+      );
+      // An empty page that does not move the cursor would be asked for again forever: it ends.
+      if (page.rows.length === 0 && page.next === cursor) return { rows: [], next: null };
+      return page;
+    },
+    close: () => Promise.resolve(),
+  };
+}
+
+export function pageReader<Row>(
+  definition: AnyBackfill<Row>,
+  ctx: Ctx,
+  size: number,
+  input: unknown,
+): PageReader<Row> {
+  if (definition.pages !== undefined) return pagesReader(definition, definition.pages, ctx, input);
+  const source = definition.source;
+  assert(
+    source !== undefined,
+    `backfill "${definition.name}" reached its pass with neither source nor pages`,
+    `pages: async ({ cursor }) => ({ rows, next }) on backfill({ name: "${definition.name}" })`,
+  );
+  return sourceReader((args) => source.call(definition, args), ctx, size);
+}

package/src/backfill-registry.ts

@@ -25,6 +25,8 @@ export type BackfillCount = (args: { readonly ctx: Ctx }) => Promise<number> | n
 /** Everything `backfill()` knows that a `JobHandle` has no field for. */
 export interface BackfillOrigin {
   readonly checksum: string;
+  /** One pass per `scope(input)`: its ledger rows are `<name>:<scope>`, never `<name>`. */
+  readonly scoped: boolean;
   /** A migration id, checked against `x_migrations` by whoever can read it. See `backfill.ts`. */
   readonly requires: string | undefined;
   /** Absent means every environment — never an implied "production only". See `backfill.ts`. */
@@ -45,6 +47,8 @@ export interface BackfillDeclaration {
   readonly requires: string | null;
   readonly environments: readonly Environment[] | null;
   readonly counts: boolean;
+  /** Its passes are keyed `<name>:<scope>`; `x db backfill` lists one row per scope it has run. */
+  readonly scoped: boolean;
 }
 
 const origin = new WeakMap<object, BackfillOrigin>();
@@ -82,6 +86,7 @@ export function declarationOf(handle: AnyJobHandle): BackfillDeclaration | undef
     requires: source.requires ?? null,
     environments: source.environments ?? null,
     counts: source.count !== undefined,
+    scoped: source.scoped,
   };
 }
 

package/src/backfill-scope.ts

@@ -8,7 +8,6 @@
 import type { Actor, Ctx } from '@ultimat3/core';
 import { runWithContext } from '@ultimat3/core';
 import { CROSS_TENANT_SCOPE, crossTenant } from '@ultimat3/entity';
-import type { BackfillInput } from './backfill';
 import type { JobTenant } from './tenant';
 import { NO_JOB_TENANT } from './tenant';
 
@@ -55,7 +54,7 @@ const withCrossTenant = (actor: Actor): Actor =>
  */
 export function withBackfillScope<T>(
   name: string,
-  tenant: JobTenant<BackfillInput>,
+  tenant: JobTenant<never>,
   ctx: Ctx,
   pass: (ctx: Ctx) => Promise<T>,
 ): Promise<T> {