@ultimat3/jobs 29.4.0 → 30.0.0

package/src/backfill-source.ts

@@ -0,0 +1,113 @@
+// Where a backfill's rows come from, and which ledger a pass writes: exactly one of `source` (a
+// local chain) and `pages` (a remote page function), and — for a scoped declaration — the key
+// `<name>:<scope(input)>` that both the ledger and the live idempotency key carry.
+
+import type { Ctx, Environment } from '@ultimat3/core';
+import { assert, describeValue } from '@ultimat3/core';
+import type { ReadBuilder } from '@ultimat3/entity';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import { validate } from '@ultimat3/schema';
+import type { BackfillBatch, BackfillPages } from './backfill-definition';
+import { BackfillSourceInvalidError } from './backfill-errors';
+import type { JobTenant } from './tenant';
+
+/**
+ * Every declaration shape as the pass reads it. Methods, so a typed declaration of either shape
+ * is assignable without a cast per field.
+ */
+export interface AnyBackfill<Row> {
+  readonly name: string;
+  readonly tenant: JobTenant<never>;
+  source?(args: { readonly ctx: Ctx }): ReadBuilder<Row>;
+  readonly pages?: BackfillPages<Row, unknown> | undefined;
+  handle(batch: BackfillBatch<Row, unknown>): Promise<void> | void;
+  count?(args: { readonly ctx: Ctx; readonly input: unknown }): Promise<number> | number;
+  readonly environments?: readonly Environment[] | undefined;
+  scope?(input: unknown): string;
+}
+
+const invalid = (backfill: string, problem: string, fix: string): BackfillSourceInvalidError =>
+  new BackfillSourceInvalidError({ backfill, problem, fix });
+
+/** Refused where it was written: both or neither source, or an input no scope can key. */
+export function assertBackfillSource(definition: {
+  readonly name: string;
+  readonly source?: unknown;
+  readonly pages?: unknown;
+  readonly input?: unknown;
+  readonly scope?: unknown;
+}): void {
+  const name = definition.name;
+  const hasSource = typeof definition.source === 'function';
+  const hasPages = typeof definition.pages === 'function';
+  if (hasSource === hasPages) {
+    throw invalid(
+      name,
+      hasSource
+        ? 'declares both source and pages, and a pass reads exactly one of them'
+        : 'declares neither source nor pages, so it has no rows to visit',
+      `pages: async ({ ctx, cursor, signal, input }) => ({ rows, next }) on backfill({ name: "${name}" }) — or source: ({ ctx }) => db.<table>.where(…) — never both`,
+    );
+  }
+  if (definition.input !== undefined && typeof definition.scope !== 'function') {
+    throw invalid(
+      name,
+      'declares an input and no scope, so every input would share one ledger row and one live pass',
+      `scope: (input) => input.accountId beside input on backfill({ name: "${name}" })`,
+    );
+  }
+}
+
+/**
+ * The ledger key and the live idempotency key. Unscoped it is the name — byte-identical to every
+ * pass recorded before scopes existed. A scope is app data, so an empty one is refused rather than
+ * colliding with the unscoped name.
+ */
+export function ledgerNameFor(
+  name: string,
+  scope: ((input: never) => string) | undefined,
+  payload: { readonly input?: unknown },
+): string {
+  if (scope === undefined) return name;
+  const key: unknown = (scope as (input: unknown) => unknown)(payload.input);
+  assert(
+    typeof key === 'string' && key.length > 0,
+    `backfill "${name}" scope(input) answered ${describeValue(key)} — a scope is a non-empty string`,
+    `scope: (input) => input.accountId on backfill({ name: "${name}" })`,
+  );
+  return `${name}:${key}`;
+}
+
+/** `{ force?, input }`, the input checked by the declaration's own schema, synchronously. */
+export function scopedPayload<I>(
+  name: string,
+  input: StandardSchemaV1<unknown, I>,
+): StandardSchemaV1<unknown, { readonly force?: boolean; readonly input: I }> {
+  return {
+    '~standard': {
+      version: 1,
+      vendor: 'ultimate-backfill',
+      validate(value: unknown) {
+        if (typeof value !== 'object' || value === null) {
+          return { issues: [{ message: `backfill "${name}" payload must be { force?, input }` }] };
+        }
+        const raw = value as { readonly force?: unknown; readonly input?: unknown };
+        if (raw.force !== undefined && typeof raw.force !== 'boolean') {
+          return { issues: [{ message: 'force must be a boolean', path: ['force'] }] };
+        }
+        const parsed = validate(input, raw.input);
+        if (parsed.issues !== undefined) {
+          return {
+            issues: parsed.issues.map((issue) => ({
+              message: issue.message,
+              path: ['input', ...(issue.path ?? [])],
+            })),
+          };
+        }
+        return {
+          value: { ...(raw.force === undefined ? {} : { force: raw.force }), input: parsed.value },
+        };
+      },
+    },
+  };
+}

package/src/backfill.ts

@@ -9,19 +9,23 @@
 // and `execute.ts` already have, and the reason the iteration, the checkpoints and the
 // `x_backfills` ledger are one file's problem rather than this one's.
 
-import type { Ctx, Environment } from '@ultimat3/core';
 import { assert } from '@ultimat3/core';
-import type { ReadBuilder } from '@ultimat3/entity';
 import { t } from '@ultimat3/schema';
+import type {
+  BackfillDefinition,
+  BackfillInput,
+  ScopedBackfillDefinition,
+  ScopedBackfillInput,
+} from './backfill-definition';
 import { backfillChecksum } from './backfill-ledger';
 import { backfillPass } from './backfill-pass';
 import { backfillPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
 import type { BackfillCount } from './backfill-registry';
 import { stampBackfill } from './backfill-registry';
-import type { DurationInput } from './clock';
+import type { AnyBackfill } from './backfill-source';
+import { assertBackfillSource, ledgerNameFor, scopedPayload } from './backfill-source';
 import type { JobHandle } from './job';
 import { job } from './job';
-import type { RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 import type { JobTenant } from './tenant';
 
@@ -33,126 +37,26 @@ import type { JobTenant } from './tenant';
  */
 export const DEFAULT_BACKFILL_BATCH = 1_000;
 
-export interface BackfillBatch<Row> {
-  /** The page `page()` would have returned here: tenancy, soft delete, projection, preloads. */
-  readonly rows: readonly Row[];
-  readonly ctx: Ctx;
-  /**
-   * The run's cancellation composed with this batch's own ceiling — the same seam `ctx.signal` is
-   * elsewhere. Past it this step may no longer write, so hand it to whatever the body calls.
-   */
-  readonly signal: AbortSignal;
-  /** 0-based position in the pass, and the step name this batch is checkpointed under. */
-  readonly index: number;
-}
-
-export interface BackfillDefinition<Row> {
-  /**
-   * REQUIRED, unlike a job's. A backfill's name is a durable key — the queue row, the step trace
-   * and the ledger of what has already been run all carry it — so it is never left to whichever
-   * export name a module happened to use.
-   */
-  readonly name: string;
-  /**
-   * REQUIRED, exactly as it is on `job()` — a backfill IS a job, so it declares the org its pass
-   * runs under rather than inheriting the worker's. A payload carries only `force`, so the two
-   * honest spellings are `tenant: () => '<org>'` for a sweep declared against one tenant, and
-   * `tenant: 'none'` for one that spans every tenant.
-   *
-   * `'none'` is where a backfill differs from a plain `job`, and the difference is forced by the
-   * shape of `source`: it hands back a LAZY chain, so every page's plan is built inside the
-   * iteration — after the declaring frame has closed — and there is nothing an author could wrap
-   * in `crossTenant(reason, fn)`. So `backfillPass` opens that scope itself, for a `'none'`
-   * declaration only (`backfill-scope.ts`). A declared tenant is handed its context untouched and
-   * every page is scoped to that org, exactly as a request would be.
-   */
-  readonly tenant: JobTenant<BackfillInput>;
-  /**
-   * The rows to visit, as a chain: `() => db.posts.where({ published: true })`, or one narrowed by
-   * the run's own context — `({ ctx }) => db.posts.where({ orgId: ctx.actor.orgId })`. Read once
-   * per attempt and never enqueued, so what a run visits cannot drift from what was declared.
-   * `orderBy` is optional — the driver's own total order ends in the primary key, which is what
-   * lets every batch resume from the last one's cursor.
-   */
-  source(args: { readonly ctx: Ctx }): ReadBuilder<Row>;
-  /**
-   * One page, in the batch's own durable step. Deliberately handed no `step`: a step name minted
-   * inside this body would have to be unique across the whole run, and the natural spelling
-   * (`step.run('rewrite', …)`) collides with itself on the second batch (`X_STEP_DUPLICATE`).
-   *
-   * At least once, like every other job body — an attempt cancelled between the last row and the
-   * checkpoint replays this page. Write through `upsertAll`, `updateWhere` or an idempotent
-   * statement; never `count + 1`.
-   */
-  handle(batch: BackfillBatch<Row>): Promise<void> | void;
-  /** Rows per statement and per step. Defaults to `DEFAULT_BACKFILL_BATCH`. */
-  readonly batch?: number;
-  /**
-   * Batches per second. Defaults to `DEFAULT_BACKFILL_RATE`, which is slow on purpose: this pass
-   * shares its pool with the requests the app is still serving. Fractions are a rate too —
-   * `rate: 0.5` is one batch every two seconds. To sweep faster raise it; there is no way to
-   * turn it off, because a backfill that saturates the pool has no correct value here.
-   */
-  readonly rate?: number;
-  readonly queue?: string;
-  readonly retry?: RetryPolicy;
-  /** Per attempt, not per pass: a resumed attempt picks up at the last checkpoint. */
-  readonly timeout?: DurationInput;
-  /**
-   * The migration this sweep needs applied first — the id `x db gen` wrote, e.g.
-   * `20260814120000_add_publish_at`. Declared DATA, and checked by whoever can read `x_migrations`
-   * (`x db backfill`): this package holds no `@ultimat3/db` dependency, and growing one so a queue
-   * could read a migration ledger would put the migration engine on tier 3's import graph.
-   *
-   * Deliberately NOT a `dependsOn` graph over other backfills. The real dependency is almost
-   * always "after code that tolerates both shapes is serving", which the framework cannot observe,
-   * so a graph would encode an ordering it has no way to be right about.
-   */
-  readonly requires?: string;
-  /**
-   * The environments this sweep may run in. Omitted means EVERY one — never an implied
-   * "cleanups are production": a staging rehearsal is correct practice, so which deploys a sweep
-   * belongs to is the app's convention and this field is only the mechanism that carries it
-   * (axiom 8). A mismatch is `X_BACKFILL_ENVIRONMENT`, refused inside the pass as well as by the
-   * CLI, because a backfill enqueued by app code never passes through a command.
-   */
-  readonly environments?: readonly Environment[];
-  /**
-   * How many rows still match — the SAME predicate `source` selects on, counted rather than read.
-   * Optional, and what it buys is that a dry run cannot lie and "did it converge" becomes
-   * arithmetic: a pass whose source is exhausted while this still answers above zero has two
-   * predicates that disagree, which is `X_BACKFILL_STALLED` and an authoring bug in any business.
-   */
-  count?(args: { readonly ctx: Ctx }): Promise<number> | number;
-}
-
-/** What one completed pass reports — bounded, so `x jobs show` can print it. */
-export interface BackfillReport {
-  readonly name: string;
-  /** Batches THIS pass handled, replayed ones included. `0` when it was skipped. */
-  readonly batches: number;
-  readonly rows: number;
-  /** True when `x_backfills` already held a completed pass under this name and `force` was not set. */
-  readonly skipped: boolean;
-  /** The completed pass the ledger answered with, when there was one. */
-  readonly previousRunId?: string | undefined;
-}
-
-/**
- * A backfill's payload is its identity plus the one decision a queue row is allowed to carry.
- * What to visit is declared, so nothing here can drift from the definition; `force` changes only
- * whether a name the ledger already records as completed runs a SECOND pass, and never what that
- * pass would do.
- */
-export interface BackfillInput {
-  /**
-   * Run even though `x_backfills` holds a completed pass under this name. The rerun is a NEW
-   * ledger row — history is never overwritten, so what each pass swept stays readable.
-   */
-  readonly force?: boolean | undefined;
-}
+export type {
+  BackfillBatch,
+  BackfillDefinition,
+  BackfillInput,
+  BackfillPage,
+  BackfillPages,
+  BackfillReport,
+  ScopedBackfillDefinition,
+  ScopedBackfillInput,
+} from './backfill-definition';
 
-export function backfill<Row>(definition: BackfillDefinition<Row>): JobHandle<BackfillInput> {
+export function backfill<Row>(definition: BackfillDefinition<Row>): JobHandle<BackfillInput>;
+export function backfill<Row, I>(
+  definition: ScopedBackfillDefinition<Row, I>,
+): JobHandle<ScopedBackfillInput<I>>;
+export function backfill<Row, I>(
+  definition: BackfillDefinition<Row> | ScopedBackfillDefinition<Row, I>,
+): JobHandle<BackfillInput> | JobHandle<ScopedBackfillInput<I>> {
+  // Before anything is registered under the name, so the corrected declaration still can be.
+  assertBackfillSource(definition);
   const size = definition.batch ?? DEFAULT_BACKFILL_BATCH;
   // Refused where it was written. `inBatches()` refuses the same number one statement in, which
   // for a backfill means a dead-lettered job and a stack trace instead of a failing build.
@@ -173,7 +77,8 @@ export function backfill<Row>(definition: BackfillDefinition<Row>): JobHandle<Ba
   // per attempt would charge every batch of every pass for a fact that is fixed at import.
   // `rate` is NOT in it, for the reason `batch` is not: pacing is a tuning knob, and changing one
   // does not make a completed sweep a different sweep.
-  const checksum = backfillChecksum(definition.source, definition.handle);
+  // `pages` is hashed where `source` would be: either is "which rows this pass visits".
+  const checksum = backfillChecksum(definition.pages ?? definition.source, definition.handle);
   // Built here rather than per attempt: the interval belongs to the table and the pool, not to
   // whichever attempt holds the run, so a retrying pass keeps the pace it was declared with.
   const pace = backfillPacer({ rate, job: definition.name });
@@ -181,33 +86,47 @@ export function backfill<Row>(definition: BackfillDefinition<Row>): JobHandle<Ba
   // Bound to the definition rather than passed bare: `count` is declared as a method, so a
   // reference torn off the object literal would run with `this` undefined the first time an
   // author writes `count: ({ ctx }) => this.something`.
-  const declaredCount = definition.count;
+  // A scoped count needs the scope's input, which `x db backfill` has none of: not stamped.
+  const declaredCount = definition.scope === undefined ? definition.count : undefined;
   const count: BackfillCount | undefined =
-    declaredCount === undefined ? undefined : (args) => declaredCount.call(definition, args);
+    declaredCount === undefined
+      ? undefined
+      : ({ ctx }) =>
+          (declaredCount as NonNullable<AnyBackfill<Row>['count']>).call(definition, {
+            ctx,
+            input: undefined,
+          });
 
-  const handle = job<BackfillInput>({
+  const scope = definition.scope;
+  const handle = job<BackfillInput & { readonly input?: unknown }>({
     name: definition.name,
-    input: t.object({ force: t.boolean.optional() }),
-    // One live run per name, forced or not. A second enqueue while the pass is still going is the
-    // same pass, and deduping it is what makes "kick it again" safe rather than a second writer on
-    // one table — which is exactly what a `force` in the key would have allowed.
-    idempotencyKey: () => definition.name,
+    // Unscoped: byte-identical to before scopes existed. Scoped: the force flag and the input.
+    input:
+      scope === undefined
+        ? t.object({ force: t.boolean.optional() })
+        : scopedPayload(definition.name, definition.input),
+    // One live run per name — per scope for a scoped one — forced or not. A second enqueue while
+    // the pass is still going is the same pass, and deduping it is what makes "kick it again" safe
+    // rather than a second writer on one table — which a `force` in the key would have allowed.
+    idempotencyKey: (payload) => ledgerNameFor(definition.name, scope, payload),
     // Forwarded verbatim, like every other job field this factory carries: a backfill that
     // declared its tenant and then ran under somebody else's would be a factory deciding authz.
-    tenant: definition.tenant,
+    tenant: definition.tenant as JobTenant<BackfillInput & { readonly input?: unknown }>,
     retry: definition.retry ?? DEFAULT_RETRY,
     ...(definition.queue === undefined ? {} : { queue: definition.queue }),
     ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
-    run: (args) => backfillPass({ definition, size, checksum, pace }, args),
+    run: (args) =>
+      backfillPass({ definition: definition as AnyBackfill<Row>, size, checksum, pace }, args),
   });
   // Stamped, never registered by the app: this is what makes a declared-but-never-enqueued sweep
   // visible to `x db backfill --pending`, and the `origin` WeakMap is `task.ts`'s mechanism rather
   // than a second one. `job()` above already refused a duplicate name, so this cannot overwrite.
   stampBackfill(handle, {
     checksum,
+    scoped: scope !== undefined,
     requires: definition.requires,
     environments: definition.environments,
     count,
   });
-  return handle;
+  return handle as JobHandle<BackfillInput>;
 }

package/src/consumer-errors.ts

@@ -0,0 +1,42 @@
+// The two `X_CONSUMER_*` codes and the classes that throw them. The registry is `error-codes.ts`;
+// this file holds only the refusals, beside the factory (`consumer.ts`) that raises the first and
+// the enqueue guard (`consumer-guard.ts`) that raises the second.
+
+import { renderFixShellArg, UltimateError } from '@ultimat3/core';
+
+/**
+ * A `consumer()` that could not be assembled correctly. Every one of these is a field whose
+ * absence has no other build error: a consumer with no cursor re-reads its stream from the top on
+ * every restart, and one with a `timeout` is killed mid-stream on a schedule nobody chose.
+ */
+export class ConsumerInvalidError extends UltimateError {
+  constructor(input: {
+    readonly consumer: string;
+    readonly problem: string;
+    readonly fix: string;
+  }) {
+    super({
+      code: 'X_CONSUMER_INVALID',
+      cause: `consumer "${input.consumer}" ${input.problem}`,
+      fix: input.fix,
+      meta: { consumer: input.consumer },
+    });
+  }
+}
+
+/**
+ * A consumer's own body enqueued the consumer. That enqueue is deduped onto the row that is
+ * running it, so it arms nothing — the trap `recipes.job.test.ts` pins. Refused instead of
+ * answered: the supervisor task already re-arms every key.
+ */
+export class ConsumerSelfEnqueueError extends UltimateError {
+  constructor(input: { readonly consumer: string }) {
+    const name = renderFixShellArg(input.consumer, "'<consumer>'");
+    super({
+      code: 'X_CONSUMER_SELF_ENQUEUE',
+      cause: `consumer "${input.consumer}" enqueued itself from inside its own run; that enqueue dedupes onto the running row and arms nothing, and the consumer's .supervisor task already re-arms every key`,
+      fix: `grep -rn ${name} apps/web --include=*.ts   # delete the .enqueue call inside the consumer's handle or open`,
+      meta: { consumer: input.consumer },
+    });
+  }
+}

package/src/consumer-guard.ts

@@ -0,0 +1,19 @@
+// Which consumer's run this async context is inside, and the refusal of that consumer enqueuing
+// itself. A self-enqueue dedupes onto the running row and arms nothing (`recipes.job.test.ts`
+// pins the trap), so it is made unreachable here rather than documented.
+
+import { asyncContext } from '@ultimat3/core';
+import { ConsumerSelfEnqueueError } from './consumer-errors';
+
+/** Core's one async-context seam: server-only, like every job body. */
+const running = asyncContext<string>('a consumer() run');
+
+/** Run `fn` as consumer `name`'s body: every enqueue of `name` inside it is refused. */
+export function asConsumer<T>(name: string, fn: () => T): T {
+  return running.run(name, fn);
+}
+
+/** Throws `X_CONSUMER_SELF_ENQUEUE` when the caller is consumer `name`'s own run. */
+export function refuseSelfEnqueue(name: string): void {
+  if (running.get() === name) throw new ConsumerSelfEnqueueError({ consumer: name });
+}

package/src/consumer-loop.ts

@@ -0,0 +1,171 @@
+// One consumer run: open the stream at the saved cursor, then turn after turn — read a batch,
+// handle it and save the cursor past it in ONE transaction, report progress, check the drain.
+// Split from `consumer.ts` for the reason `backfill-pass.ts` is split from `backfill.ts`.
+//
+// No step per batch, unlike a backfill: the cursor lives in the app's own row, committed with the
+// batch's effect, so a resumed run starts from `cursor.load` and steps would only retain history.
+
+import type { Ctx } from '@ultimat3/core';
+import { assert, logger, renderThrowable, throwIfAborted } from '@ultimat3/core';
+import { withTransaction } from '@ultimat3/db';
+import type { ConsumerDefinition, ConsumerInput, ConsumerReport } from './consumer-types';
+import type { JobRunArgs } from './job';
+
+/** Rows per transaction when a declaration names none. Each batch is one commit. */
+export const DEFAULT_CONSUMER_BATCH = 100;
+
+/**
+ * How long a partly filled batch waits for its next item before it commits what it holds. Without
+ * it a quiet stream would keep a batch in hand until `batch` items arrived — items read and never
+ * committed, for as long as the stream stays quiet.
+ */
+export const CONSUMER_LINGER_MS = 50;
+
+/** The note a consumer's progress carries, so `x jobs show` can tell its row from any job's. */
+export const CONSUMER_PROGRESS_NOTE = 'consumer';
+
+export interface ConsumerPlan<Item, C> {
+  readonly definition: ConsumerDefinition<Item, C>;
+  readonly size: number;
+}
+
+/** Answers `undefined` when `promise` has not settled within `ms`; the promise stays usable. */
+async function within<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const lapse = new Promise<undefined>((resolve) => {
+    timer = setTimeout(() => resolve(undefined), ms);
+  });
+  try {
+    return await Promise.race([promise, lapse]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Waits for `promise`, or throws the run's abort the moment the drain fires. */
+async function untilAborted<T>(promise: Promise<T>, ctx: Ctx): Promise<T> {
+  throwIfAborted(ctx);
+  let stop: (() => void) | undefined;
+  const aborted = new Promise<'aborted'>((resolve) => {
+    stop = () => resolve('aborted');
+    ctx.signal.addEventListener('abort', stop, { once: true });
+  });
+  try {
+    const first = await Promise.race([promise, aborted]);
+    if (first === 'aborted') throwIfAborted(ctx);
+    return first as T;
+  } finally {
+    if (stop !== undefined) ctx.signal.removeEventListener('abort', stop);
+  }
+}
+
+/**
+ * Groups the stream into batches of at most `size`. The pull in flight when a batch closes is
+ * carried into the next one, never dropped: an item already read is an item the next batch owns.
+ */
+function batcher<Item>(iterator: AsyncIterator<Item>, size: number, ctx: Ctx) {
+  let pending: Promise<IteratorResult<Item>> | undefined;
+  let ended = false;
+  const advance = (): Promise<IteratorResult<Item>> => {
+    if (pending === undefined) {
+      pending = iterator.next();
+      // A pull still in flight when a drain ends the run is awaited by nobody: its rejection is
+      // the stream's to report, not an unhandled one in the worker.
+      pending.catch(() => undefined);
+    }
+    return pending;
+  };
+  return {
+    inFlight: (): boolean => pending !== undefined,
+    async next(): Promise<readonly Item[] | null> {
+      if (ended) return null;
+      const first = await untilAborted(advance(), ctx);
+      pending = undefined;
+      if (first.done === true) {
+        ended = true;
+        return null;
+      }
+      const items: Item[] = [first.value];
+      while (items.length < size) {
+        const step = await within(advance(), CONSUMER_LINGER_MS);
+        if (step === undefined) break;
+        pending = undefined;
+        if (step.done === true) {
+          ended = true;
+          break;
+        }
+        items.push(step.value);
+      }
+      return items;
+    },
+  };
+}
+
+/**
+ * Closes the stream. Awaited only when no pull is in flight: an async generator queues `return()`
+ * behind a pending `next()`, so awaiting it on a quiet stream would hold a drained run open.
+ */
+async function close<Item>(iterator: AsyncIterator<Item>, inFlight: boolean, name: string) {
+  const closing = iterator.return?.();
+  if (closing === undefined) return;
+  const logged = closing.then(
+    () => undefined,
+    (error: unknown) => {
+      logger.warn('jobs.consumer.close-failed', { consumer: name, error: renderThrowable(error) });
+    },
+  );
+  if (!inFlight) await logged;
+}
+
+async function lagOf<Item, C>(
+  definition: ConsumerDefinition<Item, C>,
+  key: string,
+  cursor: C | null,
+): Promise<number> {
+  if (definition.lag === undefined) return 0;
+  const lag = await definition.lag({ key, cursor });
+  assert(
+    Number.isFinite(lag) && lag >= 0,
+    `consumer "${definition.name}" lag() returned ${String(lag)} for key "${key}" — a lag is a finite number, zero or more`,
+    `lag: ({ key, cursor }) => Math.max(0, head(key) - position(cursor)) on consumer({ name: "${definition.name}" })`,
+  );
+  return lag;
+}
+
+/** The run: until the stream ends (a normal return; the next arm reopens it) or a drain stops it. */
+export async function consumerRun<Item, C>(
+  plan: ConsumerPlan<Item, C>,
+  args: JobRunArgs<ConsumerInput>,
+): Promise<ConsumerReport> {
+  const { definition, size } = plan;
+  const { ctx } = args;
+  const { key } = args.input;
+  const signal = ctx.signal;
+  let cursor = await definition.cursor.load(key);
+  const iterator = definition.open({ key, cursor, ctx, signal })[Symbol.asyncIterator]();
+  const pull = batcher(iterator, size, ctx);
+  let batches = 0;
+  let items = 0;
+  try {
+    for (;;) {
+      // The drain check, between turns: a turn is whole or absent, so the cursor stays on the
+      // last batch that committed.
+      throwIfAborted(ctx);
+      const batch = await pull.next();
+      if (batch === null) break;
+      const next = await withTransaction(async () => {
+        const after = await definition.handle({ items: batch, key, ctx, signal });
+        await definition.cursor.save(key, after);
+        return after;
+      });
+      cursor = next;
+      batches += 1;
+      items += batch.length;
+      // `total - done` is the lag: the job row's progress is the one place a run reports itself.
+      args.progress(items, items + (await lagOf(definition, key, cursor)), CONSUMER_PROGRESS_NOTE);
+    }
+  } finally {
+    await close(iterator, pull.inFlight(), definition.name);
+  }
+  return { key, batches, items };
+}