@ultimat3/jobs 1.2.0 → 2.0.0

package/src/backfill-ledger.ts

@@ -0,0 +1,183 @@
+// `x_backfills` — what has already been swept, and whether this pass runs at all. The twin of
+// `@ultimat3/db`'s `x_migrations`, one level up: a migration ledger is keyed by migration id
+// because a migration applies once and only once, and this one is keyed by RUN because a backfill
+// may legitimately be run again. `force` therefore writes a NEW row rather than overwriting the
+// one that says what the last pass did — reruns are history, not an edit of it.
+//
+// The row is a REPORT, never a resume source. Where a resumed pass restarts is decided by the step
+// checkpoints in `backfill-pass.ts` and by nothing else; `cursor` here is what an operator reads
+// while a pass is running. A second answer to "where were we" is the one thing this file must not
+// become — the checkpoints are transactional with the work, and this row is not.
+
+import type { Clock } from '@ultimat3/core';
+import { systemClock } from '@ultimat3/core';
+import { nowMs } from './clock';
+
+/**
+ * `failed` is the LAST ATTEMPT's verdict, not the pass's: the queue owns retries, so a row here
+ * goes back to `running` when the next attempt starts. Only `completed` is terminal, and it is the
+ * one status that blocks a re-run.
+ *
+ * The runtime list is the declaration and `BackfillStatus` is derived from it, the shape
+ * `PRIMITIVE_KINDS` already has: `x db backfill --status` has to validate a string it was handed,
+ * and a second list spelled out in the CLI is a status the ledger can record and the flag rejects.
+ */
+export const BACKFILL_STATUSES = ['running', 'completed', 'failed'] as const;
+
+export type BackfillStatus = (typeof BACKFILL_STATUSES)[number];
+
+/** Narrows a string the CLI, MCP or a URL handed over. Never a cast — the list decides. */
+export const isBackfillStatus = (value: string): value is BackfillStatus =>
+  (BACKFILL_STATUSES as readonly string[]).includes(value);
+
+export interface BackfillRun {
+  /** The job run this pass belongs to, and the ledger's primary key. */
+  readonly runId: string;
+  readonly name: string;
+  readonly checksum: string;
+  readonly status: BackfillStatus;
+  /** The build that STARTED the pass — a redeploy mid-pass does not rewrite it. */
+  readonly appVersion: string;
+  readonly rows: number;
+  /** Where the pass had got to. `null` before the first batch and once it is over. */
+  readonly cursor: string | null;
+  readonly startedAt: number;
+  readonly completedAt?: number | undefined;
+}
+
+export interface BackfillFilter {
+  readonly name?: string | undefined;
+  readonly status?: BackfillStatus | undefined;
+  /**
+   * The one pass this row belongs to. A backfill's run id IS the queue row's, so `x jobs show
+   * <id>` can ask how far the sweep behind that job has got without a second lookup table.
+   */
+  readonly runId?: string | undefined;
+  readonly limit?: number | undefined;
+}
+
+/**
+ * Durable, and therefore behind the queue driver: `x_backfills` ships in the same DDL as `x_jobs`
+ * and `x_job_steps` (`driver-pg-sql.ts`), so a ledger a pass cannot write is a queue it could not
+ * have been claimed from. Optional on `JobDriver` for the same reason `introspect` is — a driver
+ * with no ledger runs backfills with no bookkeeping rather than refusing them.
+ */
+export interface BackfillLedger {
+  /** Open this run's row, or adopt the one a previous attempt of the SAME run opened. */
+  start(run: {
+    readonly runId: string;
+    readonly name: string;
+    readonly checksum: string;
+    readonly appVersion: string;
+  }): Promise<void>;
+  /** Move the row forward. Absolute position, so a replayed batch reports the same number. */
+  progress(
+    runId: string,
+    at: { readonly rows: number; readonly cursor: string | null },
+  ): Promise<void>;
+  finish(
+    runId: string,
+    at: { readonly status: 'completed' | 'failed'; readonly rows: number },
+  ): Promise<void>;
+  /** Newest first. `x db backfill --list` and the verdict below read the same method. */
+  list(filter?: BackfillFilter): Promise<readonly BackfillRun[]>;
+}
+
+/** Anything callable. `Function` is a banned type and a real signature would pin one definition. */
+type AnyFn = (...args: never[]) => unknown;
+
+/**
+ * Between the two bodies, and a byte no source text carries. Concatenating them raw would hash a
+ * BOUNDARY rather than a pair — `ab` + `c` and `a` + `bc` are one string — so a statement moving
+ * from `handle` into `source` could leave the checksum where it was.
+ */
+const CHECKSUM_SEPARATOR = '\u0000';
+
+/**
+ * What "the same backfill" means: the source text of the two functions that decide which rows a
+ * pass visits and what happens to them, which is the whole of what a completed row claims was
+ * done. `batch` is deliberately out — paging a sweep differently is a tuning change, and a
+ * checksum that moved for it would warn on every one.
+ *
+ * A code hash is fuzzier than a migration's SQL hash: a bundler that reformats a body moves it
+ * with no line of behaviour changing. That is exactly why a mismatch WARNS and never refuses,
+ * where `@ultimat3/db`'s `auditLedger` throws on the same fact — SQL text is what it applied.
+ */
+export function backfillChecksum(source: AnyFn, handle: AnyFn): string {
+  return new Bun.CryptoHasher('sha256')
+    .update(`${source.toString()}${CHECKSUM_SEPARATOR}${handle.toString()}`)
+    .digest('hex')
+    .slice(0, 32);
+}
+
+export interface BackfillVerdict {
+  readonly run: boolean;
+  /** The completed pass this verdict is about, when the ledger holds one. */
+  readonly previous?: BackfillRun | undefined;
+  /** `previous` completed under a different definition. Warned about, never refused. */
+  readonly changed: boolean;
+}
+
+/**
+ * Pure, so the pass, a test and `x db backfill` all read one decision. Only a COMPLETED row
+ * blocks: a `running` row is this pass resuming (same run) or a pass another worker holds the one
+ * live idempotency key for, and a `failed` one is an attempt the queue is about to retry.
+ */
+export function decideBackfill(
+  completed: BackfillRun | undefined,
+  checksum: string,
+  force: boolean,
+): BackfillVerdict {
+  if (completed === undefined) return { run: true, changed: false };
+  return { run: force, previous: completed, changed: completed.checksum !== checksum };
+}
+
+export function createMemoryBackfillLedger(clock: Clock = systemClock): BackfillLedger {
+  const runs = new Map<string, BackfillRun>();
+  const patch = (runId: string, fields: Partial<BackfillRun>): void => {
+    const existing = runs.get(runId);
+    if (existing !== undefined) runs.set(runId, { ...existing, ...fields });
+  };
+  return {
+    start(run) {
+      const existing = runs.get(run.runId);
+      // A retry adopts its own row: `startedAt` is when the PASS began, not this attempt, and the
+      // status goes back to `running` so a row a failed attempt marked stops claiming otherwise.
+      // `completedAt` goes with it — `finish` stamps one for `failed` too, and a running pass that
+      // kept it would report a completion time in the past on every surface that reads the row.
+      runs.set(
+        run.runId,
+        existing === undefined
+          ? { ...run, status: 'running', rows: 0, cursor: null, startedAt: nowMs(clock) }
+          : { ...existing, status: 'running', completedAt: undefined },
+      );
+      return Promise.resolve();
+    },
+    progress(runId, at) {
+      patch(runId, { rows: at.rows, cursor: at.cursor });
+      return Promise.resolve();
+    },
+    finish(runId, at) {
+      // A failure keeps its cursor — where a pass stopped is the first thing anyone asks.
+      patch(runId, {
+        status: at.status,
+        rows: at.rows,
+        completedAt: nowMs(clock),
+        ...(at.status === 'completed' ? { cursor: null } : {}),
+      });
+      return Promise.resolve();
+    },
+    list(filter = {}) {
+      // Reversed BEFORE the sort: the test clock is frozen, so two rows share a `startedAt` and a
+      // stable sort would hand back the oldest of them first under a "newest first" contract.
+      const rows = [...runs.values()]
+        .reverse()
+        .sort((a, b) => b.startedAt - a.startedAt)
+        .filter((run) => filter.name === undefined || run.name === filter.name)
+        .filter((run) => filter.status === undefined || run.status === filter.status)
+        .filter((run) => filter.runId === undefined || run.runId === filter.runId)
+        .slice(0, filter.limit ?? 100);
+      return Promise.resolve(rows);
+    },
+  };
+}

package/src/backfill-pass.ts

@@ -0,0 +1,276 @@
+// One `backfill()` pass: the iteration, its durable checkpoints, and the `x_backfills` row that
+// reports them. Split out of `backfill.ts` for the reason `execute.ts` is split out of `job.ts` —
+// that file declares a backfill, this one runs it.
+//
+// `inBatches()` reads the source one statement per page, and each page is handled inside its own
+// `step.run`, so an attempt killed mid-pass resumes on the page it stopped at instead of reading
+// the table again from the top. What a step persists is the CURSOR and the row count, never the
+// page: `steps.ts` hands a completed step's output back for the whole run, so checkpointing rows
+// would retain every row of every batch already processed until the job ended — the leak that
+// turns a backfill of a large table into an OOM.
+//
+// The ledger is the OTHER half and never a second copy of this one: a step checkpoint is written
+// in step with the work and decides where a resumed pass restarts, while the `x_backfills` row is
+// a report an operator reads and the record that a completed name has already been swept.
+
+import { appVersion, assert, logger, resolveEnvironment } from '@ultimat3/core';
+import type { BatchIterator } from '@ultimat3/entity';
+import type { BackfillDefinition, BackfillInput, BackfillReport } from './backfill';
+import { checkBackfillEnvironment } from './backfill-gate';
+import type { BackfillLedger, BackfillRun } from './backfill-ledger';
+import { decideBackfill } from './backfill-ledger';
+import type { Pacer } from './backfill-rate';
+import { withBackfillScope } from './backfill-scope';
+import { jobDriver } from './driver';
+import { BackfillStalledError } from './errors';
+import type { JobRunArgs } from './job';
+import { isStepSuspension } from './steps';
+
+/** Every batch is a step, and this is the name it is checkpointed under. See `backfillPass()`. */
+const STEP_PREFIX = 'batch:';
+
+/** What a step persists: a bounded position, never the page it came from. See the file header. */
+interface Checkpoint {
+  /** Where the next batch starts; `null` once the pass is over. */
+  readonly cursor: string | null;
+  readonly rows: number;
+}
+
+/** One live iteration and the pull that advances it. Rebuilt when the checkpoints disagree. */
+interface Iteration<Row> {
+  readonly batches: BatchIterator<Row>;
+  readonly pull: AsyncIterator<readonly Row[]>;
+}
+
+/** Everything about the backfill that is fixed at declaration, hashed, sized and paced once. */
+export interface BackfillPlan<Row> {
+  readonly definition: BackfillDefinition<Row>;
+  readonly size: number;
+  readonly checksum: string;
+  /** The declared `rate`, already an interval. Built at declaration for the reason `size` is. */
+  readonly pace: Pacer;
+}
+
+const fieldOf = (value: unknown, key: string): unknown =>
+  typeof value === 'object' && value !== null ? (value as Record<string, unknown>)[key] : undefined;
+
+/**
+ * `steps.ts` hands a completed step's output back through an unchecked `as T`, so a checkpoint
+ * READ from storage is this shape by claim and never by check. It is checked here because the
+ * failure it would otherwise cause is silent and expensive: an absent cursor is not `null`, so the
+ * loop would reopen the source at the top and walk the whole table a second time.
+ */
+function asCheckpoint(value: unknown, step: string): Checkpoint {
+  const cursor = fieldOf(value, 'cursor');
+  const rows = fieldOf(value, 'rows');
+  assert(
+    (cursor === null || typeof cursor === 'string') && typeof rows === 'number',
+    `step "${step}" replayed ${JSON.stringify(value)}, which is not a backfill checkpoint`,
+    `x jobs show <jobId> --json prints the run's steps — a run id whose "${step}" was written by something other than this backfill has to be retired, not resumed`,
+  );
+  return { cursor, rows };
+}
+
+/**
+ * The newest COMPLETED pass under this name, or nothing. A `running` row is this pass resuming or
+ * one another worker holds the single live idempotency key for, and a `failed` one is an attempt
+ * the queue is about to retry — neither is a sweep that has already happened.
+ */
+async function completedRun(
+  ledger: BackfillLedger | undefined,
+  name: string,
+): Promise<BackfillRun | undefined> {
+  if (ledger === undefined) return undefined;
+  const rows = await ledger.list({ name, status: 'completed', limit: 1 });
+  return rows[0];
+}
+
+/**
+ * Record the attempt that ended badly and let the original error through untouched: the queue's
+ * retry decision is made on it, so a bookkeeping failure that replaced it would dead-letter — or
+ * silently retry — for the wrong reason. Logged rather than swallowed, because a ledger nobody
+ * can write is worth exactly one line.
+ */
+async function markFailed(
+  ledger: BackfillLedger | undefined,
+  runId: string,
+  rows: number,
+): Promise<void> {
+  try {
+    await ledger?.finish(runId, { status: 'failed', rows });
+  } catch (error) {
+    logger.warn('jobs.backfill.ledger-failed', {
+      runId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+/**
+ * The stall detector. `count()` is the same predicate `source` selects on, so a source that ran
+ * out while the count still matches rows means the two disagree — the sweep reported success over
+ * rows nobody visited, which is an authoring bug in any business and not a condition the queue can
+ * retry its way out of. Silent without a declared `count()`: the framework will not guess a number
+ * on the author's behalf, because a dry run that lied about convergence is the failure this exists
+ * to close.
+ */
+async function assertConverged<Row>(
+  definition: BackfillDefinition<Row>,
+  ctx: JobRunArgs<BackfillInput>['ctx'],
+  swept: number,
+): Promise<void> {
+  if (definition.count === undefined) return;
+  const remaining = await definition.count({ ctx });
+  // Parsed, not trusted: `count()` is app code feeding a framework decision, and both `NaN > 0`
+  // and `-1 > 0` are FALSE — an unchecked bad number reads as "converged" and writes the completed
+  // ledger row that stops the next deploy ever re-running this sweep. The one failure mode this
+  // detector exists to close, arriving through the detector itself.
+  assert(
+    Number.isSafeInteger(remaining) && remaining >= 0,
+    `backfill "${definition.name}" count() returned ${String(remaining)} — a count is a whole number of rows, zero or more`,
+    `return the chain's own count from count() on backfill("${definition.name}"), e.g. count: ({ ctx }) => source({ ctx }).count() — a NaN or a negative reads as "nothing left" and completes the sweep`,
+  );
+  if (remaining > 0) {
+    throw new BackfillStalledError({ backfill: definition.name, remaining, swept });
+  }
+}
+
+/**
+ * The pass itself: one `step.run` per batch, named by position so a replay finds it. Completed
+ * steps are served from storage without touching the database, so a resumed attempt walks its own
+ * history to the last checkpoint and opens the source there.
+ */
+export async function backfillPass<Row>(
+  plan: BackfillPlan<Row>,
+  args: JobRunArgs<BackfillInput>,
+): Promise<BackfillReport> {
+  const { definition, size, checksum, pace } = plan;
+  const { step, runId } = args;
+  const name = definition.name;
+  // Before the ledger is even opened: a sweep this deploy may not run must leave no row saying it
+  // started. Enforced here and not only in `x db backfill`, because app code that calls
+  // `.enqueue()` directly never passes through a command — a rail only the CLI holds is a
+  // convention (axiom 3). No ledger row, so the queue's own retry/dead-letter path is what an
+  // operator reads, exactly as it is for any other permanently-failing job.
+  // `resolveEnvironment()` is asked ONLY when a declaration named environments: it throws on a
+  // typo'd `ULTIMATE_ENV`, which is core's contract for its own key — but a sweep that declared
+  // nothing must not start failing over a variable it never reads.
+  const declaredEnvironments = definition.environments;
+  if (declaredEnvironments !== undefined && declaredEnvironments.length > 0) {
+    const mismatch = checkBackfillEnvironment(name, declaredEnvironments, resolveEnvironment());
+    if (mismatch !== undefined) throw mismatch;
+  }
+  // Absent on a driver that ships no ledger, which runs the pass with no bookkeeping rather than
+  // refusing it — the same degradation `introspect` already has.
+  const ledger = jobDriver()?.backfills;
+
+  const previous = await completedRun(ledger, name);
+  const verdict = decideBackfill(previous, checksum, args.input.force === true);
+  if (!verdict.run) {
+    if (verdict.changed) {
+      // A warning and never a refusal, unlike `@ultimat3/db`'s `auditLedger`: this checksum is
+      // over function source text, which a bundler can move without a line of behaviour changing.
+      logger.warn('jobs.backfill.definition-changed', {
+        backfill: name,
+        completedAs: previous?.checksum,
+        now: checksum,
+        fix: `enqueue ${name} with { force: true } to sweep again under the new definition`,
+      });
+    }
+    logger.info('jobs.backfill.skipped', { backfill: name, completedBy: previous?.runId });
+    return {
+      name,
+      batches: 0,
+      rows: 0,
+      skipped: true,
+      ...(previous === undefined ? {} : { previousRunId: previous.runId }),
+    };
+  }
+  // The pass itself, run under the tenant the DECLARATION named. Everything below builds a plan
+  // — `source()` per page, `count()` at the end — and `scopedPlan` is applied where a plan is
+  // built, which is inside this iteration and not where the author wrote the chain. So the scope
+  // has to be opened here: `tenant: 'none'` sweeps every tenant and says so, a declared tenant is
+  // handed its context untouched. `backfill-scope.ts` says why it is not the worker's actor.
+  return withBackfillScope(
+    name,
+    definition.tenant,
+    args.ctx,
+    async (ctx): Promise<BackfillReport> => {
+      await ledger?.start({ runId, name, checksum, appVersion: appVersion() });
+
+      let cursor: string | null = null;
+      let rows = 0;
+      let batches = 0;
+      let live: Iteration<Row> | undefined;
+
+      /**
+       * The iteration positioned at `cursor`, opened on the first batch this attempt actually runs so
+       * a resumed pass sends no statement for a page it already handled.
+       *
+       * `batches.cursor` IS where the next statement starts, so comparing it with the checkpoint's is
+       * the whole staleness test — and it is not academic: `retryFromStep` re-opens ONE step in the
+       * middle of a finished run, which moves the checkpoints somewhere this iteration is not. Rebuilt
+       * from the checkpoint rather than read from wherever the old one was parked.
+       */
+      const iterate = async (): Promise<Iteration<Row>> => {
+        if (live !== undefined && live.batches.cursor === cursor) return live;
+        await live?.batches.close();
+        const opened = definition.source({ ctx }).after(cursor).inBatches(size);
+        live = { batches: opened, pull: opened[Symbol.asyncIterator]() };
+        return live;
+      };
+
+      try {
+        for (let index = 0; ; index += 1) {
+          const stepName = `${STEP_PREFIX}${index}`;
+          const checkpoint = asCheckpoint(
+            await step.run(stepName, async (signal): Promise<Checkpoint> => {
+              // INSIDE the body, which is the whole of it: a completed step is served from storage
+              // without its body running, so an attempt resuming at batch 500 replays 500 checkpoints
+              // and pays none of their pauses. Paced outside the step, a resumed pass would spend the
+              // entire throttle of everything it had already done before touching a new row.
+              //
+              // The signal is the run's cancellation composed with this step's ceiling, so a cancelled
+              // pass unwinds out of the wait instead of sitting in a timer nobody is waiting for.
+              await pace.wait({ signal, step: stepName });
+              const iteration = await iterate();
+              const next = await iteration.pull.next();
+              if (next.done === true) return { cursor: null, rows: 0 };
+              await definition.handle({ rows: next.value, ctx, signal, index });
+              return { cursor: iteration.batches.cursor, rows: next.value.length };
+            }),
+            stepName,
+          );
+          rows += checkpoint.rows;
+          cursor = checkpoint.cursor;
+          // `inBatches()` never yields an empty batch, so rows is what tells a handled page from the
+          // one step an exhausted source costs — and what keeps the exhausted step off the ledger,
+          // whose last write is `finish` either way.
+          if (checkpoint.rows > 0) {
+            batches += 1;
+            // Absolute, so a replayed batch reports the position it reported the first time.
+            await ledger?.progress(runId, { rows, cursor });
+          }
+          if (cursor === null) break;
+        }
+        // The source is exhausted; the declaration's own count is the only thing that can say whether
+        // that means the work is done. One statement per PASS, not per batch — the question is "did
+        // this converge", asked once, where the answer is finally decidable. Inside the `try` so the
+        // ledger records the attempt as `failed`: a pass that left rows behind must not write the
+        // completed row that stops the next deploy re-running it.
+        await assertConverged(definition, ctx, rows);
+      } catch (error) {
+        // Control flow, not a failure: a suspended run is parked and will be back on this step.
+        if (!isStepSuspension(error)) await markFailed(ledger, runId, rows);
+        throw error;
+      } finally {
+        // Whatever the iteration holds belongs to this attempt, and an attempt that failed, was
+        // cancelled or finished is done with it either way.
+        await live?.batches.close();
+      }
+
+      await ledger?.finish(runId, { status: 'completed', rows });
+      return { name, batches, rows, skipped: false };
+    },
+  );
+}

package/src/backfill-pending.ts

@@ -0,0 +1,131 @@
+// Declared minus completed — the diff nothing in the framework could compute, because until
+// `backfill-registry.ts` only half of it existed. `inspectBackfills` reads the ledger, so a sweep
+// that was merged and never enqueued had no row and appeared on no surface at all.
+//
+// Pure: declarations in, ledger rows in, verdict out. The reader is `x db backfill --pending`,
+// whose non-zero exit is what lets a cron or a deploy check answer "is anything unswept" without
+// parsing a table.
+
+import type { Environment } from '@ultimat3/core';
+import { checkBackfillEnvironment } from './backfill-gate';
+// `BackfillProgress` and not the driver's own `BackfillRun`: `inspectBackfills()` is the ONE
+// projection of the ledger, and this diff is its fifth reader rather than a sixth path into rows.
+import type { BackfillProgress } from './backfill-inspect';
+import type { BackfillDeclaration } from './backfill-registry';
+
+/**
+ * `excluded` is a declaration this environment may not run at all, and it is the reason the diff
+ * needs an environment: a production-only cleanup read in staging is not drift, and an alarm that
+ * fired on it would be muted within a week.
+ */
+export const BACKFILL_STATES = ['pending', 'running', 'failed', 'completed', 'excluded'] as const;
+
+export type BackfillState = (typeof BACKFILL_STATES)[number];
+
+/** One declaration, judged against the ledger. Plain JSON: absent is `null`, never `undefined`. */
+export interface BackfillStateRow {
+  readonly name: string;
+  readonly state: BackfillState;
+  readonly checksum: string;
+  /** The checksum the newest run under this name recorded, when there is one. */
+  readonly ledgerChecksum: string | null;
+  /** The completed pass ran under a different definition. Reported, never a refusal. */
+  readonly changed: boolean;
+  readonly requires: string | null;
+  readonly environments: readonly Environment[] | null;
+  readonly lastRunId: string | null;
+  readonly rows: number | null;
+}
+
+export interface BackfillPendingReport {
+  readonly environment: Environment;
+  /** Every declaration, judged. Newest-first ledger rows decide each verdict. */
+  readonly rows: readonly BackfillStateRow[];
+  /**
+   * The alarm: `pending` and `failed`, never `running` — a pass in flight is progress, and a check
+   * that went red for the duration of every sweep is a check nobody leaves wired to a deploy.
+   */
+  readonly pending: readonly BackfillStateRow[];
+  /** Ledger names no declaration carries — a sweep whose module was deleted after it ran. */
+  readonly orphaned: readonly string[];
+}
+
+/**
+ * The states the alarm is about, declared ONCE. `x db backfill --all` picks its targets by this
+ * same predicate, and a second literal there would be a second definition of "pending" — one of
+ * which would eventually be wrong while the other stayed right.
+ */
+export const PENDING_BACKFILL_STATES: readonly BackfillState[] = ['pending', 'failed'];
+
+export const isPendingBackfillState = (state: BackfillState): boolean =>
+  PENDING_BACKFILL_STATES.includes(state);
+
+/**
+ * Newest first is `BackfillLedger.list`'s contract, so within a name the first row is the newest.
+ * Grouped ONCE rather than filtered per declaration: `x db backfill --pending` reads the ledger
+ * with no limit, so a per-declaration scan is three passes over an unbounded list for every sweep
+ * the app declares.
+ */
+function groupByName(
+  runs: readonly BackfillProgress[],
+): ReadonlyMap<string, readonly BackfillProgress[]> {
+  const byName = new Map<string, BackfillProgress[]>();
+  for (const run of runs) {
+    const under = byName.get(run.name);
+    if (under === undefined) byName.set(run.name, [run]);
+    else under.push(run);
+  }
+  return byName;
+}
+
+function stateOf(
+  declaration: BackfillDeclaration,
+  under: readonly BackfillProgress[],
+  completed: BackfillProgress | undefined,
+  environment: Environment,
+): BackfillState {
+  if (
+    checkBackfillEnvironment(declaration.name, declaration.environments, environment) !== undefined
+  ) {
+    return 'excluded';
+  }
+  // A completed row anywhere in this name's history is what blocks a re-run, so it decides the
+  // state even when a later forced pass failed — `decideBackfill` reads the same fact.
+  if (completed !== undefined) return 'completed';
+  const newest = under[0];
+  if (newest === undefined) return 'pending';
+  return newest.status === 'running' ? 'running' : 'failed';
+}
+
+export function pendingBackfills(input: {
+  readonly declarations: readonly BackfillDeclaration[];
+  /** Every `x_backfills` row, newest first — `inspectBackfills`' own order, unfiltered. */
+  readonly runs: readonly BackfillProgress[];
+  readonly environment: Environment;
+}): BackfillPendingReport {
+  const byName = groupByName(input.runs);
+  const rows = input.declarations.map((declaration): BackfillStateRow => {
+    const under = byName.get(declaration.name) ?? [];
+    const newest = under[0];
+    const completed = under.find((run) => run.status === 'completed');
+    return {
+      name: declaration.name,
+      state: stateOf(declaration, under, completed, input.environment),
+      checksum: declaration.checksum,
+      ledgerChecksum: newest?.checksum ?? null,
+      changed: completed !== undefined && completed.checksum !== declaration.checksum,
+      requires: declaration.requires,
+      environments: declaration.environments,
+      lastRunId: newest?.runId ?? null,
+      rows: newest?.rows ?? null,
+    };
+  });
+  const declared = new Set(input.declarations.map((declaration) => declaration.name));
+  const orphaned = [...byName.keys()].filter((name) => !declared.has(name)).sort();
+  return {
+    environment: input.environment,
+    rows,
+    pending: rows.filter((row) => isPendingBackfillState(row.state)),
+    orphaned,
+  };
+}