@ultimat3/jobs 1.2.0 → 2.0.0

package/src/backfill-rate.ts

@@ -0,0 +1,109 @@
+// The `rate` throttle a `backfill()` sweeps under: batches per second, spent as a wait between one
+// batch and the next. A sweep shares its connection pool with the requests the app is still
+// serving, so a backfill that runs flat out is an outage of its own making — the pacer is what
+// makes "one pass over the table" background work rather than a load test.
+
+import type { Clock } from '@ultimat3/core';
+import { assert, systemClock } from '@ultimat3/core';
+import { nowMs } from './clock';
+import { JobAbortedError } from './errors';
+
+/**
+ * Batches per second, and deliberately slow: at `DEFAULT_BACKFILL_BATCH` (1,000 rows) this is
+ * 5,000 rows/sec — one statement every 200ms, so the pool spends the other 199 serving the app,
+ * and a million-row table is still swept in under four minutes. A sweep that has to go faster
+ * raises the number; there is no unthrottled mode to reach for instead.
+ */
+export const DEFAULT_BACKFILL_RATE = 5;
+
+/**
+ * Under a millisecond there is no timer to wait on — `setTimeout(0.4)` fires a millisecond later,
+ * which would leave the pass SLOWER than the rate it declared. So a rate far above what the
+ * batches can actually achieve degenerates to no wait at all, which is the point: to sweep faster
+ * you raise `rate`, never turn the pacer off.
+ */
+const RESOLUTION_MS = 1;
+
+export interface PacerOptions {
+  /**
+   * Batches per second, greater than zero and finite. `backfill()` refuses a bad one at the
+   * declaration, where it was written and with the definition's name in the message; this is the
+   * same rule at the constructor, for the callers that do not come through a declaration.
+   */
+  readonly rate: number;
+  /** The name a cancellation is reported under — one pacer belongs to exactly one backfill. */
+  readonly job: string;
+  readonly clock?: Clock | undefined;
+  /**
+   * The wait itself, injectable for the same reason the clock is: a test asserts what was ASKED
+   * for instead of spending it. Must settle when `signal` aborts rather than waiting the rest out.
+   */
+  readonly sleep?: ((ms: number, signal: AbortSignal) => Promise<void>) | undefined;
+}
+
+export interface Pacer {
+  /** Batches per second this pacer was built for. */
+  readonly rate: number;
+  /**
+   * Hold the caller until this batch's slot comes round. Rejects with `JobAbortedError` when the
+   * attempt was cancelled — before the wait or during it, never after sitting the timer out.
+   */
+  wait(args: { readonly signal: AbortSignal; readonly step?: string | undefined }): Promise<void>;
+}
+
+/** Real timers, cleared on abort so a cancelled pass leaves nothing pending behind it. */
+function timerSleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    const done = (): void => {
+      clearTimeout(timer);
+      signal.removeEventListener('abort', done);
+      resolve();
+    };
+    const timer = setTimeout(done, ms);
+    signal.addEventListener('abort', done, { once: true });
+  });
+}
+
+/**
+ * Built once, at declaration, and shared by every attempt of that backfill in this process: the
+ * rate is a property of the table being swept and the pool it is swept through, not of whichever
+ * attempt happens to hold the run.
+ */
+export function createPacer(options: PacerOptions): Pacer {
+  // Refused HERE and not only at `backfill()`: `rate: 0` makes `intervalMs` Infinity, which the
+  // timer clamps to about a millisecond — so an unvalidated zero reads as "no throttle at all",
+  // which is the one setting this module exists to make unreachable. A negative rate is the same
+  // bug with a negative wait.
+  assert(
+    Number.isFinite(options.rate) && options.rate > 0,
+    `createPacer({ rate: ${String(options.rate)} }) for "${options.job}" — a rate is batches per second, greater than zero`,
+    `pass rate: ${DEFAULT_BACKFILL_RATE} — to sweep faster raise the number, there is no unthrottled mode`,
+  );
+  const clock = options.clock ?? systemClock;
+  const sleep = options.sleep ?? timerSleep;
+  const intervalMs = 1000 / options.rate;
+  let lastAt: number | undefined;
+
+  return {
+    rate: options.rate,
+    async wait({ signal, step }) {
+      const aborted = (): JobAbortedError =>
+        new JobAbortedError({ job: options.job, ...(step === undefined ? {} : { step }) });
+      if (signal.aborted) throw aborted();
+
+      const at = nowMs(clock);
+      // The batch's own time is interval already paid, so a slow page waits for nothing and only a
+      // fast one is held back. The first batch has no previous one to be spaced from.
+      const remaining = lastAt === undefined ? 0 : intervalMs - (at - lastAt);
+      if (remaining < RESOLUTION_MS) {
+        lastAt = at;
+        return;
+      }
+      await sleep(remaining, signal);
+      // The sleeper settles early on abort, so what it means is "the wait is over", never "the
+      // slot arrived" — the attempt no longer owns the run and must not read another page.
+      if (signal.aborted) throw aborted();
+      lastAt = nowMs(clock);
+    },
+  };
+}

package/src/backfill-registry.ts

@@ -0,0 +1,108 @@
+// What the app DECLARED, as against what `x_backfills` recorded. The ledger answers "which passes
+// have run"; until this file nothing answered "which passes exist", so a cleanup that was merged
+// and never enqueued had no ledger row and was invisible on every surface — the incident where
+// four rewrites shipped and simply never happened.
+//
+// The stamp is `task.ts`'s `origin` WeakMap and deliberately not a second mechanism, and not an
+// app-side `registerBackfill()` call either: `backfill()` returns a plain `JobHandle`, so the
+// declaration's own fields have nowhere on the handle to live, and asking an app to register what
+// it already declared is the coupling axiom 8's extension model exists to refuse.
+
+import type { Ctx, Environment } from '@ultimat3/core';
+// TYPE-only, and that is what makes it safe: `backfill.ts` imports this module at runtime, and a
+// second declaration of its input here would be a second name for one shape (axiom 1).
+import type { BackfillInput } from './backfill';
+import type { AnyJobHandle, JobHandle } from './job';
+import { isJobHandle, registeredJobs } from './job';
+
+/**
+ * How many rows still match — the same predicate `source` selects on, counted rather than read.
+ * Handed a `Ctx` for the reason `source` is: a tenanted sweep counts within one org or it counts
+ * every tenant at once.
+ */
+export type BackfillCount = (args: { readonly ctx: Ctx }) => Promise<number> | number;
+
+/** Everything `backfill()` knows that a `JobHandle` has no field for. */
+export interface BackfillOrigin {
+  readonly checksum: string;
+  /** A migration id, checked against `x_migrations` by whoever can read it. See `backfill.ts`. */
+  readonly requires: string | undefined;
+  /** Absent means every environment — never an implied "production only". See `backfill.ts`. */
+  readonly environments: readonly Environment[] | undefined;
+  readonly count: BackfillCount | undefined;
+}
+
+/**
+ * One declaration as every surface reports it: plain JSON, absent as `null`, the shape
+ * `BackfillProgress` already holds for a ledger row. `counts` rather than the function itself —
+ * a declaration crosses `--json`, and "can this pass say how many rows are left" is the only
+ * thing a reader can act on.
+ */
+export interface BackfillDeclaration {
+  readonly kind: 'backfill';
+  readonly name: string;
+  readonly checksum: string;
+  readonly requires: string | null;
+  readonly environments: readonly Environment[] | null;
+  readonly counts: boolean;
+}
+
+const origin = new WeakMap<object, BackfillOrigin>();
+
+/**
+ * Called by `backfill()` and by nothing else — not exported from `src/index.ts`, for the reason
+ * `registerJob` is not: a second way to make a handle claim it is a backfill would let a plain
+ * `job()` inherit the pending diff, the gate and the deploy trigger it was never declared for.
+ */
+export function stampBackfill(handle: JobHandle<BackfillInput>, source: BackfillOrigin): void {
+  origin.set(handle, source);
+}
+
+/**
+ * Structural, exactly as `isJobHandle`/`isTaskHandle` are: a job handle plus proof `backfill()`
+ * built it. A look-alike carrying the right fields is still a job.
+ */
+export function isBackfill(value: unknown): value is JobHandle<BackfillInput> {
+  return isJobHandle(value) && origin.has(value);
+}
+
+/** What `backfill()` stamped, or nothing. The `count` function lives here and never in JSON. */
+export function backfillOrigin(handle: AnyJobHandle): BackfillOrigin | undefined {
+  return origin.get(handle);
+}
+
+/** Reads `handle.name` live, never a captured copy: registration rebinds that property in place. */
+export function declarationOf(handle: AnyJobHandle): BackfillDeclaration | undefined {
+  const source = origin.get(handle);
+  if (source === undefined) return undefined;
+  return {
+    kind: 'backfill',
+    name: handle.name,
+    checksum: source.checksum,
+    requires: source.requires ?? null,
+    environments: source.environments ?? null,
+    counts: source.count !== undefined,
+  };
+}
+
+/**
+ * Every backfill this process's modules declared, by name. Derived from `registeredJobs()` rather
+ * than from a second registry of its own: a backfill IS a job, and two registries that disagreed
+ * about one name would be two answers to "does this pass exist".
+ */
+export function registeredBackfills(): readonly BackfillDeclaration[] {
+  const declarations: BackfillDeclaration[] = [];
+  for (const handle of registeredJobs()) {
+    const declaration = declarationOf(handle);
+    if (declaration !== undefined) declarations.push(declaration);
+  }
+  return declarations;
+}
+
+/** The handle behind a declared name, for the surface that has to enqueue it. */
+export function getBackfill(name: string): JobHandle<BackfillInput> | undefined {
+  for (const handle of registeredJobs()) {
+    if (handle.name === name && isBackfill(handle)) return handle;
+  }
+  return undefined;
+}

package/src/backfill-scope.ts

@@ -0,0 +1,70 @@
+// Which sweeps run across tenants, and the capability that lets them. ONLY a `backfill()` that
+// declared `tenant: 'none'`, and only for the duration of its own pass.
+//
+// It has to be opened HERE and nowhere else: `source` hands back a lazy chain, so every page's plan
+// is built inside the iteration — long after the declaring frame closed — and `scopedPlan` is
+// applied at plan-build time. An app author holding a `ReadBuilder` has nothing to wrap.
+
+import type { Actor, Ctx } from '@ultimat3/core';
+import { runWithContext } from '@ultimat3/core';
+import { CROSS_TENANT_SCOPE, crossTenant } from '@ultimat3/entity';
+import type { BackfillInput } from './backfill';
+import type { JobTenant } from './tenant';
+import { NO_JOB_TENANT } from './tenant';
+
+/**
+ * Derived and specific, because it lands in the audit trail: `assertCrossTenant` renders it in
+ * `X_TENANCY_CROSS_DENIED` and it is what a reader sees when they ask why a plan skipped its
+ * tenant. "backfill" alone would name every sweep in the app identically.
+ */
+const reasonFor = (name: string): string =>
+  `backfill "${name}" declared tenant: 'none', so its pass sweeps every tenant's rows`;
+
+/**
+ * The capability, added to the run's own actor and to nothing else. `executeJob` already stripped
+ * the org for a `'none'` job, so this is the one fact that changes — and it changes on an actor
+ * this pass built, inside a context that dies with it.
+ */
+const withCrossTenant = (actor: Actor): Actor =>
+  actor.scopes.includes(CROSS_TENANT_SCOPE)
+    ? actor
+    : Object.freeze({ ...actor, scopes: Object.freeze([...actor.scopes, CROSS_TENANT_SCOPE]) });
+
+/**
+ * Run `pass` with the tenant guard lifted, but ONLY for a backfill that declared `tenant: 'none'`.
+ * A backfill that declared a real tenant is handed its context untouched and never sees the
+ * escape hatch — granting one to a tenanted sweep would hand every backfill in the app the
+ * capability, which is the opposite of what declaring a tenant means.
+ *
+ * **Why the grant lives here and not on the worker's actor.** The alternative is for boot to mint
+ * a worker identity carrying `tenancy:cross` (`packages/cli/src/dev-roles.ts` builds that context).
+ * That grants it to EVERY job the worker claims — including a plain `job({ tenant: 'none' })` that
+ * declared no sweep at all — and puts the decision in deployment config, where no reviewer sees it
+ * and one identity again serves every job. Here it is bounded four ways: only `backfill()`, only
+ * on an explicit `tenant: 'none'`, only for this pass, and only on a context that does not outlive
+ * it. The declaration is code in the app's own repository, carries a name the `x_backfills` ledger
+ * records, and is enumerated by `x db backfill --pending` — so the sweep that gets the capability
+ * is the one an operator can already see.
+ *
+ * `runWithContext` OUTSIDE `crossTenant`, never the other way round: `crossTenant` proves the
+ * capability against the AMBIENT actor at the call and `assertCrossTenant` proves it again for
+ * every plan built inside, so the scoped context has to be installed before either look.
+ *
+ * Nesting is safe: an app `handle` that opens its own `crossTenant(reason, fn)` replaces the reason
+ * for the plans inside it and re-proves the same capability, which this actor already carries.
+ */
+export function withBackfillScope<T>(
+  name: string,
+  tenant: JobTenant<BackfillInput>,
+  ctx: Ctx,
+  pass: (ctx: Ctx) => Promise<T>,
+): Promise<T> {
+  if (tenant !== NO_JOB_TENANT) return pass(ctx);
+  // Spread and `runWithContext` rather than `withChildContext`, which is the opposite of what
+  // `executeJob` does one frame up — and deliberately: the pass is also driven directly by tests and
+  // by tooling with no ambient context for a child to derive from, and the only fact that changes
+  // here is a SCOPE on an actor `executeJob` already built. The identity, the org and therefore
+  // every service instance are the run's own, so there is nothing for a rebuild to correct.
+  const scoped: Ctx = Object.freeze({ ...ctx, actor: withCrossTenant(ctx.actor) });
+  return runWithContext(scoped, () => crossTenant(reasonFor(name), () => pass(scoped)));
+}

package/src/backfill.ts

@@ -0,0 +1,213 @@
+// `backfill()` — one pass over every row a chain matches, declared as a `job` and NOT as a ninth
+// primitive. A backfill is durable background work with an input schema, a retry policy, an
+// idempotency key and a queue, which is the definition of a `job` — so this file is a FACTORY over
+// `job()`, exactly as `llm()` is one over `action()`. That is what gives a backfill `.enqueue()`,
+// the worker's cancellation, the dead-letter path, `x jobs show` and a manifest row without a line
+// here, and it is why nothing in the framework grows a ninth kind of thing to hold table sweeps.
+//
+// The declaration lives here and the pass lives in `backfill-pass.ts` — the same split `job.ts`
+// and `execute.ts` already have, and the reason the iteration, the checkpoints and the
+// `x_backfills` ledger are one file's problem rather than this one's.
+
+import type { Ctx, Environment } from '@ultimat3/core';
+import { assert } from '@ultimat3/core';
+import type { ReadBuilder } from '@ultimat3/entity';
+import { t } from '@ultimat3/schema';
+import { backfillChecksum } from './backfill-ledger';
+import { backfillPass } from './backfill-pass';
+import { createPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
+import type { BackfillCount } from './backfill-registry';
+import { stampBackfill } from './backfill-registry';
+import type { DurationInput } from './clock';
+import type { JobHandle } from './job';
+import { job } from './job';
+import type { RetryPolicy } from './retry';
+import { DEFAULT_RETRY } from './retry';
+import type { JobTenant } from './tenant';
+
+/**
+ * Rows per statement and per durable step. Not `entity`'s `DEFAULT_PAGE_SIZE`: that number is a
+ * page somebody scrolls, and a backfill of a million rows at 50 writes twenty thousand step rows
+ * to move the same data. One statement's worth of work that a worker can still finish inside a
+ * lease is the size that belongs here.
+ */
+export const DEFAULT_BACKFILL_BATCH = 1_000;
+
+export interface BackfillBatch<Row> {
+  /** The page `page()` would have returned here: tenancy, soft delete, projection, preloads. */
+  readonly rows: readonly Row[];
+  readonly ctx: Ctx;
+  /**
+   * The run's cancellation composed with this batch's own ceiling — the same seam `ctx.signal` is
+   * elsewhere. Past it this step may no longer write, so hand it to whatever the body calls.
+   */
+  readonly signal: AbortSignal;
+  /** 0-based position in the pass, and the step name this batch is checkpointed under. */
+  readonly index: number;
+}
+
+export interface BackfillDefinition<Row> {
+  /**
+   * REQUIRED, unlike a job's. A backfill's name is a durable key — the queue row, the step trace
+   * and the ledger of what has already been run all carry it — so it is never left to whichever
+   * export name a module happened to use.
+   */
+  readonly name: string;
+  /**
+   * REQUIRED, exactly as it is on `job()` — a backfill IS a job, so it declares the org its pass
+   * runs under rather than inheriting the worker's. A payload carries only `force`, so the two
+   * honest spellings are `tenant: () => '<org>'` for a sweep declared against one tenant, and
+   * `tenant: 'none'` for one that spans every tenant.
+   *
+   * `'none'` is where a backfill differs from a plain `job`, and the difference is forced by the
+   * shape of `source`: it hands back a LAZY chain, so every page's plan is built inside the
+   * iteration — after the declaring frame has closed — and there is nothing an author could wrap
+   * in `crossTenant(reason, fn)`. So `backfillPass` opens that scope itself, for a `'none'`
+   * declaration only (`backfill-scope.ts`). A declared tenant is handed its context untouched and
+   * every page is scoped to that org, exactly as a request would be.
+   */
+  readonly tenant: JobTenant<BackfillInput>;
+  /**
+   * The rows to visit, as a chain: `() => db.posts.where({ published: true })`, or one narrowed by
+   * the run's own context — `({ ctx }) => db.posts.where({ orgId: ctx.actor.orgId })`. Read once
+   * per attempt and never enqueued, so what a run visits cannot drift from what was declared.
+   * `orderBy` is optional — the driver's own total order ends in the primary key, which is what
+   * lets every batch resume from the last one's cursor.
+   */
+  source(args: { readonly ctx: Ctx }): ReadBuilder<Row>;
+  /**
+   * One page, in the batch's own durable step. Deliberately handed no `step`: a step name minted
+   * inside this body would have to be unique across the whole run, and the natural spelling
+   * (`step.run('rewrite', …)`) collides with itself on the second batch (`X_STEP_DUPLICATE`).
+   *
+   * At least once, like every other job body — an attempt cancelled between the last row and the
+   * checkpoint replays this page. Write through `upsertAll`, `updateWhere` or an idempotent
+   * statement; never `count + 1`.
+   */
+  handle(batch: BackfillBatch<Row>): Promise<void> | void;
+  /** Rows per statement and per step. Defaults to `DEFAULT_BACKFILL_BATCH`. */
+  readonly batch?: number;
+  /**
+   * Batches per second. Defaults to `DEFAULT_BACKFILL_RATE`, which is slow on purpose: this pass
+   * shares its pool with the requests the app is still serving. Fractions are a rate too —
+   * `rate: 0.5` is one batch every two seconds. To sweep faster raise it; there is no way to
+   * turn it off, because a backfill that saturates the pool has no correct value here.
+   */
+  readonly rate?: number;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  /** Per attempt, not per pass: a resumed attempt picks up at the last checkpoint. */
+  readonly timeout?: DurationInput;
+  /**
+   * The migration this sweep needs applied first — the id `x db gen` wrote, e.g.
+   * `20260814120000_add_publish_at`. Declared DATA, and checked by whoever can read `x_migrations`
+   * (`x db backfill`): this package holds no `@ultimat3/db` dependency, and growing one so a queue
+   * could read a migration ledger would put the migration engine on tier 3's import graph.
+   *
+   * Deliberately NOT a `dependsOn` graph over other backfills. The real dependency is almost
+   * always "after code that tolerates both shapes is serving", which the framework cannot observe,
+   * so a graph would encode an ordering it has no way to be right about.
+   */
+  readonly requires?: string;
+  /**
+   * The environments this sweep may run in. Omitted means EVERY one — never an implied
+   * "cleanups are production": a staging rehearsal is correct practice, so which deploys a sweep
+   * belongs to is the app's convention and this field is only the mechanism that carries it
+   * (axiom 8). A mismatch is `X_BACKFILL_ENVIRONMENT`, refused inside the pass as well as by the
+   * CLI, because a backfill enqueued by app code never passes through a command.
+   */
+  readonly environments?: readonly Environment[];
+  /**
+   * How many rows still match — the SAME predicate `source` selects on, counted rather than read.
+   * Optional, and what it buys is that a dry run cannot lie and "did it converge" becomes
+   * arithmetic: a pass whose source is exhausted while this still answers above zero has two
+   * predicates that disagree, which is `X_BACKFILL_STALLED` and an authoring bug in any business.
+   */
+  count?(args: { readonly ctx: Ctx }): Promise<number> | number;
+}
+
+/** What one completed pass reports — bounded, so `x jobs show` can print it. */
+export interface BackfillReport {
+  readonly name: string;
+  /** Batches THIS pass handled, replayed ones included. `0` when it was skipped. */
+  readonly batches: number;
+  readonly rows: number;
+  /** True when `x_backfills` already held a completed pass under this name and `force` was not set. */
+  readonly skipped: boolean;
+  /** The completed pass the ledger answered with, when there was one. */
+  readonly previousRunId?: string | undefined;
+}
+
+/**
+ * A backfill's payload is its identity plus the one decision a queue row is allowed to carry.
+ * What to visit is declared, so nothing here can drift from the definition; `force` changes only
+ * whether a name the ledger already records as completed runs a SECOND pass, and never what that
+ * pass would do.
+ */
+export interface BackfillInput {
+  /**
+   * Run even though `x_backfills` holds a completed pass under this name. The rerun is a NEW
+   * ledger row — history is never overwritten, so what each pass swept stays readable.
+   */
+  readonly force?: boolean | undefined;
+}
+
+export function backfill<Row>(definition: BackfillDefinition<Row>): JobHandle<BackfillInput> {
+  const size = definition.batch ?? DEFAULT_BACKFILL_BATCH;
+  // Refused where it was written. `inBatches()` refuses the same number one statement in, which
+  // for a backfill means a dead-lettered job and a stack trace instead of a failing build.
+  assert(
+    Number.isSafeInteger(size) && size >= 1,
+    `backfill "${definition.name}" declares batch: ${String(size)} — a batch is a whole number of rows, at least one`,
+    `set batch: ${DEFAULT_BACKFILL_BATCH} on backfill("${definition.name}") — the rows one statement reads and one durable step handles`,
+  );
+  const rate = definition.rate ?? DEFAULT_BACKFILL_RATE;
+  // Refused in the same voice and for the same reason as `batch` above — except that an unpaced
+  // sweep is not a dead-lettered job but a saturated pool, which the app finds out about first.
+  assert(
+    Number.isFinite(rate) && rate > 0,
+    `backfill "${definition.name}" declares rate: ${String(rate)} — a rate is batches per second, greater than zero`,
+    `set rate: ${DEFAULT_BACKFILL_RATE} on backfill("${definition.name}"), or leave it out — to sweep faster raise the number, there is no unthrottled mode`,
+  );
+  // Hashed once, at declaration: the definition cannot change while the process runs, and a hash
+  // per attempt would charge every batch of every pass for a fact that is fixed at import.
+  // `rate` is NOT in it, for the reason `batch` is not: pacing is a tuning knob, and changing one
+  // does not make a completed sweep a different sweep.
+  const checksum = backfillChecksum(definition.source, definition.handle);
+  // Built here rather than per attempt: the interval belongs to the table and the pool, not to
+  // whichever attempt holds the run, so a retrying pass keeps the pace it was declared with.
+  const pace = createPacer({ rate, job: definition.name });
+
+  // Bound to the definition rather than passed bare: `count` is declared as a method, so a
+  // reference torn off the object literal would run with `this` undefined the first time an
+  // author writes `count: ({ ctx }) => this.something`.
+  const declaredCount = definition.count;
+  const count: BackfillCount | undefined =
+    declaredCount === undefined ? undefined : (args) => declaredCount.call(definition, args);
+
+  const handle = job<BackfillInput>({
+    name: definition.name,
+    input: t.object({ force: t.boolean.optional() }),
+    // One live run per name, forced or not. A second enqueue while the pass is still going is the
+    // same pass, and deduping it is what makes "kick it again" safe rather than a second writer on
+    // one table — which is exactly what a `force` in the key would have allowed.
+    idempotencyKey: () => definition.name,
+    // Forwarded verbatim, like every other job field this factory carries: a backfill that
+    // declared its tenant and then ran under somebody else's would be a factory deciding authz.
+    tenant: definition.tenant,
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    run: (args) => backfillPass({ definition, size, checksum, pace }, args),
+  });
+  // Stamped, never registered by the app: this is what makes a declared-but-never-enqueued sweep
+  // visible to `x db backfill --pending`, and the `origin` WeakMap is `task.ts`'s mechanism rather
+  // than a second one. `job()` above already refused a duplicate name, so this cannot overwrite.
+  stampBackfill(handle, {
+    checksum,
+    requires: definition.requires,
+    environments: definition.environments,
+    count,
+  });
+  return handle;
+}

package/src/driver-memory.ts

@@ -4,12 +4,15 @@
 
 import type { Clock } from '@ultimat3/core';
 import { assert, systemClock, uuid } from '@ultimat3/core';
+import type { BackfillLedger } from './backfill-ledger';
+import { createMemoryBackfillLedger } from './backfill-ledger';
 import { nowMs } from './clock';
 import type {
   ClaimedJob,
   ClaimOptions,
   EnqueueRequest,
   EnqueueResult,
+  HeartbeatOptions,
   JobDriver,
   JobFilter,
   JobIntrospection,
@@ -19,12 +22,18 @@ import type {
 } from './driver';
 import { DEFAULT_QUEUE } from './driver';
 import { JobDuplicateError } from './errors';
+import type { LeaseStore } from './leases';
+import { createMemoryLeaseStore } from './leases';
 import type { StepStore } from './steps';
 import { createMemoryStepStore } from './steps';
 
 export interface MemoryDriverOptions {
   readonly clock?: Clock;
   readonly steps?: StepStore;
+  /** Injectable for the same reason `steps` is: two drivers in one test sharing one ledger. */
+  readonly backfills?: BackfillLedger;
+  /** Injectable so two drivers in one test can share one set of fleet slots. */
+  readonly leases?: LeaseStore;
 }
 
 const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
@@ -32,11 +41,19 @@ const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
 export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver {
   const clock = options.clock ?? systemClock;
   const steps = options.steps ?? createMemoryStepStore();
+  const backfills = options.backfills ?? createMemoryBackfillLedger(clock);
+  const leases =
+    options.leases ?? createMemoryLeaseStore(options.clock === undefined ? {} : { clock });
   const jobs = new Map<string, JobRecord>();
 
-  const liveByKey = (key: string): JobRecord | undefined => {
+  // Keyed by NAME and key, exactly as `x_jobs_name_idempotency_live_idx` is. A global key
+  // namespace let two unrelated jobs that derived the same natural key dedupe against each
+  // other: the second enqueue returned the first's id and its work never ran.
+  const liveByKey = (name: string, key: string): JobRecord | undefined => {
     for (const record of jobs.values()) {
-      if (record.idempotencyKey === key && LIVE_STATES.has(record.state)) return record;
+      if (record.name === name && record.idempotencyKey === key && LIVE_STATES.has(record.state)) {
+        return record;
+      }
     }
     return undefined;
   };
@@ -56,7 +73,10 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
         .filter((record) => filter.queue === undefined || record.queue === filter.queue)
         .filter((record) => filter.name === undefined || record.name === filter.name)
         .filter((record) => filter.state === undefined || record.state === filter.state)
-        .sort((a, b) => a.createdAt - b.createdAt)
+        // NEWEST first, as `createPgDriver`'s `order by created_at desc` is. Ascending here meant
+        // `x jobs ls` answered one thing against `x dev` and the opposite in production — and,
+        // because the limit is applied after the sort, a default page of the hundred OLDEST rows.
+        .sort((a, b) => b.createdAt - a.createdAt)
         .slice(0, filter.limit ?? 100);
       return Promise.resolve(rows);
     },
@@ -83,15 +103,28 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
       const next = jobs.get(jobId);
       return next ?? record;
     },
+    cancel(jobId, reason) {
+      const existing = jobs.get(jobId);
+      // `state !== 'done'`, mirroring `SQL_CANCEL`: a job that already finished has nothing to
+      // stop, and cancelling it would rewrite a terminal row an operator is reading as success.
+      if (existing === undefined || existing.state === 'done') return Promise.resolve(undefined);
+      update(jobId, {
+        state: 'cancelled',
+        ...(reason === undefined ? {} : { lastError: reason }),
+      });
+      return Promise.resolve(jobs.get(jobId));
+    },
   };
 
   return {
     name: 'memory',
     steps,
+    backfills,
+    leases,
     introspect,
 
     enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
-      const existing = liveByKey(request.idempotencyKey);
+      const existing = liveByKey(request.name, request.idempotencyKey);
       if (existing !== undefined) {
         if (request.onConflict === 'error') {
           throw new JobDuplicateError({
@@ -119,6 +152,8 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
         createdAt: at,
         updatedAt: at,
         ...(request.tenantId === undefined ? {} : { tenantId: request.tenantId }),
+        ...(request.traceparent === undefined ? {} : { traceparent: request.traceparent }),
+        ...(request.enqueuedBy === undefined ? {} : { enqueuedBy: request.enqueuedBy }),
       };
       jobs.set(record.id, record);
       return Promise.resolve({ id: record.id, runId: record.runId, deduped: false });
@@ -156,30 +191,47 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
       return Promise.resolve(out);
     },
 
+    // Both settlements are FENCED on `running`, as `SQL_ACK`/`SQL_NACK` are: an ack from a worker
+    // whose job was cancelled — or whose lease lapsed and whose job another worker re-claimed —
+    // would otherwise overwrite a row it no longer owns.
     ack(jobId: string): Promise<void> {
+      if (jobs.get(jobId)?.state !== 'running') return Promise.resolve();
       update(jobId, { state: 'done' });
       return Promise.resolve();
     },
 
     nack(jobId: string, nackOptions: NackOptions): Promise<void> {
       const record = jobs.get(jobId);
-      if (record === undefined) return Promise.resolve();
+      if (record === undefined || record.state !== 'running') return Promise.resolve();
       const at = nowMs(clock);
       const counts = nackOptions.countsAsAttempt !== false;
       const patch: Partial<JobRecord> = {
         state: nackOptions.deadLetter === true ? 'dead' : counts ? 'ready' : 'suspended',
         runAt: at + nackOptions.delayMs,
-        // A suspension must not burn an attempt, or a 3-day sleep dead-letters the run.
-        attempt: counts ? record.attempt : record.attempt - 1,
+        // A suspension must not burn an attempt, or a 3-day sleep dead-letters the run. Floored
+        // where `SQL_NACK` floors it (`greatest(attempt - 1, 0)`): the fence above is what keeps
+        // the decrement paired with a claim today, so this is the guard that survives the fence
+        // being read as the only one.
+        attempt: counts ? record.attempt : Math.max(0, record.attempt - 1),
         ...(nackOptions.error === undefined ? {} : { lastError: nackOptions.error }),
       };
       update(jobId, patch);
       return Promise.resolve();
     },
 
-    heartbeat(jobId: string, heartbeatOptions): Promise<void> {
+    heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
+      const record = jobs.get(jobId);
+      // The same predicate `SQL_HEARTBEAT` carries. `false` is how an external cancel reaches a
+      // job that is already running: the worker's next renewal misses and the attempt is aborted.
+      if (
+        record === undefined ||
+        record.state !== 'running' ||
+        (heartbeatOptions.workerId !== undefined && record.claimedBy !== heartbeatOptions.workerId)
+      ) {
+        return Promise.resolve(false);
+      }
       update(jobId, { visibleAt: nowMs(clock) + heartbeatOptions.visibilityTimeoutMs });
-      return Promise.resolve();
+      return Promise.resolve(true);
     },
 
     stats(): Promise<readonly QueueStats[]> {