@ultimat3/jobs 23.0.0 → 25.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "23.0.0",
+  "version": "25.0.0",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -25,17 +25,17 @@
     "LICENSE"
   ],
   "engines": {
-    "bun": ">=1.4.0"
+    "bun": ">=1.4.2"
   },
   "scripts": {
     "typecheck": "tsc --noEmit -p tsconfig.json",
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "23.0.0",
-    "@ultimat3/db": "23.0.0",
-    "@ultimat3/entity": "23.0.0",
-    "@ultimat3/schema": "23.0.0",
-    "@ultimat3/time": "23.0.0"
+    "@ultimat3/core": "25.0.0",
+    "@ultimat3/db": "25.0.0",
+    "@ultimat3/entity": "25.0.0",
+    "@ultimat3/schema": "25.0.0",
+    "@ultimat3/time": "25.0.0"
   }
 }

package/src/backfill-errors.ts

@@ -5,7 +5,7 @@
 // declared in ONE place; only the classes that throw them live here, beside `backfill-ledger.ts`,
 // `backfill-pending.ts` and `backfill-registry.ts`.
 
-import { UltimateError } from '@ultimat3/core';
+import { renderFixShellArg, UltimateError } from '@ultimat3/core';
 
 /**
  * The seven backfill codes below all answer one question — "why is this sweep not running?" — and
@@ -64,7 +64,8 @@ export class BackfillEnvironmentError extends UltimateError {
       fix:
         target === undefined
           ? 'x db backfill --pending --json'
-          : `ULTIMATE_ENV=${target} x db backfill ${input.backfill} --write --json`,
+          : // Both are the declaration's own strings and the class is public: screened (sweep 1c).
+            `ULTIMATE_ENV=${renderFixShellArg(target, "'<environment>'")} x db backfill ${renderFixShellArg(input.backfill, "'<backfill>'")} --write --json`,
     });
   }
 }

package/src/backfill-ledger.ts

@@ -10,7 +10,7 @@
 // become — the checkpoints are transactional with the work, and this row is not.
 
 import type { Clock } from '@ultimat3/core';
-import { finiteCount, systemClock } from '@ultimat3/core';
+import { describeValue, finiteCount, systemClock, UltimateError } from '@ultimat3/core';
 import { nowMs } from './clock';
 
 /**
@@ -56,6 +56,27 @@ export interface BackfillFilter {
   readonly limit?: number | undefined;
 }
 
+/** What Postgres prints for a `uuid` — the one spelling both ledgers store identically. */
+const RUN_ID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/**
+ * `BackfillFilter.runId`, refused by BOTH ledgers' `list` when it is not a uuid. `run_id` is a
+ * `uuid` column: Postgres raised a raw `22P02` where the memory ledger answered an empty page, so
+ * a mistyped id read as "no such pass" under `x dev` and as a database fault in production. The
+ * shape `outbox.ts` holds an enqueue's `runId` to — lowercase, so the two never disagree on case.
+ */
+export function listedRunId(subject: string, runId: unknown): string | undefined {
+  if (runId === undefined) return undefined;
+  if (typeof runId === 'string' && RUN_ID_SHAPE.test(runId)) return runId;
+  throw new UltimateError({
+    code: 'X_ID_INVALID',
+    // `describeValue`, never the value: what arrives here wrong is as often a key as a typo.
+    cause: `${subject} backfills.list was filtered by runId ${describeValue(runId)}, and a run id is a lowercase uuid (8-4-4-4-12 hex) — x_backfills.run_id is a uuid column`,
+    fix: 'x db backfill --list --json',
+    meta: { option: 'runId' },
+  });
+}
+
 /**
  * Durable, and therefore behind the queue driver: `x_backfills` ships in the same DDL as `x_jobs`
  * and `x_job_steps` (`driver-pg-sql.ts`), so a ledger a pass cannot write is a queue it could not
@@ -132,7 +153,7 @@ export function decideBackfill(
   return { run: force, previous: completed, changed: completed.checksum !== checksum };
 }
 
-export function createMemoryBackfillLedger(clock: Clock = systemClock): BackfillLedger {
+export function memoryBackfillLedger(clock: Clock = systemClock): BackfillLedger {
   const runs = new Map<string, BackfillRun>();
   const patch = (runId: string, fields: Partial<BackfillRun>): void => {
     const existing = runs.get(runId);
@@ -169,6 +190,7 @@ export function createMemoryBackfillLedger(clock: Clock = systemClock): Backfill
     },
     // `async`, so a refused limit REJECTS here as it does on the pg ledger — see `driver-memory.ts`.
     async list(filter = {}) {
+      const runId = listedRunId('the memory ledger', filter.runId);
       // Reversed BEFORE the sort: the test clock is frozen, so two rows share a `startedAt` and a
       // stable sort would hand back the oldest of them first under a "newest first" contract.
       const rows = [...runs.values()]
@@ -176,7 +198,7 @@ export function createMemoryBackfillLedger(clock: Clock = systemClock): Backfill
         .sort((a, b) => b.startedAt - a.startedAt)
         .filter((run) => filter.name === undefined || run.name === filter.name)
         .filter((run) => filter.status === undefined || run.status === filter.status)
-        .filter((run) => filter.runId === undefined || run.runId === filter.runId)
+        .filter((run) => runId === undefined || run.runId === runId)
         .slice(0, finiteCount('the backfill ledger list', 'limit', filter.limit ?? 100));
       return rows;
     },

package/src/backfill-pass.ts

@@ -164,7 +164,13 @@ export async function backfillPass<Row>(
   // refusing it — the same degradation `introspect` already has.
   const ledger = jobDriver()?.backfills;
 
-  const previous = await completedRun(ledger, name);
+  const completed = await completedRun(ledger, name);
+  // Its OWN row is not a previous pass: a worker that died between `ledger.finish` and the ack is
+  // handed the same run again, and skipping there reported `{ skipped: true, previousRunId:
+  // <itself> }` for a pass that swept. It falls through and replays — every batch is served from
+  // its checkpoint, nothing is read or handled twice, and the report is the one it first made.
+  const redelivered = completed?.runId === runId;
+  const previous = redelivered ? undefined : completed;
   const verdict = decideBackfill(previous, checksum, args.input.force === true);
   if (!verdict.run) {
     if (verdict.changed) {
@@ -196,7 +202,14 @@ export async function backfillPass<Row>(
     definition.tenant,
     args.ctx,
     async (ctx): Promise<BackfillReport> => {
-      await ledger?.start({ runId, name, checksum, appVersion: appVersion() });
+      // A redelivered pass leaves its completed row ALONE until a batch body actually runs (an
+      // operator's `retryFromStep` re-opens one): replaying checkpoints sweeps nothing, so there is
+      // nothing to record and no convergence to re-ask — rows that started matching since would
+      // otherwise turn a finished pass into a failed one.
+      let recording = !redelivered;
+      const open = (): Promise<void> | undefined =>
+        ledger?.start({ runId, name, checksum, appVersion: appVersion() });
+      if (recording) await open();
 
       let cursor: string | null = null;
       let rows = 0;
@@ -242,6 +255,10 @@ export async function backfillPass<Row>(
               const iteration = await iterate();
               const next = await iteration.pull.next();
               if (next.done === true) return { cursor: null, rows: 0 };
+              if (!recording) {
+                recording = true;
+                await open();
+              }
               await definition.handle({ rows: next.value, ctx, signal, index });
               swept = true;
               return { cursor: iteration.batches.cursor, rows: next.value.length };
@@ -271,10 +288,10 @@ export async function backfillPass<Row>(
         // this converge", asked once, where the answer is finally decidable. Inside the `try` so the
         // ledger records the attempt as `failed`: a pass that left rows behind must not write the
         // completed row that stops the next deploy re-running it.
-        await assertConverged(definition, ctx, rows);
+        if (recording) await assertConverged(definition, ctx, rows);
       } catch (error) {
         // Control flow, not a failure: a suspended run is parked and will be back on this step.
-        if (!isStepSuspension(error)) await markFailed(ledger, runId, rows);
+        if (!isStepSuspension(error) && recording) await markFailed(ledger, runId, rows);
         throw error;
       } finally {
         // Whatever the iteration holds belongs to this attempt, and an attempt that failed, was
@@ -282,7 +299,7 @@ export async function backfillPass<Row>(
         await live?.batches.close();
       }
 
-      await ledger?.finish(runId, { status: 'completed', rows });
+      if (recording) await ledger?.finish(runId, { status: 'completed', rows });
       return { name, batches, rows, skipped: false };
     },
   );

package/src/backfill-rate.ts

@@ -69,14 +69,14 @@ function timerSleep(ms: number, signal: AbortSignal): Promise<void> {
  * rate is a property of the table being swept and the pool it is swept through, not of whichever
  * attempt happens to hold the run.
  */
-export function createPacer(options: PacerOptions): Pacer {
+export function backfillPacer(options: PacerOptions): Pacer {
   // Refused HERE and not only at `backfill()`: `rate: 0` makes `intervalMs` Infinity, which the
   // timer clamps to about a millisecond — so an unvalidated zero reads as "no throttle at all",
   // which is the one setting this module exists to make unreachable. A negative rate is the same
   // bug with a negative wait.
   assert(
     Number.isFinite(options.rate) && options.rate > 0,
-    `createPacer({ rate: ${String(options.rate)} }) for "${options.job}" — a rate is batches per second, greater than zero`,
+    `backfillPacer({ rate: ${String(options.rate)} }) for "${options.job}" — a rate is batches per second, greater than zero`,
     `pass rate: ${DEFAULT_BACKFILL_RATE} — to sweep faster raise the number, there is no unthrottled mode`,
   );
   const clock = options.clock ?? systemClock;

package/src/backfill.ts

@@ -15,7 +15,7 @@ import type { ReadBuilder } from '@ultimat3/entity';
 import { t } from '@ultimat3/schema';
 import { backfillChecksum } from './backfill-ledger';
 import { backfillPass } from './backfill-pass';
-import { createPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
+import { backfillPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
 import type { BackfillCount } from './backfill-registry';
 import { stampBackfill } from './backfill-registry';
 import type { DurationInput } from './clock';
@@ -176,7 +176,7 @@ export function backfill<Row>(definition: BackfillDefinition<Row>): JobHandle<Ba
   const checksum = backfillChecksum(definition.source, definition.handle);
   // Built here rather than per attempt: the interval belongs to the table and the pool, not to
   // whichever attempt holds the run, so a retrying pass keeps the pace it was declared with.
-  const pace = createPacer({ rate, job: definition.name });
+  const pace = backfillPacer({ rate, job: definition.name });
 
   // Bound to the definition rather than passed bare: `count` is declared as a method, so a
   // reference torn off the object literal would run with `this` undefined the first time an

package/src/claim-exhausted.ts

@@ -0,0 +1,76 @@
+// The claim round's half of a burial: a row the claim settled `dead` (`ClaimOptions.onExhausted`)
+// has no body to end it and no settle to announce it, so the pass that buried it says so — one log
+// line an operator can alert on, and the job's `onSettled`, exactly as a dead letter a worker
+// settled itself would get.
+
+import type { Ctx } from '@ultimat3/core';
+import { logger, renderThrowable } from '@ultimat3/core';
+import type { JobRecord } from './driver';
+import { getJob } from './job';
+import { announceSettled } from './settled';
+
+export interface AnnounceExhaustedOptions {
+  /** What `onExhausted` handed this pass — every row the claim buried, as it now stands. */
+  readonly exhausted: readonly JobRecord[];
+  readonly workerId: string;
+  /** The worker's own context: what the job's `onSettled` is scoped from, as a body would be. */
+  readonly context: () => Ctx;
+}
+
+/** What one pass buried, by how each row ended — what the worker's own counters take. */
+export interface ExhaustedCounts {
+  readonly deadLettered: number;
+  readonly dropped: number;
+}
+
+/**
+ * Announce every row one claim pass buried, and answer how each ended. Once per row, because the
+ * claim buries a row once: the statement that settled it is the only one that hands it to
+ * `onExhausted`. A row buried `failed` is a job declaring `retry.deadLetter: false`
+ * (`ClaimOptions.dropExhausted`) — `dropped`, as its own nack would have ended it. Never rejects:
+ * `announceSettled` owns a hook's failure, and a `context()` that throws is logged.
+ */
+export async function announceExhausted(
+  options: AnnounceExhaustedOptions,
+): Promise<ExhaustedCounts> {
+  let dropped = 0;
+  for (const record of options.exhausted) {
+    const error = record.lastError ?? '';
+    const outcome = record.state === 'failed' ? 'dropped' : 'dead-lettered';
+    if (outcome === 'dropped') dropped += 1;
+    // `error`, not `warn`: nobody recovered from this one, and a job that outlives every worker
+    // that takes it is a crash loop somebody has to look at.
+    logger.error('jobs.claim.exhausted', {
+      workerId: options.workerId,
+      job: record.name,
+      jobId: record.id,
+      attempt: record.attempt,
+      maxAttempts: record.maxAttempts,
+      outcome,
+      error,
+    });
+    const handle = getJob(record.name);
+    if (handle === undefined) continue;
+    let ctx: Ctx;
+    try {
+      ctx = options.context();
+    } catch (raised) {
+      // The app's own function, asked AFTER the row was settled: thrown, it failed the round.
+      logger.error('jobs.claim.exhausted-unannounced', {
+        workerId: options.workerId,
+        job: record.name,
+        jobId: record.id,
+        error: renderThrowable(raised),
+      });
+      continue;
+    }
+    await announceSettled({
+      handle,
+      claimed: record,
+      ctx,
+      // No code: nothing was thrown, and `JobFailed.code` is never invented.
+      settlement: { outcome, error, code: undefined },
+    });
+  }
+  return { deadLettered: options.exhausted.length - dropped, dropped };
+}

package/src/counters.ts

@@ -45,7 +45,7 @@ export interface MemoryCounters {
 }
 
 /** The memory driver's counters. Same buckets, same tiers, same fold as `x_job_counters`. */
-export function createMemoryCounters(): MemoryCounters {
+export function memoryCounters(): MemoryCounters {
   const buckets = new Map<string, JobCounter>();
   const keyOf = (job: string, bucketMs: number, bucketStart: number): string =>
     `${bucketMs}\u0000${bucketStart}\u0000${job}`;

package/src/driver-memory-operator.ts

@@ -7,7 +7,7 @@ import type { Clock } from '@ultimat3/core';
 import { assert, finiteOption } from '@ultimat3/core';
 import { nowMs } from './clock';
 import type { MemoryCounters } from './counters';
-import { createMemoryCounters } from './counters';
+import { memoryCounters } from './counters';
 import type { JobRecord } from './driver';
 import { REQUEUEABLE_STATES } from './driver';
 import { signalEnqueued } from './enqueue-signal';
@@ -92,9 +92,9 @@ function pauseSet(clock: Clock): {
   };
 }
 
-export function createMemoryOperator(state: MemoryQueueState): MemoryOperator {
+export function memoryOperator(state: MemoryQueueState): MemoryOperator {
   const { jobs, clock } = state;
-  const counters = createMemoryCounters();
+  const counters = memoryCounters();
   const queues = pauseSet(clock);
   const tasks = pauseSet(clock);
   const workers = new Map<string, WorkerRecord & { readonly expiresAt: number }>();
@@ -134,7 +134,7 @@ export function createMemoryOperator(state: MemoryQueueState): MemoryOperator {
       const before =
         filter.before === undefined
           ? undefined
-          : parseJobCursor(filter.before, 'the memory driver list');
+          : parseJobCursor(filter.before, 'the memory driver list', 'before');
       const { idPrefix, createdFrom, createdTo, tenantId } = filter;
       const matching = [...jobs.values()]
         .filter((record) => filter.queue === undefined || record.queue === filter.queue)
@@ -168,16 +168,38 @@ export function createMemoryOperator(state: MemoryQueueState): MemoryOperator {
         `requeueMany was asked for ${filter.state} jobs, and only a finished job can be requeued`,
         "pass state: 'dead', 'failed', 'cancelled' or 'done' to requeueMany",
       );
-      const result = await bulk(filter, (record) => {
-        // A key a live job holds stays where it is — the single requeue's `X_JOB_DUPLICATE`,
-        // answered as "not this one" so one conflict does not fail a thousand rows.
-        if (state.liveHolder(record) !== undefined) return false;
-        state.settle(record.id, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
-        return true;
-      });
+      // A key a live job holds stays where it is — the single requeue's `X_JOB_DUPLICATE`,
+      // answered as "not this one" so one conflict does not fail a thousand rows.
+      const free = (record: JobRecord): boolean => state.liveHolder(record) === undefined;
+      // `SQL_JOB_REQUEUE_MANY`'s candidates: no live holder, and ONE row per idempotency namespace
+      // — the oldest — both decided BEFORE the bound. Decided inside `apply`, a held row spent a
+      // slot of `MAX_BULK_ROWS` and was then skipped: a thousand held rows ahead of one free row
+      // answered `affected: 0` here and `affected: 1` on Postgres.
+      const oldest = new Map<string, JobRecord>();
+      for (const record of jobs.values()) {
+        if (!inBulk(record, filter) || !free(record)) continue;
+        const key = JSON.stringify([record.name, record.tenantId ?? '', record.idempotencyKey]);
+        const seen = oldest.get(key);
+        if (seen === undefined || newerThan(seen, record)) oldest.set(key, record);
+      }
+      const candidates = new Set([...oldest.values()].map((record) => record.id));
+      const result = await bulk(
+        filter,
+        (record) => {
+          if (!free(record)) return false;
+          state.settle(record.id, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
+          return true;
+        },
+        (record) => candidates.has(record.id),
+      );
       // The pg driver's `SQL_WAKE`, in-process: an operator's repair starts now.
       if (result.affected > 0) signalEnqueued(filter.queue);
-      return result;
+      // What a second call could still move: rows of the filter whose key no live job holds. A
+      // row requeued just now holds its key, so its dead siblings left with it.
+      const remaining = [...jobs.values()].filter(
+        (record) => inBulk(record, filter) && free(record),
+      ).length;
+      return { affected: result.affected, remaining };
     },
 
     async promoteMany(filter) {

package/src/driver-memory.ts

@@ -3,9 +3,9 @@
 // real claim/ack/nack paths rather than a mock that always succeeds.
 
 import type { Clock } from '@ultimat3/core';
-import { assert, finiteCount, systemClock, uuid } from '@ultimat3/core';
+import { finiteCount, systemClock, uuidV7 } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
-import { createMemoryBackfillLedger } from './backfill-ledger';
+import { memoryBackfillLedger } from './backfill-ledger';
 import { nowMs } from './clock';
 import { nackOutcome } from './counters';
 import type {
@@ -25,21 +25,24 @@ import {
   assertClaimBounds,
   assertClaimQueues,
   DEFAULT_QUEUE,
+  LEASE_LAPSED_FINAL_ATTEMPT,
+  LIVE_STATES,
   nackState,
   REQUEUEABLE_STATES,
 } from './driver';
-import { createMemoryOperator } from './driver-memory-operator';
+import { memoryOperator } from './driver-memory-operator';
 import { signalEnqueued } from './enqueue-signal';
-import { JobDuplicateError } from './errors';
-import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
+import { JobDuplicateError, LeaseLostError } from './errors';
+import { JobNotFoundError, JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
 import type { JobIntrospection } from './introspection';
 import { MAX_ERROR_STACK_LENGTH } from './introspection';
 import type { LeaseStore } from './leases';
-import { createMemoryLeaseStore } from './leases';
+import { memoryLeaseStore } from './leases';
+import { isFinalAttempt } from './retry';
 import type { StepStore } from './steps';
-import { createMemoryStepStore } from './steps-memory';
+import { memoryStepStore } from './steps-memory';
 
-export interface MemoryDriverOptions {
+export interface MemoryJobDriverOptions {
   readonly clock?: Clock;
   readonly steps?: StepStore;
   /** Injectable for the same reason `steps` is: two drivers in one test sharing one ledger. */
@@ -48,8 +51,6 @@ export interface MemoryDriverOptions {
   readonly leases?: LeaseStore;
 }
 
-const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
-
 /**
  * The in-memory driver's own type: `JobDriver` with `close` REQUIRED.
  *
@@ -61,12 +62,25 @@ const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
  */
 export type MemoryJobDriver = JobDriver & { close(): Promise<void> };
 
-export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJobDriver {
+export function memoryJobDriver(options: MemoryJobDriverOptions = {}): MemoryJobDriver {
   const clock = options.clock ?? systemClock;
-  const steps = options.steps ?? createMemoryStepStore();
-  const backfills = options.backfills ?? createMemoryBackfillLedger(clock);
-  const leases =
-    options.leases ?? createMemoryLeaseStore(options.clock === undefined ? {} : { clock });
+  const stepStore = options.steps ?? memoryStepStore();
+  // `SQL_STEP_PUT`'s fence: a write made under a claim lands only while that claim holds the row.
+  const steps: StepStore = {
+    ...stepStore,
+    put(record, by) {
+      const row = by === undefined ? undefined : jobs.get(by.jobId);
+      if (
+        by !== undefined &&
+        !(row?.state === 'running' && row.claimedBy === by.workerId && row.claim === by.claim)
+      ) {
+        return Promise.reject(new LeaseLostError({ job: by.job, jobId: by.jobId }));
+      }
+      return stepStore.put(record);
+    },
+  };
+  const backfills = options.backfills ?? memoryBackfillLedger(clock);
+  const leases = options.leases ?? memoryLeaseStore(options.clock === undefined ? {} : { clock });
   const jobs = new Map<string, JobRecord>();
 
   // Keyed by NAME, TENANT and key, exactly as `x_jobs_name_tenant_idempotency_live_idx` is. A
@@ -104,14 +118,22 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
    * lease deadline — which `x jobs show` prints, and which is the very pair the claim scan's
    * lease-expiry branch reads to decide a row was abandoned.
    */
-  const settle = (id: string, patch: Partial<JobRecord>): void => {
+  const settle = (id: string, patch: Partial<JobRecord>, drop?: 'stack'): void => {
     const existing = jobs.get(id);
     if (existing === undefined) return;
     const { visibleAt: _visibleAt, claimedBy: _claimedBy, ...released } = existing;
-    jobs.set(id, { ...released, ...patch, updatedAt: nowMs(clock) });
+    // `drop: 'stack'` is `SQL_NACK`'s `case when $5 is null then last_error_stack else $7 end`
+    // with no stack bound: a NEW failure replaces the old one's stack even with nothing — the key
+    // is removed, never set to `undefined`, as a null column is absent from a pg record.
+    const { lastErrorStack: _stack, ...unstacked } = released;
+    jobs.set(id, {
+      ...(drop === 'stack' ? unstacked : released),
+      ...patch,
+      updatedAt: nowMs(clock),
+    });
   };
 
-  const operator = createMemoryOperator({
+  const operator = memoryOperator({
     jobs,
     steps,
     clock,
@@ -145,13 +167,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       return rows;
     },
     async requeue(jobId, requeueOptions) {
-      const existing = jobs.get(jobId);
-      assert(
-        existing !== undefined,
-        `no job ${jobId} in the memory driver`,
-        'requeue a job id returned by enqueue() or stats() — the memory driver holds no state across processes, so an id from another run will not resolve',
-      );
-      const record = existing;
+      const record = jobs.get(jobId);
+      // The one answer both drivers give an id nobody queued — it was `X_INVARIANT` here.
+      if (record === undefined) throw new JobNotFoundError({ jobId, driver: 'memory' });
       // The two refusals `SQL_JOB_REQUEUE` and `SQL_JOB_LIVE_HOLDER` give, before anything moves.
       if (!REQUEUEABLE_STATES.has(record.state)) {
         throw new JobNotRequeueableError({ jobId, state: record.state });
@@ -161,7 +179,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         throw requeueKeyTaken({ ...record, holderId: holder.id });
       }
       if (requeueOptions?.fromStep !== undefined) {
-        // The target step and every step that started AFTER it — `SQL_STEPS_FROM`, whose header
+        // The target step and every step that started AFTER it — `SQL_JOB_REQUEUE`, whose header
         // says why a tie is kept. Earlier steps stay memoized; a later one replaying the old run's
         // result is what this prevents.
         const all = await steps.list(record.runId);
@@ -182,9 +200,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
     },
     cancel(jobId, reason) {
       const existing = jobs.get(jobId);
-      // `state !== 'done'`, mirroring `SQL_CANCEL`: a job that already finished has nothing to
-      // stop, and cancelling it would rewrite a terminal row an operator is reading as success.
-      if (existing === undefined || existing.state === 'done') return Promise.resolve(undefined);
+      // LIVE rows only, mirroring `SQL_CANCEL`: a job that already finished has nothing to stop.
+      // The fence was `state !== 'done'`, which let a cancel REWRITE a dead letter — the record of
+      // a failure, and its `lastError`, became the record of an operator's typo.
+      if (existing === undefined || !LIVE_STATES.has(existing.state)) {
+        return Promise.resolve(undefined);
+      }
       // `settle`, not `update`: a cancellation RELEASES the claim, and `SQL_CANCEL` writes
       // `visible_at = null, claimed_by = null` with the state. Left stamped, a cancelled row named
       // the worker still holding it and carried that attempt's lease deadline — the pair
@@ -230,7 +251,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       const at = nowMs(clock);
       const runAt = request.runAt ?? at;
       const record: JobRecord = {
-        id: request.id ?? uuid(),
+        id: request.id ?? uuidV7(),
         name: request.name,
         queue: request.queue || DEFAULT_QUEUE,
         // Through JSON, exactly as the pg driver binds it (`JSON.stringify(request.input ?? null)`):
@@ -238,7 +259,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         // queue in production never stores, so a job that read one passed here and failed there.
         input: JSON.parse(JSON.stringify(request.input ?? null)) as unknown,
         idempotencyKey: request.idempotencyKey,
-        runId: request.runId ?? uuid(),
+        runId: request.runId ?? uuidV7(),
         attempt: 0,
         maxAttempts: request.maxAttempts,
         state: runAt > at ? 'delayed' : 'ready',
@@ -276,7 +297,24 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         .slice(0, claimOptions.limit);
 
       const out: ClaimedJob[] = [];
+      const buried: JobRecord[] = [];
       for (const record of claimable) {
+        // `SQL_CLAIM`'s second arm. A `running` row reaches this loop only with a lapsed lease, and
+        // one that lapsed on its FINAL attempt has nothing left to be re-delivered for: handing
+        // it out was an attempt past `maxAttempts`, every lease, forever — a job that kills its
+        // worker took one per visibility timeout and never reached the dead-letter queue.
+        if (
+          record.state === 'running' &&
+          isFinalAttempt({ attempts: record.maxAttempts }, record.attempt)
+        ) {
+          // `$5` of the statement: a job declaring `retry.deadLetter: false` is dropped.
+          const state = claimOptions.dropExhausted?.includes(record.name) ? 'failed' : 'dead';
+          settle(record.id, { state, lastError: LEASE_LAPSED_FINAL_ATTEMPT }, 'stack');
+          operator.counters.add(record.name, state, 0, at);
+          const dead = jobs.get(record.id);
+          if (dead !== undefined) buried.push(dead);
+          continue;
+        }
         const claimed: ClaimedJob = {
           ...record,
           state: 'running',
@@ -290,7 +328,8 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         jobs.set(record.id, claimed);
         out.push(claimed);
       }
-      return Promise.resolve(out);
+      if (buried.length > 0) claimOptions.onExhausted?.(buried);
+      return out;
     },
 
     // Both settlements are FENCED on `running` AND on the claimer, as `SQL_ACK`/`SQL_NACK` are: an
@@ -301,9 +340,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       const record = heldBy(jobId, by);
       if (record === undefined) return Promise.resolve(false);
       settle(jobId, { state: 'done' });
-      if (by.counted !== false) {
-        operator.counters.add(record.name, 'done', by.durationMs ?? 0, nowMs(clock));
-      }
+      operator.counters.add(record.name, 'done', by.durationMs ?? 0, nowMs(clock));
       return Promise.resolve(true);
     },
 
@@ -323,11 +360,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         // being read as the only one.
         attempt: counts ? record.attempt : Math.max(0, record.attempt - 1),
         ...(nackOptions.error === undefined ? {} : { lastError: nackOptions.error }),
-        ...(nackOptions.stack === undefined
+        // The stack belongs to the ERROR beside it, as `SQL_NACK` binds it: no error, no stack.
+        ...(nackOptions.error === undefined || nackOptions.stack === undefined
           ? {}
           : { lastErrorStack: nackOptions.stack.slice(0, MAX_ERROR_STACK_LENGTH) }),
       };
-      settle(jobId, patch);
+      settle(jobId, patch, nackOptions.error === undefined ? undefined : 'stack');
       const outcome = nackOutcome(nackOptions);
       if (outcome !== undefined) {
         operator.counters.add(record.name, outcome, nackOptions.durationMs ?? 0, at);
@@ -382,7 +420,11 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         else if (record.state === 'dead') next.dead += 1;
         byQueue.set(record.queue, next);
       }
-      return Promise.resolve([...byQueue.values()].sort((a, b) => a.queue.localeCompare(b.queue)));
+      // Code units, as `collate "C"` orders them in `SQL_STATS` — never `localeCompare`, which
+      // answers by the runtime's locale and so differently from the database on the same names.
+      return Promise.resolve(
+        [...byQueue.values()].sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0)),
+      );
     },
 
     close(): Promise<void> {