@ultimat3/jobs 22.14.0 → 23.0.0

package/src/counters.ts

@@ -0,0 +1,105 @@
+// The per-job history counters: which bucket a settle lands in, what it adds, and how buckets
+// age from one tier to the next. The arithmetic lives here ONCE — the memory store below runs it
+// directly and the pg statements (`driver-pg-operator-sql.ts`) spell the same floor in SQL, so a
+// count read under `x dev` and one read in production are the same number for the same settles.
+
+import type { NackOptions } from './driver';
+import { nackState } from './driver';
+import type { CounterBucketMs, CounterOutcome, CounterTotals, JobCounter } from './introspection';
+import { COUNTER_BUCKET_MS, COUNTER_TIERS } from './introspection';
+
+/** The instant a bucket of `bucketMs` containing `atMs` opens. */
+export const counterBucketStart = (atMs: number, bucketMs: number): number =>
+  Math.floor(atMs / bucketMs) * bucketMs;
+
+/**
+ * What a nack adds, or nothing. A shed, a suspension and a drained attempt are handed back
+ * UNCOUNTED and are not history: counting them would make every `step.sleep` a failure.
+ */
+export function nackOutcome(options: NackOptions): CounterOutcome | undefined {
+  const state = nackState(options);
+  if (state === 'dead') return 'dead';
+  if (state === 'failed') return 'failed';
+  return state === 'ready' && options.countsAsAttempt !== false ? 'retried' : undefined;
+}
+
+type Counts = Omit<JobCounter, 'job' | 'bucketStart' | 'bucketMs'>;
+
+const ZERO: Counts = Object.freeze({ done: 0, retried: 0, failed: 0, dead: 0, durationMs: 0 });
+
+const sum = (a: Counts, b: Counts): Counts => ({
+  done: a.done + b.done,
+  retried: a.retried + b.retried,
+  failed: a.failed + b.failed,
+  dead: a.dead + b.dead,
+  durationMs: a.durationMs + b.durationMs,
+});
+
+const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
+export interface MemoryCounters {
+  add(job: string, outcome: CounterOutcome, durationMs: number, atMs: number): void;
+  list(query: { readonly job: string; readonly sinceMs: number }): readonly JobCounter[];
+  totals(sinceMs: number): readonly CounterTotals[];
+  rollup(atMs: number): number;
+}
+
+/** The memory driver's counters. Same buckets, same tiers, same fold as `x_job_counters`. */
+export function createMemoryCounters(): MemoryCounters {
+  const buckets = new Map<string, JobCounter>();
+  const keyOf = (job: string, bucketMs: number, bucketStart: number): string =>
+    `${bucketMs}\u0000${bucketStart}\u0000${job}`;
+
+  const merge = (
+    job: string,
+    bucketMs: CounterBucketMs,
+    bucketStart: number,
+    add: Counts,
+  ): void => {
+    const key = keyOf(job, bucketMs, bucketStart);
+    const current = buckets.get(key);
+    buckets.set(key, { job, bucketMs, bucketStart, ...sum(current ?? ZERO, add) });
+  };
+
+  return {
+    add(job, outcome, durationMs, atMs) {
+      const start = counterBucketStart(atMs, COUNTER_BUCKET_MS);
+      merge(job, COUNTER_BUCKET_MS, start, { ...ZERO, [outcome]: 1, durationMs });
+    },
+    list({ job, sinceMs }) {
+      return [...buckets.values()]
+        .filter((bucket) => bucket.job === job && bucket.bucketStart >= sinceMs)
+        .sort((a, b) => a.bucketStart - b.bucketStart || a.bucketMs - b.bucketMs);
+    },
+    totals(sinceMs) {
+      const byJob = new Map<string, Counts>();
+      for (const bucket of buckets.values()) {
+        if (bucket.bucketStart < sinceMs) continue;
+        byJob.set(bucket.job, sum(byJob.get(bucket.job) ?? ZERO, bucket));
+      }
+      return [...byJob.entries()]
+        .sort(([a], [b]) => byName(a, b))
+        .map(([job, counts]) => ({ job, ...sum(ZERO, counts) }));
+    },
+    rollup(atMs) {
+      let moved = 0;
+      for (const [index, tier] of COUNTER_TIERS.entries()) {
+        const next = COUNTER_TIERS[index + 1];
+        for (const [key, bucket] of [...buckets]) {
+          if (bucket.bucketMs !== tier.bucketMs || bucket.bucketStart >= atMs - tier.keepMs)
+            continue;
+          buckets.delete(key);
+          moved += 1;
+          if (next === undefined) continue;
+          merge(
+            bucket.job,
+            next.bucketMs,
+            counterBucketStart(bucket.bucketStart, next.bucketMs),
+            bucket,
+          );
+        }
+      }
+      return moved;
+    },
+  };
+}

package/src/describe.ts

@@ -3,6 +3,7 @@
 // clock- or iteration-order-dependent in this shape is a spurious change on every build.
 
 import { toJsonSchema } from '@ultimat3/schema';
+import type { WhenBusy } from './concurrency';
 import type { BackoffStrategy, RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 
@@ -22,6 +23,21 @@ export interface JobDescriptor {
    * is app data, so a descriptor carrying it would put customer ids in `x.manifest.json`.
    */
   readonly idempotent: boolean;
+  /**
+   * The declared fleet-wide cap, or `null` for none. `keyed: true` says `limit` holds per
+   * `concurrency.key(input)` rather than for the whole job, and `whenBusy` is what a claim over
+   * it does — `null` for a plain number, which always waits. The KEY never crosses, for
+   * `idempotent`'s reason: it is computed from an input. `x jobs show <id>` reports one run's.
+   */
+  readonly concurrency: JobConcurrencyDescriptor | null;
+  /** Whether the job declares an `onSettled` hook — what is told how each run ended. */
+  readonly onSettled: boolean;
+}
+
+export interface JobConcurrencyDescriptor {
+  readonly limit: number;
+  readonly keyed: boolean;
+  readonly whenBusy: WhenBusy | null;
 }
 
 /**
@@ -40,6 +56,10 @@ export interface DescribableJob {
    * which is the exact wrong answer the `/_x` jobs panel used to give for every job.
    */
   readonly idempotencyKeyFor: unknown;
+  /** `JobHandle.concurrency` and `.whenBusy` — the cap as `job()` resolved it. */
+  readonly concurrency: number | undefined;
+  readonly whenBusy: WhenBusy | undefined;
+  readonly declaresOnSettled: boolean;
 }
 
 export function describeJob(handle: DescribableJob): JobDescriptor {
@@ -55,6 +75,15 @@ export function describeJob(handle: DescribableJob): JobDescriptor {
     // not statically knowable. `inspect(name)` reports the steps an actual run recorded.
     steps: [],
     idempotent: typeof handle.idempotencyKeyFor === 'function',
+    concurrency:
+      handle.concurrency === undefined
+        ? null
+        : {
+            limit: handle.concurrency,
+            keyed: handle.whenBusy !== undefined,
+            whenBusy: handle.whenBusy ?? null,
+          },
+    onSettled: handle.declaresOnSettled,
   };
 }
 

package/src/driver-memory-operator.ts

@@ -0,0 +1,297 @@
+// The memory driver's operator surface: paging, bulk, pause, promote, the worker registry,
+// progress and counters, over the SAME map the claim loop reads. Apart from `driver-memory.ts`
+// because that file is the six-method queue; this is everything `JobIntrospection` added, written
+// to answer exactly as the pg statements in `driver-pg-operator-sql.ts` do.
+
+import type { Clock } from '@ultimat3/core';
+import { assert, finiteOption } from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { MemoryCounters } from './counters';
+import { createMemoryCounters } from './counters';
+import type { JobRecord } from './driver';
+import { REQUEUEABLE_STATES } from './driver';
+import { signalEnqueued } from './enqueue-signal';
+import { JobNotRemovableError } from './errors-operator';
+import type {
+  BulkFilter,
+  BulkResult,
+  JobFilter,
+  JobIntrospection,
+  PausedName,
+  TaskFire,
+  WorkerRecord,
+} from './introspection';
+import {
+  MAX_BULK_ROWS,
+  MAX_TASK_FIRES,
+  MAX_WORKER_IN_FLIGHT,
+  PROMOTABLE_STATES,
+  pageLimit,
+  parseJobCursor,
+} from './introspection';
+import type { StepStore } from './steps';
+
+export interface MemoryQueueState {
+  readonly jobs: Map<string, JobRecord>;
+  readonly steps: StepStore;
+  readonly clock: Clock;
+  /** The LIVE row holding this record's `(name, tenant, key)`, other than itself. */
+  liveHolder(record: JobRecord): JobRecord | undefined;
+  /** Rewrite a row and release its claim — the driver's own `settle`. */
+  settle(id: string, patch: Partial<JobRecord>): void;
+}
+
+export type MemoryOperatorMembers = Omit<
+  JobIntrospection,
+  'job' | 'deadLetters' | 'requeue' | 'cancel'
+>;
+
+export interface MemoryOperator {
+  readonly members: MemoryOperatorMembers;
+  readonly counters: MemoryCounters;
+  /** The claim scan's question: is this queue held back? */
+  isQueuePaused(queue: string): boolean;
+}
+
+/** Newest first by `(createdAt, id)` — the keyset `SQL_JOB_LIST` orders and seeks by. */
+const newestFirst = (a: JobRecord, b: JobRecord): number =>
+  b.createdAt - a.createdAt || (a.id < b.id ? 1 : a.id > b.id ? -1 : 0);
+
+const inBulk = (record: JobRecord, filter: BulkFilter): boolean =>
+  record.state === filter.state &&
+  (filter.queue === undefined || record.queue === filter.queue) &&
+  (filter.name === undefined || record.name === filter.name) &&
+  (filter.tenantId === undefined || record.tenantId === filter.tenantId);
+
+type Keyset = Pick<JobRecord, 'createdAt' | 'id'>;
+
+/** `a` strictly newer than `b` by `(createdAt, id)`, in the keyset's own order. */
+const newerThan = (a: Keyset, b: Keyset): boolean =>
+  a.createdAt > b.createdAt || (a.createdAt === b.createdAt && a.id > b.id);
+
+const byName = (a: PausedName, b: PausedName): number =>
+  a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
+
+function pauseSet(clock: Clock): {
+  pause(name: string): void;
+  resume(name: string): void;
+  has(name: string): boolean;
+  list(): readonly PausedName[];
+} {
+  const paused = new Map<string, number>();
+  return {
+    // The FIRST pause is the one recorded, as `on conflict do nothing` keeps it in pg.
+    pause: (name) => {
+      if (!paused.has(name)) paused.set(name, nowMs(clock));
+    },
+    resume: (name) => {
+      paused.delete(name);
+    },
+    has: (name) => paused.has(name),
+    list: () => [...paused].map(([name, pausedAt]) => ({ name, pausedAt })).sort(byName),
+  };
+}
+
+export function createMemoryOperator(state: MemoryQueueState): MemoryOperator {
+  const { jobs, clock } = state;
+  const counters = createMemoryCounters();
+  const queues = pauseSet(clock);
+  const tasks = pauseSet(clock);
+  const workers = new Map<string, WorkerRecord & { readonly expiresAt: number }>();
+  const fires = new Map<string, TaskFire>();
+
+  // `eligible` is the verb's own predicate, applied BEFORE the bound as the pg statement's `where`
+  // is applied before its `limit`: rows the verb would skip never spend the bound.
+  const bulk = async (
+    filter: BulkFilter,
+    apply: (record: JobRecord) => Promise<boolean> | boolean,
+    eligible: (record: JobRecord) => boolean = () => true,
+  ): Promise<BulkResult> => {
+    let affected = 0;
+    const matching = [...jobs.values()].filter(
+      (record) => inBulk(record, filter) && eligible(record),
+    );
+    for (const record of matching.sort(newestFirst).reverse().slice(0, MAX_BULK_ROWS)) {
+      if (await apply(record)) affected += 1;
+    }
+    const remaining = [...jobs.values()].filter((record) => inBulk(record, filter)).length;
+    return { affected, remaining };
+  };
+
+  const members: MemoryOperatorMembers = {
+    // `async`, so a refused bound REJECTS here exactly as it does on the pg driver.
+    async list(filter: JobFilter = {}) {
+      const limit = pageLimit('the memory driver list', filter.limit);
+      assert(
+        filter.after === undefined || filter.before === undefined,
+        'list was asked for a page both after and before a cursor',
+        'pass one of after: jobCursor(lastRow) or before: jobCursor(firstRow)',
+      );
+      const after =
+        filter.after === undefined
+          ? undefined
+          : parseJobCursor(filter.after, 'the memory driver list');
+      const before =
+        filter.before === undefined
+          ? undefined
+          : parseJobCursor(filter.before, 'the memory driver list');
+      const { idPrefix, createdFrom, createdTo, tenantId } = filter;
+      const matching = [...jobs.values()]
+        .filter((record) => filter.queue === undefined || record.queue === filter.queue)
+        .filter((record) => filter.name === undefined || record.name === filter.name)
+        .filter((record) => filter.state === undefined || record.state === filter.state)
+        .filter((record) => tenantId === undefined || record.tenantId === tenantId)
+        .filter((record) => idPrefix === undefined || record.id.startsWith(idPrefix))
+        .filter((record) => createdFrom === undefined || record.createdAt >= createdFrom)
+        .filter((record) => createdTo === undefined || record.createdAt < createdTo)
+        .filter((record) => after === undefined || newerThan(after, record))
+        .filter((record) => before === undefined || newerThan(record, before))
+        .sort(newestFirst);
+      // Backwards: the rows NEAREST the cursor, which are the oldest of the newer ones.
+      return before === undefined
+        ? matching.slice(0, limit)
+        : matching.slice(Math.max(0, matching.length - limit));
+    },
+
+    async remove(jobId) {
+      const record = jobs.get(jobId);
+      if (record === undefined) return undefined;
+      if (record.state === 'running') throw new JobNotRemovableError({ jobId });
+      jobs.delete(jobId);
+      await state.steps.clear(record.runId);
+      return record;
+    },
+
+    async requeueMany(filter) {
+      assert(
+        REQUEUEABLE_STATES.has(filter.state),
+        `requeueMany was asked for ${filter.state} jobs, and only a finished job can be requeued`,
+        "pass state: 'dead', 'failed', 'cancelled' or 'done' to requeueMany",
+      );
+      const result = await bulk(filter, (record) => {
+        // A key a live job holds stays where it is — the single requeue's `X_JOB_DUPLICATE`,
+        // answered as "not this one" so one conflict does not fail a thousand rows.
+        if (state.liveHolder(record) !== undefined) return false;
+        state.settle(record.id, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
+        return true;
+      });
+      // The pg driver's `SQL_WAKE`, in-process: an operator's repair starts now.
+      if (result.affected > 0) signalEnqueued(filter.queue);
+      return result;
+    },
+
+    async promoteMany(filter) {
+      assert(
+        PROMOTABLE_STATES.has(filter.state),
+        `promoteMany was asked for ${filter.state} jobs, and only a job waiting on its run time can be promoted`,
+        "pass state: 'delayed' or 'ready' to promoteMany",
+      );
+      const at = nowMs(clock);
+      const result = await bulk(
+        filter,
+        (record) => {
+          jobs.set(record.id, { ...record, state: 'ready', runAt: at, updatedAt: at });
+          return true;
+        },
+        (record) => record.runAt > at,
+      );
+      if (result.affected > 0) signalEnqueued(filter.queue);
+      // Rows of the filter still waiting — a due `ready` row has nothing left to promote.
+      const waiting = [...jobs.values()].filter(
+        (record) => inBulk(record, filter) && record.runAt > at,
+      ).length;
+      return { affected: result.affected, remaining: waiting };
+    },
+
+    async removeMany(filter) {
+      if (filter.state === 'running') throw new JobNotRemovableError({});
+      return bulk(filter, async (record) => {
+        jobs.delete(record.id);
+        await state.steps.clear(record.runId);
+        return true;
+      });
+    },
+
+    promote(jobId) {
+      const record = jobs.get(jobId);
+      const at = nowMs(clock);
+      const waiting =
+        record !== undefined &&
+        (record.state === 'ready' || record.state === 'delayed') &&
+        record.runAt > at;
+      if (record === undefined || !waiting) return Promise.resolve(undefined);
+      const next: JobRecord = { ...record, state: 'ready', runAt: at, updatedAt: at };
+      jobs.set(jobId, next);
+      signalEnqueued(next.queue);
+      return Promise.resolve(next);
+    },
+
+    pauseQueue: (queue) => Promise.resolve(queues.pause(queue)),
+    resumeQueue(queue) {
+      queues.resume(queue);
+      signalEnqueued(queue);
+      return Promise.resolve();
+    },
+    pausedQueues: () => Promise.resolve(queues.list()),
+    pauseTask: (task) => Promise.resolve(tasks.pause(task)),
+    resumeTask: (task) => Promise.resolve(tasks.resume(task)),
+    pausedTasks: () => Promise.resolve(tasks.list()),
+
+    recordTaskFire({ task, occurrenceMs }) {
+      fires.set(task, { task, occurrenceMs, firedAt: nowMs(clock) });
+      return Promise.resolve();
+    },
+    taskFires: () =>
+      Promise.resolve(
+        [...fires.values()]
+          // Code units, as `collate "C"` orders them in `SQL_TASK_FIRES`.
+          .sort((a, b) => (a.task < b.task ? -1 : a.task > b.task ? 1 : 0))
+          .slice(0, MAX_TASK_FIRES),
+      ),
+
+    announceWorker(worker, ttlMs) {
+      const at = nowMs(clock);
+      workers.set(worker.id, {
+        ...worker,
+        queues: [...worker.queues],
+        inFlight: worker.inFlight.slice(0, MAX_WORKER_IN_FLIGHT),
+        heartbeatAt: at,
+        expiresAt: at + finiteOption('the memory worker registry', 'ttlMs', ttlMs),
+      });
+      return Promise.resolve();
+    },
+    forgetWorker(workerId) {
+      workers.delete(workerId);
+      return Promise.resolve();
+    },
+    workers() {
+      const at = nowMs(clock);
+      // Expired by reading, never by cleanup: a killed worker announces nothing and is gone.
+      for (const [id, worker] of workers) if (worker.expiresAt <= at) workers.delete(id);
+      return Promise.resolve(
+        [...workers.values()]
+          .map(({ expiresAt: _expiresAt, ...worker }) => worker)
+          .sort((a, b) => a.startedAt - b.startedAt || (a.id < b.id ? -1 : 1)),
+      );
+    },
+
+    recordProgress(jobId, by, progress) {
+      const record = jobs.get(jobId);
+      // The settle's fence: progress from a claim the row no longer carries is dropped.
+      if (
+        record?.state === 'running' &&
+        record.claimedBy === by.workerId &&
+        record.claim === by.claim
+      ) {
+        jobs.set(jobId, { ...record, progress });
+      }
+      return Promise.resolve();
+    },
+
+    counters: (query) => Promise.resolve(counters.list(query)),
+    counterTotals: (sinceMs) => Promise.resolve(counters.totals(sinceMs)),
+    rollupCounters: () => Promise.resolve(counters.rollup(nowMs(clock))),
+  };
+
+  return { members, counters, isQueuePaused: (queue) => queues.has(queue) };
+}

package/src/driver-memory.ts

@@ -7,22 +7,33 @@ import { assert, finiteCount, systemClock, uuid } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
 import { createMemoryBackfillLedger } from './backfill-ledger';
 import { nowMs } from './clock';
+import { nackOutcome } from './counters';
 import type {
+  AckOptions,
   ClaimedJob,
+  ClaimIdentity,
   ClaimOptions,
   EnqueueRequest,
   EnqueueResult,
   HeartbeatOptions,
   JobDriver,
-  JobFilter,
-  JobIntrospection,
   JobRecord,
   NackOptions,
   QueueStats,
 } from './driver';
-import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE, REQUEUEABLE_STATES } from './driver';
+import {
+  assertClaimBounds,
+  assertClaimQueues,
+  DEFAULT_QUEUE,
+  nackState,
+  REQUEUEABLE_STATES,
+} from './driver';
+import { createMemoryOperator } from './driver-memory-operator';
+import { signalEnqueued } from './enqueue-signal';
 import { JobDuplicateError } from './errors';
 import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
+import type { JobIntrospection } from './introspection';
+import { MAX_ERROR_STACK_LENGTH } from './introspection';
 import type { LeaseStore } from './leases';
 import { createMemoryLeaseStore } from './leases';
 import type { StepStore } from './steps';
@@ -100,25 +111,32 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
     jobs.set(id, { ...released, ...patch, updatedAt: nowMs(clock) });
   };
 
+  const operator = createMemoryOperator({
+    jobs,
+    steps,
+    clock,
+    settle,
+    liveHolder: (record) => {
+      const holder = liveByKey(record.name, record.idempotencyKey, record.tenantId);
+      return holder === undefined || holder.id === record.id ? undefined : holder;
+    },
+  });
+
+  /** The row, when this worker still holds it `running` — the fence both settles share. */
+  const heldBy = (jobId: string, by: ClaimIdentity): JobRecord | undefined => {
+    const record = jobs.get(jobId);
+    return record?.state === 'running' &&
+      record.claimedBy === by.workerId &&
+      record.claim === by.claim
+      ? record
+      : undefined;
+  };
+
   const introspect: JobIntrospection = {
     job(jobId) {
       return Promise.resolve(jobs.get(jobId));
     },
-    // `async` for the reason `claim` is: a refused bound must REJECT here exactly as it does on the
-    // pg driver, and a synchronous throw out of a method typed `Promise<…>` is a second answer to
-    // one question.
-    async list(filter: JobFilter = {}) {
-      const rows = [...jobs.values()]
-        .filter((record) => filter.queue === undefined || record.queue === filter.queue)
-        .filter((record) => filter.name === undefined || record.name === filter.name)
-        .filter((record) => filter.state === undefined || record.state === filter.state)
-        // NEWEST first, as `createPgDriver`'s `order by created_at desc` is. Ascending here meant
-        // `x jobs ls` answered one thing against `x dev` and the opposite in production — and,
-        // because the limit is applied after the sort, a default page of the hundred OLDEST rows.
-        .sort((a, b) => b.createdAt - a.createdAt)
-        .slice(0, finiteCount('the memory driver list', 'limit', filter.limit ?? 100));
-      return rows;
-    },
+    ...operator.members,
     async deadLetters(limit = 100) {
       const rows = [...jobs.values()]
         .filter((record) => record.state === 'dead')
@@ -158,6 +176,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       }
       // `settle` releases the claim — `SQL_JOB_REQUEUE` writes `claimed_by = null` too.
       settle(jobId, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
+      signalEnqueued(record.queue);
       const next = jobs.get(jobId);
       return next ?? record;
     },
@@ -190,6 +209,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
     // `Promise<…>` is a second answer to one question — caught by different code, and an
     // unhandled exception rather than a settled promise wherever the caller holds the promise.
     async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
+      // A caller-allocated id that already names a row is that row's publish, repeated: the same
+      // job, whatever state it has reached — `SQL_ENQUEUE`'s `not exists` is the pg half.
+      const published = request.id === undefined ? undefined : jobs.get(request.id);
+      if (published !== undefined) {
+        return { id: published.id, runId: published.runId, deduped: true };
+      }
       const existing = liveByKey(request.name, request.idempotencyKey, request.tenantId);
       if (existing !== undefined) {
         if (request.onConflict === 'error') {
@@ -205,10 +230,13 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       const at = nowMs(clock);
       const runAt = request.runAt ?? at;
       const record: JobRecord = {
-        id: uuid(),
+        id: request.id ?? uuid(),
         name: request.name,
         queue: request.queue || DEFAULT_QUEUE,
-        input: request.input,
+        // Through JSON, exactly as the pg driver binds it (`JSON.stringify(request.input ?? null)`):
+        // a live reference kept a `Date`, an `undefined` member and a non-enumerable property the
+        // queue in production never stores, so a job that read one passed here and failed there.
+        input: JSON.parse(JSON.stringify(request.input ?? null)) as unknown,
         idempotencyKey: request.idempotencyKey,
         runId: request.runId ?? uuid(),
         attempt: 0,
@@ -235,6 +263,8 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       const wanted = new Set(claimOptions.queues);
       const claimable = [...jobs.values()]
         .filter((record) => wanted.has(record.queue))
+        // A paused queue is never claimed — `SQL_CLAIM`'s `not exists` over `x_job_pauses`.
+        .filter((record) => !operator.isQueuePaused(record.queue))
         .filter((record) => {
           if (record.runAt > at) return false;
           if (record.state === 'ready' || record.state === 'delayed') return true;
@@ -252,6 +282,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
           state: 'running',
           attempt: record.attempt + 1,
           claimedBy: claimOptions.workerId,
+          claim: (record.claim ?? 0) + 1,
           claimedAt: at,
           visibleAt: at + claimOptions.visibilityTimeoutMs,
           updatedAt: at,
@@ -262,29 +293,29 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       return Promise.resolve(out);
     },
 
-    // Both settlements are FENCED on `running`, as `SQL_ACK`/`SQL_NACK` are: an ack from a worker
-    // whose job was cancelled — or whose lease lapsed and whose job another worker re-claimed —
-    // would otherwise overwrite a row it no longer owns.
-    ack(jobId: string): Promise<void> {
-      if (jobs.get(jobId)?.state !== 'running') return Promise.resolve();
+    // Both settlements are FENCED on `running` AND on the claimer, as `SQL_ACK`/`SQL_NACK` are: an
+    // ack from a worker whose job was cancelled — or whose lease lapsed and whose job another
+    // worker re-claimed — would otherwise overwrite a row it no longer owns. The counter moves in
+    // the same step as the row, so the two cannot disagree.
+    ack(jobId: string, by: AckOptions): Promise<boolean> {
+      const record = heldBy(jobId, by);
+      if (record === undefined) return Promise.resolve(false);
       settle(jobId, { state: 'done' });
-      return Promise.resolve();
+      if (by.counted !== false) {
+        operator.counters.add(record.name, 'done', by.durationMs ?? 0, nowMs(clock));
+      }
+      return Promise.resolve(true);
     },
 
-    nack(jobId: string, nackOptions: NackOptions): Promise<void> {
-      const record = jobs.get(jobId);
-      if (record === undefined || record.state !== 'running') return Promise.resolve();
+    nack(jobId: string, nackOptions: NackOptions): Promise<boolean> {
+      const record = heldBy(jobId, nackOptions);
+      if (record === undefined) return Promise.resolve(false);
       const at = nowMs(clock);
       const counts = nackOptions.countsAsAttempt !== false;
       const patch: Partial<JobRecord> = {
         // `park`, never `counts`: parking is what leaves the ready bucket, and burning an attempt
         // is a separate fact. A shed sets neither and stays `ready`, which is what it is.
-        state:
-          nackOptions.deadLetter === true
-            ? 'dead'
-            : nackOptions.park === true
-              ? 'suspended'
-              : 'ready',
+        state: nackState(nackOptions),
         runAt: at + nackOptions.delayMs,
         // A suspension must not burn an attempt, or a 3-day sleep dead-letters the run. Floored
         // where `SQL_NACK` floors it (`greatest(attempt - 1, 0)`): the fence above is what keeps
@@ -292,9 +323,16 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         // being read as the only one.
         attempt: counts ? record.attempt : Math.max(0, record.attempt - 1),
         ...(nackOptions.error === undefined ? {} : { lastError: nackOptions.error }),
+        ...(nackOptions.stack === undefined
+          ? {}
+          : { lastErrorStack: nackOptions.stack.slice(0, MAX_ERROR_STACK_LENGTH) }),
       };
       settle(jobId, patch);
-      return Promise.resolve();
+      const outcome = nackOutcome(nackOptions);
+      if (outcome !== undefined) {
+        operator.counters.add(record.name, outcome, nackOptions.durationMs ?? 0, at);
+      }
+      return Promise.resolve(true);
     },
 
     heartbeat(jobId: string, heartbeatOptions: HeartbeatOptions): Promise<boolean> {
@@ -304,7 +342,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       if (
         record === undefined ||
         record.state !== 'running' ||
-        (heartbeatOptions.workerId !== undefined && record.claimedBy !== heartbeatOptions.workerId)
+        (heartbeatOptions.workerId !== undefined &&
+          record.claimedBy !== heartbeatOptions.workerId) ||
+        (heartbeatOptions.claim !== undefined && record.claim !== heartbeatOptions.claim)
       ) {
         return Promise.resolve(false);
       }
@@ -322,6 +362,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
           delayed: 0,
           running: 0,
           suspended: 0,
+          failed: 0,
           dead: 0,
           oldestReadyMs: 0,
         };
@@ -337,6 +378,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         } else if (waiting) next.delayed += 1;
         else if (record.state === 'running') next.running += 1;
         else if (record.state === 'suspended') next.suspended += 1;
+        else if (record.state === 'failed') next.failed += 1;
         else if (record.state === 'dead') next.dead += 1;
         byQueue.set(record.queue, next);
       }