@ultimat3/jobs 23.0.0 → 24.0.0

package/src/driver-memory.ts

@@ -3,7 +3,7 @@
 // real claim/ack/nack paths rather than a mock that always succeeds.
 
 import type { Clock } from '@ultimat3/core';
-import { assert, finiteCount, systemClock, uuid } from '@ultimat3/core';
+import { finiteCount, systemClock, uuid } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
 import { createMemoryBackfillLedger } from './backfill-ledger';
 import { nowMs } from './clock';
@@ -25,17 +25,20 @@ import {
   assertClaimBounds,
   assertClaimQueues,
   DEFAULT_QUEUE,
+  LEASE_LAPSED_FINAL_ATTEMPT,
+  LIVE_STATES,
   nackState,
   REQUEUEABLE_STATES,
 } from './driver';
 import { createMemoryOperator } from './driver-memory-operator';
 import { signalEnqueued } from './enqueue-signal';
-import { JobDuplicateError } from './errors';
-import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
+import { JobDuplicateError, LeaseLostError } from './errors';
+import { JobNotFoundError, JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
 import type { JobIntrospection } from './introspection';
 import { MAX_ERROR_STACK_LENGTH } from './introspection';
 import type { LeaseStore } from './leases';
 import { createMemoryLeaseStore } from './leases';
+import { isFinalAttempt } from './retry';
 import type { StepStore } from './steps';
 import { createMemoryStepStore } from './steps-memory';
 
@@ -48,8 +51,6 @@ export interface MemoryDriverOptions {
   readonly leases?: LeaseStore;
 }
 
-const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
-
 /**
  * The in-memory driver's own type: `JobDriver` with `close` REQUIRED.
  *
@@ -63,7 +64,21 @@ export type MemoryJobDriver = JobDriver & { close(): Promise<void> };
 
 export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJobDriver {
   const clock = options.clock ?? systemClock;
-  const steps = options.steps ?? createMemoryStepStore();
+  const stepStore = options.steps ?? createMemoryStepStore();
+  // `SQL_STEP_PUT`'s fence: a write made under a claim lands only while that claim holds the row.
+  const steps: StepStore = {
+    ...stepStore,
+    put(record, by) {
+      const row = by === undefined ? undefined : jobs.get(by.jobId);
+      if (
+        by !== undefined &&
+        !(row?.state === 'running' && row.claimedBy === by.workerId && row.claim === by.claim)
+      ) {
+        return Promise.reject(new LeaseLostError({ job: by.job, jobId: by.jobId }));
+      }
+      return stepStore.put(record);
+    },
+  };
   const backfills = options.backfills ?? createMemoryBackfillLedger(clock);
   const leases =
     options.leases ?? createMemoryLeaseStore(options.clock === undefined ? {} : { clock });
@@ -104,11 +119,19 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
    * lease deadline — which `x jobs show` prints, and which is the very pair the claim scan's
    * lease-expiry branch reads to decide a row was abandoned.
    */
-  const settle = (id: string, patch: Partial<JobRecord>): void => {
+  const settle = (id: string, patch: Partial<JobRecord>, drop?: 'stack'): void => {
     const existing = jobs.get(id);
     if (existing === undefined) return;
     const { visibleAt: _visibleAt, claimedBy: _claimedBy, ...released } = existing;
-    jobs.set(id, { ...released, ...patch, updatedAt: nowMs(clock) });
+    // `drop: 'stack'` is `SQL_NACK`'s `case when $5 is null then last_error_stack else $7 end`
+    // with no stack bound: a NEW failure replaces the old one's stack even with nothing — the key
+    // is removed, never set to `undefined`, as a null column is absent from a pg record.
+    const { lastErrorStack: _stack, ...unstacked } = released;
+    jobs.set(id, {
+      ...(drop === 'stack' ? unstacked : released),
+      ...patch,
+      updatedAt: nowMs(clock),
+    });
   };
 
   const operator = createMemoryOperator({
@@ -145,13 +168,9 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       return rows;
     },
     async requeue(jobId, requeueOptions) {
-      const existing = jobs.get(jobId);
-      assert(
-        existing !== undefined,
-        `no job ${jobId} in the memory driver`,
-        'requeue a job id returned by enqueue() or stats() — the memory driver holds no state across processes, so an id from another run will not resolve',
-      );
-      const record = existing;
+      const record = jobs.get(jobId);
+      // The one answer both drivers give an id nobody queued — it was `X_INVARIANT` here.
+      if (record === undefined) throw new JobNotFoundError({ jobId, driver: 'memory' });
       // The two refusals `SQL_JOB_REQUEUE` and `SQL_JOB_LIVE_HOLDER` give, before anything moves.
       if (!REQUEUEABLE_STATES.has(record.state)) {
         throw new JobNotRequeueableError({ jobId, state: record.state });
@@ -161,7 +180,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         throw requeueKeyTaken({ ...record, holderId: holder.id });
       }
       if (requeueOptions?.fromStep !== undefined) {
-        // The target step and every step that started AFTER it — `SQL_STEPS_FROM`, whose header
+        // The target step and every step that started AFTER it — `SQL_JOB_REQUEUE`, whose header
         // says why a tie is kept. Earlier steps stay memoized; a later one replaying the old run's
         // result is what this prevents.
         const all = await steps.list(record.runId);
@@ -182,9 +201,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
     },
     cancel(jobId, reason) {
       const existing = jobs.get(jobId);
-      // `state !== 'done'`, mirroring `SQL_CANCEL`: a job that already finished has nothing to
-      // stop, and cancelling it would rewrite a terminal row an operator is reading as success.
-      if (existing === undefined || existing.state === 'done') return Promise.resolve(undefined);
+      // LIVE rows only, mirroring `SQL_CANCEL`: a job that already finished has nothing to stop.
+      // The fence was `state !== 'done'`, which let a cancel REWRITE a dead letter — the record of
+      // a failure, and its `lastError`, became the record of an operator's typo.
+      if (existing === undefined || !LIVE_STATES.has(existing.state)) {
+        return Promise.resolve(undefined);
+      }
       // `settle`, not `update`: a cancellation RELEASES the claim, and `SQL_CANCEL` writes
       // `visible_at = null, claimed_by = null` with the state. Left stamped, a cancelled row named
       // the worker still holding it and carried that attempt's lease deadline — the pair
@@ -276,7 +298,24 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         .slice(0, claimOptions.limit);
 
       const out: ClaimedJob[] = [];
+      const buried: JobRecord[] = [];
       for (const record of claimable) {
+        // `SQL_CLAIM`'s second arm. A `running` row reaches this loop only with a lapsed lease, and
+        // one that lapsed on its FINAL attempt has nothing left to be re-delivered for: handing
+        // it out was an attempt past `maxAttempts`, every lease, forever — a job that kills its
+        // worker took one per visibility timeout and never reached the dead-letter queue.
+        if (
+          record.state === 'running' &&
+          isFinalAttempt({ attempts: record.maxAttempts }, record.attempt)
+        ) {
+          // `$5` of the statement: a job declaring `retry.deadLetter: false` is dropped.
+          const state = claimOptions.dropExhausted?.includes(record.name) ? 'failed' : 'dead';
+          settle(record.id, { state, lastError: LEASE_LAPSED_FINAL_ATTEMPT }, 'stack');
+          operator.counters.add(record.name, state, 0, at);
+          const dead = jobs.get(record.id);
+          if (dead !== undefined) buried.push(dead);
+          continue;
+        }
         const claimed: ClaimedJob = {
           ...record,
           state: 'running',
@@ -290,7 +329,8 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         jobs.set(record.id, claimed);
         out.push(claimed);
       }
-      return Promise.resolve(out);
+      if (buried.length > 0) claimOptions.onExhausted?.(buried);
+      return out;
     },
 
     // Both settlements are FENCED on `running` AND on the claimer, as `SQL_ACK`/`SQL_NACK` are: an
@@ -323,11 +363,12 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         // being read as the only one.
         attempt: counts ? record.attempt : Math.max(0, record.attempt - 1),
         ...(nackOptions.error === undefined ? {} : { lastError: nackOptions.error }),
-        ...(nackOptions.stack === undefined
+        // The stack belongs to the ERROR beside it, as `SQL_NACK` binds it: no error, no stack.
+        ...(nackOptions.error === undefined || nackOptions.stack === undefined
           ? {}
           : { lastErrorStack: nackOptions.stack.slice(0, MAX_ERROR_STACK_LENGTH) }),
       };
-      settle(jobId, patch);
+      settle(jobId, patch, nackOptions.error === undefined ? undefined : 'stack');
       const outcome = nackOutcome(nackOptions);
       if (outcome !== undefined) {
         operator.counters.add(record.name, outcome, nackOptions.durationMs ?? 0, at);
@@ -382,7 +423,11 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         else if (record.state === 'dead') next.dead += 1;
         byQueue.set(record.queue, next);
       }
-      return Promise.resolve([...byQueue.values()].sort((a, b) => a.queue.localeCompare(b.queue)));
+      // Code units, as `collate "C"` orders them in `SQL_STATS` — never `localeCompare`, which
+      // answers by the runtime's locale and so differently from the database on the same names.
+      return Promise.resolve(
+        [...byQueue.values()].sort((a, b) => (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0)),
+      );
     },
 
     close(): Promise<void> {

package/src/driver-nats.ts

@@ -3,6 +3,7 @@
 // a durable pull consumer per worker pool (`fetch` == claim, `ack`/`nak` map 1:1),
 // `ack_wait` as the visibility timeout, and a KV bucket for step records.
 
+import { NotImplementedError } from '@ultimat3/core';
 import type {
   ClaimedJob,
   ClaimOptions,
@@ -13,7 +14,6 @@ import type {
   NackOptions,
   QueueStats,
 } from './driver';
-import { JobsNotImplementedError } from './errors';
 import type { StepRecord, StepStore } from './steps';
 
 // Names the seam that actually replaces the stub, and NOTHING ELSE — the two other repairs this
@@ -34,7 +34,10 @@ const FIX =
   'call setJobDriver(createPgDriver()) at boot instead of this driver; nothing needs moving first, because enqueue here refuses too, so no job was ever written to it';
 
 const unavailable = (method: string): never => {
-  throw new JobsNotImplementedError({ feature: `nats jobs driver (${method})`, fix: FIX });
+  throw new NotImplementedError({
+    cause: `nats jobs driver (${method}) is not implemented`,
+    fix: FIX,
+  });
 };
 
 const natsStepStore = (): StepStore => ({

package/src/driver-pg-jobs-sql.ts

@@ -104,14 +104,35 @@ select ${JOB_ROW_COLUMNS}
  * (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
  * or raced to live in between. `woke` announces the row in the statement that made it claimable
  * (`driver-pg-wake-sql.ts`): `x jobs retry` is a job starting now, not at an idle worker's poll.
+ *
+ * $2 is `fromStep`, or null — and the steps are dropped HERE, off the row this statement moved.
+ * They were a statement of their own, sent first: a requeue that then matched nothing (the row
+ * raced to live) had already deleted a running job's steps, and a crash between the two left a
+ * dead letter with its steps gone and no retry queued.
+ *
+ * "From that step onward": the step `fromStep` names and every step that started AFTER it.
+ * Deleting the named step alone left later steps replaying results computed from the old run.
+ * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
+ * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
  */
 export const SQL_JOB_REQUEUE = `
-update x_jobs
-   set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
-       claimed_by = null, visible_at = null
- where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
-returning ${JOB_ROW_COLUMNS},
-          pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
+with requeued as (
+  update x_jobs
+     set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
+         claimed_by = null, visible_at = null
+   where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
+  returning ${JOB_ROW_COLUMNS},
+            pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
+), dropped as (
+  delete from x_job_steps s
+   using requeued r
+   where $2::text is not null
+     and s.run_id = r.run_id
+     and (s.name = $2::text
+          or s.started_at > (select t.started_at from x_job_steps t
+                              where t.run_id = r.run_id and t.name = $2::text))
+)
+select ${JOB_ROW_NAMES}, woke from requeued
 `.trim();
 
 /**
@@ -124,16 +145,3 @@ select id from x_jobs
    and id <> $4 and state in ('ready', 'delayed', 'running', 'suspended')
  limit 1
 `.trim();
-
-/**
- * "From that step onward": the step `fromStep` names and every step that started AFTER it.
- * Deleting the named step alone left later steps replaying results computed from the old run.
- * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
- * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
- */
-export const SQL_STEPS_FROM = `
-delete from x_job_steps
- where run_id = $1
-   and (name = $2
-        or started_at > (select started_at from x_job_steps where run_id = $1 and name = $2))
-`.trim();

package/src/driver-pg-operator-sql.ts

@@ -15,15 +15,18 @@ const bulkMatch = (alias = ''): string =>
   `${alias}state = $1 and ($2::text is null or ${alias}queue = $2) and ($3::text is null or ${alias}name = $3) and ($5::text is null or ${alias}tenant_id = $5)`;
 
 /**
- * Re-queue up to $4 finished rows. `distinct on` the idempotency namespace, because two dead rows
- * of one key would both go `ready` and the second would break the live index — and `not exists`
- * a live holder, the single requeue's `X_JOB_DUPLICATE`, answered here as "not this row".
- * `matching` is counted in the same statement, before the update, so the caller's `remaining` is
- * `matching - affected` with no second read.
+ * Re-queue up to $4 finished rows. `eligible` is every row of the filter no LIVE job holds the key
+ * of — the single requeue's `X_JOB_DUPLICATE`, answered here as "not this row" — and `distinct on`
+ * the idempotency namespace picks one of them per key, because two dead rows of one key would both
+ * go `ready` and the second would break the live index.
+ *
+ * `remaining` is what a second call could still move, counted in the same statement off the same
+ * snapshot: the eligible rows whose key this call did not just take. A held row was counted too,
+ * so a caller told to "call again until it is zero" never stopped.
  */
 export const SQL_JOB_REQUEUE_MANY = `
-with candidates as (
-  select distinct on (j.name, coalesce(j.tenant_id, ''), j.idempotency_key) j.id
+with eligible as (
+  select j.id, j.name, coalesce(j.tenant_id, '') as tenant, j.idempotency_key, j.created_at
     from x_jobs j
    where ${bulkMatch('j.')}
      and not exists (
@@ -33,7 +36,10 @@ with candidates as (
           and l.idempotency_key = j.idempotency_key
           and l.state in ${LIVE}
      )
-   order by j.name, coalesce(j.tenant_id, ''), j.idempotency_key, j.created_at
+), candidates as (
+  select distinct on (name, tenant, idempotency_key) id, name, tenant, idempotency_key
+    from eligible
+   order by name, tenant, idempotency_key, created_at
    limit $4
 ), moved as (
   update x_jobs
@@ -43,7 +49,13 @@ with candidates as (
   returning id
 )
 select (select count(*) from moved)::int as affected,
-       (select count(*) from x_jobs where ${bulkMatch()})::int as matching
+       (select count(*) from eligible e
+         where not exists (
+           select 1 from candidates c
+             join moved m on m.id = c.id
+            where c.name = e.name and c.tenant = e.tenant
+              and c.idempotency_key = e.idempotency_key
+         ))::int as remaining
 `.trim();
 
 /** Delete up to $4 rows and their step records. `running` is refused before this is sent. */
@@ -267,17 +279,25 @@ select task_name,
  * `woken` announces each queue that received a job, once, when the statement commits — read in
  * the final select because a plain-select CTE nothing reads is never run.
  *
- * $1 task, $2 occurrence (ms), $3 the jobs as a JSON array.
+ * The fence is the OCCURRENCE and the watermark may land past it: `run-once` runs the earliest
+ * missed occurrence and drops the rest, and dropping is the watermark moving beyond them. That was
+ * a second statement behind this one, and a crash between the two fired a second "one catch-up".
+ * `greatest` ignores a null, so with no `$4` the watermark lands on the occurrence — and a `$4`
+ * behind the occurrence can never leave the watermark short of what just fired.
+ *
+ * $1 task, $2 occurrence (ms), $3 the jobs as a JSON array, $4 where the watermark lands (ms) or
+ * null for the occurrence itself.
  */
 export const SQL_SCHEDULER_FIRE = `
 with moved as (
   insert into x_scheduler_state
     (task_name, last_fired_at, updated_at, fired_occurrence_at, fired_at)
-  values ($1, to_timestamp($2::bigint / 1000.0), now(), to_timestamp($2::bigint / 1000.0), now())
+  values ($1, to_timestamp(greatest($2::bigint, $4::bigint) / 1000.0), now(),
+          to_timestamp($2::bigint / 1000.0), now())
   on conflict (task_name) do update
      set last_fired_at = excluded.last_fired_at, updated_at = now(),
          fired_occurrence_at = excluded.fired_occurrence_at, fired_at = excluded.fired_at
-   where x_scheduler_state.last_fired_at < excluded.last_fired_at
+   where x_scheduler_state.last_fired_at < excluded.fired_occurrence_at
   returning task_name
 ), queued as (
   insert into x_jobs

package/src/driver-pg-operator.ts

@@ -3,9 +3,9 @@
 // that file is the claim/settle driver and sits near its size ceiling; this is the read-and-repair
 // half a dashboard and `x jobs` call.
 
+import type { PgExecutor } from '@ultimat3/core';
 import { assert, finiteOption } from '@ultimat3/core';
 import { REQUEUEABLE_STATES } from './driver';
-import type { PgExecutor } from './driver-pg';
 import { SQL_JOB_LIST, SQL_JOB_LIST_BEFORE } from './driver-pg-jobs-sql';
 import {
   SQL_COUNTER_DROP,
@@ -126,7 +126,14 @@ export function pgOperator(
       );
       const limit = pageLimit('the pg driver list', filter.limit);
       const raw = filter.before ?? filter.after;
-      const cursor = raw === undefined ? undefined : parseJobCursor(raw, 'the pg driver list');
+      const cursor =
+        raw === undefined
+          ? undefined
+          : parseJobCursor(
+              raw,
+              'the pg driver list',
+              filter.before === undefined ? 'after' : 'before',
+            );
       const rows = await exec().query<JobRow>(
         filter.before === undefined ? SQL_JOB_LIST : SQL_JOB_LIST_BEFORE,
         [
@@ -162,7 +169,16 @@ export function pgOperator(
         `requeueMany was asked for ${filter.state} jobs, and only a finished job can be requeued`,
         "pass state: 'dead', 'failed', 'cancelled' or 'done' to requeueMany",
       );
-      const result = await bulk(SQL_JOB_REQUEUE_MANY, filter);
+      // Its own read, not `bulk`: the statement answers `remaining` itself — the rows a second
+      // call could move — where `matching - affected` counted the held rows it never will.
+      const rows = await exec().query<{ affected: Count; remaining: Count }>(SQL_JOB_REQUEUE_MANY, [
+        filter.state,
+        filter.queue ?? null,
+        filter.name ?? null,
+        MAX_BULK_ROWS,
+        filter.tenantId ?? null,
+      ]);
+      const result = { affected: num(rows[0]?.affected), remaining: num(rows[0]?.remaining) };
       if (result.affected > 0) await wake(filter.queue);
       return result;
     },

package/src/driver-pg-rows.ts

@@ -94,7 +94,7 @@ export function toJobRecord(row: JobRow): JobRecord {
     attempt: row.attempt,
     maxAttempts: row.max_attempts,
     state: statusIn<JobState>(JOB_STATES, isJobState, {
-      table: 'ultimate_jobs',
+      table: 'x_jobs',
       column: 'state',
       value: row.state,
     }),
@@ -146,7 +146,7 @@ export function toStepRecord(row: StepRow): StepRecord {
     runId: row.run_id,
     name: row.name,
     status: statusIn<StepStatus>(STEP_STATUSES, isStepStatus, {
-      table: 'ultimate_job_steps',
+      table: 'x_job_steps',
       column: 'status',
       value: row.status,
     }),
@@ -168,7 +168,7 @@ export function toBackfillRun(row: BackfillRow): BackfillRun {
     name: row.name,
     checksum: row.checksum,
     status: statusIn(BACKFILL_STATUSES, isBackfillStatus, {
-      table: 'ultimate_backfills',
+      table: 'x_backfills',
       column: 'status',
       value: row.status,
     }),

package/src/driver-pg-sql.ts

@@ -19,7 +19,6 @@ export {
   SQL_JOB_LIST,
   SQL_JOB_LIVE_HOLDER,
   SQL_JOB_REQUEUE,
-  SQL_STEPS_FROM,
 } from './driver-pg-jobs-sql';
 
 export {
@@ -32,8 +31,23 @@ export {
 } from './driver-pg-outbox-sql';
 export { SQL_ACK, SQL_NACK } from './driver-pg-settle-sql';
 
+import { LEASE_LAPSED_FINAL_ATTEMPT } from './driver';
 import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
 import { notifyJobReady } from './driver-pg-wake-sql';
+import { COUNTER_BUCKET_MS } from './introspection';
+
+/** What both arms of the claim return: a `JobRow`, read off the updated row `j`. */
+const CLAIM_ROW_COLUMNS = `j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
+          j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
+          j.traceparent, j.enqueued_by,
+          (extract(epoch from j.run_at)     * 1000)::bigint as run_at,
+          (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
+          (extract(epoch from j.created_at) * 1000)::bigint as created_at,
+          (extract(epoch from j.updated_at) * 1000)::bigint as updated_at`;
+
+const CLAIM_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
+       state, tenant_id, last_error, claimed_by, claims, traceparent, enqueued_by, run_at,
+       visible_at, created_at, updated_at`;
 
 /**
  * The insert, and its wake: `woke` is the notification's own column, evaluated once per inserted
@@ -85,10 +99,19 @@ select id, run_id from x_jobs
  * lock and throughput collapses to one worker. `visible_at` in the predicate reclaims leases
  * abandoned by a crashed worker. `cancelled` is absent from every branch by construction — a
  * cancelled row is terminal, so it is never handed to a worker again.
+ *
+ * TWO arms over one pick. A `running` row is picked only with a lapsed lease, and one that lapsed
+ * on its FINAL attempt (`attempt >= max_attempts` — `isFinalAttempt`, `retry.ts`) is `buried`:
+ * settled `dead` here, counted, and answered with its new state so the driver reports it instead
+ * of handing it out. Re-claimed, it was an attempt past `max_attempts` every lease, forever — a
+ * job that kills its worker took one per visibility timeout and never reached the dead letters.
+ * `claimed_by` is cleared with the lease, so the body that stalled rather than died cannot settle
+ * the row it lost. A buried row spends a slot of `$2`: the pass after it claims what it displaced.
+ * `$5` names the jobs declaring `retry.deadLetter: false`: their rows are buried `failed`.
  */
 export const SQL_CLAIM = `
-with claimed as (
-  select id
+with picked as (
+  select id, (state = 'running' and attempt >= max_attempts) as exhausted
     from x_jobs
    where queue = any($1::text[])
      and run_at <= now()
@@ -102,35 +125,56 @@ with claimed as (
    order by run_at
    limit $2
      for update skip locked
+), buried as (
+  update x_jobs j
+     set state            = case when j.name = any($5::text[]) then 'failed' else 'dead' end,
+         visible_at       = null,
+         claimed_by       = null,
+         last_error       = '${LEASE_LAPSED_FINAL_ATTEMPT}',
+         last_error_stack = null,
+         updated_at       = now()
+    from picked p
+   where j.id = p.id and p.exhausted
+  returning ${CLAIM_ROW_COLUMNS}
+), counted as (
+  insert into x_job_counters (job, bucket_ms, bucket_start, dead, failed, duration_ms)
+  select name, ${COUNTER_BUCKET_MS}, date_trunc('minute', now()),
+         (count(*) filter (where state = 'dead'))::int,
+         (count(*) filter (where state = 'failed'))::int, 0
+    from buried
+   group by name
+  on conflict (job, bucket_ms, bucket_start) do update
+     set dead   = x_job_counters.dead + excluded.dead,
+         failed = x_job_counters.failed + excluded.failed
+), claimed as (
+  update x_jobs j
+     set state      = 'running',
+         attempt    = j.attempt + 1,
+         claims     = j.claims + 1,
+         claimed_by = $3,
+         visible_at = now() + ($4::bigint * interval '1 millisecond'),
+         updated_at = now()
+    from picked p
+   where j.id = p.id and not p.exhausted
+  returning ${CLAIM_ROW_COLUMNS}
 )
-update x_jobs j
-   set state      = 'running',
-       attempt    = j.attempt + 1,
-       claims     = j.claims + 1,
-       claimed_by = $3,
-       visible_at = now() + ($4::bigint * interval '1 millisecond'),
-       updated_at = now()
-  from claimed c
- where j.id = c.id
-returning j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
-          j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
-          j.traceparent, j.enqueued_by,
-          (extract(epoch from j.run_at)     * 1000)::bigint as run_at,
-          (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
-          (extract(epoch from j.created_at) * 1000)::bigint as created_at,
-          (extract(epoch from j.updated_at) * 1000)::bigint as updated_at
+select ${CLAIM_ROW_NAMES} from claimed
+union all
+select ${CLAIM_ROW_NAMES} from buried
 `.trim();
 
 /**
- * Stop a job from outside. `state <> 'done'` and not `state = 'ready'`: the runaway backfill this
- * exists for is `running`, and a job that already finished has nothing to stop. The worker holding
- * it learns on its next heartbeat, which no longer matches `state = 'running'`.
+ * Stop a job from outside — a LIVE one. The runaway backfill this exists for is `running`; the
+ * worker holding it learns on its next heartbeat, which no longer matches `state = 'running'`.
+ * Fenced on the four live states and not on `state <> 'done'`, which let a cancel REWRITE a dead
+ * letter: `dead / "card declined"` became `cancelled / "oops wrong id"` and left the dead-letter
+ * queue. A finished row is how the job ended, and stays that.
  */
 export const SQL_CANCEL = `
 update x_jobs
    set state = 'cancelled', visible_at = null, claimed_by = null,
        last_error = coalesce($2, last_error), updated_at = now()
- where id = $1 and state <> 'done'
+ where id = $1 and state in ('ready', 'delayed', 'running', 'suspended')
 returning ${JOB_ROW_COLUMNS}
 `.trim();
 
@@ -167,7 +211,7 @@ select queue,
          (where state in ('ready', 'delayed') and run_at <= now()), 0) * 1000   as oldest_ready_ms
   from x_jobs
  group by queue
- order by queue
+ order by queue collate "C"
 `.trim();
 
 /**
@@ -324,7 +368,17 @@ select id, name, payload, correlation_key,
  limit $2
 `.trim();
 
-export const SQL_EVENT_PURGE = `delete from x_job_events where expires_at <= statement_timestamp()`;
+/**
+ * The sweep, COUNTED in the statement: `PgExecutor` answers rows and never a command tag, and
+ * `returning` alone would ship every deleted id back to count them. The statement's own time, as
+ * every other event statement reads — an unpurged row costs `find` a filter, never a wrong answer.
+ */
+export const SQL_EVENT_PURGE_COUNTED = `
+with gone as (
+  delete from x_job_events where expires_at <= statement_timestamp() returning 1
+)
+select count(*)::int as removed from gone
+`.trim();
 
 export const SQL_STEP_GET = `
 select run_id, name, status, output, attempts, error,
@@ -351,18 +405,33 @@ select run_id, name, status, output, attempts, error,
   from x_job_steps where run_id = $1 order by started_at
 `.trim();
 
+/**
+ * FENCED on the claim when one is given, exactly as `SQL_ACK` is: `running`, the same worker, the
+ * same claim ordinal. The runner's own fence is an in-process signal, and it cannot stop a body in
+ * ANOTHER process — a worker stalled past its lease wrote its step over the re-claimer's. A null
+ * `$12` is an unfenced write: a transfer between drivers, with no row on this one to hold yet.
+ * Answers one row when the write landed, none when the fence refused it.
+ *
+ * $12 the job id or null, $13 the worker, $14 the claim.
+ */
 export const SQL_STEP_PUT = `
 insert into x_job_steps
   (run_id, name, status, output, started_at, completed_at, wake_at, event,
    correlation_key, attempts, error)
-values ($1, $2, $3, $4::jsonb, to_timestamp($5 / 1000.0),
-        case when $6::bigint is null then null else to_timestamp($6 / 1000.0) end,
-        case when $7::bigint is null then null else to_timestamp($7 / 1000.0) end,
-        $8, $9, $10, $11)
+select $1::uuid, $2, $3, $4::jsonb, to_timestamp($5::bigint / 1000.0),
+       case when $6::bigint is null then null else to_timestamp($6::bigint / 1000.0) end,
+       case when $7::bigint is null then null else to_timestamp($7::bigint / 1000.0) end,
+       $8, $9, $10::int, $11
+ where $12::uuid is null
+    or exists (
+      select 1 from x_jobs
+       where id = $12::uuid and state = 'running' and claimed_by = $13 and claims = $14::int
+    )
 on conflict (run_id, name) do update
    set status = excluded.status, output = excluded.output,
        completed_at = excluded.completed_at, wake_at = excluded.wake_at,
        attempts = excluded.attempts, error = excluded.error
+returning 1 as written
 `.trim();
 
 /**