@ultimat3/jobs 23.0.0 → 24.0.0

package/src/driver-pg.ts

@@ -4,9 +4,10 @@
 // one lease instead of one job. The statements themselves live in `driver-pg-sql.ts`, the schema
 // they run against in `driver-pg-ddl.ts`, and the row-to-record decoding in `driver-pg-rows.ts`.
 
-import type { Clock } from '@ultimat3/core';
-import { finiteCount, systemClock, uuid } from '@ultimat3/core';
+import type { Clock, PgExecutor } from '@ultimat3/core';
+import { finiteCount, renderCauseValue, systemClock, uuid } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
+import { listedRunId } from './backfill-ledger';
 import { nowMs } from './clock';
 import { nackOutcome } from './counters';
 import type {
@@ -57,32 +58,17 @@ import {
   SQL_STEP_GET,
   SQL_STEP_LIST,
   SQL_STEP_PUT,
-  SQL_STEPS_FROM,
   SQL_TRY_ADVISORY_LOCK,
 } from './driver-pg-sql';
-import { DriverUnavailableError, JobDuplicateError } from './errors';
-import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
+import { DriverUnavailableError, JobDuplicateError, LeaseLostError } from './errors';
+import { JobNotFoundError, JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
 import type { JobIntrospection } from './introspection';
 import { MAX_ERROR_STACK_LENGTH } from './introspection';
 import type { HeldLease, LeaseStore } from './leases';
 import type { StepStore } from './steps';
 
-/**
- * The one thing this driver needs from the DB layer, declared structurally so this package needs
- * no database CLIENT: `@ultimat3/db` is imported for `expectedQueryLoop` — the marker that tells
- * the N+1 detector a step write is one-per-step by design (`steps.ts`) — and never for a
- * connection.
- *
- * **Not satisfied by `Bun.sql`** — verified against Bun 1.4.0: `Bun.sql.query` is `undefined`.
- * `Bun.sql` is a tagged template whose positional form is `unsafe`, so a `{ executor: Bun.sql }`
- * would `TypeError` on the first claim. What satisfies it is a one-line adapter over a client that
- * already speaks `(text, values)` — `@ultimat3/cli`'s `pgExecutorFor(client)` is the framework's
- * own, wrapping `@ultimat3/db`'s `DbClient.query({ text, values })` — and a `DbTx`, which is a
- * client on the transaction's own connection.
- */
-export interface PgExecutor {
-  query<R>(sql: string, params: readonly unknown[]): Promise<readonly R[]>;
-}
+/** How often `enqueue` sends its insert before it calls a refusal an index fault, not a race. */
+const ENQUEUE_ATTEMPTS = 2;
 
 export interface PgDriverOptions {
   readonly executor?: PgExecutor;
@@ -107,8 +93,8 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
       const row = rows[0];
       return row === undefined ? undefined : toStepRecord(row);
     },
-    async put(record) {
-      await exec().query(SQL_STEP_PUT, [
+    async put(record, by) {
+      const written = await exec().query(SQL_STEP_PUT, [
         record.runId,
         record.name,
         record.status,
@@ -120,7 +106,14 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
         record.correlationKey ?? null,
         record.attempts,
         record.error ?? null,
+        by?.jobId ?? null,
+        by?.workerId ?? null,
+        by?.claim ?? null,
       ]);
+      // No row back is the fence refusing: the claim this write was made under is gone.
+      if (by !== undefined && written.length === 0) {
+        throw new LeaseLostError({ job: by.job, jobId: by.jobId });
+      }
     },
     async list(runId) {
       const rows = await exec().query<StepRow>(SQL_STEP_LIST, [runId]);
@@ -147,10 +140,12 @@ function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
       await exec().query(SQL_BACKFILL_FINISH, [runId, at.status, at.rows]);
     },
     async list(filter = {}) {
+      // Before the statement: `$3::uuid` raises a raw 22P02 for anything the screen refuses.
+      const runId = listedRunId('the pg ledger', filter.runId);
       const rows = await exec().query<BackfillRow>(SQL_BACKFILL_LIST, [
         filter.name ?? null,
         filter.status ?? null,
-        filter.runId ?? null,
+        runId ?? null,
         finiteCount('the pg driver list', 'limit', filter.limit ?? 100),
       ]);
       return rows.map(toBackfillRun);
@@ -226,13 +221,9 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
     },
     async requeue(jobId, requeueOptions) {
       const current = await job(jobId);
-      if (current === undefined) {
-        throw new DriverUnavailableError({
-          driver: 'pg',
-          cause: `job ${jobId} does not exist`,
-          fix: 'x jobs ls --state dead --json',
-        });
-      }
+      // The one answer both drivers give an id nobody queued — it was `X_DRIVER_UNAVAILABLE`
+      // here, from a driver that was answering perfectly well.
+      if (current === undefined) throw new JobNotFoundError({ jobId, driver: 'pg' });
       // Both refusals BEFORE anything is deleted or updated — a refused requeue changes nothing.
       if (!REQUEUEABLE_STATES.has(current.state)) {
         throw new JobNotRequeueableError({ jobId, state: current.state });
@@ -245,10 +236,12 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
       ]);
       const holder = holders[0];
       if (holder !== undefined) throw requeueKeyTaken({ ...current, holderId: holder.id });
-      if (requeueOptions?.fromStep !== undefined) {
-        await exec().query(SQL_STEPS_FROM, [current.runId, requeueOptions.fromStep]);
-      }
-      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [jobId]);
+      // ONE statement moves the row and drops its steps: nothing is deleted for a requeue that
+      // did not land.
+      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [
+        jobId,
+        requeueOptions?.fromStep ?? null,
+      ]);
       const row = rows[0];
       // Read as finished a moment ago and live now: another requeue won the race.
       if (row === undefined) throw new JobNotRequeueableError({ jobId, state: 'live' });
@@ -270,7 +263,7 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
 
     async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
       const runAt = request.runAt ?? nowMs(clock);
-      const rows = await exec().query<{ id: string; run_id: string }>(SQL_ENQUEUE, [
+      const params = [
         request.id ?? uuid(),
         request.name,
         request.queue || DEFAULT_QUEUE,
@@ -282,49 +275,56 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         request.tenantId ?? null,
         request.traceparent ?? null,
         request.enqueuedBy ?? null,
-      ]);
-      const inserted = rows[0];
-      if (inserted !== undefined) {
-        return { id: inserted.id, runId: inserted.run_id, deduped: false };
-      }
+      ];
+      // TWICE at most. The insert and the lookup below are two statements, and the live holder
+      // of the key can settle between them: the insert met it, the lookup did not. That is a key
+      // that just came free, so the insert is sent once more and lands. A second miss is not a
+      // race any more — see the refusal.
+      for (let attempt = 1; attempt <= ENQUEUE_ATTEMPTS; attempt += 1) {
+        const rows = await exec().query<{ id: string; run_id: string }>(SQL_ENQUEUE, params);
+        const inserted = rows[0];
+        if (inserted !== undefined) {
+          return { id: inserted.id, runId: inserted.run_id, deduped: false };
+        }
 
-      // Nothing inserted under an id the CALLER allocated: the outbox publishing one staged row a
-      // second time. The job its first publish made is the answer, whatever state it has reached.
-      if (request.id !== undefined) {
-        const published = await exec().query<{ id: string; run_id: string }>(
-          SQL_OUTBOX_PUBLISHED_JOB,
-          [request.id],
-        );
-        const made = published[0];
-        if (made !== undefined) return { id: made.id, runId: made.run_id, deduped: true };
-      }
+        // Nothing inserted under an id the CALLER allocated: the outbox publishing one staged row
+        // a second time. The job its first publish made is the answer, whatever state it reached.
+        if (request.id !== undefined) {
+          const published = await exec().query<{ id: string; run_id: string }>(
+            SQL_OUTBOX_PUBLISHED_JOB,
+            [request.id],
+          );
+          const made = published[0];
+          if (made !== undefined) return { id: made.id, runId: made.run_id, deduped: true };
+        }
 
-      // `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this idempotency
-      // key. Both are in the lookup because both are in the index — without the name this returned
-      // whichever other job derived the same natural key; without the tenant it returned another
-      // TENANT's row, so the caller's work silently never ran AND the caller was handed an id it
-      // has no right to, on a surface (`cancel`) that takes an id with no tenant predicate.
-      const existing = await exec().query<{ id: string; run_id: string }>(SQL_FIND_LIVE_BY_KEY, [
-        request.name,
-        request.idempotencyKey,
-        request.tenantId ?? null,
-      ]);
-      const found = existing[0];
-      if (found === undefined) {
-        throw new DriverUnavailableError({
-          driver: 'pg',
-          cause: `enqueue of "${request.name}" was rejected but no live row holds its idempotency key`,
-          fix: 'x db migrate   # reapplies SQL_JOBS_TABLE, whose x_jobs_name_tenant_idempotency_live_idx is what this lookup reads',
-        });
-      }
-      if (request.onConflict === 'error') {
-        throw new JobDuplicateError({
-          job: request.name,
-          idempotencyKey: request.idempotencyKey,
-          existingId: found.id,
-        });
+        // `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this
+        // idempotency key. Both are in the lookup because both are in the index — without the
+        // name this returned whichever other job derived the same natural key; without the tenant
+        // it returned another TENANT's row, so the caller's work silently never ran AND the caller
+        // was handed an id it has no right to, on a surface (`cancel`) that takes an id with no
+        // tenant predicate.
+        const existing = await exec().query<{ id: string; run_id: string }>(SQL_FIND_LIVE_BY_KEY, [
+          request.name,
+          request.idempotencyKey,
+          request.tenantId ?? null,
+        ]);
+        const found = existing[0];
+        if (found === undefined) continue;
+        if (request.onConflict === 'error') {
+          throw new JobDuplicateError({
+            job: request.name,
+            idempotencyKey: request.idempotencyKey,
+            existingId: found.id,
+          });
+        }
+        return { id: found.id, runId: found.run_id, deduped: true };
       }
-      return { id: found.id, runId: found.run_id, deduped: true };
+      throw new DriverUnavailableError({
+        driver: 'pg',
+        cause: `enqueue of ${renderCauseValue(request.name)} was rejected ${ENQUEUE_ATTEMPTS} times running and no live row holds its idempotency key either time — one miss is a holder that settled between the insert and the lookup, and is retried; two is an index the lookup does not agree with, and x db migrate reapplies SQL_JOBS_TABLE, whose x_jobs_name_tenant_idempotency_live_idx is what the lookup reads`,
+        fix: 'x db migrate',
+      });
     },
 
     async claim(claimOptions: ClaimOptions): Promise<readonly ClaimedJob[]> {
@@ -335,18 +335,23 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         claimOptions.limit,
         claimOptions.workerId,
         claimOptions.visibilityTimeoutMs,
+        claimOptions.dropExhausted ?? [],
       ]);
       const at = nowMs(clock);
-      return rows.map((row) => {
-        const record = toJobRecord(row);
-        return {
+      // Both arms of the statement answer, told apart by the state each wrote: `running` is a
+      // claim, `dead` a row the claim buried — reported, never handed out as work.
+      const records = rows.map(toJobRecord);
+      const buried = records.filter((record) => record.state !== 'running');
+      if (buried.length > 0) claimOptions.onExhausted?.(buried);
+      return records
+        .filter((record) => record.state === 'running')
+        .map((record) => ({
           ...record,
           claimedBy: claimOptions.workerId,
           claim: record.claim ?? 0,
           claimedAt: at,
           visibleAt: record.visibleAt ?? at + claimOptions.visibilityTimeoutMs,
-        };
-      });
+        }));
     },
 
     // Both settles answer whether they LANDED: no row back is a settle from a worker that no

package/src/driver-redis.ts

@@ -6,6 +6,7 @@
 // (XADD / XREADGROUP / XACK), `XAUTOCLAIM` for the visibility timeout, a ZSET for delayed
 // and suspended runs, and step records in a hash keyed by run id.
 
+import { NotImplementedError } from '@ultimat3/core';
 import type {
   ClaimedJob,
   ClaimOptions,
@@ -16,7 +17,6 @@ import type {
   NackOptions,
   QueueStats,
 } from './driver';
-import { JobsNotImplementedError } from './errors';
 import type { StepRecord, StepStore } from './steps';
 
 // Names the seam that actually replaces the stub, and NOTHING ELSE — the two other repairs this
@@ -37,7 +37,10 @@ const FIX =
   'call setJobDriver(createPgDriver()) at boot instead of this driver; nothing needs moving first, because enqueue here refuses too, so no job was ever written to it';
 
 const unavailable = (method: string): never => {
-  throw new JobsNotImplementedError({ feature: `redis jobs driver (${method})`, fix: FIX });
+  throw new NotImplementedError({
+    cause: `redis jobs driver (${method}) is not implemented`,
+    fix: FIX,
+  });
 };
 
 const redisStepStore = (): StepStore => ({

package/src/driver.ts

@@ -44,6 +44,27 @@ export const REQUEUEABLE_STATES: ReadonlySet<JobState> = new Set<JobState>([
   'failed',
 ]);
 
+/**
+ * The states a row is still WORK in — queued, held or parked. One set, read by the idempotency
+ * namespace (a key is taken only while a live row holds it) and by `cancel`, which stops live work
+ * and refuses everything else: cancelling a dead letter rewrote the record of a failure into the
+ * record of an operator's decision. The pg statements spell the same four.
+ */
+export const LIVE_STATES: ReadonlySet<JobState> = new Set<JobState>([
+  'ready',
+  'delayed',
+  'running',
+  'suspended',
+]);
+
+/**
+ * What a row's `lastError` says when the CLAIM buried it: its lease lapsed on the attempt that was
+ * its last, so nothing ever settled it and nothing may run it again. Fixed text, written by the
+ * claim statement itself — no apostrophe, because it is spliced into SQL as a literal.
+ */
+export const LEASE_LAPSED_FINAL_ATTEMPT =
+  'the lease lapsed on the final attempt: the worker holding this job stopped renewing it and never settled it, and no attempts remain';
+
 /** Narrows a state read back off a queue row. Never a cast — the list decides. */
 export const isJobState = (value: string): value is JobState =>
   (JOB_STATES as readonly string[]).includes(value);
@@ -144,6 +165,21 @@ export interface ClaimOptions {
   /** Lease length. A worker that dies without ack makes the job claimable again after this. */
   readonly visibilityTimeoutMs: number;
   readonly workerId: string;
+  /**
+   * Told the rows this pass BURIED instead of claiming: a lease that lapsed on the row's final
+   * attempt is settled `dead` by the claim itself (`LEASE_LAPSED_FINAL_ATTEMPT`) and is never
+   * handed out, so a job whose worker dies on every attempt reaches the dead-letter queue rather
+   * than killing a worker per lease, forever. Called at most once per pass, after the statement
+   * landed, with the rows as they now stand — what the claim round announces `onSettled` from
+   * (`announceExhausted`). Omit it and the rows are buried all the same.
+   */
+  readonly onExhausted?: (dead: readonly JobRecord[]) => void;
+  /**
+   * Job NAMES whose final failure is `failed`, never `dead` — the ones declaring
+   * `retry.deadLetter: false`. The row carries `max_attempts` and not the policy, so the caller
+   * that holds the registry says which rows a burial DROPS; a name not listed is dead-lettered.
+   */
+  readonly dropExhausted?: readonly string[];
 }
 
 export interface ClaimedJob extends JobRecord {
@@ -316,7 +352,7 @@ export const DEFAULT_VISIBILITY_TIMEOUT_MS = 30_000;
 
 let ambient: JobDriver | undefined;
 
-/** Set once at boot from `app.config.ts`. Roles share one driver instance per process. */
+/** Set once at boot over the database binding, never a config key. One instance per process. */
 export function setJobDriver(driver: JobDriver): void {
   ambient = driver;
 }

package/src/errors-operator.ts

@@ -77,14 +77,22 @@ export class JobOnSettledFailedError extends UltimateError {
  * walks past it. `MAX_JOB_PAGE` is passed in — `introspection.ts` owns it and imports this file.
  */
 export class JobPageInvalidError extends UltimateError {
-  constructor(input: { subject: string; maxPage: number; limit?: number }) {
+  constructor(input: {
+    subject: string;
+    maxPage: number;
+    limit?: number;
+    /** Which cursor was refused. Default `after`. */
+    cursor?: 'after' | 'before';
+  }) {
+    const cursor = input.cursor ?? 'after';
+    const source = cursor === 'after' ? 'jobCursor(lastRow)' : 'jobCursor(firstRow)';
     super({
       code: 'X_JOB_PAGE_INVALID',
       cause:
         input.limit === undefined
-          ? `${input.subject} was handed an after cursor that no previous page produced`
-          : `${input.subject} limit is ${input.limit}, over MAX_JOB_PAGE (${input.maxPage}) — the rows one page may answer`,
-      fix: `list({ limit: ${input.maxPage}, after: jobCursor(lastRow) })   # lastRow is the last row of the page before; omit after for the first page, and stop at a page shorter than its limit`,
+          ? `${input.subject} was handed ${cursor === 'after' ? 'an after' : 'a before'} cursor that no page produced — ${cursor}: ${source} of a page this driver answered is the only one it reads; omit it for the first page`
+          : `${input.subject} limit is ${input.limit}, over MAX_JOB_PAGE (${input.maxPage}) — the rows one page may answer; walk further with after: jobCursor(lastRow), and stop at a page shorter than its limit`,
+      fix: `x jobs ls --limit ${renderFixShellArg(String(input.maxPage), '200')} --json`,
       meta: {
         maxPage: input.maxPage,
         ...(input.limit === undefined ? {} : { limit: input.limit }),

package/src/errors-requeue.ts

@@ -1,7 +1,8 @@
-// Single responsibility: the two refusals `requeue` — `x jobs retry` — gives before it moves a
-// row. Its own module because `errors.ts` sits at the size ceiling; the codes are registered there.
+// Single responsibility: the three refusals `requeue` — `x jobs retry` — gives before it moves a
+// row, raised identically by `driver-memory.ts` and `driver-pg.ts`: one answer for both drivers is
+// the point, and neither driver imports the other. The codes are registered in `errors.ts`.
 
-import { renderFixShellArg, UltimateError } from '@ultimat3/core';
+import { renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
 import { JobDuplicateError } from './errors';
 
 /**
@@ -19,6 +20,23 @@ export class JobNotRequeueableError extends UltimateError {
   }
 }
 
+/**
+ * `requeue` of an id the queue does not hold. ONE answer for both drivers: memory raised
+ * `X_INVARIANT` and pg `X_DRIVER_UNAVAILABLE` — a driver that is perfectly available — so a caller
+ * could not tell a mistyped id from an outage, and tested against one answer on `x dev` and met
+ * the other in production.
+ */
+export class JobNotFoundError extends UltimateError {
+  constructor(input: { jobId: string; driver: string }) {
+    super({
+      code: 'X_JOB_NOT_FOUND',
+      cause: `the "${input.driver}" queue holds no job with id ${renderCauseValue(input.jobId)} — a mistyped id, a job removed since, or an id from another environment`,
+      fix: 'x jobs ls --json',
+      meta: { jobId: input.jobId },
+    });
+  }
+}
+
 /** The error a requeue gives when a LIVE job already holds the requeued row's key. */
 export const requeueKeyTaken = (record: {
   readonly id: string;

package/src/errors.ts

@@ -43,12 +43,13 @@ export const JOB_OWNED_ERROR_CODES = [
   'X_JOB_NOT_PROMOTABLE',
   'X_JOB_PAGE_INVALID',
   'X_JOB_ON_SETTLED_FAILED',
+  'X_JOB_NOT_FOUND',
 ] as const;
 
 /**
- * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. `JobsNotImplementedError`,
- * `JobAbortedError` and `JobDrainedError` below throw them; jobs keeps no title for any of the
- * three, because the copy this file used to hold was a second title that nothing would have failed
+ * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. The first is thrown as
+ * core's own `NotImplementedError` (`inspect.ts`, the redis/nats stubs), the other two by
+ * `JobAbortedError` and `JobDrainedError` below; jobs keeps no title for any of the three, because the copy this file used to hold was a second title that nothing would have failed
  * on once core's changed. Listed here all the same, so `JobErrorCode` can name every code a job
  * can see — `X_DRAINING` was thrown for a day before it was, and the type said it could not be.
  */
@@ -100,6 +101,7 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
   X_JOB_NOT_PROMOTABLE: 'the job is not waiting on its run time',
   X_JOB_PAGE_INVALID: 'a job list page was asked for outside its bounds',
   X_JOB_ON_SETTLED_FAILED: 'the onSettled hook failed after the run was settled',
+  X_JOB_NOT_FOUND: 'the queue holds no job with this id',
 };
 
 // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -454,16 +456,6 @@ export class OutboxNoTxError extends UltimateError {
   }
 }
 
-export class JobsNotImplementedError extends UltimateError {
-  constructor(input: { feature: string; fix: string }) {
-    super({
-      code: 'X_NOT_IMPLEMENTED',
-      cause: `${input.feature} is declared but not implemented in @ultimat3/jobs`,
-      fix: input.fix,
-    });
-  }
-}
-
 /**
  * `claim({ queues: [] })`, refused by every driver rather than answered by each.
  *

package/src/events-pg.ts

@@ -6,16 +6,16 @@
 // Stored and not broadcast, exactly as the memory bus is: a step that suspends at 12:00 and
 // resumes at 12:00:30 must still see an event published at 12:00:10.
 
-import { finiteOption, logger, renderThrowable, uuid } from '@ultimat3/core';
+import type { PgExecutor } from '@ultimat3/core';
+import { finiteOption, logger, uuid } from '@ultimat3/core';
 import type { DurationInput } from './clock';
 import { finiteDurationMs } from './clock';
-import type { PgExecutor } from './driver-pg';
 import {
   SQL_EVENT_FIND,
   SQL_EVENT_LIST,
   SQL_EVENT_NOW,
   SQL_EVENT_PUBLISH,
-  SQL_EVENT_PURGE,
+  SQL_EVENT_PURGE_COUNTED,
 } from './driver-pg-sql';
 import type { EventBus } from './events';
 
@@ -40,13 +40,6 @@ export interface PgEventBusOptions {
   readonly listLimit?: number;
 }
 
-/**
- * `purgeExpired()` is SYNCHRONOUS in `EventBus` because the memory bus can be — it walks a Map.
- * Over SQL the delete is a round trip, so this fires it and answers 0: the count is a diagnostic
- * the memory bus offers and this one cannot, and blocking a step's resume on a housekeeping
- * DELETE would be a far worse trade than an unanswered number. The index on `(name, published_at)`
- * plus `expires_at > now()` in `find` means an unpurged row costs a filter, never a wrong answer.
- */
 export function createPgEventBus(options: PgEventBusOptions): EventBus {
   // TWO screens, for the reason `events.ts` states: `defaultTtl` is the constructor's knob and
   // `ttl` is the publish call's, so one screen over `ttl ?? defaultTtl` names the wrong one for
@@ -59,14 +52,11 @@ export function createPgEventBus(options: PgEventBusOptions): EventBus {
   const listLimit = finiteOption('the pg event bus', 'listLimit', options.listLimit ?? 1_000);
   const exec = options.executor;
 
-  const purgeExpired = (): number => {
-    void exec.query(SQL_EVENT_PURGE, []).catch((error: unknown) => {
-      // Housekeeping never costs a publish: an unpurged row is filtered out of every read.
-      logger.warn('jobs.event.purge-failed', {
-        error: renderThrowable(error),
-      });
-    });
-    return 0;
+  // Awaited and never fired-and-forgotten: it was `void`ed with nothing calling it, so the table
+  // only grew. A failure REJECTS — the caller is the retention sweep's step, which retries.
+  const purgeExpired = async (): Promise<number> => {
+    const rows = await exec.query<{ removed: number | string }>(SQL_EVENT_PURGE_COUNTED, []);
+    return Number(rows[0]?.removed ?? 0);
   };
 
   return {

package/src/events.ts

@@ -6,6 +6,7 @@ import type { Clock } from '@ultimat3/core';
 import { finiteOption, logger, systemClock, uuid } from '@ultimat3/core';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
+import type { PurgeTarget } from './purge';
 import type { EventLookup } from './steps';
 
 export interface JobEvent {
@@ -31,16 +32,14 @@ export interface EventBus extends EventLookup {
    * split asks, and the reason `eventPrompt()` refuses the in-memory bus outside development.
    */
   readonly stored: boolean;
-  /**
-   * The bus's OWN clock, in epoch ms: what a publish issued now would be stamped with. A consumer
-   * that wants "published after I asked" takes its "asked at" from here and never from its own
-   * process — `publishedAt` is this clock's, and two clocks compared is a skew between pods
-   * deciding which answers count.
-   */
-  now(): Promise<number>;
   publish(name: string, payload: unknown, options?: PublishOptions): Promise<JobEvent>;
   list(name?: string): Promise<readonly JobEvent[]>;
-  purgeExpired(): number;
+  /**
+   * Delete every event past its expiry and answer how many went. Measured on the bus's own clock,
+   * like every other instant it holds. The memory bus also sweeps on each publish; the stored one
+   * has no caller but the retention sweep (`eventsPurgeTarget`).
+   */
+  purgeExpired(): Promise<number>;
   size(): number;
 }
 
@@ -136,11 +135,23 @@ export function createMemoryEventBus(options: MemoryEventBusOptions = {}): Event
       return Promise.resolve(all);
     },
 
-    purgeExpired,
+    purgeExpired: () => Promise.resolve(purgeExpired()),
     size: () => events.size,
   };
 }
 
+/** The table the stored bus keeps — the sweep's durable step key, log field and report name. */
+export const EVENTS_PURGE_TARGET = 'x_job_events';
+
+/**
+ * The bus as one table of the retention sweep (`purge()`). `x_job_events` takes a row per publish
+ * and an expired one is only FILTERED by `find`, so with no sweep the table grows for the life of
+ * the deployment. The sweep's `nowMs` is not passed on: a bus holds its own clock.
+ */
+export function eventsPurgeTarget(bus: EventBus): PurgeTarget {
+  return { name: EVENTS_PURGE_TARGET, purgeExpired: () => bus.purgeExpired() };
+}
+
 let ambientBus: EventBus = createMemoryEventBus();
 
 /**

package/src/execute.ts

@@ -151,6 +151,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     runId: claimed.runId,
     jobName: handle.name,
     store: driver.steps,
+    fence: { job: handle.name, jobId: claimed.id, ...claimOf(claimed) },
     signal,
     // The DECLARED per-step ceiling and event poll. Passed here or nowhere: this is the only
     // production construction of a runner, so a `StepRunnerOptions` field it omits is a feature

package/src/index.ts

@@ -11,7 +11,7 @@ import './register';
 // the tier both halves can reach, because this package signs a delivery, `@ultimat3/http` verifies
 // one, and neither may import the other. Re-exported here so a `job` file needs one import rather
 // than two — the same reason `t` is re-exported above.
-export type { WebhookMacInput, WebhookSigningInput } from '@ultimat3/core';
+export type { PgExecutor, WebhookMacInput, WebhookSigningInput } from '@ultimat3/core';
 export {
   isCanonicalWebhookField,
   WEBHOOK_FIELD_MAX,
@@ -84,6 +84,8 @@ export {
   isBackfill,
   registeredBackfills,
 } from './backfill-registry';
+export type { AnnounceExhaustedOptions, ExhaustedCounts } from './claim-exhausted';
+export { announceExhausted } from './claim-exhausted';
 export type { JobConcurrency, KeyedConcurrency, WhenBusy } from './concurrency';
 export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
 export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
@@ -112,6 +114,8 @@ export {
   isJobState,
   JOB_STATES,
   jobDriver,
+  LEASE_LAPSED_FINAL_ATTEMPT,
+  LIVE_STATES,
   nackState,
   resetJobDriver,
   setJobDriver,
@@ -120,7 +124,7 @@ export type { MemoryDriverOptions, MemoryJobDriver } from './driver-memory';
 export { createMemoryDriver } from './driver-memory';
 export type { NatsDriverOptions } from './driver-nats';
 export { createNatsDriver } from './driver-nats';
-export type { PgDriverOptions, PgExecutor } from './driver-pg';
+export type { PgDriverOptions } from './driver-pg';
 export { createPgDriver, createPgLeader } from './driver-pg';
 export {
   SQL_COUNTER_DROP,
@@ -190,7 +194,6 @@ export {
   JobNotCancellableError,
   JobRowStatusUnknownError,
   JobSlotLostError,
-  JobsNotImplementedError,
   JobTenantRequiredError,
   JobTimeoutError,
   LeaseLostError,
@@ -210,11 +213,13 @@ export {
   JobOnSettledFailedError,
   JobPageInvalidError,
 } from './errors-operator';
-export { JobNotRequeueableError } from './errors-requeue';
+export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
 export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
 export {
   createMemoryEventBus,
+  EVENTS_PURGE_TARGET,
   eventBus,
+  eventsPurgeTarget,
   publishEvent,
   resetEventBus,
   setEventBus,
@@ -377,6 +382,7 @@ export { ON_SETTLED_ATTEMPTS } from './settled';
 export type {
   EventLookup,
   StepApi,
+  StepFence,
   StepRecord,
   StepRunner,
   StepRunnerOptions,