@ultimat3/jobs 23.0.0 → 25.0.0

package/src/driver-pg.ts

@@ -4,9 +4,10 @@
 // one lease instead of one job. The statements themselves live in `driver-pg-sql.ts`, the schema
 // they run against in `driver-pg-ddl.ts`, and the row-to-record decoding in `driver-pg-rows.ts`.
 
-import type { Clock } from '@ultimat3/core';
-import { finiteCount, systemClock, uuid } from '@ultimat3/core';
+import type { Clock, PgExecutor } from '@ultimat3/core';
+import { finiteCount, renderCauseValue, systemClock, uuidV7 } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
+import { listedRunId } from './backfill-ledger';
 import { nowMs } from './clock';
 import { nackOutcome } from './counters';
 import type {
@@ -57,34 +58,19 @@ import {
   SQL_STEP_GET,
   SQL_STEP_LIST,
   SQL_STEP_PUT,
-  SQL_STEPS_FROM,
   SQL_TRY_ADVISORY_LOCK,
 } from './driver-pg-sql';
-import { DriverUnavailableError, JobDuplicateError } from './errors';
-import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
+import { DriverUnavailableError, JobDuplicateError, LeaseLostError } from './errors';
+import { JobNotFoundError, JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
 import type { JobIntrospection } from './introspection';
 import { MAX_ERROR_STACK_LENGTH } from './introspection';
 import type { HeldLease, LeaseStore } from './leases';
 import type { StepStore } from './steps';
 
-/**
- * The one thing this driver needs from the DB layer, declared structurally so this package needs
- * no database CLIENT: `@ultimat3/db` is imported for `expectedQueryLoop` — the marker that tells
- * the N+1 detector a step write is one-per-step by design (`steps.ts`) — and never for a
- * connection.
- *
- * **Not satisfied by `Bun.sql`** — verified against Bun 1.4.0: `Bun.sql.query` is `undefined`.
- * `Bun.sql` is a tagged template whose positional form is `unsafe`, so a `{ executor: Bun.sql }`
- * would `TypeError` on the first claim. What satisfies it is a one-line adapter over a client that
- * already speaks `(text, values)` — `@ultimat3/cli`'s `pgExecutorFor(client)` is the framework's
- * own, wrapping `@ultimat3/db`'s `DbClient.query({ text, values })` — and a `DbTx`, which is a
- * client on the transaction's own connection.
- */
-export interface PgExecutor {
-  query<R>(sql: string, params: readonly unknown[]): Promise<readonly R[]>;
-}
+/** How often `enqueue` sends its insert before it calls a refusal an index fault, not a race. */
+const ENQUEUE_ATTEMPTS = 2;
 
-export interface PgDriverOptions {
+export interface PostgresJobDriverOptions {
   readonly executor?: PgExecutor;
   readonly clock?: Clock;
 }
@@ -95,20 +81,21 @@ function resolveExecutor(injected: PgExecutor | undefined): PgExecutor {
     driver: 'pg',
     // `Bun.sql` is named nowhere in this function and never was: there is no ambient fallback to
     // be "not configured". An executor is injected by the boot or the driver has none.
-    cause: 'createPgDriver() was called with no executor, and this driver has no ambient fallback',
-    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(createMemoryDriver())',
+    cause:
+      'postgresJobDriver() was called with no executor, and this driver has no ambient fallback',
+    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(memoryJobDriver())',
   });
 }
 
-function pgStepStore(exec: () => PgExecutor): StepStore {
+function postgresStepStore(exec: () => PgExecutor): StepStore {
   return {
     async get(runId, name) {
       const rows = await exec().query<StepRow>(SQL_STEP_GET, [runId, name]);
       const row = rows[0];
       return row === undefined ? undefined : toStepRecord(row);
     },
-    async put(record) {
-      await exec().query(SQL_STEP_PUT, [
+    async put(record, by) {
+      const written = await exec().query(SQL_STEP_PUT, [
         record.runId,
         record.name,
         record.status,
@@ -120,7 +107,14 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
         record.correlationKey ?? null,
         record.attempts,
         record.error ?? null,
+        by?.jobId ?? null,
+        by?.workerId ?? null,
+        by?.claim ?? null,
       ]);
+      // No row back is the fence refusing: the claim this write was made under is gone.
+      if (by !== undefined && written.length === 0) {
+        throw new LeaseLostError({ job: by.job, jobId: by.jobId });
+      }
     },
     async list(runId) {
       const rows = await exec().query<StepRow>(SQL_STEP_LIST, [runId]);
@@ -135,7 +129,7 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
   };
 }
 
-function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
+function postgresBackfillLedger(exec: () => PgExecutor): BackfillLedger {
   return {
     async start(run) {
       await exec().query(SQL_BACKFILL_START, [run.runId, run.name, run.checksum, run.appVersion]);
@@ -147,10 +141,12 @@ function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
       await exec().query(SQL_BACKFILL_FINISH, [runId, at.status, at.rows]);
     },
     async list(filter = {}) {
+      // Before the statement: `$3::uuid` raises a raw 22P02 for anything the screen refuses.
+      const runId = listedRunId('the pg ledger', filter.runId);
       const rows = await exec().query<BackfillRow>(SQL_BACKFILL_LIST, [
         filter.name ?? null,
         filter.status ?? null,
-        filter.runId ?? null,
+        runId ?? null,
         finiteCount('the pg driver list', 'limit', filter.limit ?? 100),
       ]);
       return rows.map(toBackfillRun);
@@ -162,7 +158,7 @@ function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
  * Fleet-wide slots over `x_job_leases`. Every decision is ONE statement — the `(lease_key, slot)`
  * primary key is what serialises two workers, so nothing here reads a count and then acts on it.
  */
-function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
+function postgresLeaseStore(exec: () => PgExecutor): LeaseStore {
   return {
     async acquire(key, limit, ttlMs, holder) {
       if (limit <= 0) return undefined;
@@ -201,7 +197,7 @@ function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
   };
 }
 
-export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
+export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDriver {
   const clock = options.clock ?? systemClock;
   let executor: PgExecutor | undefined;
   const exec = (): PgExecutor => {
@@ -226,13 +222,9 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
     },
     async requeue(jobId, requeueOptions) {
       const current = await job(jobId);
-      if (current === undefined) {
-        throw new DriverUnavailableError({
-          driver: 'pg',
-          cause: `job ${jobId} does not exist`,
-          fix: 'x jobs ls --state dead --json',
-        });
-      }
+      // The one answer both drivers give an id nobody queued — it was `X_DRIVER_UNAVAILABLE`
+      // here, from a driver that was answering perfectly well.
+      if (current === undefined) throw new JobNotFoundError({ jobId, driver: 'pg' });
       // Both refusals BEFORE anything is deleted or updated — a refused requeue changes nothing.
       if (!REQUEUEABLE_STATES.has(current.state)) {
         throw new JobNotRequeueableError({ jobId, state: current.state });
@@ -245,10 +237,12 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
       ]);
       const holder = holders[0];
       if (holder !== undefined) throw requeueKeyTaken({ ...current, holderId: holder.id });
-      if (requeueOptions?.fromStep !== undefined) {
-        await exec().query(SQL_STEPS_FROM, [current.runId, requeueOptions.fromStep]);
-      }
-      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [jobId]);
+      // ONE statement moves the row and drops its steps: nothing is deleted for a requeue that
+      // did not land.
+      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [
+        jobId,
+        requeueOptions?.fromStep ?? null,
+      ]);
       const row = rows[0];
       // Read as finished a moment ago and live now: another requeue won the race.
       if (row === undefined) throw new JobNotRequeueableError({ jobId, state: 'live' });
@@ -263,68 +257,75 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
 
   return {
     name: 'pg',
-    steps: pgStepStore(exec),
-    backfills: pgBackfillLedger(exec),
-    leases: pgLeaseStore(exec),
+    steps: postgresStepStore(exec),
+    backfills: postgresBackfillLedger(exec),
+    leases: postgresLeaseStore(exec),
     introspect,
 
     async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
       const runAt = request.runAt ?? nowMs(clock);
-      const rows = await exec().query<{ id: string; run_id: string }>(SQL_ENQUEUE, [
-        request.id ?? uuid(),
+      const params = [
+        request.id ?? uuidV7(),
         request.name,
         request.queue || DEFAULT_QUEUE,
         JSON.stringify(request.input ?? null),
         request.idempotencyKey,
-        request.runId ?? uuid(),
+        request.runId ?? uuidV7(),
         request.maxAttempts,
         runAt,
         request.tenantId ?? null,
         request.traceparent ?? null,
         request.enqueuedBy ?? null,
-      ]);
-      const inserted = rows[0];
-      if (inserted !== undefined) {
-        return { id: inserted.id, runId: inserted.run_id, deduped: false };
-      }
+      ];
+      // TWICE at most. The insert and the lookup below are two statements, and the live holder
+      // of the key can settle between them: the insert met it, the lookup did not. That is a key
+      // that just came free, so the insert is sent once more and lands. A second miss is not a
+      // race any more — see the refusal.
+      for (let attempt = 1; attempt <= ENQUEUE_ATTEMPTS; attempt += 1) {
+        const rows = await exec().query<{ id: string; run_id: string }>(SQL_ENQUEUE, params);
+        const inserted = rows[0];
+        if (inserted !== undefined) {
+          return { id: inserted.id, runId: inserted.run_id, deduped: false };
+        }
 
-      // Nothing inserted under an id the CALLER allocated: the outbox publishing one staged row a
-      // second time. The job its first publish made is the answer, whatever state it has reached.
-      if (request.id !== undefined) {
-        const published = await exec().query<{ id: string; run_id: string }>(
-          SQL_OUTBOX_PUBLISHED_JOB,
-          [request.id],
-        );
-        const made = published[0];
-        if (made !== undefined) return { id: made.id, runId: made.run_id, deduped: true };
-      }
+        // Nothing inserted under an id the CALLER allocated: the outbox publishing one staged row
+        // a second time. The job its first publish made is the answer, whatever state it reached.
+        if (request.id !== undefined) {
+          const published = await exec().query<{ id: string; run_id: string }>(
+            SQL_OUTBOX_PUBLISHED_JOB,
+            [request.id],
+          );
+          const made = published[0];
+          if (made !== undefined) return { id: made.id, runId: made.run_id, deduped: true };
+        }
 
-      // `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this idempotency
-      // key. Both are in the lookup because both are in the index — without the name this returned
-      // whichever other job derived the same natural key; without the tenant it returned another
-      // TENANT's row, so the caller's work silently never ran AND the caller was handed an id it
-      // has no right to, on a surface (`cancel`) that takes an id with no tenant predicate.
-      const existing = await exec().query<{ id: string; run_id: string }>(SQL_FIND_LIVE_BY_KEY, [
-        request.name,
-        request.idempotencyKey,
-        request.tenantId ?? null,
-      ]);
-      const found = existing[0];
-      if (found === undefined) {
-        throw new DriverUnavailableError({
-          driver: 'pg',
-          cause: `enqueue of "${request.name}" was rejected but no live row holds its idempotency key`,
-          fix: 'x db migrate   # reapplies SQL_JOBS_TABLE, whose x_jobs_name_tenant_idempotency_live_idx is what this lookup reads',
-        });
-      }
-      if (request.onConflict === 'error') {
-        throw new JobDuplicateError({
-          job: request.name,
-          idempotencyKey: request.idempotencyKey,
-          existingId: found.id,
-        });
+        // `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this
+        // idempotency key. Both are in the lookup because both are in the index — without the
+        // name this returned whichever other job derived the same natural key; without the tenant
+        // it returned another TENANT's row, so the caller's work silently never ran AND the caller
+        // was handed an id it has no right to, on a surface (`cancel`) that takes an id with no
+        // tenant predicate.
+        const existing = await exec().query<{ id: string; run_id: string }>(SQL_FIND_LIVE_BY_KEY, [
+          request.name,
+          request.idempotencyKey,
+          request.tenantId ?? null,
+        ]);
+        const found = existing[0];
+        if (found === undefined) continue;
+        if (request.onConflict === 'error') {
+          throw new JobDuplicateError({
+            job: request.name,
+            idempotencyKey: request.idempotencyKey,
+            existingId: found.id,
+          });
+        }
+        return { id: found.id, runId: found.run_id, deduped: true };
       }
-      return { id: found.id, runId: found.run_id, deduped: true };
+      throw new DriverUnavailableError({
+        driver: 'pg',
+        cause: `enqueue of ${renderCauseValue(request.name)} was rejected ${ENQUEUE_ATTEMPTS} times running and no live row holds its idempotency key either time — one miss is a holder that settled between the insert and the lookup, and is retried; two is an index the lookup does not agree with, and x db migrate reapplies SQL_JOBS_TABLE, whose x_jobs_name_tenant_idempotency_live_idx is what the lookup reads`,
+        fix: 'x db migrate',
+      });
     },
 
     async claim(claimOptions: ClaimOptions): Promise<readonly ClaimedJob[]> {
@@ -335,18 +336,23 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         claimOptions.limit,
         claimOptions.workerId,
         claimOptions.visibilityTimeoutMs,
+        claimOptions.dropExhausted ?? [],
       ]);
       const at = nowMs(clock);
-      return rows.map((row) => {
-        const record = toJobRecord(row);
-        return {
+      // Both arms of the statement answer, told apart by the state each wrote: `running` is a
+      // claim, `dead` a row the claim buried — reported, never handed out as work.
+      const records = rows.map(toJobRecord);
+      const buried = records.filter((record) => record.state !== 'running');
+      if (buried.length > 0) claimOptions.onExhausted?.(buried);
+      return records
+        .filter((record) => record.state === 'running')
+        .map((record) => ({
           ...record,
           claimedBy: claimOptions.workerId,
           claim: record.claim ?? 0,
           claimedAt: at,
           visibleAt: record.visibleAt ?? at + claimOptions.visibilityTimeoutMs,
-        };
-      });
+        }));
     },
 
     // Both settles answer whether they LANDED: no row back is a settle from a worker that no
@@ -357,7 +363,6 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         by.workerId,
         Math.round(by.durationMs ?? 0),
         by.claim,
-        by.counted !== false,
       ]);
       return rows.length > 0;
     },
@@ -432,14 +437,14 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
  * key would need a second `release()` — hence the `held` guard below, which makes repeated
  * `acquire()` calls (the scheduler renews every round) a no-op rather than a leak.
  *
- * `@ultimat3/realtime`'s `PgAdvisoryLock` is the shape that gets this right: it opens a connection
+ * `@ultimat3/realtime`'s `postgresAdvisoryLock()` is the shape that gets this right: it opens a connection
  * of its own and *is* the lock. This package holds no wire protocol, so the executor it is handed
- * is whatever boot built — which is a pool. **Use `createPgLeaseLeader` instead** unless you can
+ * is whatever boot built — which is a pool. **Use `postgresLeaseLeader` instead** unless you can
  * prove the executor is a single dedicated session.
  */
-export function createPgLeader(
+export function postgresLeader(
   lockKey: number,
-  options: PgDriverOptions = {},
+  options: PostgresJobDriverOptions = {},
 ): { acquire(): Promise<boolean>; release(): Promise<void>; readonly renewEveryMs: number } {
   const exec = (): PgExecutor => resolveExecutor(options.executor);
   let held = false;

package/src/driver.ts

@@ -2,11 +2,11 @@
 // Six methods and no more: claim/ack/nack with a visibility timeout is the smallest set that
 // survives a worker crash.
 //
-// This header used to say "switching backends is a config line" (`As of 2026-08`). There is no
-// such config line: `JobsConfig.driver` has no reader anywhere and boot always builds
-// `createPgDriver`. `pg` and `memory` are the two that exist; `redis` and `nats` are honest
-// `X_NOT_IMPLEMENTED` stubs. What IS true is the second half — swapping the driver is
-// `setJobDriver(other)` and ZERO job-code change — and that is what the interface buys.
+// There is no config line that picks a backend: boot always builds `postgresJobDriver()`, and a
+// stale `jobs.driver` in `app.config.ts` is refused by `defineConfig` (25.0.0; it was read by
+// nothing since 5.0.0). `postgresJobDriver` and `memoryJobDriver` are the two that exist — 25.0.0
+// deleted the all-throw `redis` and `nats` stubs. Swapping the driver is `setJobDriver(other)` and
+// ZERO job-code change, and that is what the interface buys.
 
 import { finiteCount, finiteOption } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
@@ -44,6 +44,27 @@ export const REQUEUEABLE_STATES: ReadonlySet<JobState> = new Set<JobState>([
   'failed',
 ]);
 
+/**
+ * The states a row is still WORK in — queued, held or parked. One set, read by the idempotency
+ * namespace (a key is taken only while a live row holds it) and by `cancel`, which stops live work
+ * and refuses everything else: cancelling a dead letter rewrote the record of a failure into the
+ * record of an operator's decision. The pg statements spell the same four.
+ */
+export const LIVE_STATES: ReadonlySet<JobState> = new Set<JobState>([
+  'ready',
+  'delayed',
+  'running',
+  'suspended',
+]);
+
+/**
+ * What a row's `lastError` says when the CLAIM buried it: its lease lapsed on the attempt that was
+ * its last, so nothing ever settled it and nothing may run it again. Fixed text, written by the
+ * claim statement itself — no apostrophe, because it is spliced into SQL as a literal.
+ */
+export const LEASE_LAPSED_FINAL_ATTEMPT =
+  'the lease lapsed on the final attempt: the worker holding this job stopped renewing it and never settled it, and no attempts remain';
+
 /** Narrows a state read back off a queue row. Never a cast — the list decides. */
 export const isJobState = (value: string): value is JobState =>
   (JOB_STATES as readonly string[]).includes(value);
@@ -136,7 +157,7 @@ export interface EnqueueResult {
 export interface ClaimOptions {
   /**
    * The queues this pass may take work from, by name. **At least one, and an empty list is
-   * refused** (`X_JOB_CLAIM_QUEUES_EMPTY`) — see `assertClaimQueues`. `createWorker` passes exactly
+   * refused** (`X_JOB_CLAIM_QUEUES_EMPTY`) — see `assertClaimQueues`. `jobWorker` passes exactly
    * one per pass, which is what keeps a slow queue from starving the others.
    */
   readonly queues: readonly string[];
@@ -144,6 +165,21 @@ export interface ClaimOptions {
   /** Lease length. A worker that dies without ack makes the job claimable again after this. */
   readonly visibilityTimeoutMs: number;
   readonly workerId: string;
+  /**
+   * Told the rows this pass BURIED instead of claiming: a lease that lapsed on the row's final
+   * attempt is settled `dead` by the claim itself (`LEASE_LAPSED_FINAL_ATTEMPT`) and is never
+   * handed out, so a job whose worker dies on every attempt reaches the dead-letter queue rather
+   * than killing a worker per lease, forever. Called at most once per pass, after the statement
+   * landed, with the rows as they now stand — what the claim round announces `onSettled` from
+   * (`announceExhausted`). Omit it and the rows are buried all the same.
+   */
+  readonly onExhausted?: (dead: readonly JobRecord[]) => void;
+  /**
+   * Job NAMES whose final failure is `failed`, never `dead` — the ones declaring
+   * `retry.deadLetter: false`. The row carries `max_attempts` and not the policy, so the caller
+   * that holds the registry says which rows a burial DROPS; a name not listed is dead-lettered.
+   */
+  readonly dropExhausted?: readonly string[];
 }
 
 export interface ClaimedJob extends JobRecord {
@@ -185,14 +221,8 @@ export interface SettleBy extends ClaimIdentity {
   readonly durationMs?: number;
 }
 
-export interface AckOptions extends SettleBy {
-  /**
-   * False when the row is finished WITHOUT its body having run here — `x jobs drain` moving it to
-   * another driver. It is settled `done` and adds nothing to the job's counters: a job that was
-   * moved is not a job that completed.
-   */
-  readonly counted?: boolean;
-}
+/** An ack is a settle by the claimer: every ack is a completed run and adds to the counters. */
+export type AckOptions = SettleBy;
 
 export interface NackOptions extends SettleBy {
   /** Delay before the job becomes claimable again. */
@@ -301,7 +331,7 @@ export interface JobDriver {
    * Optional, like `introspect` and `backfills`: fleet-wide slot counting, which is the only thing
    * that can make `job.concurrency` mean what its docstring says. The in-process limiter is a fast
    * path over ONE heap and is multiplied by the replica count; this is the gate that is not.
-   * A driver without one can only hold the cap per process, so `createWorker().start()` THROWS
+   * A driver without one can only hold the cap per process, so `jobWorker().start()` THROWS
    * `X_JOB_CONCURRENCY_UNENFORCEABLE` (`worker.ts`) naming every registered job that declared
    * `concurrency` — refused rather than logged, because a documented guarantee that silently does
    * nothing is worse than either alternative.
@@ -316,7 +346,7 @@ export const DEFAULT_VISIBILITY_TIMEOUT_MS = 30_000;
 
 let ambient: JobDriver | undefined;
 
-/** Set once at boot from `app.config.ts`. Roles share one driver instance per process. */
+/** Set once at boot over the database binding, never a config key. One instance per process. */
 export function setJobDriver(driver: JobDriver): void {
   ambient = driver;
 }

package/src/errors-concurrency.ts

@@ -85,7 +85,7 @@ export class ConcurrencyUnenforceableError extends UltimateError {
     super({
       code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
       cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
-      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(createPgDriver()) at boot — the pg driver is the one with a lease store`,
+      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(postgresJobDriver()) at boot — the pg driver is the one with a lease store`,
     });
   }
 }

package/src/errors-operator.ts

@@ -77,14 +77,22 @@ export class JobOnSettledFailedError extends UltimateError {
  * walks past it. `MAX_JOB_PAGE` is passed in — `introspection.ts` owns it and imports this file.
  */
 export class JobPageInvalidError extends UltimateError {
-  constructor(input: { subject: string; maxPage: number; limit?: number }) {
+  constructor(input: {
+    subject: string;
+    maxPage: number;
+    limit?: number;
+    /** Which cursor was refused. Default `after`. */
+    cursor?: 'after' | 'before';
+  }) {
+    const cursor = input.cursor ?? 'after';
+    const source = cursor === 'after' ? 'jobCursor(lastRow)' : 'jobCursor(firstRow)';
     super({
       code: 'X_JOB_PAGE_INVALID',
       cause:
         input.limit === undefined
-          ? `${input.subject} was handed an after cursor that no previous page produced`
-          : `${input.subject} limit is ${input.limit}, over MAX_JOB_PAGE (${input.maxPage}) — the rows one page may answer`,
-      fix: `list({ limit: ${input.maxPage}, after: jobCursor(lastRow) })   # lastRow is the last row of the page before; omit after for the first page, and stop at a page shorter than its limit`,
+          ? `${input.subject} was handed ${cursor === 'after' ? 'an after' : 'a before'} cursor that no page produced — ${cursor}: ${source} of a page this driver answered is the only one it reads; omit it for the first page`
+          : `${input.subject} limit is ${input.limit}, over MAX_JOB_PAGE (${input.maxPage}) — the rows one page may answer; walk further with after: jobCursor(lastRow), and stop at a page shorter than its limit`,
+      fix: `x jobs ls --limit ${renderFixShellArg(String(input.maxPage), '200')} --json`,
       meta: {
         maxPage: input.maxPage,
         ...(input.limit === undefined ? {} : { limit: input.limit }),

package/src/errors-requeue.ts

@@ -1,7 +1,8 @@
-// Single responsibility: the two refusals `requeue` — `x jobs retry` — gives before it moves a
-// row. Its own module because `errors.ts` sits at the size ceiling; the codes are registered there.
+// Single responsibility: the three refusals `requeue` — `x jobs retry` — gives before it moves a
+// row, raised identically by `driver-memory.ts` and `driver-pg.ts`: one answer for both drivers is
+// the point, and neither driver imports the other. The codes are registered in `errors.ts`.
 
-import { renderFixShellArg, UltimateError } from '@ultimat3/core';
+import { renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
 import { JobDuplicateError } from './errors';
 
 /**
@@ -19,6 +20,23 @@ export class JobNotRequeueableError extends UltimateError {
   }
 }
 
+/**
+ * `requeue` of an id the queue does not hold. ONE answer for both drivers: memory raised
+ * `X_INVARIANT` and pg `X_DRIVER_UNAVAILABLE` — a driver that is perfectly available — so a caller
+ * could not tell a mistyped id from an outage, and tested against one answer on `x dev` and met
+ * the other in production.
+ */
+export class JobNotFoundError extends UltimateError {
+  constructor(input: { jobId: string; driver: string }) {
+    super({
+      code: 'X_JOB_NOT_FOUND',
+      cause: `the "${input.driver}" queue holds no job with id ${renderCauseValue(input.jobId)} — a mistyped id, a job removed since, or an id from another environment`,
+      fix: 'x jobs ls --json',
+      meta: { jobId: input.jobId },
+    });
+  }
+}
+
 /** The error a requeue gives when a LIVE job already holds the requeued row's key. */
 export const requeueKeyTaken = (record: {
   readonly id: string;

package/src/errors.ts

@@ -43,12 +43,13 @@ export const JOB_OWNED_ERROR_CODES = [
   'X_JOB_NOT_PROMOTABLE',
   'X_JOB_PAGE_INVALID',
   'X_JOB_ON_SETTLED_FAILED',
+  'X_JOB_NOT_FOUND',
 ] as const;
 
 /**
- * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. `JobsNotImplementedError`,
- * `JobAbortedError` and `JobDrainedError` below throw them; jobs keeps no title for any of the
- * three, because the copy this file used to hold was a second title that nothing would have failed
+ * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. The first is thrown as
+ * core's own `NotImplementedError` (`inspect.ts`), the other two by
+ * `JobAbortedError` and `JobDrainedError` below; jobs keeps no title for any of the three, because the copy this file used to hold was a second title that nothing would have failed
  * on once core's changed. Listed here all the same, so `JobErrorCode` can name every code a job
  * can see — `X_DRAINING` was thrown for a day before it was, and the type said it could not be.
  */
@@ -100,6 +101,7 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
   X_JOB_NOT_PROMOTABLE: 'the job is not waiting on its run time',
   X_JOB_PAGE_INVALID: 'a job list page was asked for outside its bounds',
   X_JOB_ON_SETTLED_FAILED: 'the onSettled hook failed after the run was settled',
+  X_JOB_NOT_FOUND: 'the queue holds no job with this id',
 };
 
 // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -432,13 +434,13 @@ export class JobNotCancellableError extends UltimateError {
   }
 }
 
-/** The driver has no `introspect.cancel`. The redis/nats stubs, and any hand-rolled driver. */
+/** The driver has no `introspect.cancel` — a hand-rolled driver that ships no introspection. */
 export class CancelUnsupportedError extends UltimateError {
   constructor(input: { driver: string }) {
     super({
       code: 'X_JOB_NOT_CANCELLABLE',
       cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
-      fix: 'call setJobDriver(createPgDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
+      fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
     });
   }
 }
@@ -454,22 +456,12 @@ export class OutboxNoTxError extends UltimateError {
   }
 }
 
-export class JobsNotImplementedError extends UltimateError {
-  constructor(input: { feature: string; fix: string }) {
-    super({
-      code: 'X_NOT_IMPLEMENTED',
-      cause: `${input.feature} is declared but not implemented in @ultimat3/jobs`,
-      fix: input.fix,
-    });
-  }
-}
-
 /**
  * `claim({ queues: [] })`, refused by every driver rather than answered by each.
  *
  * The two shipped drivers had two meanings for it — every queue on `driver-memory.ts`, the
  * `default` queue on `driver-pg.ts` — and `ClaimOptions.queues` documented neither. Nothing reached
- * it (`createWorker` passes exactly one queue per pass), so the divergence could only ever be found
+ * it (`jobWorker` passes exactly one queue per pass), so the divergence could only ever be found
  * by an embedder in production, and each meaning is silently wrong in the other's deployment: one
  * takes work this worker was never configured for, the other drains nothing and reads as an idle
  * queue. There is no third meaning to pick — an empty list is a caller that has not said what to
@@ -480,7 +472,7 @@ export class ClaimQueuesEmptyError extends UltimateError {
     super({
       code: 'X_JOB_CLAIM_QUEUES_EMPTY',
       cause: `the ${driver} driver was asked to claim with queues: [] — an empty list names no queue, and "every queue" and "the default queue" are different deployments`,
-      fix: "pass the queue by name — driver.claim({ queues: ['default'], limit, visibilityTimeoutMs, workerId }) — or read the list from config.jobs.queues, which is what createWorker walks one queue at a time",
+      fix: "pass the queue by name — driver.claim({ queues: ['default'], limit, visibilityTimeoutMs, workerId }) — or read the list from config.jobs.queues, which is what jobWorker walks one queue at a time",
       meta: { driver },
     });
   }