@ultimat3/jobs 2.0.0 → 4.0.0

package/src/errors.ts

@@ -1,6 +1,6 @@
 // The X_* codes owned by @ultimat3/jobs. Every one names the command or code change that
 // fixes it — a job failure an agent cannot act on is a job failure that gets retried forever.
-import { registerErrorCodes, UltimateError } from '@ultimat3/core';
+import { registerErrorCodes, registerErrorRetry, UltimateError } from '@ultimat3/core';
 
 /** Codes this package declares and owns. */
 export const JOB_OWNED_ERROR_CODES = [
@@ -23,6 +23,8 @@ export const JOB_OWNED_ERROR_CODES = [
   'X_BACKFILL_RUNNING',
   'X_BACKFILL_STALLED',
   'X_BACKFILL_UNKNOWN',
+  'X_JOB_ROW_STATUS_UNKNOWN',
+  'X_ACTION_JOB_UNBRIDGED',
 ] as const;
 
 /**
@@ -58,6 +60,8 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
   X_BACKFILL_RUNNING: 'a pass under this name is already live',
   X_BACKFILL_STALLED: 'the sweep ended with rows its own count still matches',
   X_BACKFILL_UNKNOWN: 'no declaration carries this backfill name',
+  X_JOB_ROW_STATUS_UNKNOWN: 'a queue row carries a status this build does not know',
+  X_ACTION_JOB_UNBRIDGED: 'an action projection was registered as a job',
 };
 
 // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -66,7 +70,31 @@ registerErrorCodes(
   Object.fromEntries(Object.entries(JOB_ERROR_TITLES).map(([code, title]) => [code, { title }])),
 );
 
-const docsFor = (code: JobErrorCode): string => `https://ultimate.dev/errors/${code}`;
+/**
+ * The codes of this package's that can be thrown INSIDE a job body, classified — `executeJob`
+ * reads this, so a `terminal` one dead-letters on the attempt it happened instead of spending the
+ * whole policy on an answer that cannot change. Same rule every package uses: retryable means the
+ * same code, run again, has a real chance of a different answer.
+ *
+ * Two are deliberately absent. `X_JOB_LEASE_LOST` and `X_JOB_SLOT_LOST` mean the row is somebody
+ * else's now, so this attempt's verdict is not this attempt's to give: dead-lettering would settle
+ * a job another worker is running. They keep the attempt-count path, which ends in the queue
+ * re-delivering — the honest outcome for "we stopped owning it".
+ */
+registerErrorRetry({
+  X_JOB_TIMEOUT: 'retryable',
+  X_DRIVER_UNAVAILABLE: 'retryable',
+  // A second `step.run` under one name is a defect in the handler, replayed identically forever.
+  X_STEP_DUPLICATE: 'terminal',
+  // A sweep whose source ran dry while its own count still matches rows: the next attempt resumes
+  // at the cursor that just ran dry and diverges again.
+  X_BACKFILL_STALLED: 'terminal',
+  X_BACKFILL_ENVIRONMENT: 'terminal',
+  X_BACKFILL_APPLIED: 'terminal',
+});
+
+/** Shared with `backfill-errors.ts`, which holds the seven `X_BACKFILL_*` classes. */
+export const docsFor = (code: JobErrorCode): string => `https://ultimate.dev/errors/${code}`;
 
 /** An enqueue collided with a live job holding the same idempotency key under `onConflict: 'error'`. */
 export class JobDuplicateError extends UltimateError {
@@ -101,6 +129,57 @@ export class JobNameTakenError extends UltimateError {
   }
 }
 
+/**
+ * A `text` status column holds a value outside the vocabulary this build compiled.
+ *
+ * Refused rather than passed through, because the alternative is what used to happen: the three
+ * decoders in `driver-pg-rows.ts` cast the column, and `stepRun`'s `existing?.status ===
+ * 'completed'` then read false for the laundered value and RE-EXECUTED the step. A second charge
+ * is a worse answer than a failed attempt, and "an unrecognised fact is never a satisfied one" is
+ * the rule the rest of the framework already follows.
+ *
+ * Almost always a NEWER deploy's row, not corruption: a status string only reaches the table
+ * because some version of this framework wrote it. A rolling deploy that only ADDS a status is
+ * safe in the normal direction — the new build knows every old value — and it is the old build
+ * reading the new build's row that lands here, on that one job, loudly.
+ */
+export class JobRowStatusUnknownError extends UltimateError {
+  constructor(input: { table: string; column: string; value: string; known: readonly string[] }) {
+    super({
+      code: 'X_JOB_ROW_STATUS_UNKNOWN',
+      cause:
+        `${input.table}.${input.column} holds "${input.value}", which this build does not know — ` +
+        `it reads ${input.known.join(', ')}`,
+      fix: `x jobs show --json   # then drain the older workers: a status this build cannot read was almost certainly written by a newer deploy`,
+      docs: docsFor('X_JOB_ROW_STATUS_UNKNOWN'),
+    });
+  }
+}
+
+/**
+ * `registerJobs()` was handed `someAction.job()`.
+ *
+ * That call answers an `ActionJobHandle` — `kind: 'action-job'`, deliberately a different literal
+ * from `'job'` — which is the four fields `job()` takes, not a job. It cannot be one: `action` and
+ * `jobs` are both tier 3, so neither may import the other, and only `job()` seats a handle the
+ * queue, the worker and the manifest accept.
+ *
+ * Refused BY NAME rather than skipped, which is what used to happen. `registerJobs(module)` is
+ * handed a whole module namespace, so silently ignoring a constant or a helper exported beside a
+ * job is right — but ignoring this one meant `registerJobs({ publishPost: publishPost.job() })`
+ * registered nothing, returned `[]`, and the job never ran, with nothing failing anywhere.
+ */
+export class ActionJobUnbridgedError extends UltimateError {
+  constructor(input: { export: string; job: string }) {
+    super({
+      code: 'X_ACTION_JOB_UNBRIDGED',
+      cause: `export "${input.export}" is the action projection "${input.job}", which is not a job handle and cannot be registered as one`,
+      fix: `wrap it: agentJob(${input.export}, { name: '${input.export}', tenant, retry }) from @ultimat3/ai — that composes job() and returns a handle the queue accepts`,
+      docs: docsFor('X_ACTION_JOB_UNBRIDGED'),
+    });
+  }
+}
+
 /** Two steps in one run share a name, so replay cannot tell their persisted results apart. */
 export class StepDuplicateError extends UltimateError {
   constructor(input: { job: string; step: string }) {
@@ -311,134 +390,6 @@ export class OutboxNoTxError extends UltimateError {
   }
 }
 
-/**
- * The seven backfill codes below all answer one question — "why is this sweep not running?" — and
- * each is here because it sends the reader somewhere different: run it, force it, change
- * environment, migrate first, wait, fix the predicates, or fix the name. A code that shared a fix
- * line with another one would be code inflation, which is why `X_BACKFILL_WRITE_UNCONFIRMED` was
- * considered and rejected: a dry run that wrote nothing did exactly what it was asked to.
- *
- * Every `fix` below is ONE line something can execute — a shell command, or the edit to make.
- * Never a command with prose appended: a `fix:` is copied and run verbatim, so a trailing clause
- * turns a working command into a syntax error at the one moment the reader is following it
- * literally. Explanations belong in `cause`, which is read and never run.
- */
-
-/**
- * Declared and never completed. The alarm the framework did not have: an author could
- * `x g backfill`, merge and deploy, and nothing anywhere said the pass had not run.
- */
-export class BackfillPendingError extends UltimateError {
-  constructor(input: { backfill: string; environment: string }) {
-    super({
-      code: 'X_BACKFILL_PENDING',
-      cause: `backfill "${input.backfill}" is declared and x_backfills holds no completed pass for it in ${input.environment}`,
-      fix: `x db backfill ${input.backfill} --write --json`,
-      docs: docsFor('X_BACKFILL_PENDING'),
-    });
-  }
-}
-
-/** Already swept. A rerun is legitimate, so this names the flag rather than refusing outright. */
-export class BackfillAppliedError extends UltimateError {
-  constructor(input: { backfill: string; runId: string; completedAt: string }) {
-    super({
-      code: 'X_BACKFILL_APPLIED',
-      cause: `backfill "${input.backfill}" completed as run ${input.runId} at ${input.completedAt}; a forced rerun writes a NEW ledger row and never edits that one`,
-      fix: `x db backfill ${input.backfill} --write --force --json`,
-      docs: docsFor('X_BACKFILL_APPLIED'),
-    });
-  }
-}
-
-/**
- * The declaration names the environments it belongs to and this is not one. Declared DATA, never a
- * hardcoded "cleanups are production": a staging rehearsal is correct practice, so which
- * environments a sweep belongs to is the app's convention and this is only the mechanism carrying
- * it (axiom 8).
- */
-export class BackfillEnvironmentError extends UltimateError {
-  constructor(input: { backfill: string; environment: string; declared: readonly string[] }) {
-    // The first declared environment, because the fix has to be ONE runnable line and the list is
-    // ordered by the author. The empty case cannot arise from `checkBackfillEnvironment`, which
-    // treats an empty list as "every environment" — but this constructor is public, so it answers
-    // with the command that lists what IS declared rather than an `ULTIMATE_ENV=undefined`.
-    const target = input.declared[0];
-    super({
-      code: 'X_BACKFILL_ENVIRONMENT',
-      cause: `backfill "${input.backfill}" declares environments: ${input.declared.join(', ')} and this process resolved ${input.environment} — add "${input.environment}" to that list if this deploy should sweep too`,
-      fix:
-        target === undefined
-          ? 'x db backfill --pending --json'
-          : `ULTIMATE_ENV=${target} x db backfill ${input.backfill} --write --json`,
-      docs: docsFor('X_BACKFILL_ENVIRONMENT'),
-    });
-  }
-}
-
-/**
- * `requires` names a migration the ledger has not applied. Checked where `x_migrations` is
- * readable — this package holds no `@ultimat3/db` dependency and growing one to read a ledger
- * would put the migration engine on the tier-3 queue's import graph.
- */
-export class BackfillMigrationPendingError extends UltimateError {
-  constructor(input: { backfill: string; migration: string }) {
-    super({
-      code: 'X_BACKFILL_MIGRATION_PENDING',
-      cause: `backfill "${input.backfill}" requires migration ${input.migration}, which x_migrations does not record as applied`,
-      fix: 'x db migrate --json',
-      docs: docsFor('X_BACKFILL_MIGRATION_PENDING'),
-    });
-  }
-}
-
-/**
- * The enqueue deduped: one live pass per name, so this run is the one already going. Distinct from
- * `X_BACKFILL_APPLIED`, which is a pass that finished — the response there is `--force`, and the
- * response here is to look at the run that is holding the key.
- */
-export class BackfillRunningError extends UltimateError {
-  constructor(input: { backfill: string; jobId: string }) {
-    super({
-      code: 'X_BACKFILL_RUNNING',
-      cause: `backfill "${input.backfill}" already has a live pass queued as ${input.jobId}, and one name holds one live pass; its step trace names the batch it is on, and a pass that is not advancing is a worker that lost its lease`,
-      fix: `x jobs show ${input.jobId} --json`,
-      docs: docsFor('X_BACKFILL_RUNNING'),
-    });
-  }
-}
-
-/**
- * The source ran out of rows and the declaration's own `count()` still matches some. Two
- * predicates that disagree is an authoring bug in any business — the sweep reported success over
- * rows it never visited — so the pass fails rather than writing a completed row nobody can trust.
- */
-export class BackfillStalledError extends UltimateError {
-  constructor(input: { backfill: string; remaining: number; swept: number }) {
-    super({
-      code: 'X_BACKFILL_STALLED',
-      cause: `backfill "${input.backfill}" swept ${input.swept} rows, exhausted its source, and count() still matches ${input.remaining} — a WHERE the sweep narrows and the count does not is what leaves rows behind`,
-      fix: `make count() select on exactly what source() selects on in backfill("${input.backfill}")`,
-      docs: docsFor('X_BACKFILL_STALLED'),
-    });
-  }
-}
-
-/** A name no declaration in this app carries — a typo, or a backfill whose module was deleted. */
-export class BackfillUnknownError extends UltimateError {
-  constructor(input: { backfill: string; known: readonly string[] }) {
-    super({
-      code: 'X_BACKFILL_UNKNOWN',
-      cause:
-        input.known.length === 0
-          ? `no backfill named "${input.backfill}" is declared, and this app declares none at all`
-          : `no backfill named "${input.backfill}" is declared (declared: ${input.known.join(', ')})`,
-      fix: 'x db backfill --pending --json',
-      docs: docsFor('X_BACKFILL_UNKNOWN'),
-    });
-  }
-}
-
 export class JobsNotImplementedError extends UltimateError {
   constructor(input: { feature: string; fix: string }) {
     super({

package/src/execute.ts

@@ -1,5 +1,7 @@
 // One claimed job run to completion, suspension or failure and settled with the driver — the
-// single execution path the worker loop and `x jobs run` share. It owns the run's deadline, and
+// single execution path, shared by the worker loop (`worker-run.ts`) and by the one caller outside
+// this package, `@ultimat3/testing`'s job fixture. There is no `x jobs run` to share it with: the
+// subcommands are `ls`, `show`, `retry`, `cancel`, `drain`. It owns the run's deadline, and
 // a deadline here means CANCEL: the nack that follows makes the job claimable again, so a body
 // still running past it would be a second copy of one job, racing the attempt that replaced it.
 
@@ -18,7 +20,8 @@ import type { ClaimedJob, JobDriver } from './driver';
 import { JobAbortedError, JobTimeoutError } from './errors';
 import { eventBus } from './events';
 import type { AnyJobHandle } from './job';
-import { nextRetry } from './retry';
+import type { JobStopReason } from './retry-classification';
+import { nextRetryForError, recordedFailure } from './retry-classification';
 import type { EventLookup, StepRecord } from './steps';
 import { createStepRunner, isStepSuspension } from './steps';
 import { jobRunActor } from './tenant';
@@ -69,6 +72,8 @@ export interface JobExecution {
   readonly durationMs: number;
   readonly resumeAt?: number;
   readonly error?: string;
+  /** Why this attempt was the last. Absent while the job is still being retried. */
+  readonly stopReason?: JobStopReason;
   readonly steps: readonly StepRecord[];
   readonly replayed: readonly string[];
 }
@@ -84,7 +89,8 @@ export interface ExecuteJobOptions {
 
 /**
  * Run one claimed job to completion, suspension or failure, and settle it with the driver.
- * Shared by the worker loop and `x jobs run` so both take exactly the same code path.
+ * Shared by the worker loop and by `@ultimat3/testing`'s job fixture, so a job under test takes
+ * exactly the code path the worker takes.
  */
 export async function executeJob(options: ExecuteJobOptions): Promise<JobExecution> {
   const { driver, claimed, handle } = options;
@@ -105,6 +111,11 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     jobName: handle.name,
     store: driver.steps,
     signal,
+    // The DECLARED per-step ceiling and event poll. Passed here or nowhere: this is the only
+    // production construction of a runner, so a `StepRunnerOptions` field it omits is a feature
+    // no `job()` can reach — which both of these were until 2026-08.
+    ...(handle.stepTimeoutMs === undefined ? {} : { stepTimeoutMs: handle.stepTimeoutMs }),
+    ...(handle.eventPollMs === undefined ? {} : { eventPollMs: handle.eventPollMs }),
     ...(options.clock === undefined ? {} : { clock: options.clock }),
     events: options.events ?? eventBus(),
   });
@@ -151,8 +162,10 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
   } catch (error) {
     if (isStepSuspension(error)) {
       const delayMs = Math.max(0, error.resumeAt - nowMs(options.clock));
-      // countsAsAttempt: false — parking a run is not a failure.
-      await driver.nack(claimed.id, { delayMs, countsAsAttempt: false });
+      // `park: true` is the suspension itself — the row leaves the ready bucket — and
+      // `countsAsAttempt: false` only says not to burn an attempt on it. A limiter shed passes the
+      // second and not the first: it is a job still waiting, and it belongs in `queue_depth`.
+      await driver.nack(claimed.id, { delayMs, countsAsAttempt: false, park: true });
       return settle({
         outcome: 'suspended',
         jobId: claimed.id,
@@ -166,10 +179,16 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     }
 
     const message = error instanceof Error ? error.message : String(error);
-    const decision = nextRetry(handle.retry, claimed.attempt);
+    // The ERROR decides too, not only the attempt count. A `terminal` code — a rotated password,
+    // a schema mismatch, a permission denial — fails identically on every remaining attempt, so
+    // spending them is a queue slot, a provider bill and, at a site that locks an account after
+    // three wrong passwords, the framework destroying what it was asked to read. A code nobody
+    // classified keeps the attempt-count path exactly as it was.
+    const decision = nextRetryForError(handle.retry, claimed.attempt, error);
+    const stop = decision.stoppedBy;
     await driver.nack(claimed.id, {
       delayMs: decision.delayMs,
-      error: message,
+      error: recordedFailure(message, decision),
       countsAsAttempt: true,
       deadLetter: !decision.retry && decision.deadLetter,
     });
@@ -178,13 +197,16 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
       jobId: claimed.id,
       attempt: claimed.attempt,
       retry: decision.retry,
+      // "stopped because terminal" and "stopped because the attempts ran out" are different
+      // incidents with the same `retry: false`, and only one of them is fixed by raising attempts.
+      ...(stop === undefined ? {} : { stop }),
       error: message,
     });
     // This package's ONE error-reporting call site, and it is here rather than in the loop because
     // this is the only frame that still holds the thrown value — the loop sees a message string.
     // A retry is a failure the framework recovered from, so it is a `warning`; a dead letter is
-    // one nobody recovered from. `x jobs run` takes this path too, which is the point: one
-    // execution path means one place a failed job can become visible.
+    // one nobody recovered from. A job driven by `@ultimat3/testing`'s fixture takes this path
+    // too, which is the point: one execution path means one place a failed job becomes visible.
     reportError(error, {
       source: 'job',
       severity: decision.retry ? 'warning' : 'error',
@@ -195,6 +217,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
           runId: claimed.runId,
           attempt: claimed.attempt,
           retry: decision.retry,
+          ...(stop === undefined ? {} : { stop }),
         },
       },
     });
@@ -205,6 +228,7 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
       attempt: claimed.attempt,
       durationMs: nowMs(options.clock) - startedAt,
       error: message,
+      ...(stop === undefined ? {} : { stopReason: stop }),
       steps: [],
       replayed: [],
     });

package/src/heartbeat.ts

@@ -8,6 +8,7 @@ import { logger, recordLeaseLost } from '@ultimat3/core';
 import { nowMs } from './clock';
 import type { ClaimedJob, JobDriver } from './driver';
 import { LeaseLostError } from './errors';
+import { startRenewalTimer } from './renewal-timer';
 
 export interface LeaseHeartbeatOptions {
   /** Only `heartbeat` is used — a lease renews itself and settles nothing. */
@@ -49,14 +50,8 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
   let renewedAt = now();
   let renewing = false;
   let lost = false;
-  let timer: ReturnType<typeof setInterval> | undefined;
   const gone = new AbortController();
 
-  const stop = (): void => {
-    if (timer !== undefined) clearInterval(timer);
-    timer = undefined;
-  };
-
   const lapsed = (): boolean => now() - renewedAt >= visibilityTimeoutMs;
 
   /**
@@ -65,9 +60,14 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
    * ticked once per interval would count intervals, not jobs.
    */
   const reportLost = (error?: unknown, reason?: 'expired' | 'not-ours'): void => {
-    if (lost) return;
+    // `stopped()` as well as `lost`, and it is the difference between a page and a fact: a clean
+    // completion acks the row out of `running` and `worker-run.ts` stops the heartbeat, so a
+    // renewal already in flight comes back `false` for a job that FINISHED. Reported, that is
+    // `jobs.lease.lost` at error plus `recordLeaseLost(queue)` — the one signal meaning the queue
+    // re-delivered a job this process was still running — raised for a run nobody re-delivered.
+    if (lost || timer.stopped()) return;
     lost = true;
-    stop();
+    timer.stop();
     logger.error('jobs.lease.lost', {
       workerId,
       job: claimed.name,
@@ -86,7 +86,7 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
   };
 
   const renew = async (): Promise<void> => {
-    if (lost) return;
+    if (lost || timer.stopped()) return;
     // Expiry is decided BEFORE the driver is asked, because the failure that loses a lease most
     // quietly is the one that never answers: a heartbeat hung on a dead connection neither
     // resolves nor rejects, so a check that ran only on rejection would never run at all.
@@ -100,6 +100,10 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
     renewing = true;
     try {
       const held = await options.driver.heartbeat(claimed.id, { visibilityTimeoutMs, workerId });
+      // Re-read AFTER the await, never only before it: the whole point of the flag is the answer
+      // that lands past `stop()`. Every branch below decides something about a lease this process
+      // may no longer be running under.
+      if (timer.stopped()) return;
       // The driver answered, and it said the row is not ours. That is a DIFFERENT fact from an
       // expired window and the only one an operator can cause on purpose: `x jobs cancel` writes
       // a terminal state, and the renewal that misses it is what tells this attempt to stop. It
@@ -138,9 +142,7 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
     }
   };
 
-  timer = setInterval(() => {
-    void renew();
-  }, options.intervalMs);
+  const timer = startRenewalTimer(options.intervalMs, renew);
 
-  return { renew, lost: () => lost, signal: gone.signal, stop };
+  return { renew, lost: () => lost, signal: gone.signal, stop: () => timer.stop() };
 }

package/src/index.ts

@@ -17,6 +17,15 @@ export type {
   BackfillReport,
 } from './backfill';
 export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
+export {
+  BackfillAppliedError,
+  BackfillEnvironmentError,
+  BackfillMigrationPendingError,
+  BackfillPendingError,
+  BackfillRunningError,
+  BackfillStalledError,
+  BackfillUnknownError,
+} from './backfill-errors';
 export type { BackfillGate, BackfillGateInput } from './backfill-gate';
 export { checkBackfillEnvironment, gateBackfill } from './backfill-gate';
 export type { BackfillProgress } from './backfill-inspect';
@@ -77,11 +86,13 @@ export type {
 export {
   DEFAULT_QUEUE,
   DEFAULT_VISIBILITY_TIMEOUT_MS,
+  isJobState,
+  JOB_STATES,
   jobDriver,
   resetJobDriver,
   setJobDriver,
 } from './driver';
-export type { MemoryDriverOptions } from './driver-memory';
+export type { MemoryDriverOptions, MemoryJobDriver } from './driver-memory';
 export { createMemoryDriver } from './driver-memory';
 export type { NatsDriverOptions } from './driver-nats';
 export { createNatsDriver } from './driver-nats';
@@ -107,6 +118,7 @@ export {
   SQL_NACK,
   SQL_OUTBOX_CLAIM,
   SQL_OUTBOX_MARK_PUBLISHED,
+  SQL_OUTBOX_RELEASE,
   SQL_OUTBOX_STAGE,
   SQL_OUTBOX_TABLE,
   SQL_SCHEDULER_STATE_GET,
@@ -120,13 +132,7 @@ export type { RedisDriverOptions } from './driver-redis';
 export { createRedisDriver } from './driver-redis';
 export type { JobErrorCode } from './errors';
 export {
-  BackfillAppliedError,
-  BackfillEnvironmentError,
-  BackfillMigrationPendingError,
-  BackfillPendingError,
-  BackfillRunningError,
-  BackfillStalledError,
-  BackfillUnknownError,
+  ActionJobUnbridgedError,
   CancelUnsupportedError,
   ConcurrencyUnenforceableError,
   DriverUnavailableError,
@@ -138,6 +144,7 @@ export {
   JobMaxAttemptsError,
   JobNameTakenError,
   JobNotCancellableError,
+  JobRowStatusUnknownError,
   JobSlotLostError,
   JobsNotImplementedError,
   JobTenantRequiredError,
@@ -191,6 +198,7 @@ export {
 export type {
   EnqueueOptions,
   JobsFacade,
+  MemoryOutboxOptions,
   MemoryOutboxStore,
   OutboxDeps,
   OutboxRecord,
@@ -207,11 +215,16 @@ export {
   resetJobsFacade,
   setJobsFacade,
 } from './outbox';
+// One definition of the lease, consumed by both stores — a memory default and a pg default that
+// could drift are two answers to "how long is a claim mine for", and the shorter one duplicates.
+export { DEFAULT_OUTBOX_CLAIM_LEASE_MS } from './outbox-lease';
 export type { PgOutboxOptions } from './outbox-pg';
 export { createPgOutboxStore } from './outbox-pg';
 
 export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
 export { backoffDelayMs, DEFAULT_RETRY, nextRetry, retrySchedule } from './retry';
+export type { JobRetryDecision, JobStopReason } from './retry-classification';
+export { classifyThrown, nextRetryForError } from './retry-classification';
 export type {
   CronResolver,
   DispatchedOccurrence,
@@ -241,8 +254,10 @@ export type {
 export {
   createMemoryStepStore,
   createStepRunner,
+  isStepStatus,
   isStepSuspension,
   MAX_TRACE_NAMES,
+  STEP_STATUSES,
   StepSuspension,
 } from './steps';
 export type {

package/src/job.ts

@@ -80,6 +80,22 @@ export interface JobDefinition<I> {
    */
   readonly concurrency?: number;
   readonly timeout?: DurationInput;
+  /**
+   * Ceiling for ONE `step.run`, where `timeout` is the ceiling for the whole attempt. Folded into
+   * the signal the step body is handed, so a body reads one signal and sees whichever deadline
+   * lands first — and it ABORTS before it rejects, because the attempt that replaces this one is
+   * claimable the moment the nack lands.
+   *
+   * Declared here and nowhere else: the runner has implemented this ceiling since 1.0 and no
+   * declaration could ask for it, which is a documented guarantee that does nothing.
+   */
+  readonly stepTimeout?: DurationInput;
+  /**
+   * How long a `step.waitForEvent` parks between polls. Default 30s. Lower it for a wait a user
+   * is watching; the step suspends for exactly this long each time, so it is also the resolution
+   * of the resume, never a busy loop.
+   */
+  readonly eventPoll?: DurationInput;
   run(args: JobRunArgs<I>): Promise<unknown>;
 }
 
@@ -114,6 +130,10 @@ export interface JobHandle<I = unknown> {
   readonly retry: RetryPolicy;
   readonly concurrency: number | undefined;
   readonly timeoutMs: number | undefined;
+  /** The declared per-step ceiling in ms; `executeJob` hands it to the step runner. */
+  readonly stepTimeoutMs: number | undefined;
+  /** The declared event-poll interval in ms; `undefined` leaves the runner's 30s default. */
+  readonly eventPollMs: number | undefined;
   readonly input: StandardSchemaV1<unknown, I>;
   parse(raw: unknown): I;
   idempotencyKeyFor(input: I): string;
@@ -183,6 +203,28 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
     `set a whole concurrency of 1 or more on job("${name}"), or omit the field for no cap at all`,
   );
 
+  const stepTimeoutMs =
+    definition.stepTimeout === undefined ? undefined : toMs(definition.stepTimeout);
+  const eventPollMs = definition.eventPoll === undefined ? undefined : toMs(definition.eventPoll);
+  // `withStepTimeout` reads `<= 0` as "no ceiling at all" and a poll of zero is a suspension that
+  // resumes immediately, forever. Both are an author who asked for a limit and got the opposite,
+  // so they are refused where they are written — the same answer `concurrency: 0` gets.
+  //
+  // FINITE, not merely positive: `> 0` admits `Infinity`, which is the same defect spelled the
+  // other way. `eventPoll: Infinity` parks a waiting step and schedules the poll that would wake
+  // it for never; `stepTimeout: Infinity` is a ceiling no step can reach. `NaN` fails `> 0` on its
+  // own, and is covered here so the predicate says what it means rather than passing by accident.
+  assert(
+    stepTimeoutMs === undefined || (Number.isFinite(stepTimeoutMs) && stepTimeoutMs > 0),
+    `job "${name}" declares stepTimeout ${String(definition.stepTimeout)}, which is no ceiling at all`,
+    `set a finite positive stepTimeout on job("${name}") — "30s" or 30_000 — or omit the field for no per-step ceiling`,
+  );
+  assert(
+    eventPollMs === undefined || (Number.isFinite(eventPollMs) && eventPollMs > 0),
+    `job "${name}" declares eventPoll ${String(definition.eventPoll)}, which parks a waiting step for no time at all`,
+    `set a finite positive eventPoll on job("${name}") — "5s" or 5_000 — or omit the field for the 30s default`,
+  );
+
   const handle: JobHandle<I> = {
     kind: 'job',
     name,
@@ -190,6 +232,8 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
     retry: { ...DEFAULT_RETRY, ...definition.retry },
     concurrency: definition.concurrency,
     timeoutMs: definition.timeout === undefined ? undefined : toMs(definition.timeout),
+    stepTimeoutMs,
+    eventPollMs,
     input: definition.input,
     parse(raw: unknown): I {
       return parse(definition.input, raw) as I;
@@ -311,8 +355,18 @@ export function getJob(name: string): AnyJobHandle | undefined {
   return registry.get(name);
 }
 
+/**
+ * Code-unit compare, never `localeCompare`. This list is projected into `x.manifest.json`, which
+ * both tracked apps COMMIT and `x verify`'s drift step diffs byte for byte — and `localeCompare`
+ * with no locale argument answers from the runtime's ICU default and collation version, so the
+ * same source could sort two ways on two machines. `@ultimat3/http`'s `describeRoutes` states the
+ * same rule; the comparator is restated rather than imported because `http` is not below this
+ * package on the tier table.
+ */
+const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
 export function registeredJobs(): readonly AnyJobHandle[] {
-  return [...registry.values()].sort((a, b) => a.name.localeCompare(b.name));
+  return [...registry.values()].sort((a, b) => byName(a.name, b.name));
 }
 
 export function resetJobs(): void {

package/src/metrics.ts

@@ -18,7 +18,7 @@ import { gauge } from '@ultimat3/core';
 /** Seconds and not milliseconds: every Prometheus duration is seconds, and the alert is `> 300`. */
 export const queueOldestReady: Gauge = gauge('queue_oldest_ready_seconds', {
   unit: 's',
-  description: 'Age of the oldest claimable job, by queue — 0 when the queue is empty',
+  description: 'Age of the oldest job that is ready and due, by queue — 0 when none is',
 });
 
 export const queueDeadJobs: Gauge = gauge('queue_dead_jobs', {

package/src/outbox-lease.ts

@@ -0,0 +1,29 @@
+// The claim lease's one definition and its one normalisation. Both outbox stores read it, because
+// a lease the memory store defaults and the pg store validates is two answers to "how long is a
+// claim mine for" — and the shorter of the two is a row published twice.
+
+import { assert } from '@ultimat3/core';
+
+/**
+ * How long a claimed row stays its claimant's. Long enough that no healthy pass loses a batch it
+ * is still publishing, short enough that a relay killed mid-batch does not strand one for minutes.
+ */
+export const DEFAULT_OUTBOX_CLAIM_LEASE_MS = 30_000;
+
+/**
+ * Refused where it is written, the way `concurrency: 0` and `stepTimeout: 0` are. A lease of `0`
+ * expires before `claim()` resolves, so every relay reclaims every row on every tick and the lease
+ * buys nothing; a fractional one is compared against `now()` in Postgres and against whole ms
+ * here; `Infinity` never expires, so the rows of a relay that died are stranded forever — the one
+ * failure the lease exists to bound. `X_INVARIANT` because this is a caller-argument check with no
+ * dedicated code, the generic `@ultimat3/db` already borrows for the same shape.
+ */
+export function resolveClaimLeaseMs(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_OUTBOX_CLAIM_LEASE_MS;
+  assert(
+    Number.isInteger(value) && value > 0,
+    `outbox claimLeaseMs is ${String(value)}, which is not a positive whole number of milliseconds`,
+    'pass a positive whole claimLeaseMs: 30_000 — createPgOutboxStore({ executor, txExecutor, claimLeaseMs: 30_000 }) — or omit the field for the 30s default',
+  );
+  return value;
+}