@ultimat3/jobs 24.0.0 → 25.0.0

package/src/driver-pg.ts

@@ -5,7 +5,7 @@
 // they run against in `driver-pg-ddl.ts`, and the row-to-record decoding in `driver-pg-rows.ts`.
 
 import type { Clock, PgExecutor } from '@ultimat3/core';
-import { finiteCount, renderCauseValue, systemClock, uuid } from '@ultimat3/core';
+import { finiteCount, renderCauseValue, systemClock, uuidV7 } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
 import { listedRunId } from './backfill-ledger';
 import { nowMs } from './clock';
@@ -70,7 +70,7 @@ import type { StepStore } from './steps';
 /** How often `enqueue` sends its insert before it calls a refusal an index fault, not a race. */
 const ENQUEUE_ATTEMPTS = 2;
 
-export interface PgDriverOptions {
+export interface PostgresJobDriverOptions {
   readonly executor?: PgExecutor;
   readonly clock?: Clock;
 }
@@ -81,12 +81,13 @@ function resolveExecutor(injected: PgExecutor | undefined): PgExecutor {
     driver: 'pg',
     // `Bun.sql` is named nowhere in this function and never was: there is no ambient fallback to
     // be "not configured". An executor is injected by the boot or the driver has none.
-    cause: 'createPgDriver() was called with no executor, and this driver has no ambient fallback',
-    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(createMemoryDriver())',
+    cause:
+      'postgresJobDriver() was called with no executor, and this driver has no ambient fallback',
+    fix: 'set DATABASE_URL in .env so the boot builds one — x db migrate then x dev — or hand this process a queue with no database: setJobDriver(memoryJobDriver())',
   });
 }
 
-function pgStepStore(exec: () => PgExecutor): StepStore {
+function postgresStepStore(exec: () => PgExecutor): StepStore {
   return {
     async get(runId, name) {
       const rows = await exec().query<StepRow>(SQL_STEP_GET, [runId, name]);
@@ -128,7 +129,7 @@ function pgStepStore(exec: () => PgExecutor): StepStore {
   };
 }
 
-function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
+function postgresBackfillLedger(exec: () => PgExecutor): BackfillLedger {
   return {
     async start(run) {
       await exec().query(SQL_BACKFILL_START, [run.runId, run.name, run.checksum, run.appVersion]);
@@ -157,7 +158,7 @@ function pgBackfillLedger(exec: () => PgExecutor): BackfillLedger {
  * Fleet-wide slots over `x_job_leases`. Every decision is ONE statement — the `(lease_key, slot)`
  * primary key is what serialises two workers, so nothing here reads a count and then acts on it.
  */
-function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
+function postgresLeaseStore(exec: () => PgExecutor): LeaseStore {
   return {
     async acquire(key, limit, ttlMs, holder) {
       if (limit <= 0) return undefined;
@@ -196,7 +197,7 @@ function pgLeaseStore(exec: () => PgExecutor): LeaseStore {
   };
 }
 
-export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
+export function postgresJobDriver(options: PostgresJobDriverOptions = {}): JobDriver {
   const clock = options.clock ?? systemClock;
   let executor: PgExecutor | undefined;
   const exec = (): PgExecutor => {
@@ -256,20 +257,20 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
 
   return {
     name: 'pg',
-    steps: pgStepStore(exec),
-    backfills: pgBackfillLedger(exec),
-    leases: pgLeaseStore(exec),
+    steps: postgresStepStore(exec),
+    backfills: postgresBackfillLedger(exec),
+    leases: postgresLeaseStore(exec),
     introspect,
 
     async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
       const runAt = request.runAt ?? nowMs(clock);
       const params = [
-        request.id ?? uuid(),
+        request.id ?? uuidV7(),
         request.name,
         request.queue || DEFAULT_QUEUE,
         JSON.stringify(request.input ?? null),
         request.idempotencyKey,
-        request.runId ?? uuid(),
+        request.runId ?? uuidV7(),
         request.maxAttempts,
         runAt,
         request.tenantId ?? null,
@@ -362,7 +363,6 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         by.workerId,
         Math.round(by.durationMs ?? 0),
         by.claim,
-        by.counted !== false,
       ]);
       return rows.length > 0;
     },
@@ -437,14 +437,14 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
  * key would need a second `release()` — hence the `held` guard below, which makes repeated
  * `acquire()` calls (the scheduler renews every round) a no-op rather than a leak.
  *
- * `@ultimat3/realtime`'s `PgAdvisoryLock` is the shape that gets this right: it opens a connection
+ * `@ultimat3/realtime`'s `postgresAdvisoryLock()` is the shape that gets this right: it opens a connection
  * of its own and *is* the lock. This package holds no wire protocol, so the executor it is handed
- * is whatever boot built — which is a pool. **Use `createPgLeaseLeader` instead** unless you can
+ * is whatever boot built — which is a pool. **Use `postgresLeaseLeader` instead** unless you can
  * prove the executor is a single dedicated session.
  */
-export function createPgLeader(
+export function postgresLeader(
   lockKey: number,
-  options: PgDriverOptions = {},
+  options: PostgresJobDriverOptions = {},
 ): { acquire(): Promise<boolean>; release(): Promise<void>; readonly renewEveryMs: number } {
   const exec = (): PgExecutor => resolveExecutor(options.executor);
   let held = false;

package/src/driver.ts

@@ -2,11 +2,11 @@
 // Six methods and no more: claim/ack/nack with a visibility timeout is the smallest set that
 // survives a worker crash.
 //
-// This header used to say "switching backends is a config line" (`As of 2026-08`). There is no
-// such config line: `JobsConfig.driver` has no reader anywhere and boot always builds
-// `createPgDriver`. `pg` and `memory` are the two that exist; `redis` and `nats` are honest
-// `X_NOT_IMPLEMENTED` stubs. What IS true is the second half — swapping the driver is
-// `setJobDriver(other)` and ZERO job-code change — and that is what the interface buys.
+// There is no config line that picks a backend: boot always builds `postgresJobDriver()`, and a
+// stale `jobs.driver` in `app.config.ts` is refused by `defineConfig` (25.0.0; it was read by
+// nothing since 5.0.0). `postgresJobDriver` and `memoryJobDriver` are the two that exist — 25.0.0
+// deleted the all-throw `redis` and `nats` stubs. Swapping the driver is `setJobDriver(other)` and
+// ZERO job-code change, and that is what the interface buys.
 
 import { finiteCount, finiteOption } from '@ultimat3/core';
 import type { BackfillLedger } from './backfill-ledger';
@@ -157,7 +157,7 @@ export interface EnqueueResult {
 export interface ClaimOptions {
   /**
    * The queues this pass may take work from, by name. **At least one, and an empty list is
-   * refused** (`X_JOB_CLAIM_QUEUES_EMPTY`) — see `assertClaimQueues`. `createWorker` passes exactly
+   * refused** (`X_JOB_CLAIM_QUEUES_EMPTY`) — see `assertClaimQueues`. `jobWorker` passes exactly
    * one per pass, which is what keeps a slow queue from starving the others.
    */
   readonly queues: readonly string[];
@@ -221,14 +221,8 @@ export interface SettleBy extends ClaimIdentity {
   readonly durationMs?: number;
 }
 
-export interface AckOptions extends SettleBy {
-  /**
-   * False when the row is finished WITHOUT its body having run here — `x jobs drain` moving it to
-   * another driver. It is settled `done` and adds nothing to the job's counters: a job that was
-   * moved is not a job that completed.
-   */
-  readonly counted?: boolean;
-}
+/** An ack is a settle by the claimer: every ack is a completed run and adds to the counters. */
+export type AckOptions = SettleBy;
 
 export interface NackOptions extends SettleBy {
   /** Delay before the job becomes claimable again. */
@@ -337,7 +331,7 @@ export interface JobDriver {
    * Optional, like `introspect` and `backfills`: fleet-wide slot counting, which is the only thing
    * that can make `job.concurrency` mean what its docstring says. The in-process limiter is a fast
    * path over ONE heap and is multiplied by the replica count; this is the gate that is not.
-   * A driver without one can only hold the cap per process, so `createWorker().start()` THROWS
+   * A driver without one can only hold the cap per process, so `jobWorker().start()` THROWS
    * `X_JOB_CONCURRENCY_UNENFORCEABLE` (`worker.ts`) naming every registered job that declared
    * `concurrency` — refused rather than logged, because a documented guarantee that silently does
    * nothing is worse than either alternative.

package/src/errors-concurrency.ts

@@ -85,7 +85,7 @@ export class ConcurrencyUnenforceableError extends UltimateError {
     super({
       code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
       cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
-      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(createPgDriver()) at boot — the pg driver is the one with a lease store`,
+      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or call setJobDriver(postgresJobDriver()) at boot — the pg driver is the one with a lease store`,
     });
   }
 }

package/src/errors.ts

@@ -48,7 +48,7 @@ export const JOB_OWNED_ERROR_CODES = [
 
 /**
  * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. The first is thrown as
- * core's own `NotImplementedError` (`inspect.ts`, the redis/nats stubs), the other two by
+ * core's own `NotImplementedError` (`inspect.ts`), the other two by
  * `JobAbortedError` and `JobDrainedError` below; jobs keeps no title for any of the three, because the copy this file used to hold was a second title that nothing would have failed
  * on once core's changed. Listed here all the same, so `JobErrorCode` can name every code a job
  * can see — `X_DRAINING` was thrown for a day before it was, and the type said it could not be.
@@ -434,13 +434,13 @@ export class JobNotCancellableError extends UltimateError {
   }
 }
 
-/** The driver has no `introspect.cancel`. The redis/nats stubs, and any hand-rolled driver. */
+/** The driver has no `introspect.cancel` — a hand-rolled driver that ships no introspection. */
 export class CancelUnsupportedError extends UltimateError {
   constructor(input: { driver: string }) {
     super({
       code: 'X_JOB_NOT_CANCELLABLE',
       cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
-      fix: 'call setJobDriver(createPgDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
+      fix: 'call setJobDriver(postgresJobDriver()) at boot — only the pg driver implements introspect.cancel — then: x jobs cancel <id> --json',
     });
   }
 }
@@ -461,7 +461,7 @@ export class OutboxNoTxError extends UltimateError {
  *
  * The two shipped drivers had two meanings for it — every queue on `driver-memory.ts`, the
  * `default` queue on `driver-pg.ts` — and `ClaimOptions.queues` documented neither. Nothing reached
- * it (`createWorker` passes exactly one queue per pass), so the divergence could only ever be found
+ * it (`jobWorker` passes exactly one queue per pass), so the divergence could only ever be found
  * by an embedder in production, and each meaning is silently wrong in the other's deployment: one
  * takes work this worker was never configured for, the other drains nothing and reads as an idle
  * queue. There is no third meaning to pick — an empty list is a caller that has not said what to
@@ -472,7 +472,7 @@ export class ClaimQueuesEmptyError extends UltimateError {
     super({
       code: 'X_JOB_CLAIM_QUEUES_EMPTY',
       cause: `the ${driver} driver was asked to claim with queues: [] — an empty list names no queue, and "every queue" and "the default queue" are different deployments`,
-      fix: "pass the queue by name — driver.claim({ queues: ['default'], limit, visibilityTimeoutMs, workerId }) — or read the list from config.jobs.queues, which is what createWorker walks one queue at a time",
+      fix: "pass the queue by name — driver.claim({ queues: ['default'], limit, visibilityTimeoutMs, workerId }) — or read the list from config.jobs.queues, which is what jobWorker walks one queue at a time",
       meta: { driver },
     });
   }

package/src/events-pg.ts

@@ -7,7 +7,7 @@
 // resumes at 12:00:30 must still see an event published at 12:00:10.
 
 import type { PgExecutor } from '@ultimat3/core';
-import { finiteOption, logger, uuid } from '@ultimat3/core';
+import { finiteOption, logger, uuidV7 } from '@ultimat3/core';
 import type { DurationInput } from './clock';
 import { finiteDurationMs } from './clock';
 import {
@@ -28,7 +28,7 @@ interface EventRow {
   readonly expires_at: number | string;
 }
 
-export interface PgEventBusOptions {
+export interface PostgresEventBusOptions {
   /**
    * The only thing this bus is built from. There is no `clock`: every instant it writes or
    * compares is the database's (`SQL_EVENT_PUBLISH`), and a process clock here was the defect.
@@ -40,7 +40,7 @@ export interface PgEventBusOptions {
   readonly listLimit?: number;
 }
 
-export function createPgEventBus(options: PgEventBusOptions): EventBus {
+export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
   // TWO screens, for the reason `events.ts` states: `defaultTtl` is the constructor's knob and
   // `ttl` is the publish call's, so one screen over `ttl ?? defaultTtl` names the wrong one for
   // whichever value actually arrived.
@@ -67,7 +67,7 @@ export function createPgEventBus(options: PgEventBusOptions): EventBus {
     },
 
     async publish(name, payload, publishOptions = {}) {
-      const id = uuid();
+      const id = uuidV7();
       const ttlMs =
         publishOptions.ttl === undefined
           ? defaultTtlMs

package/src/events.ts

@@ -3,7 +3,7 @@
 // 12:00:10, so a fire-and-forget emitter would silently strand every waiting run.
 
 import type { Clock } from '@ultimat3/core';
-import { finiteOption, logger, systemClock, uuid } from '@ultimat3/core';
+import { finiteOption, logger, systemClock, uuidV7 } from '@ultimat3/core';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
 import type { PurgeTarget } from './purge';
@@ -49,7 +49,7 @@ export interface MemoryEventBusOptions {
   readonly maxEvents?: number;
 }
 
-export function createMemoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
+export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
   const clock = options.clock ?? systemClock;
   // TWO screens, because these are two knobs: the default is declared at construction and belongs
   // to whoever built the bus, `ttl` rides the publish CALL. One screen over `ttl ?? defaultTtl`
@@ -83,7 +83,7 @@ export function createMemoryEventBus(options: MemoryEventBusOptions = {}): Event
       purgeExpired();
       const at = nowMs(clock);
       const event: JobEvent = {
-        id: uuid(),
+        id: uuidV7(),
         name,
         payload,
         publishedAt: at,
@@ -152,10 +152,10 @@ export function eventsPurgeTarget(bus: EventBus): PurgeTarget {
   return { name: EVENTS_PURGE_TARGET, purgeExpired: () => bus.purgeExpired() };
 }
 
-let ambientBus: EventBus = createMemoryEventBus();
+let ambientBus: EventBus = memoryEventBus();
 
 /**
- * **Install `createPgEventBus({ executor })` here in any deployment with more than one process.**
+ * **Install `postgresEventBus({ executor })` here in any deployment with more than one process.**
  * The default above is one process's heap: the pod that publishes and the pod that resumes are
  * never the same one, so a webhook landing on web-3 strands a run on worker-7 until its 24h
  * timeout dead-letters it, with nothing logged before then. This line used to promise a
@@ -176,7 +176,7 @@ export function eventBus(): EventBus {
  * wait before that test has asked anything — call this between tests.
  */
 export function resetEventBus(): void {
-  ambientBus = createMemoryEventBus();
+  ambientBus = memoryEventBus();
 }
 
 /** The one function app code calls to unblock a waiting step. */

package/src/execute.ts

@@ -26,12 +26,17 @@ import type { AnyJobHandle } from './job';
 import { createProgressReporter } from './progress';
 import { isFinalAttempt } from './retry';
 import type { JobStopReason } from './retry-classification';
-import { failureForRow, nextRetryForError, recordedFailure } from './retry-classification';
+import {
+  failureForRow,
+  nextRetryForError,
+  rateLimitDeferralMs,
+  recordedFailure,
+} from './retry-classification';
 import { raceTimeout } from './run-deadline';
 import { createRunSignal } from './run-signal';
 import { announceSettled, settledCode } from './settled';
 import type { EventLookup, StepRecord } from './steps';
-import { createStepRunner, isStepSuspension } from './steps';
+import { isStepSuspension, stepRunner } from './steps';
 import { jobRunActor } from './tenant';
 
 /**
@@ -57,7 +62,7 @@ export type JobOutcome =
 const NEVER_ABORTED = new AbortController().signal;
 
 /**
- * `Ctx.signal` is non-optional in the type and `createContext` always sets it — but a context can
+ * `Ctx.signal` is non-optional in the type and `ctxOf` always sets it — but a context can
  * still arrive across a cast (`@ultimat3/http`'s `asCtx`, a test's `{} as Ctx`) without one, and
  * a job that crashed on a missing field would be a far worse answer than a job with no caller to
  * follow. Read it, do not assume it.
@@ -68,7 +73,7 @@ function callerSignal(ctx: Ctx): AbortSignal {
 
 /**
  * The same defensive read, for the same reason: `Ctx.actor` is non-optional in the type and
- * `createContext` always sets it, but `WorkerOptions.context()` is the app's own function and a
+ * `ctxOf` always sets it, but `WorkerOptions.context()` is the app's own function and a
  * cast context (`{} as Ctx`) reaches here without one. Anonymous is the honest stand-in — it
  * carries no org, so the job's declared tenant is the only thing that can put one on the run.
  */
@@ -147,12 +152,15 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     signal,
     services: callerServices(options.ctx),
   });
-  const runner = createStepRunner({
+  const runner = stepRunner({
     runId: claimed.runId,
     jobName: handle.name,
     store: driver.steps,
     fence: { job: handle.name, jobId: claimed.id, ...claimOf(claimed) },
     signal,
+    // This attempt's OWN end — the deadline, the `finally` below — apart from the run's signal,
+    // whose first reason may be the drain's: a drained step still records, an ended attempt never.
+    ended: cancel.signal,
     // The DECLARED per-step ceiling and event poll. Passed here or nowhere: this is the only
     // production construction of a runner, so a `StepRunnerOptions` field it omits is a feature
     // no `job()` can reach — which both of these were until 2026-08.
@@ -310,6 +318,39 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
         replayed: [],
       });
     }
+    // A rate-limit refusal is "not yet": rescheduled for its Retry-After with the attempt
+    // UNCOUNTED, like a suspension — but left in the ready bucket (no park): it is a job still
+    // waiting, and it belongs in `queue_depth`. Counted, a backlog of rate-limited `llm()` jobs was
+    // dead-lettered for having waited on a bucket that was only ever going to refill.
+    const deferMs = rateLimitDeferralMs(handle.retry, claimed.attempt, error);
+    if (deferMs !== undefined) {
+      landed(
+        await driver.nack(claimed.id, {
+          ...by(),
+          delayMs: deferMs,
+          error: failureForRow(error),
+          countsAsAttempt: false,
+        }),
+        'retried',
+      );
+      logger.info('jobs.attempt.rate_limited', {
+        job: handle.name,
+        jobId: claimed.id,
+        attempt: claimed.attempt,
+        delayMs: deferMs,
+      });
+      return settle({
+        outcome: 'retried',
+        jobId: claimed.id,
+        job: handle.name,
+        attempt: claimed.attempt,
+        durationMs: nowMs(options.clock) - startedAt,
+        error: message,
+        resumeAt: nowMs(options.clock) + deferMs,
+        steps: [],
+        replayed: [],
+      });
+    }
     // The ERROR decides too, not only the attempt count. A `terminal` code — a rotated password,
     // a schema mismatch, a permission denial — fails identically on every remaining attempt, so
     // spending them is a queue slot, a provider bill and, at a site that locks an account after

package/src/export.ts

@@ -20,7 +20,7 @@ import { assert, systemClock } from '@ultimat3/core';
 import type { ReadBuilder } from '@ultimat3/entity';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import type { Pacer } from './backfill-rate';
-import { createPacer } from './backfill-rate';
+import { backfillPacer } from './backfill-rate';
 import type { DurationInput } from './clock';
 import { nowMs } from './clock';
 import type { ExportFormat, ExportRecord } from './export-format';
@@ -177,7 +177,7 @@ export function exportRows<Row, I>(definition: ExportDefinition<Row, I>): JobHan
   const pace =
     definition.rate === undefined
       ? undefined
-      : createPacer({ rate: definition.rate, job: definition.name, clock });
+      : backfillPacer({ rate: definition.rate, job: definition.name, clock });
 
   // Resolved per write, never here: this line runs at the app's import, before its storage exists.
   const planned: PlannedExport<Row, I> = {

package/src/heartbeat.ts

@@ -98,6 +98,11 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
     // request per tick onto the connection that is already the thing failing.
     if (renewing) return;
     renewing = true;
+    // Stamped BEFORE the call: the driver writes the new lease at the statement, so the window it
+    // bought starts no later than this. Stamped at the reply instead, a slow answer would stretch
+    // this process's view of the lease past the server's by the reply's latency — and a second
+    // worker re-claims the row while this one still believes it holds it.
+    const sentAt = now();
     try {
       const held = await options.driver.heartbeat(claimed.id, {
         visibilityTimeoutMs,
@@ -121,14 +126,14 @@ export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartb
       }
       // Asked again AFTER the call, because a renewal that SUCCEEDS late is still late: an
       // event-loop stall or a driver that answered at the end of a connect timeout can land this
-      // past the window it was renewing, and `renewedAt = now()` there would restart the clock on
+      // past the window it was renewing, and renewing there would restart the clock on
       // a lease the queue has already re-delivered — the loss hidden by the very call meant to
       // prevent it.
       if (lapsed()) {
         reportLost();
         return;
       }
-      renewedAt = now();
+      renewedAt = sentAt;
     } catch (error) {
       // One failed renewal is not a lost lease: there is a whole visibility window left to land
       // the next one, and the default interval gives three tries inside it. Say it at warn and