@ultimat3/jobs 19.1.3 → 19.3.1

package/CLAUDE.md

@@ -4,7 +4,9 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
 
 ## Boundary
 
-- May import: `core`, `schema`, `entity`, `policy`, `cache`, `time`. Never `http`, `render`, `ui`.
+- May import: `core`, `schema`, `entity`, `policy`, `cache`, `time` — and `db`, for
+  `expectedQueryLoop` ONLY: `steps.ts` declares the per-step write one-per-step to the N+1
+  detector there, and no client is ever taken from it. Never `http`, `render`, `ui`.
 - Consumers: `action` (`<job>.enqueue`, via the ambient jobs facade), `cli`, `mcp`, `admin`.
 - External deps: none. Postgres access goes through the injected `PgExecutor`.
 
@@ -265,6 +267,16 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
   standstill, so start -> stop -> start stacked a second registration retaining a stopped worker's
   driver, and the next process-wide drain ran all of them. `start()` refuses while draining for the
   same reason: a claim loop back on a driver the drain is about to close.
+- **The outbox relay drains in the same two phases, `As of 2026-09`.** It registered NO hook: on
+  SIGTERM it went on claiming and publishing through every phase of the drain, and the only thing
+  that ever stopped it was `RunningRoles.stop()` in `x dev`'s release path — a caller a signal can
+  skip. Two consequences, both silent: a lease stamped on rows nothing on this pod will run (stranded
+  for the visibility window), and a row past `driver.enqueue` but short of `markPublished` published
+  a second time on the next boot, which the idempotency key collapses only while the first job is
+  still live. `accept` clears the interval and returns; `close` awaits the pass in flight under
+  `settleAllBy(…, reason.deadlineAt)`; both unregisters come back in the teardown's `finally`. The
+  poll timer is `unref`ed for `renewal-timer.ts`'s reason — a 200ms interval refed holds the event
+  loop open past every phase and makes SIGKILL the exit.
 - **A claimed job is counted with core's `beginWork()`, so the DRAIN does the waiting**
   (`As of 2026-08-23`). The wait for in-flight jobs belongs to the phase between `accept` and
   `inflight`, which exists for exactly this and is where `@ultimat3/http` already puts a request —
@@ -905,7 +917,8 @@ picture from the other side.
 | `register.ts` | `registerJobs`/`registerTasks` over a module namespace + the registrar announcements. Skips a non-job in silence — a module namespace is full of helpers — EXCEPT an `@ultimat3/action` projection (`kind: 'action-job'`), which is `X_ACTION_JOB_UNBRIDGED` |
 | `describe.ts` | the JSON projection one handle emits; `describeJobs()` is a map over it |
 | `steps.ts` | `StepStore`, `StepApi`, memoized-replay executor, `StepSuspension` |
-| `outbox.ts` | staging in a `Tx`, the relay, the ambient `JobsFacade` slot |
+| `outbox.ts` | staging in a `Tx`, the store seam, the ambient `JobsFacade` slot |
+| `outbox-relay.ts` | the relay: the poll timer, one pass, and its TWO shutdown hooks. Split off at `outbox.ts`'s 500-line ceiling |
 | `outbox-pg.ts` | `createPgOutboxStore` — `stage()` on the caller's OWN connection, claim on the pool |
 | `outbox-lease.ts` | the claim lease's one definition and its one normalisation, for both stores |
 | `leases.ts` | `LeaseStore` — fleet-wide slots, the memory one, `jobLeaseKey` |

package/README.md

@@ -603,6 +603,10 @@ things have to be true in a process:
 | the facade is installed | `setJobsFacade(createJobsFacade({ store, driver }, currentTx))` |
 | the relay is running | `createOutboxRelay({ store, driver }).start()` |
 
+`start()` registers the same two shutdown hooks `createWorker` does — `accept` stops polling,
+`close` waits out the pass in flight under the drain's deadline — and `stop()` hands both back.
+`drainOnShutdown: false` opts out, for a caller that drives its own teardown.
+
 with `store = createPgOutboxStore({ executor, txExecutor })`. `txExecutor` is what makes it
 transactional: `stage()` runs on the CALLER'S connection, never the pool. With nothing installed,
 `jobsFacade()` answers a fallback whose `currentTx` is `() => undefined` and every enqueue

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "19.1.3",
+  "version": "19.3.1",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -32,9 +32,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "19.1.3",
-    "@ultimat3/entity": "19.1.3",
-    "@ultimat3/schema": "19.1.3",
-    "@ultimat3/time": "19.1.3"
+    "@ultimat3/core": "19.3.1",
+    "@ultimat3/db": "19.3.1",
+    "@ultimat3/entity": "19.3.1",
+    "@ultimat3/schema": "19.3.1",
+    "@ultimat3/time": "19.3.1"
   }
 }

package/src/driver-memory.ts

@@ -25,7 +25,7 @@ import { JobDuplicateError } from './errors';
 import type { LeaseStore } from './leases';
 import { createMemoryLeaseStore } from './leases';
 import type { StepStore } from './steps';
-import { createMemoryStepStore } from './steps';
+import { createMemoryStepStore } from './steps-memory';
 
 export interface MemoryDriverOptions {
   readonly clock?: Clock;
@@ -146,7 +146,11 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
       // `state !== 'done'`, mirroring `SQL_CANCEL`: a job that already finished has nothing to
       // stop, and cancelling it would rewrite a terminal row an operator is reading as success.
       if (existing === undefined || existing.state === 'done') return Promise.resolve(undefined);
-      update(jobId, {
+      // `settle`, not `update`: a cancellation RELEASES the claim, and `SQL_CANCEL` writes
+      // `visible_at = null, claimed_by = null` with the state. Left stamped, a cancelled row named
+      // the worker still holding it and carried that attempt's lease deadline — the pair
+      // `x jobs show` prints, and the pair the claim scan reads to decide a row was abandoned.
+      settle(jobId, {
         state: 'cancelled',
         ...(reason === undefined ? {} : { lastError: reason }),
       });
@@ -161,7 +165,11 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
     leases,
     introspect,
 
-    enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
+    // `async` for the reason `claim`, `list` and `deadLetters` are: `onConflict: 'error'` REJECTS
+    // here exactly as the pg driver's does, and a synchronous throw out of a method typed
+    // `Promise<…>` is a second answer to one question — caught by different code, and an
+    // unhandled exception rather than a settled promise wherever the caller holds the promise.
+    async enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
       const existing = liveByKey(request.name, request.idempotencyKey, request.tenantId);
       if (existing !== undefined) {
         if (request.onConflict === 'error') {
@@ -171,7 +179,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
             existingId: existing.id,
           });
         }
-        return Promise.resolve({ id: existing.id, runId: existing.runId, deduped: true });
+        return { id: existing.id, runId: existing.runId, deduped: true };
       }
 
       const at = nowMs(clock);
@@ -194,7 +202,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         ...(request.enqueuedBy === undefined ? {} : { enqueuedBy: request.enqueuedBy }),
       };
       jobs.set(record.id, record);
-      return Promise.resolve({ id: record.id, runId: record.runId, deduped: false });
+      return { id: record.id, runId: record.runId, deduped: false };
     },
 
     // `async`, so an empty queue list REJECTS here exactly as it does on the pg driver: a

package/src/driver-nats.ts

@@ -16,13 +16,22 @@ import type {
 import { JobsNotImplementedError } from './errors';
 import type { StepRecord, StepStore } from './steps';
 
-// Names the seam that actually replaces the stub, plus the runnable command for whatever is
-// already queued. NOT `jobs: { driver }` in app.config.ts, which this line said until 2026-08-20:
-// `JobsConfig.driver` has no reader anywhere (see `driver.ts`'s header), so that edit repairs
-// nothing and the reader is sent back to the same throw. #223 removes the field.
+// Names the seam that actually replaces the stub, and NOTHING ELSE — the two other repairs this
+// line has carried were both unrunnable.
+//
+// NOT `jobs: { driver }` in app.config.ts, which it said until 2026-08-20: the field had no
+// reader anywhere, so that edit repaired nothing and sent the reader back to the same throw. It
+// is deleted now.
+//
+// NOT `x jobs drain --to memory` either, which it said until 2026-09, and that one was a route
+// into data loss: the target is a Map inside the command's own process, so the drain acked every
+// durable row and lost the copy at exit. `x jobs` refuses that value by name now. There is no
+// drain to run in its place, and the reason is in this file — `enqueue` below refuses too, so
+// nothing was ever queued onto this driver and the queue is untouched.
+//
 // The nats driver lands in v2; there is no flag that turns this one on.
 const FIX =
-  'call setJobDriver(createPgDriver()) at boot instead of this driver, then move what is already queued: x jobs drain --to memory --json';
+  'call setJobDriver(createPgDriver()) at boot instead of this driver; nothing needs moving first, because enqueue here refuses too, so no job was ever written to it';
 
 const unavailable = (method: string): never => {
   throw new JobsNotImplementedError({ feature: `nats jobs driver (${method})`, fix: FIX });

package/src/driver-pg.ts

@@ -54,8 +54,10 @@ import type { HeldLease, LeaseStore } from './leases';
 import type { StepStore } from './steps';
 
 /**
- * The one thing this driver needs from the DB layer, declared structurally so this package can
- * depend on no database package at all.
+ * The one thing this driver needs from the DB layer, declared structurally so this package needs
+ * no database CLIENT: `@ultimat3/db` is imported for `expectedQueryLoop` — the marker that tells
+ * the N+1 detector a step write is one-per-step by design (`steps.ts`) — and never for a
+ * connection.
  *
  * **Not satisfied by `Bun.sql`** — verified against Bun 1.4.0: `Bun.sql.query` is `undefined`.
  * `Bun.sql` is a tagged template whose positional form is `unsafe`, so a `{ executor: Bun.sql }`

package/src/driver-redis.ts

@@ -19,13 +19,22 @@ import type {
 import { JobsNotImplementedError } from './errors';
 import type { StepRecord, StepStore } from './steps';
 
-// Names the seam that actually replaces the stub, plus the runnable command for whatever is
-// already queued. NOT `jobs: { driver }` in app.config.ts, which this line said until 2026-08-20:
-// `JobsConfig.driver` has no reader anywhere (see `driver.ts`'s header), so that edit repairs
-// nothing and the reader is sent back to the same throw. #223 removes the field.
+// Names the seam that actually replaces the stub, and NOTHING ELSE — the two other repairs this
+// line has carried were both unrunnable.
+//
+// NOT `jobs: { driver }` in app.config.ts, which it said until 2026-08-20: the field had no
+// reader anywhere, so that edit repaired nothing and sent the reader back to the same throw. It
+// is deleted now.
+//
+// NOT `x jobs drain --to memory` either, which it said until 2026-09, and that one was a route
+// into data loss: the target is a Map inside the command's own process, so the drain acked every
+// durable row and lost the copy at exit. `x jobs` refuses that value by name now. There is no
+// drain to run in its place, and the reason is in this file — `enqueue` below refuses too, so
+// nothing was ever queued onto this driver and the queue is untouched.
+//
 // The redis driver lands in v2; there is no flag that turns this one on.
 const FIX =
-  'call setJobDriver(createPgDriver()) at boot instead of this driver, then move what is already queued: x jobs drain --to memory --json';
+  'call setJobDriver(createPgDriver()) at boot instead of this driver; nothing needs moving first, because enqueue here refuses too, so no job was ever written to it';
 
 const unavailable = (method: string): never => {
   throw new JobsNotImplementedError({ feature: `redis jobs driver (${method})`, fix: FIX });

package/src/index.ts

@@ -233,14 +233,11 @@ export type {
   MemoryOutboxStore,
   OutboxDeps,
   OutboxRecord,
-  OutboxRelay,
   OutboxStore,
-  RelayOptions,
 } from './outbox';
 export {
   createJobsFacade,
   createMemoryOutboxStore,
-  createOutboxRelay,
   enqueueInTx,
   jobsFacade,
   resetJobsFacade,
@@ -251,6 +248,8 @@ export {
 export { DEFAULT_OUTBOX_CLAIM_LEASE_MS } from './outbox-lease';
 export type { PgOutboxOptions } from './outbox-pg';
 export { createPgOutboxStore } from './outbox-pg';
+export type { OutboxRelay, RelayOptions } from './outbox-relay';
+export { createOutboxRelay } from './outbox-relay';
 export type {
   PurgeDefinition,
   PurgeInput,
@@ -290,7 +289,6 @@ export type {
   WaitForEventOptions,
 } from './steps';
 export {
-  createMemoryStepStore,
   createStepRunner,
   isStepStatus,
   isStepSuspension,
@@ -298,6 +296,7 @@ export {
   STEP_STATUSES,
   StepSuspension,
 } from './steps';
+export { createMemoryStepStore } from './steps-memory';
 export type {
   CatchUpPolicy,
   TaskDefinition,

package/src/outbox-relay.ts

@@ -0,0 +1,221 @@
+// The outbox relay: the loop that turns a COMMITTED `x_outbox` row into a queued job. Split from
+// `outbox.ts` — that file owns the store seam, the staging and the facade; this one owns a timer, a
+// pass and the process drain around them.
+//
+// At-least-once by construction: publish, THEN mark published. A crash between the two re-publishes,
+// which the idempotency key collapses — the opposite order would lose jobs.
+
+import { assert, logger, onShutdown, renderThrowable } from '@ultimat3/core';
+import { nowMs } from './clock';
+import { settleAllBy } from './drain-wait';
+import type { OutboxDeps } from './outbox';
+
+export interface RelayOptions extends OutboxDeps {
+  readonly batchSize?: number;
+  readonly intervalMs?: number;
+  /**
+   * Register the two shutdown hooks. `false` only for a relay whose caller drives the teardown
+   * itself — a test, or a script that owns the process. The same knob `WorkerOptions` carries, and
+   * the default is the same: a loop nobody stops on SIGTERM keeps leasing rows this pod will never
+   * publish.
+   */
+  readonly drainOnShutdown?: boolean;
+}
+
+export interface OutboxRelay {
+  /** One pass. Returns how many rows were published. Call it directly in tests. */
+  tick(): Promise<number>;
+  start(): void;
+  /**
+   * Stop polling and WAIT OUT the pass in flight, the way `worker.stop()` waits out its rounds and
+   * `scheduler.stop()` its dispatch. A pass is a publish followed by a `markPublished`, and a
+   * caller that returned between the two closed the database under the row it was about to mark:
+   * re-published next boot at best, a rejection against a closed pool at worst.
+   */
+  stop(deadlineAt?: number): Promise<void>;
+  pending(): Promise<number>;
+}
+
+/**
+ * At-least-once by construction: publish, THEN mark published. A crash between the two
+ * re-publishes, which the idempotency key collapses — the opposite order would lose jobs.
+ */
+export function createOutboxRelay(options: RelayOptions): OutboxRelay {
+  const batchSize = options.batchSize ?? 100;
+  const intervalMs = options.intervalMs ?? 200;
+  // Refused, never clamped — `worker-options.ts` carries the reason. `setInterval(fn, NaN)` reads
+  // the delay as 0 and `claim(NaN)` slices `(0, NaN)`: a relay that spins and publishes nothing.
+  assert(
+    Number.isSafeInteger(batchSize) && batchSize >= 1,
+    `createOutboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
+    'pass a finite batchSize to createOutboxRelay(...), or omit it for the default 100',
+  );
+  assert(
+    Number.isFinite(intervalMs) && intervalMs >= 0,
+    `createOutboxRelay intervalMs is ${String(intervalMs)}, which setInterval reads as 0 — the relay would spin, not poll`,
+    'pass a finite intervalMs to createOutboxRelay(...), or omit it for the default 200',
+  );
+  let timer: ReturnType<typeof setInterval> | undefined;
+  let running = false;
+  /** The pass in flight, so `stop()` joins it instead of returning underneath it. */
+  let pass: Promise<void> | undefined;
+  /** `'draining'` is the state in which a timer that somehow fires still claims nothing. */
+  let state: 'idle' | 'running' | 'draining' | 'stopped' = 'idle';
+  let releaseShutdownHooks: (() => void)[] = [];
+  let stopping: Promise<void> | undefined;
+
+  const tick = async (): Promise<number> => {
+    const batch = await options.store.claim(batchSize);
+    let published = 0;
+    for (const record of batch) {
+      try {
+        await options.driver.enqueue({
+          name: record.job,
+          queue: record.queue,
+          input: record.input,
+          idempotencyKey: record.idempotencyKey,
+          maxAttempts: record.maxAttempts,
+          runAt: record.runAt,
+          ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
+          ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
+          ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
+        });
+        // The claim's own token goes back with the mark. Without it a relay whose lease lapsed
+        // mid-stall retires a row the relay that reclaimed it has not published yet — the row is
+        // gone and nothing publishes it.
+        await options.store.markPublished(record.id, nowMs(options.clock), record.claimedBy);
+        published += 1;
+      } catch (error) {
+        // STOP the batch. `claim()` returns rows in `staged_at` order and the loop used to log
+        // and continue, which published every LATER row past the one that failed — so an app
+        // that stages `createInvoice` then `chargeCard` in one transaction could have the charge
+        // run first. The row stays unpublished and the next tick starts again from it; a
+        // permanently poisoned row wedges its queue, which is visible in `pending()` and is the
+        // correct trade against silently reordering committed work.
+        logger.warn('jobs.outbox.publish-failed', {
+          job: record.job,
+          id: record.id,
+          published,
+          remaining: batch.length - published,
+          error: renderThrowable(error),
+        });
+        // Hand the rest of the batch back rather than sit on a claim nobody is publishing. The
+        // claim is a lease now, so without this a single pool timeout parks every committed row
+        // behind it for the whole lease window instead of for one poll interval.
+        await options.store.release?.(
+          batch.slice(published).map((row) => row.id),
+          record.claimedBy,
+        );
+        break;
+      }
+    }
+    return published;
+  };
+
+  /**
+   * The `accept` phase, and all of it: flip the state, clear the timer, return. Every hook behind
+   * this one — `@ultimat3/http`'s "stop listening", the sync node's "stop upgrading" — then runs
+   * with the whole budget still in hand, which is what a wait parked here costs them.
+   */
+  const stopAccepting = (): void => {
+    if (timer !== undefined) clearInterval(timer);
+    timer = undefined;
+    if (state === 'running') state = 'draining';
+  };
+
+  const teardown = async (deadlineAt?: number): Promise<void> => {
+    stopAccepting();
+    try {
+      // The pass in flight is a `driver.enqueue` and the `markPublished` behind it. Abandoning
+      // between the two republishes the row on the next boot — which the idempotency key collapses
+      // only while the first job is still live — so it is waited out here, in `close`, under the
+      // deadline the hook was handed. `undefined` on a manual `stop()`: a caller that asked has no
+      // budget to spend, and nothing else in this process is waiting on the answer.
+      const settled = await settleAllBy(pass === undefined ? [] : [pass], deadlineAt);
+      if (!settled) {
+        logger.warn('jobs.outbox.drain-abandoned', {
+          fix: 'raise the drain budget past one publish — configureLifecycle({ deadlineMs: 60_000 }) — and set terminationGracePeriodSeconds to at least as many seconds',
+        });
+      }
+    } finally {
+      // Whatever the wait did, this relay is done, and the hooks go back. One left registered
+      // publishes through a stopped relay on the next process-wide drain — against a driver
+      // already closed — and keeps this closure, its store and its driver alive with it.
+      state = 'stopped';
+      for (const release of releaseShutdownHooks) release();
+      releaseShutdownHooks = [];
+    }
+  };
+
+  const stop = async (deadlineAt?: number): Promise<void> => {
+    // Answered immediately once this relay is done: the teardown always REACHES 'stopped' (its
+    // wait is bounded on the SIGTERM path and the state is set in a `finally`), so a caller
+    // landing after an abandoned drain gets an answer rather than joining a settled lifetime ago.
+    if (state === 'stopped') return;
+    // One teardown, joined rather than repeated: a SIGTERM landing on a manual stop waits out the
+    // same pass instead of returning underneath it. Cleared as it settles, so a relay started
+    // again stops again rather than joining a promise that settled a lifetime ago.
+    stopping ??= teardown(deadlineAt).finally(() => {
+      stopping = undefined;
+    });
+    await stopping;
+  };
+
+  return {
+    tick,
+    start() {
+      if (timer !== undefined) return;
+      // Only from a standstill, the worker's rule: a start mid-drain would re-arm the poll on a
+      // store the drain is about to leave, and stack a second pair of hooks on the one still
+      // running — the first pair's unregisters are held in a single slot and would be dropped.
+      if (state === 'draining') return;
+      state = 'running';
+      // TWO hooks, for the two phases — the shape `worker.ts` and `scheduler.ts` hold. This loop
+      // had NONE: on SIGTERM it went on claiming and publishing through every phase, stamping
+      // leases on rows nothing on this pod would run, and `RunningRoles.stop()` was the only thing
+      // that ever stopped it — a caller a signal can skip entirely.
+      //
+      // Both unregisters are kept, never discarded: the teardown hands them back, so
+      // start -> stop -> start holds one pair rather than one per start.
+      if (options.drainOnShutdown !== false) {
+        releaseShutdownHooks = [
+          onShutdown('jobs.outbox.accept', stopAccepting, { phase: 'accept' }),
+          onShutdown('jobs.outbox', (reason) => stop(reason.deadlineAt), { phase: 'close' }),
+        ];
+      }
+      timer = setInterval(() => {
+        // Re-read on every tick: "stop claiming" means this timer too, and a callback already on
+        // the event loop when the interval was cleared must not open a claim behind the drain.
+        if (running || state !== 'running') return;
+        running = true;
+        // `.catch` before `.finally`, the shape every other loop in this package uses. `tick()`
+        // guards each publish but not `store.claim()` — one pool timeout during a failover
+        // rejects here unobserved, and Bun's default for an unhandled rejection is to end the
+        // process, taking every staged, unpublished row with it.
+        //
+        // Kept rather than discarded, because `stop()` awaits exactly this chain: the publish and
+        // the `markPublished` behind it are one pass, and a teardown that returned between them
+        // closed the database under the row it was about to mark. The chain carries its own
+        // `catch`, so a caller that does not await still gets no unhandled rejection.
+        pass = tick()
+          .then((): void => undefined)
+          .catch((error: unknown) => {
+            logger.error('jobs.outbox.tick-failed', {
+              error: renderThrowable(error),
+            });
+          })
+          .finally(() => {
+            running = false;
+            pass = undefined;
+          });
+      }, intervalMs);
+      // Never the thing keeping a drained process alive — the rule `renewal-timer.ts` and
+      // `lifecycle-deadline.ts` both state for their own timers. A poll every 200ms holds the
+      // event loop open past every phase of the shutdown, and the kubelet's SIGKILL becomes the
+      // exit; the hooks above are what stop this loop, not the process refusing to end.
+      timer.unref?.();
+    },
+    stop,
+    pending: () => options.store.pendingCount(),
+  };
+}

package/src/outbox.ts

@@ -24,14 +24,7 @@
 // test and `x dev` must enqueue with nothing wired — but it is a fallback, not the guarantee.
 
 import type { Clock } from '@ultimat3/core';
-import {
-  assert,
-  currentSpanContext,
-  logger,
-  renderThrowable,
-  traceparent,
-  uuid,
-} from '@ultimat3/core';
+import { currentSpanContext, traceparent, uuid } from '@ultimat3/core';
 import type { Tx } from '@ultimat3/entity';
 import { nowMs } from './clock';
 import type { EnqueueResult, JobDriver } from './driver';
@@ -364,136 +357,6 @@ export function resetJobsFacade(): void {
   ambient = undefined;
 }
 
-export interface RelayOptions extends OutboxDeps {
-  readonly batchSize?: number;
-  readonly intervalMs?: number;
-}
-
-export interface OutboxRelay {
-  /** One pass. Returns how many rows were published. Call it directly in tests. */
-  tick(): Promise<number>;
-  start(): void;
-  /**
-   * Stop polling and WAIT OUT the pass in flight, the way `worker.stop()` waits out its rounds and
-   * `scheduler.stop()` its dispatch. A pass is a publish followed by a `markPublished`, and a
-   * caller that returned between the two closed the database under the row it was about to mark:
-   * re-published next boot at best, a rejection against a closed pool at worst.
-   */
-  stop(): Promise<void>;
-  pending(): Promise<number>;
-}
-
-/**
- * At-least-once by construction: publish, THEN mark published. A crash between the two
- * re-publishes, which the idempotency key collapses — the opposite order would lose jobs.
- */
-export function createOutboxRelay(options: RelayOptions): OutboxRelay {
-  const batchSize = options.batchSize ?? 100;
-  const intervalMs = options.intervalMs ?? 200;
-  // Refused, never clamped — `worker-options.ts` carries the reason. `setInterval(fn, NaN)` reads
-  // the delay as 0 and `claim(NaN)` slices `(0, NaN)`: a relay that spins and publishes nothing.
-  assert(
-    Number.isSafeInteger(batchSize) && batchSize >= 1,
-    `createOutboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
-    'pass a finite batchSize to createOutboxRelay(...), or omit it for the default 100',
-  );
-  assert(
-    Number.isFinite(intervalMs) && intervalMs >= 0,
-    `createOutboxRelay intervalMs is ${String(intervalMs)}, which setInterval reads as 0 — the relay would spin, not poll`,
-    'pass a finite intervalMs to createOutboxRelay(...), or omit it for the default 200',
-  );
-  let timer: ReturnType<typeof setInterval> | undefined;
-  let running = false;
-  /** The pass in flight, so `stop()` joins it instead of returning underneath it. */
-  let pass: Promise<void> | undefined;
-
-  const tick = async (): Promise<number> => {
-    const batch = await options.store.claim(batchSize);
-    let published = 0;
-    for (const record of batch) {
-      try {
-        await options.driver.enqueue({
-          name: record.job,
-          queue: record.queue,
-          input: record.input,
-          idempotencyKey: record.idempotencyKey,
-          maxAttempts: record.maxAttempts,
-          runAt: record.runAt,
-          ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
-          ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
-          ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
-        });
-        // The claim's own token goes back with the mark. Without it a relay whose lease lapsed
-        // mid-stall retires a row the relay that reclaimed it has not published yet — the row is
-        // gone and nothing publishes it.
-        await options.store.markPublished(record.id, nowMs(options.clock), record.claimedBy);
-        published += 1;
-      } catch (error) {
-        // STOP the batch. `claim()` returns rows in `staged_at` order and the loop used to log
-        // and continue, which published every LATER row past the one that failed — so an app
-        // that stages `createInvoice` then `chargeCard` in one transaction could have the charge
-        // run first. The row stays unpublished and the next tick starts again from it; a
-        // permanently poisoned row wedges its queue, which is visible in `pending()` and is the
-        // correct trade against silently reordering committed work.
-        logger.warn('jobs.outbox.publish-failed', {
-          job: record.job,
-          id: record.id,
-          published,
-          remaining: batch.length - published,
-          error: renderThrowable(error),
-        });
-        // Hand the rest of the batch back rather than sit on a claim nobody is publishing. The
-        // claim is a lease now, so without this a single pool timeout parks every committed row
-        // behind it for the whole lease window instead of for one poll interval.
-        await options.store.release?.(
-          batch.slice(published).map((row) => row.id),
-          record.claimedBy,
-        );
-        break;
-      }
-    }
-    return published;
-  };
-
-  return {
-    tick,
-    start() {
-      if (timer !== undefined) return;
-      timer = setInterval(() => {
-        if (running) return;
-        running = true;
-        // `.catch` before `.finally`, the shape every other loop in this package uses. `tick()`
-        // guards each publish but not `store.claim()` — one pool timeout during a failover
-        // rejects here unobserved, and Bun's default for an unhandled rejection is to end the
-        // process, taking every staged, unpublished row with it.
-        //
-        // Kept rather than discarded, because `stop()` awaits exactly this chain: the publish and
-        // the `markPublished` behind it are one pass, and a teardown that returned between them
-        // closed the database under the row it was about to mark. The chain carries its own
-        // `catch`, so a caller that does not await still gets no unhandled rejection.
-        pass = tick()
-          .then((): void => undefined)
-          .catch((error: unknown) => {
-            logger.error('jobs.outbox.tick-failed', {
-              error: renderThrowable(error),
-            });
-          })
-          .finally(() => {
-            running = false;
-            pass = undefined;
-          });
-      }, intervalMs);
-    },
-    async stop() {
-      if (timer !== undefined) clearInterval(timer);
-      timer = undefined;
-      // Awaited AFTER the interval is cleared, so no further pass can start behind this one.
-      await pass;
-    },
-    pending: () => options.store.pendingCount(),
-  };
-}
-
 // The outbox's SQL moved to `driver-pg-sql.ts`, where every statement this package runs lives —
 // and, more to the point, where `SQL_JOBS_TABLE` is: `x_outbox` was declared here, in a constant
 // no boot code applied, which is the whole reason the outbox was documented and never created.

package/src/scheduler.ts

@@ -139,8 +139,10 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     resolveCron(handle.cron, { tz: handle.tz, from: from ?? new Date(nowMs(options.clock)) });
 
   /**
-   * Occurrences in `(after, until]`. Walking forward from the last fire is what makes
-   * catch-up possible at all — a scheduler that only knows "now" cannot know what it missed.
+   * Occurrences in `(after, until]`, the first `maxCatchUp` of them. Walking forward from the
+   * last fire is what makes catch-up possible at all — a scheduler that only knows "now" cannot
+   * know what it missed. TRUNCATED, so its last element is the tenth occurrence after the
+   * watermark and not the latest one missed; `latestOccurrenceBy` answers that question.
    */
   const occurrencesSince = (
     handle: TaskHandle,
@@ -158,6 +160,31 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     return out;
   };
 
+  /**
+   * The latest occurrence at or before `until`, given one is known to lie in `(after, until]`.
+   *
+   * The resolver only answers "the first occurrence strictly after an instant", and walking it
+   * forward from the watermark is bounded by `maxCatchUp` — which is how `skip` came to dispatch
+   * the tenth minute after a three-hour outage and then the twentieth, one per tick, for a policy
+   * whose whole promise is ONE dispatch (measured: twenty `catchUp=true` dispatches a second apart
+   * for a minute cron down 14:23–17:34). So the latest is found by bisection over the instant the
+   * resolver is asked from, not by walking: `next(x) <= until` is monotone in `x`, the invariant
+   * is `next(lo) <= until < next(hi)`, and at `hi - lo === 1` the one occurrence in `(lo, until]`
+   * is `next(lo)`. About 25 resolver calls for a three-hour gap and 35 for a year, whatever the
+   * cron's period — never one per missed minute.
+   */
+  const latestOccurrenceBy = (handle: TaskHandle, after: number, until: number): number => {
+    const nextAfter = (from: number): number => nextRunFor(handle, new Date(from)).getTime();
+    let lo = after;
+    let hi = until;
+    while (hi - lo > 1) {
+      const mid = lo + Math.floor((hi - lo) / 2);
+      if (nextAfter(mid) <= until) lo = mid;
+      else hi = mid;
+    }
+    return nextAfter(lo);
+  };
+
   const dispatch = async (
     handle: TaskHandle,
     occurrenceMs: number,
@@ -242,8 +269,13 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
       if (due.length === 0) continue;
 
       if (handle.catchUp === 'skip') {
-        const latest = due[due.length - 1];
-        if (latest !== undefined) dispatched.push(await dispatch(handle, latest, due.length > 1));
+        // The real latest occurrence, never `due`'s last element: that one is `maxCatchUp` steps
+        // past the watermark, and dispatching it leaves the watermark there — so the next tick
+        // found the next ten still due and fired again, until the walk reached `at`. The
+        // occurrence key stays honest (this IS the occurrence the payload is for), and the
+        // watermark `dispatch` leaves is that occurrence — nothing at or before `at` is due past it.
+        const latest = latestOccurrenceBy(handle, last, at);
+        dispatched.push(await dispatch(handle, latest, due.length > 1));
         continue;
       }
       if (handle.catchUp === 'run-once') {

package/src/steps-memory.ts

@@ -0,0 +1,39 @@
+// The in-memory `StepStore`: what `createMemoryDriver` and every runner test persist steps into.
+// Split from `steps.ts` at the file-size ceiling, along the seam the pg driver already draws —
+// `driver-pg.ts` holds the Postgres store, this file the map-backed one, `steps.ts` the runner
+// both are handed to. Same contract, and `steps.test.ts` is where the contract is pinned.
+
+import type { StepRecord, StepStore } from './steps';
+
+export function createMemoryStepStore(): StepStore {
+  const byRun = new Map<string, Map<string, StepRecord>>();
+  const runOf = (runId: string): Map<string, StepRecord> => {
+    let run = byRun.get(runId);
+    if (run === undefined) {
+      run = new Map();
+      byRun.set(runId, run);
+    }
+    return run;
+  };
+  return {
+    get(runId, name) {
+      return Promise.resolve(byRun.get(runId)?.get(name));
+    },
+    put(record) {
+      runOf(record.runId).set(record.name, record);
+      return Promise.resolve();
+    },
+    list(runId) {
+      const records = [...(byRun.get(runId)?.values() ?? [])];
+      return Promise.resolve(records.sort((a, b) => a.startedAt - b.startedAt));
+    },
+    del(runId, name) {
+      byRun.get(runId)?.delete(name);
+      return Promise.resolve();
+    },
+    clear(runId) {
+      byRun.delete(runId);
+      return Promise.resolve();
+    },
+  };
+}

package/src/steps.ts

@@ -8,6 +8,7 @@
 
 import type { Clock } from '@ultimat3/core';
 import { finiteOption, logger, renderThrowable } from '@ultimat3/core';
+import { expectedQueryLoop } from '@ultimat3/db';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
 import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
@@ -113,39 +114,6 @@ export function isStepSuspension(error: unknown): error is StepSuspension {
   return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
 }
 
-export function createMemoryStepStore(): StepStore {
-  const byRun = new Map<string, Map<string, StepRecord>>();
-  const runOf = (runId: string): Map<string, StepRecord> => {
-    let run = byRun.get(runId);
-    if (run === undefined) {
-      run = new Map();
-      byRun.set(runId, run);
-    }
-    return run;
-  };
-  return {
-    get(runId, name) {
-      return Promise.resolve(byRun.get(runId)?.get(name));
-    },
-    put(record) {
-      runOf(record.runId).set(record.name, record);
-      return Promise.resolve();
-    },
-    list(runId) {
-      const records = [...(byRun.get(runId)?.values() ?? [])];
-      return Promise.resolve(records.sort((a, b) => a.startedAt - b.startedAt));
-    },
-    del(runId, name) {
-      byRun.get(runId)?.delete(name);
-      return Promise.resolve();
-    },
-    clear(runId) {
-      byRun.delete(runId);
-      return Promise.resolve();
-    },
-  };
-}
-
 export interface StepRunnerOptions {
   readonly runId: string;
   readonly jobName: string;
@@ -257,6 +225,21 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
 
   const cancelled = (): boolean => runSignal.aborted;
 
+  /**
+   * The statement itself, declared deliberate to the N+1 detector. One write per step IS the
+   * design this file's header states — each step completes at its own instant and its output has
+   * to be durable before the next one starts, so five steps are five `SQL_STEP_PUT`s that no
+   * batch could replace. Without the declaration `x dev` warned `X_N_PLUS_ONE_WRITE` on every
+   * job of five or more steps, a verdict against the framework's own persistence that an app
+   * could neither fix nor silence. The scope ends with the write: the hydrating `list` and the
+   * job's own statements are judged as before.
+   */
+  const persist = (record: StepRecord): Promise<void> =>
+    expectedQueryLoop(
+      'a durable step is written the instant it completes, one statement per step by design',
+      () => store.put(record),
+    );
+
   /**
    * EVERY write this runner makes, and the one place the cancellation is enforced. A step result
    * from a cancelled attempt is a write onto the attempt that replaced it: the deadline nacked
@@ -265,7 +248,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
    */
   const put = async (record: StepRecord): Promise<void> => {
     if (cancelled()) throw new JobAbortedError({ job: jobName, step: record.name });
-    await store.put(record);
+    await persist(record);
     remember(record);
   };
 
@@ -329,7 +312,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
           attempts,
           error: renderThrowable(error),
         };
-        await store.put(failure);
+        await persist(failure);
         remember(failure);
       }
       throw error;

package/src/webhook.ts

@@ -19,6 +19,10 @@ import type { Clock, Ctx } from '@ultimat3/core';
 import {
   finiteOption,
   isCanonicalWebhookField,
+  // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
+  // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
+  // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
+  isRetryableStatus,
   renderThrowable,
   systemClock,
   WEBHOOK_FIELD_MAX,
@@ -164,10 +168,6 @@ const retryAfterSeconds = (response: Response): number | undefined => {
   return Number(header.trim());
 };
 
-/** A status the same request, unchanged, can still land on. Everything else is somebody's edit. */
-const isRetryableStatus = (status: number): boolean =>
-  status >= 500 || status === 408 || status === 425 || status === 429;
-
 export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
   const clock = definition.clock ?? systemClock;
   const disableAfter = finiteOption(