@ultimat3/jobs 19.2.0 → 19.3.2

package/src/outbox-relay.ts

@@ -0,0 +1,221 @@
+// The outbox relay: the loop that turns a COMMITTED `x_outbox` row into a queued job. Split from
+// `outbox.ts` — that file owns the store seam, the staging and the facade; this one owns a timer, a
+// pass and the process drain around them.
+//
+// At-least-once by construction: publish, THEN mark published. A crash between the two re-publishes,
+// which the idempotency key collapses — the opposite order would lose jobs.
+
+import { assert, logger, onShutdown, renderThrowable } from '@ultimat3/core';
+import { nowMs } from './clock';
+import { settleAllBy } from './drain-wait';
+import type { OutboxDeps } from './outbox';
+
+export interface RelayOptions extends OutboxDeps {
+  readonly batchSize?: number;
+  readonly intervalMs?: number;
+  /**
+   * Register the two shutdown hooks. `false` only for a relay whose caller drives the teardown
+   * itself — a test, or a script that owns the process. The same knob `WorkerOptions` carries, and
+   * the default is the same: a loop nobody stops on SIGTERM keeps leasing rows this pod will never
+   * publish.
+   */
+  readonly drainOnShutdown?: boolean;
+}
+
+export interface OutboxRelay {
+  /** One pass. Returns how many rows were published. Call it directly in tests. */
+  tick(): Promise<number>;
+  start(): void;
+  /**
+   * Stop polling and WAIT OUT the pass in flight, the way `worker.stop()` waits out its rounds and
+   * `scheduler.stop()` its dispatch. A pass is a publish followed by a `markPublished`, and a
+   * caller that returned between the two closed the database under the row it was about to mark:
+   * re-published next boot at best, a rejection against a closed pool at worst.
+   */
+  stop(deadlineAt?: number): Promise<void>;
+  pending(): Promise<number>;
+}
+
+/**
+ * At-least-once by construction: publish, THEN mark published. A crash between the two
+ * re-publishes, which the idempotency key collapses — the opposite order would lose jobs.
+ */
+export function createOutboxRelay(options: RelayOptions): OutboxRelay {
+  const batchSize = options.batchSize ?? 100;
+  const intervalMs = options.intervalMs ?? 200;
+  // Refused, never clamped — `worker-options.ts` carries the reason. `setInterval(fn, NaN)` reads
+  // the delay as 0 and `claim(NaN)` slices `(0, NaN)`: a relay that spins and publishes nothing.
+  assert(
+    Number.isSafeInteger(batchSize) && batchSize >= 1,
+    `createOutboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
+    'pass a finite batchSize to createOutboxRelay(...), or omit it for the default 100',
+  );
+  assert(
+    Number.isFinite(intervalMs) && intervalMs >= 0,
+    `createOutboxRelay intervalMs is ${String(intervalMs)}, which setInterval reads as 0 — the relay would spin, not poll`,
+    'pass a finite intervalMs to createOutboxRelay(...), or omit it for the default 200',
+  );
+  let timer: ReturnType<typeof setInterval> | undefined;
+  let running = false;
+  /** The pass in flight, so `stop()` joins it instead of returning underneath it. */
+  let pass: Promise<void> | undefined;
+  /** `'draining'` is the state in which a timer that somehow fires still claims nothing. */
+  let state: 'idle' | 'running' | 'draining' | 'stopped' = 'idle';
+  let releaseShutdownHooks: (() => void)[] = [];
+  let stopping: Promise<void> | undefined;
+
+  const tick = async (): Promise<number> => {
+    const batch = await options.store.claim(batchSize);
+    let published = 0;
+    for (const record of batch) {
+      try {
+        await options.driver.enqueue({
+          name: record.job,
+          queue: record.queue,
+          input: record.input,
+          idempotencyKey: record.idempotencyKey,
+          maxAttempts: record.maxAttempts,
+          runAt: record.runAt,
+          ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
+          ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
+          ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
+        });
+        // The claim's own token goes back with the mark. Without it a relay whose lease lapsed
+        // mid-stall retires a row the relay that reclaimed it has not published yet — the row is
+        // gone and nothing publishes it.
+        await options.store.markPublished(record.id, nowMs(options.clock), record.claimedBy);
+        published += 1;
+      } catch (error) {
+        // STOP the batch. `claim()` returns rows in `staged_at` order and the loop used to log
+        // and continue, which published every LATER row past the one that failed — so an app
+        // that stages `createInvoice` then `chargeCard` in one transaction could have the charge
+        // run first. The row stays unpublished and the next tick starts again from it; a
+        // permanently poisoned row wedges its queue, which is visible in `pending()` and is the
+        // correct trade against silently reordering committed work.
+        logger.warn('jobs.outbox.publish-failed', {
+          job: record.job,
+          id: record.id,
+          published,
+          remaining: batch.length - published,
+          error: renderThrowable(error),
+        });
+        // Hand the rest of the batch back rather than sit on a claim nobody is publishing. The
+        // claim is a lease now, so without this a single pool timeout parks every committed row
+        // behind it for the whole lease window instead of for one poll interval.
+        await options.store.release?.(
+          batch.slice(published).map((row) => row.id),
+          record.claimedBy,
+        );
+        break;
+      }
+    }
+    return published;
+  };
+
+  /**
+   * The `accept` phase, and all of it: flip the state, clear the timer, return. Every hook behind
+   * this one — `@ultimat3/http`'s "stop listening", the sync node's "stop upgrading" — then runs
+   * with the whole budget still in hand, which is what a wait parked here costs them.
+   */
+  const stopAccepting = (): void => {
+    if (timer !== undefined) clearInterval(timer);
+    timer = undefined;
+    if (state === 'running') state = 'draining';
+  };
+
+  const teardown = async (deadlineAt?: number): Promise<void> => {
+    stopAccepting();
+    try {
+      // The pass in flight is a `driver.enqueue` and the `markPublished` behind it. Abandoning
+      // between the two republishes the row on the next boot — which the idempotency key collapses
+      // only while the first job is still live — so it is waited out here, in `close`, under the
+      // deadline the hook was handed. `undefined` on a manual `stop()`: a caller that asked has no
+      // budget to spend, and nothing else in this process is waiting on the answer.
+      const settled = await settleAllBy(pass === undefined ? [] : [pass], deadlineAt);
+      if (!settled) {
+        logger.warn('jobs.outbox.drain-abandoned', {
+          fix: 'raise the drain budget past one publish — configureLifecycle({ deadlineMs: 60_000 }) — and set terminationGracePeriodSeconds to at least as many seconds',
+        });
+      }
+    } finally {
+      // Whatever the wait did, this relay is done, and the hooks go back. One left registered
+      // publishes through a stopped relay on the next process-wide drain — against a driver
+      // already closed — and keeps this closure, its store and its driver alive with it.
+      state = 'stopped';
+      for (const release of releaseShutdownHooks) release();
+      releaseShutdownHooks = [];
+    }
+  };
+
+  const stop = async (deadlineAt?: number): Promise<void> => {
+    // Answered immediately once this relay is done: the teardown always REACHES 'stopped' (its
+    // wait is bounded on the SIGTERM path and the state is set in a `finally`), so a caller
+    // landing after an abandoned drain gets an answer rather than joining a settled lifetime ago.
+    if (state === 'stopped') return;
+    // One teardown, joined rather than repeated: a SIGTERM landing on a manual stop waits out the
+    // same pass instead of returning underneath it. Cleared as it settles, so a relay started
+    // again stops again rather than joining a promise that settled a lifetime ago.
+    stopping ??= teardown(deadlineAt).finally(() => {
+      stopping = undefined;
+    });
+    await stopping;
+  };
+
+  return {
+    tick,
+    start() {
+      if (timer !== undefined) return;
+      // Only from a standstill, the worker's rule: a start mid-drain would re-arm the poll on a
+      // store the drain is about to leave, and stack a second pair of hooks on the one still
+      // running — the first pair's unregisters are held in a single slot and would be dropped.
+      if (state === 'draining') return;
+      state = 'running';
+      // TWO hooks, for the two phases — the shape `worker.ts` and `scheduler.ts` hold. This loop
+      // had NONE: on SIGTERM it went on claiming and publishing through every phase, stamping
+      // leases on rows nothing on this pod would run, and `RunningRoles.stop()` was the only thing
+      // that ever stopped it — a caller a signal can skip entirely.
+      //
+      // Both unregisters are kept, never discarded: the teardown hands them back, so
+      // start -> stop -> start holds one pair rather than one per start.
+      if (options.drainOnShutdown !== false) {
+        releaseShutdownHooks = [
+          onShutdown('jobs.outbox.accept', stopAccepting, { phase: 'accept' }),
+          onShutdown('jobs.outbox', (reason) => stop(reason.deadlineAt), { phase: 'close' }),
+        ];
+      }
+      timer = setInterval(() => {
+        // Re-read on every tick: "stop claiming" means this timer too, and a callback already on
+        // the event loop when the interval was cleared must not open a claim behind the drain.
+        if (running || state !== 'running') return;
+        running = true;
+        // `.catch` before `.finally`, the shape every other loop in this package uses. `tick()`
+        // guards each publish but not `store.claim()` — one pool timeout during a failover
+        // rejects here unobserved, and Bun's default for an unhandled rejection is to end the
+        // process, taking every staged, unpublished row with it.
+        //
+        // Kept rather than discarded, because `stop()` awaits exactly this chain: the publish and
+        // the `markPublished` behind it are one pass, and a teardown that returned between them
+        // closed the database under the row it was about to mark. The chain carries its own
+        // `catch`, so a caller that does not await still gets no unhandled rejection.
+        pass = tick()
+          .then((): void => undefined)
+          .catch((error: unknown) => {
+            logger.error('jobs.outbox.tick-failed', {
+              error: renderThrowable(error),
+            });
+          })
+          .finally(() => {
+            running = false;
+            pass = undefined;
+          });
+      }, intervalMs);
+      // Never the thing keeping a drained process alive — the rule `renewal-timer.ts` and
+      // `lifecycle-deadline.ts` both state for their own timers. A poll every 200ms holds the
+      // event loop open past every phase of the shutdown, and the kubelet's SIGKILL becomes the
+      // exit; the hooks above are what stop this loop, not the process refusing to end.
+      timer.unref?.();
+    },
+    stop,
+    pending: () => options.store.pendingCount(),
+  };
+}

package/src/outbox.ts

@@ -24,14 +24,7 @@
 // test and `x dev` must enqueue with nothing wired — but it is a fallback, not the guarantee.
 
 import type { Clock } from '@ultimat3/core';
-import {
-  assert,
-  currentSpanContext,
-  logger,
-  renderThrowable,
-  traceparent,
-  uuid,
-} from '@ultimat3/core';
+import { currentSpanContext, traceparent, uuid } from '@ultimat3/core';
 import type { Tx } from '@ultimat3/entity';
 import { nowMs } from './clock';
 import type { EnqueueResult, JobDriver } from './driver';
@@ -364,136 +357,6 @@ export function resetJobsFacade(): void {
   ambient = undefined;
 }
 
-export interface RelayOptions extends OutboxDeps {
-  readonly batchSize?: number;
-  readonly intervalMs?: number;
-}
-
-export interface OutboxRelay {
-  /** One pass. Returns how many rows were published. Call it directly in tests. */
-  tick(): Promise<number>;
-  start(): void;
-  /**
-   * Stop polling and WAIT OUT the pass in flight, the way `worker.stop()` waits out its rounds and
-   * `scheduler.stop()` its dispatch. A pass is a publish followed by a `markPublished`, and a
-   * caller that returned between the two closed the database under the row it was about to mark:
-   * re-published next boot at best, a rejection against a closed pool at worst.
-   */
-  stop(): Promise<void>;
-  pending(): Promise<number>;
-}
-
-/**
- * At-least-once by construction: publish, THEN mark published. A crash between the two
- * re-publishes, which the idempotency key collapses — the opposite order would lose jobs.
- */
-export function createOutboxRelay(options: RelayOptions): OutboxRelay {
-  const batchSize = options.batchSize ?? 100;
-  const intervalMs = options.intervalMs ?? 200;
-  // Refused, never clamped — `worker-options.ts` carries the reason. `setInterval(fn, NaN)` reads
-  // the delay as 0 and `claim(NaN)` slices `(0, NaN)`: a relay that spins and publishes nothing.
-  assert(
-    Number.isSafeInteger(batchSize) && batchSize >= 1,
-    `createOutboxRelay batchSize is ${String(batchSize)} — a batch is a whole number of staged rows, at least one`,
-    'pass a finite batchSize to createOutboxRelay(...), or omit it for the default 100',
-  );
-  assert(
-    Number.isFinite(intervalMs) && intervalMs >= 0,
-    `createOutboxRelay intervalMs is ${String(intervalMs)}, which setInterval reads as 0 — the relay would spin, not poll`,
-    'pass a finite intervalMs to createOutboxRelay(...), or omit it for the default 200',
-  );
-  let timer: ReturnType<typeof setInterval> | undefined;
-  let running = false;
-  /** The pass in flight, so `stop()` joins it instead of returning underneath it. */
-  let pass: Promise<void> | undefined;
-
-  const tick = async (): Promise<number> => {
-    const batch = await options.store.claim(batchSize);
-    let published = 0;
-    for (const record of batch) {
-      try {
-        await options.driver.enqueue({
-          name: record.job,
-          queue: record.queue,
-          input: record.input,
-          idempotencyKey: record.idempotencyKey,
-          maxAttempts: record.maxAttempts,
-          runAt: record.runAt,
-          ...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
-          ...(record.traceparent === undefined ? {} : { traceparent: record.traceparent }),
-          ...(record.enqueuedBy === undefined ? {} : { enqueuedBy: record.enqueuedBy }),
-        });
-        // The claim's own token goes back with the mark. Without it a relay whose lease lapsed
-        // mid-stall retires a row the relay that reclaimed it has not published yet — the row is
-        // gone and nothing publishes it.
-        await options.store.markPublished(record.id, nowMs(options.clock), record.claimedBy);
-        published += 1;
-      } catch (error) {
-        // STOP the batch. `claim()` returns rows in `staged_at` order and the loop used to log
-        // and continue, which published every LATER row past the one that failed — so an app
-        // that stages `createInvoice` then `chargeCard` in one transaction could have the charge
-        // run first. The row stays unpublished and the next tick starts again from it; a
-        // permanently poisoned row wedges its queue, which is visible in `pending()` and is the
-        // correct trade against silently reordering committed work.
-        logger.warn('jobs.outbox.publish-failed', {
-          job: record.job,
-          id: record.id,
-          published,
-          remaining: batch.length - published,
-          error: renderThrowable(error),
-        });
-        // Hand the rest of the batch back rather than sit on a claim nobody is publishing. The
-        // claim is a lease now, so without this a single pool timeout parks every committed row
-        // behind it for the whole lease window instead of for one poll interval.
-        await options.store.release?.(
-          batch.slice(published).map((row) => row.id),
-          record.claimedBy,
-        );
-        break;
-      }
-    }
-    return published;
-  };
-
-  return {
-    tick,
-    start() {
-      if (timer !== undefined) return;
-      timer = setInterval(() => {
-        if (running) return;
-        running = true;
-        // `.catch` before `.finally`, the shape every other loop in this package uses. `tick()`
-        // guards each publish but not `store.claim()` — one pool timeout during a failover
-        // rejects here unobserved, and Bun's default for an unhandled rejection is to end the
-        // process, taking every staged, unpublished row with it.
-        //
-        // Kept rather than discarded, because `stop()` awaits exactly this chain: the publish and
-        // the `markPublished` behind it are one pass, and a teardown that returned between them
-        // closed the database under the row it was about to mark. The chain carries its own
-        // `catch`, so a caller that does not await still gets no unhandled rejection.
-        pass = tick()
-          .then((): void => undefined)
-          .catch((error: unknown) => {
-            logger.error('jobs.outbox.tick-failed', {
-              error: renderThrowable(error),
-            });
-          })
-          .finally(() => {
-            running = false;
-            pass = undefined;
-          });
-      }, intervalMs);
-    },
-    async stop() {
-      if (timer !== undefined) clearInterval(timer);
-      timer = undefined;
-      // Awaited AFTER the interval is cleared, so no further pass can start behind this one.
-      await pass;
-    },
-    pending: () => options.store.pendingCount(),
-  };
-}
-
 // The outbox's SQL moved to `driver-pg-sql.ts`, where every statement this package runs lives —
 // and, more to the point, where `SQL_JOBS_TABLE` is: `x_outbox` was declared here, in a constant
 // no boot code applied, which is the whole reason the outbox was documented and never created.

package/src/scheduler.ts

@@ -139,8 +139,10 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     resolveCron(handle.cron, { tz: handle.tz, from: from ?? new Date(nowMs(options.clock)) });
 
   /**
-   * Occurrences in `(after, until]`. Walking forward from the last fire is what makes
-   * catch-up possible at all — a scheduler that only knows "now" cannot know what it missed.
+   * Occurrences in `(after, until]`, the first `maxCatchUp` of them. Walking forward from the
+   * last fire is what makes catch-up possible at all — a scheduler that only knows "now" cannot
+   * know what it missed. TRUNCATED, so its last element is the tenth occurrence after the
+   * watermark and not the latest one missed; `latestOccurrenceBy` answers that question.
    */
   const occurrencesSince = (
     handle: TaskHandle,
@@ -158,6 +160,31 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     return out;
   };
 
+  /**
+   * The latest occurrence at or before `until`, given one is known to lie in `(after, until]`.
+   *
+   * The resolver only answers "the first occurrence strictly after an instant", and walking it
+   * forward from the watermark is bounded by `maxCatchUp` — which is how `skip` came to dispatch
+   * the tenth minute after a three-hour outage and then the twentieth, one per tick, for a policy
+   * whose whole promise is ONE dispatch (measured: twenty `catchUp=true` dispatches a second apart
+   * for a minute cron down 14:23–17:34). So the latest is found by bisection over the instant the
+   * resolver is asked from, not by walking: `next(x) <= until` is monotone in `x`, the invariant
+   * is `next(lo) <= until < next(hi)`, and at `hi - lo === 1` the one occurrence in `(lo, until]`
+   * is `next(lo)`. About 25 resolver calls for a three-hour gap and 35 for a year, whatever the
+   * cron's period — never one per missed minute.
+   */
+  const latestOccurrenceBy = (handle: TaskHandle, after: number, until: number): number => {
+    const nextAfter = (from: number): number => nextRunFor(handle, new Date(from)).getTime();
+    let lo = after;
+    let hi = until;
+    while (hi - lo > 1) {
+      const mid = lo + Math.floor((hi - lo) / 2);
+      if (nextAfter(mid) <= until) lo = mid;
+      else hi = mid;
+    }
+    return nextAfter(lo);
+  };
+
   const dispatch = async (
     handle: TaskHandle,
     occurrenceMs: number,
@@ -242,8 +269,13 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
       if (due.length === 0) continue;
 
       if (handle.catchUp === 'skip') {
-        const latest = due[due.length - 1];
-        if (latest !== undefined) dispatched.push(await dispatch(handle, latest, due.length > 1));
+        // The real latest occurrence, never `due`'s last element: that one is `maxCatchUp` steps
+        // past the watermark, and dispatching it leaves the watermark there — so the next tick
+        // found the next ten still due and fired again, until the walk reached `at`. The
+        // occurrence key stays honest (this IS the occurrence the payload is for), and the
+        // watermark `dispatch` leaves is that occurrence — nothing at or before `at` is due past it.
+        const latest = latestOccurrenceBy(handle, last, at);
+        dispatched.push(await dispatch(handle, latest, due.length > 1));
         continue;
       }
       if (handle.catchUp === 'run-once') {

package/src/steps-memory.ts

@@ -0,0 +1,39 @@
+// The in-memory `StepStore`: what `createMemoryDriver` and every runner test persist steps into.
+// Split from `steps.ts` at the file-size ceiling, along the seam the pg driver already draws —
+// `driver-pg.ts` holds the Postgres store, this file the map-backed one, `steps.ts` the runner
+// both are handed to. Same contract, and `steps.test.ts` is where the contract is pinned.
+
+import type { StepRecord, StepStore } from './steps';
+
+export function createMemoryStepStore(): StepStore {
+  const byRun = new Map<string, Map<string, StepRecord>>();
+  const runOf = (runId: string): Map<string, StepRecord> => {
+    let run = byRun.get(runId);
+    if (run === undefined) {
+      run = new Map();
+      byRun.set(runId, run);
+    }
+    return run;
+  };
+  return {
+    get(runId, name) {
+      return Promise.resolve(byRun.get(runId)?.get(name));
+    },
+    put(record) {
+      runOf(record.runId).set(record.name, record);
+      return Promise.resolve();
+    },
+    list(runId) {
+      const records = [...(byRun.get(runId)?.values() ?? [])];
+      return Promise.resolve(records.sort((a, b) => a.startedAt - b.startedAt));
+    },
+    del(runId, name) {
+      byRun.get(runId)?.delete(name);
+      return Promise.resolve();
+    },
+    clear(runId) {
+      byRun.delete(runId);
+      return Promise.resolve();
+    },
+  };
+}

package/src/steps.ts

@@ -8,6 +8,7 @@
 
 import type { Clock } from '@ultimat3/core';
 import { finiteOption, logger, renderThrowable } from '@ultimat3/core';
+import { expectedQueryLoop } from '@ultimat3/db';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
 import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
@@ -113,39 +114,6 @@ export function isStepSuspension(error: unknown): error is StepSuspension {
   return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
 }
 
-export function createMemoryStepStore(): StepStore {
-  const byRun = new Map<string, Map<string, StepRecord>>();
-  const runOf = (runId: string): Map<string, StepRecord> => {
-    let run = byRun.get(runId);
-    if (run === undefined) {
-      run = new Map();
-      byRun.set(runId, run);
-    }
-    return run;
-  };
-  return {
-    get(runId, name) {
-      return Promise.resolve(byRun.get(runId)?.get(name));
-    },
-    put(record) {
-      runOf(record.runId).set(record.name, record);
-      return Promise.resolve();
-    },
-    list(runId) {
-      const records = [...(byRun.get(runId)?.values() ?? [])];
-      return Promise.resolve(records.sort((a, b) => a.startedAt - b.startedAt));
-    },
-    del(runId, name) {
-      byRun.get(runId)?.delete(name);
-      return Promise.resolve();
-    },
-    clear(runId) {
-      byRun.delete(runId);
-      return Promise.resolve();
-    },
-  };
-}
-
 export interface StepRunnerOptions {
   readonly runId: string;
   readonly jobName: string;
@@ -257,6 +225,21 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
 
   const cancelled = (): boolean => runSignal.aborted;
 
+  /**
+   * The statement itself, declared deliberate to the N+1 detector. One write per step IS the
+   * design this file's header states — each step completes at its own instant and its output has
+   * to be durable before the next one starts, so five steps are five `SQL_STEP_PUT`s that no
+   * batch could replace. Without the declaration `x dev` warned `X_N_PLUS_ONE_WRITE` on every
+   * job of five or more steps, a verdict against the framework's own persistence that an app
+   * could neither fix nor silence. The scope ends with the write: the hydrating `list` and the
+   * job's own statements are judged as before.
+   */
+  const persist = (record: StepRecord): Promise<void> =>
+    expectedQueryLoop(
+      'a durable step is written the instant it completes, one statement per step by design',
+      () => store.put(record),
+    );
+
   /**
    * EVERY write this runner makes, and the one place the cancellation is enforced. A step result
    * from a cancelled attempt is a write onto the attempt that replaced it: the deadline nacked
@@ -265,7 +248,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
    */
   const put = async (record: StepRecord): Promise<void> => {
     if (cancelled()) throw new JobAbortedError({ job: jobName, step: record.name });
-    await store.put(record);
+    await persist(record);
     remember(record);
   };
 
@@ -329,7 +312,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
           attempts,
           error: renderThrowable(error),
         };
-        await store.put(failure);
+        await persist(failure);
         remember(failure);
       }
       throw error;

package/src/webhook.ts

@@ -19,6 +19,10 @@ import type { Clock, Ctx } from '@ultimat3/core';
 import {
   finiteOption,
   isCanonicalWebhookField,
+  // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
+  // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
+  // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
+  isRetryableStatus,
   renderThrowable,
   systemClock,
   WEBHOOK_FIELD_MAX,
@@ -164,10 +168,6 @@ const retryAfterSeconds = (response: Response): number | undefined => {
   return Number(header.trim());
 };
 
-/** A status the same request, unchanged, can still land on. Everything else is somebody's edit. */
-const isRetryableStatus = (status: number): boolean =>
-  status >= 500 || status === 408 || status === 425 || status === 429;
-
 export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
   const clock = definition.clock ?? systemClock;
   const disableAfter = finiteOption(

package/src/worker-run.ts

@@ -27,6 +27,12 @@ export interface RunClaimedOptions {
   readonly heartbeatIntervalMs: number;
   readonly clock?: Clock;
   readonly events?: EventLookup;
+  /**
+   * The worker's drain, composed into every run it starts: aborted with a `JobDrainedError` when
+   * the process is going away, so the body hears it on `ctx.signal` — the one seam it already
+   * reads — before core's in-flight wait starts spending the budget on it.
+   */
+  readonly drain?: AbortSignal;
 }
 
 /** A name this deploy does not know, parked rather than failed — almost always a deploy skew. */
@@ -95,8 +101,10 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
     // controller this worker owns rather than `AbortSignal.any`, for two reasons: it is handed BACK
     // when the run settles (an app whose `context()` carries a process-lifetime signal was
     // accumulating one composite per job), and the worker can abort it itself — which is the only
-    // way a fleet slot taken by somebody else reaches the body running under it.
-    runSignal = createRunSignal([base.signal, heartbeat.signal]);
+    // way a fleet slot taken by somebody else reaches the body running under it. The worker's
+    // drain is the third source: SIGTERM reaches the body through the same signal, carrying the
+    // `X_DRAINING` reason `executeJob` reads to hand the attempt back uncounted.
+    runSignal = createRunSignal([base.signal, heartbeat.signal, options.drain]);
     const signal = runSignal;
     const ctx: Ctx = { ...base, signal: signal.signal };
 

package/src/worker-types.ts

@@ -0,0 +1,56 @@
+// The `worker` role's public contract: what `createWorker` takes, what it hands back, and what
+// `stats()` reports. Apart from `worker.ts` because that file's job is the claim loop and the
+// drain, and a contract three files import should not sit under 500 lines of loop.
+
+import type { Clock, Ctx } from '@ultimat3/core';
+import type { JobDriver, QueueStats } from './driver';
+import type { JobExecution } from './execute';
+import type { Limiter } from './limits';
+import type { EventLookup } from './steps';
+
+export interface WorkerOptions {
+  readonly driver: JobDriver;
+  /** Queues this process serves. Default `['default']`. */
+  readonly queues?: readonly string[];
+  /** Slots per queue. A number applies to every queue. */
+  readonly concurrency?: number | Readonly<Record<string, number>>;
+  readonly limiter?: Limiter;
+  readonly clock?: Clock;
+  readonly events?: EventLookup;
+  /** Supplies the ambient Ctx for a job run; the app wires ALS + tenant here. */
+  readonly context: () => Ctx;
+  readonly visibilityTimeoutMs?: number;
+  readonly pollIntervalMs?: number;
+  readonly heartbeatIntervalMs?: number;
+  readonly workerId?: string;
+  /** Default true. Registers a SIGTERM drain via `onShutdown`. */
+  readonly drainOnShutdown?: boolean;
+}
+
+export interface WorkerStats {
+  readonly workerId: string;
+  readonly queues: readonly string[];
+  readonly state: 'idle' | 'running' | 'draining' | 'stopped';
+  readonly inFlight: number;
+  readonly processed: number;
+  readonly failed: number;
+  readonly suspended: number;
+  readonly deadLettered: number;
+  /** Attempts this worker's drain cut short and handed back uncounted — see `JobDrainedError`. */
+  readonly interrupted: number;
+  readonly queueDepth: readonly QueueStats[];
+}
+
+export interface Worker {
+  start(): void;
+  /** One claim+run round. Returns jobs processed. Tests drive this instead of the timer. */
+  tick(): Promise<readonly JobExecution[]>;
+  /**
+   * Stop claiming, wait for every job this worker holds, close the driver. Unbounded, and it
+   * aborts nothing: a caller that asked wants its work finished. SIGTERM takes the other path —
+   * the shutdown hooks `start()` registers abort every held run's `ctx.signal` and wait under
+   * the lifecycle's deadline.
+   */
+  stop(reason?: string): Promise<void>;
+  stats(): Promise<WorkerStats>;
+}