@ultimat3/jobs 23.0.0 → 25.0.0

package/src/scheduler-pg.ts

@@ -3,11 +3,10 @@
 // both failures are silent — a redeployed pod arms to tomorrow and never reports the 03:00 run the
 // pod it replaced dropped, and two pods in a rolling update both dispatch every task.
 
-import type { Clock } from '@ultimat3/core';
-import { finiteOption, uuid } from '@ultimat3/core';
+import type { Clock, PgExecutor } from '@ultimat3/core';
+import { finiteOption, uuidV7 } from '@ultimat3/core';
 import { nowMs } from './clock';
 import { DEFAULT_QUEUE } from './driver';
-import type { PgExecutor } from './driver-pg';
 import { SQL_SCHEDULER_FIRE } from './driver-pg-operator-sql';
 import {
   SQL_LEADER_ACQUIRE,
@@ -25,7 +24,7 @@ import { fireThroughDriver } from './scheduler-state';
  * whose watermark dies with its process has none of them — it takes the arming branch on every
  * boot and drops every occurrence between the two processes with nothing logged.
  */
-export function pgSchedulerState(executor: PgExecutor): SchedulerState {
+export function postgresSchedulerState(executor: PgExecutor): SchedulerState {
   const state: SchedulerState = {
     async lastFiredAt(taskName) {
       const rows = await executor.query<{ last_fired_at: number | string | null }>(
@@ -42,12 +41,11 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
       // One statement only when the queue IS this database. A host that swapped the driver keeps
       // the watermark here and its jobs elsewhere, and no statement spans the two.
       if (driver.name !== 'pg') return fireThroughDriver(state, driver, fire);
-      const ids = fire.jobs.map(() => uuid());
+      const ids = fire.jobs.map(() => uuidV7());
       const rows = await executor.query<{
         fired: number | string;
         id: string | null;
         run_id: string | null;
-        idempotency_key: string | null;
       }>(SQL_SCHEDULER_FIRE, [
         fire.task,
         fire.occurrenceMs,
@@ -58,23 +56,28 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
             queue: request.queue || DEFAULT_QUEUE,
             input: request.input ?? null,
             idempotency_key: request.idempotencyKey,
-            run_id: request.runId ?? uuid(),
+            run_id: request.runId ?? uuidV7(),
             max_attempts: request.maxAttempts,
           })),
         ),
+        // Where the watermark lands when that is past the occurrence (`run-once`): ONE statement.
+        fire.watermarkMs ?? null,
       ]);
       if (Number(rows[0]?.fired ?? 0) === 0) return undefined;
+      // Keyed on the id THIS statement minted (`ids[index]`), never on the idempotency key: the
+      // key is unique per (name, tenant, key), so two jobs of one occurrence may share it, and a
+      // map by key told both callers the second one's id — where the memory store tells each its own.
       const queued = new Map(
         rows.flatMap((row) =>
-          row.id === null || row.run_id === null || row.idempotency_key === null
+          row.id === null || row.run_id === null
             ? []
-            : [[row.idempotency_key, { id: row.id, runId: row.run_id }] as const],
+            : [[row.id.toLowerCase(), { id: row.id, runId: row.run_id }] as const],
         ),
       );
       // A job absent from the answer met a LIVE row of its key — a fire from before this
       // statement existed, still in flight. It is that row's work, so it is reported deduped.
-      return fire.jobs.map((request) => {
-        const inserted = queued.get(request.idempotencyKey);
+      return fire.jobs.map((_, index) => {
+        const inserted = queued.get(String(ids[index]).toLowerCase());
         return inserted === undefined
           ? { id: '', runId: '', deduped: true }
           : { ...inserted, deduped: false };
@@ -84,7 +87,7 @@ export function pgSchedulerState(executor: PgExecutor): SchedulerState {
   return state;
 }
 
-export interface PgLeaseLeaderOptions {
+export interface PostgresLeaseLeaderOptions {
   readonly executor: PgExecutor;
   /** One key per elected role. Default `'scheduler'`. */
   readonly lockKey?: string;
@@ -106,9 +109,9 @@ export const LEASE_RENEWALS_PER_TTL = 3;
 
 /**
  * Leader election as an expiring row, which is what makes it correct on the executor this package
- * is actually handed: a POOL. `createPgLeader`'s `pg_try_advisory_lock` is session-scoped, and a
+ * is actually handed: a POOL. `postgresLeader`'s `pg_try_advisory_lock` is session-scoped, and a
  * session on a pool ends the moment the connection goes back — so every node reads itself as
- * leader and a rolling update double-fires every task. `@ultimat3/realtime`'s `PgAdvisoryLock`
+ * leader and a rolling update double-fires every task. `@ultimat3/realtime`'s `postgresAdvisoryLock()`
  * solves the same problem by owning its connection; this package holds no wire protocol, so it
  * solves it with a row instead.
  *
@@ -117,9 +120,9 @@ export const LEASE_RENEWALS_PER_TTL = 3;
  * with nothing to clean up, which is the one property the advisory lock had and a plain
  * `insert ... on conflict do nothing` would not.
  */
-export function createPgLeaseLeader(options: PgLeaseLeaderOptions): LeaderElection {
+export function postgresLeaseLeader(options: PostgresLeaseLeaderOptions): LeaderElection {
   const lockKey = options.lockKey ?? 'scheduler';
-  const holder = options.holder ?? `scheduler-${uuid()}`;
+  const holder = options.holder ?? `scheduler-${uuidV7()}`;
   const ttlMs = finiteOption(
     'the pg scheduler lease',
     'ttlMs',

package/src/scheduler-state.ts

@@ -9,6 +9,13 @@ import type { EnqueueRequest, EnqueueResult, JobDriver } from './driver';
 export interface ScheduledFire {
   readonly task: string;
   readonly occurrenceMs: number;
+  /**
+   * Where the watermark LANDS, when that is past the occurrence: `run-once` runs the earliest
+   * missed occurrence and drops the rest, and dropping is moving the watermark past them — in the
+   * same write, or a crash between the two fires a second "one catch-up". The fence is still the
+   * occurrence. Omitted, the watermark lands on the occurrence.
+   */
+  readonly watermarkMs?: number;
   readonly jobs: readonly EnqueueRequest[];
 }
 
@@ -32,7 +39,7 @@ export interface SchedulerState {
 /**
  * The fire for a store that cannot share a statement with the queue: the watermark is checked,
  * the jobs go through `driver.enqueue`, and the watermark moves last. Not atomic across a crash —
- * which is why `pgSchedulerState` does not use it against the pg driver — but exact for the
+ * which is why `postgresSchedulerState` does not use it against the pg driver — but exact for the
  * memory pair, where nothing can fail between the steps and no second process exists.
  */
 export async function fireThroughDriver(
@@ -44,14 +51,14 @@ export async function fireThroughDriver(
   if (last !== undefined && last >= fire.occurrenceMs) return undefined;
   const results: EnqueueResult[] = [];
   for (const request of fire.jobs) results.push(await driver.enqueue(request));
-  await state.markFired(fire.task, fire.occurrenceMs);
+  await state.markFired(fire.task, fire.watermarkMs ?? fire.occurrenceMs);
   // What an operator reads as "last fired". After the watermark, so a record never names an
   // occurrence a crash would fire again.
   await driver.introspect?.recordTaskFire({ task: fire.task, occurrenceMs: fire.occurrenceMs });
   return results;
 }
 
-export function createMemorySchedulerState(): SchedulerState {
+export function memorySchedulerState(): SchedulerState {
   const fired = new Map<string, number>();
   const state: SchedulerState = {
     lastFiredAt: (taskName) => Promise.resolve(fired.get(taskName)),

package/src/scheduler.ts

@@ -2,7 +2,7 @@
 // it owes and enqueues their jobs. The `task` primitive it reads lives in `task.ts`.
 //
 // Exactly one node dispatches per tick, enforced by leader election. Multi-node that is an
-// EXPIRING LEASE ROW in `x_scheduler_leader` (`createPgLeaseLeader`), never an advisory lock: an
+// EXPIRING LEASE ROW in `x_scheduler_leader` (`postgresLeaseLeader`), never an advisory lock: an
 // advisory lock is held by the SESSION, not by this process — it outlives every transaction and is
 // released only by an explicit unlock, the pool's reset on release, or the connection dying, and
 // the next round may run on a different connection. So a node can neither renew it nor prove it
@@ -31,7 +31,7 @@ import type { LeaderElection } from './scheduler-leader';
 import { soleLeader } from './scheduler-leader';
 import { defaultCronResolver, latestOccurrence, occurrencesIn } from './scheduler-occurrences';
 import type { SchedulerState } from './scheduler-state';
-import { createMemorySchedulerState } from './scheduler-state';
+import { memorySchedulerState } from './scheduler-state';
 import type { TaskHandle, TaskJobResult } from './task';
 import { registeredTasks } from './task';
 
@@ -89,11 +89,11 @@ export interface Scheduler {
   nextRunFor(handle: TaskHandle, from?: Date): Date;
 }
 
-export function createScheduler(options: SchedulerOptions): Scheduler {
-  const schedulerState = options.state ?? createMemorySchedulerState();
+export function jobScheduler(options: SchedulerOptions): Scheduler {
+  const schedulerState = options.state ?? memorySchedulerState();
   const resolveCron = options.cron ?? defaultCronResolver;
   const tickIntervalMs = finiteOption(
-    'createScheduler',
+    'jobScheduler',
     'tickIntervalMs',
     options.tickIntervalMs ?? 1_000,
   );
@@ -144,6 +144,7 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     handle: TaskHandle,
     occurrenceMs: number,
     catchUp: boolean,
+    watermarkMs?: number,
   ): Promise<DispatchedOccurrence | undefined> => {
     // The occurrence, not `at`: a catch-up dispatch runs long after the instant it fires for,
     // and the payload has to describe the occurrence the email/report claims to be about.
@@ -166,6 +167,7 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
     const results = await schedulerState.fire(options.driver, {
       task: handle.name,
       occurrenceMs,
+      ...(watermarkMs === undefined ? {} : { watermarkMs }),
       jobs: requests,
     });
     if (results === undefined) {
@@ -323,16 +325,13 @@ export function createScheduler(options: SchedulerOptions): Scheduler {
         if (handle.catchUp === 'run-once') {
           const first = due[0];
           if (first !== undefined) {
-            const ran = fired(await dispatch(handle, first, due.length > 1));
-            // `dispatch` leaves the watermark on the occurrence it RAN — the earliest missed one
-            // here — so the next round found occurrences 2..n still due and fired the second, then
-            // the third, one per tick until the backlog drained: 24 nightly digests a second apart
-            // after a day down. "One catch-up" means the rest are DROPPED, and dropping an
-            // occurrence is moving the watermark past it. `at` rather than the last element of
-            // `due`, which `maxCatchUp` truncates: every occurrence at or before `at` is missed by
-            // definition, and this policy fires none of them.
-            if (first !== at) await schedulerState.markFired(handle.name, at);
-            if (ran) remember(handle, Math.max(first, at));
+            // "One catch-up" means the rest are DROPPED, and dropping an occurrence is moving the
+            // watermark past it: left on the occurrence that RAN, the next round found 2..n still
+            // due and fired one per tick — 24 nightly digests a second apart after a day down.
+            // `at` rather than the last element of `due`, which `maxCatchUp` truncates. It rides
+            // the FIRE (`watermarkMs`): a `markFired` behind it was a second write, and a scheduler
+            // that died between the two handed its successor another catch-up.
+            if (fired(await dispatch(handle, first, due.length > 1, at))) remember(handle, at);
           }
           continue;
         }

package/src/steps-memory.ts

@@ -1,11 +1,11 @@
-// The in-memory `StepStore`: what `createMemoryDriver` and every runner test persist steps into.
+// The in-memory `StepStore`: what `memoryJobDriver` and every runner test persist steps into.
 // Split from `steps.ts` at the file-size ceiling, along the seam the pg driver already draws —
 // `driver-pg.ts` holds the Postgres store, this file the map-backed one, `steps.ts` the runner
 // both are handed to. Same contract, and `steps.test.ts` is where the contract is pinned.
 
 import type { StepRecord, StepStore } from './steps';
 
-export function createMemoryStepStore(): StepStore {
+export function memoryStepStore(): StepStore {
   const byRun = new Map<string, Map<string, StepRecord>>();
   const runOf = (runId: string): Map<string, StepRecord> => {
     let run = byRun.get(runId);

package/src/steps-suspension.ts

@@ -0,0 +1,46 @@
+// A run that unwinds on purpose: the `StepSuspension` a `sleep` or `waitForEvent` throws, and the
+// marker a timed-out wait persists. Split from `steps.ts` at the file-size ceiling — that file is
+// the runner; this is the control flow the worker reads off what the runner threw.
+
+/**
+ * Control flow, not a failure: it must never be logged as an error or counted as an attempt.
+ * Branded by string rather than `instanceof` so two copies of this module still agree.
+ */
+export class StepSuspension extends Error {
+  static readonly brand = 'ultimate.jobs.suspension';
+  readonly brand: string = StepSuspension.brand;
+  readonly step: string;
+  readonly resumeAt: number;
+  readonly reason: 'sleep' | 'event';
+
+  constructor(input: { step: string; resumeAt: number; reason: 'sleep' | 'event' }) {
+    super(`step "${input.step}" suspended until ${new Date(input.resumeAt).toISOString()}`);
+    this.name = 'StepSuspension';
+    this.step = input.step;
+    this.resumeAt = input.resumeAt;
+    this.reason = input.reason;
+  }
+}
+
+/**
+ * What a timed-out `waitForEvent` persists, and maps back to `undefined` on replay. `undefined`
+ * itself cannot survive a store — the pg driver writes `JSON.stringify(output ?? null)` — so the
+ * timeout replayed as `null` and `evt === undefined` read as "an event arrived". A namespaced key
+ * rather than a bare `{ timedOut: true }`, so no event payload can be mistaken for the marker.
+ */
+export const WAIT_TIMED_OUT = Object.freeze<Record<string, true>>({
+  '~ultimate.waitTimedOut': true,
+});
+
+export function isWaitTimedOut(output: unknown): boolean {
+  return (
+    typeof output === 'object' &&
+    output !== null &&
+    Object.keys(output).length === 1 &&
+    (output as Record<string, unknown>)['~ultimate.waitTimedOut'] === true
+  );
+}
+
+export function isStepSuspension(error: unknown): error is StepSuspension {
+  return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
+}

package/src/steps-timeout.ts

@@ -0,0 +1,33 @@
+// A step's own ceiling: the race between a step body and `stepTimeout`. Split from `steps.ts` at
+// the file-size ceiling — that file is the runner, and this is the one timer it arms.
+
+/**
+ * A step's own ceiling. It ABORTS before it rejects, in that order: `run()` fails the step on this
+ * rejection and the job retries, so a body still holding a socket open past the deadline would be
+ * racing the attempt that replaced it. Cancelling first is the only thing that can stop it.
+ */
+export function withStepTimeout<T>(
+  work: Promise<T> | T,
+  timeoutMs: number | undefined,
+  deadline: AbortController,
+  error: () => Error,
+): Promise<T> {
+  if (timeoutMs === undefined || timeoutMs <= 0) return Promise.resolve(work);
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => {
+      const failure = error();
+      deadline.abort(failure);
+      reject(failure);
+    }, timeoutMs);
+    Promise.resolve(work).then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (cause) => {
+        clearTimeout(timer);
+        reject(cause);
+      },
+    );
+  });
+}

package/src/steps.ts

@@ -7,12 +7,21 @@
 // catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
 
 import type { Clock } from '@ultimat3/core';
-import { finiteOption, logger, renderThrowable } from '@ultimat3/core';
+import { finiteOption, isUltimateError, logger, renderThrowable } from '@ultimat3/core';
 import { expectedQueryLoop } from '@ultimat3/db';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
 import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
 import { createRunSignal } from './run-signal';
+import {
+  isStepSuspension,
+  isWaitTimedOut,
+  StepSuspension,
+  WAIT_TIMED_OUT,
+} from './steps-suspension';
+import { withStepTimeout } from './steps-timeout';
+
+export { isStepSuspension, StepSuspension } from './steps-suspension';
 
 /**
  * The runtime list is the declaration and `StepStatus` is derived from it, the shape
@@ -45,9 +54,23 @@ export interface StepRecord {
   readonly error?: string;
 }
 
+/** The claim a step write is made under: `claimOf(claimed)`, plus the row it names. */
+export interface StepFence {
+  readonly job: string;
+  readonly jobId: string;
+  readonly workerId: string;
+  readonly claim: number;
+}
+
 export interface StepStore {
   get(runId: string, name: string): Promise<StepRecord | undefined>;
-  put(record: StepRecord): Promise<void>;
+  /**
+   * With `by`, the write lands only while that claim still holds the row (`running`, same worker,
+   * same claim ordinal — `SQL_ACK`'s fence) and is `X_JOB_LEASE_LOST` otherwise: the in-process
+   * signal cannot stop a body in ANOTHER process whose lease lapsed. Without it the write is
+   * unfenced — a transfer (`x jobs drain`) or a runner driven by hand, with no row to hold.
+   */
+  put(record: StepRecord, by?: StepFence): Promise<void>;
   list(runId: string): Promise<readonly StepRecord[]>;
   del(runId: string, name: string): Promise<void>;
   clear(runId: string): Promise<void>;
@@ -55,6 +78,13 @@ export interface StepStore {
 
 /** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
 export interface EventLookup {
+  /**
+   * The bus's OWN clock, in epoch ms: what a publish issued now would be stamped with. A consumer
+   * that wants "published after I asked" takes its "asked at" from here and never from its own
+   * process — `publishedAt` is this clock's, and two clocks compared is a skew between pods
+   * deciding which answers count. `waitForEvent` stamps a NEW wait with it.
+   */
+  now(): Promise<number>;
   find(
     event: string,
     correlationKey: string | undefined,
@@ -90,53 +120,12 @@ export interface StepApi {
   ): Promise<T | undefined>;
 }
 
-/**
- * Control flow, not a failure: it must never be logged as an error or counted as an attempt.
- * Branded by string rather than `instanceof` so two copies of this module still agree.
- */
-export class StepSuspension extends Error {
-  static readonly brand = 'ultimate.jobs.suspension';
-  readonly brand: string = StepSuspension.brand;
-  readonly step: string;
-  readonly resumeAt: number;
-  readonly reason: 'sleep' | 'event';
-
-  constructor(input: { step: string; resumeAt: number; reason: 'sleep' | 'event' }) {
-    super(`step "${input.step}" suspended until ${new Date(input.resumeAt).toISOString()}`);
-    this.name = 'StepSuspension';
-    this.step = input.step;
-    this.resumeAt = input.resumeAt;
-    this.reason = input.reason;
-  }
-}
-
-/**
- * What a timed-out `waitForEvent` persists, and maps back to `undefined` on replay. `undefined`
- * itself cannot survive a store — the pg driver writes `JSON.stringify(output ?? null)` — so the
- * timeout replayed as `null` and `evt === undefined` read as "an event arrived". A namespaced key
- * rather than a bare `{ timedOut: true }`, so no event payload can be mistaken for the marker.
- */
-const WAIT_TIMED_OUT = Object.freeze<Record<string, true>>({
-  '~ultimate.waitTimedOut': true,
-});
-
-function isWaitTimedOut(output: unknown): boolean {
-  return (
-    typeof output === 'object' &&
-    output !== null &&
-    Object.keys(output).length === 1 &&
-    (output as Record<string, unknown>)['~ultimate.waitTimedOut'] === true
-  );
-}
-
-export function isStepSuspension(error: unknown): error is StepSuspension {
-  return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
-}
-
 export interface StepRunnerOptions {
   readonly runId: string;
   readonly jobName: string;
   readonly store: StepStore;
+  /** The claim this attempt runs under — every write is fenced on it. `executeJob` passes it. */
+  readonly fence?: StepFence;
   readonly clock?: Clock;
   readonly events?: EventLookup;
   /** How long a waiting step stays parked between event polls. Default 30s. */
@@ -149,6 +138,13 @@ export interface StepRunnerOptions {
    * store belongs to whichever attempt has it now.
    */
   readonly signal?: AbortSignal;
+  /**
+   * Aborted the moment the ATTEMPT is over — settled, or past its job deadline. `signal` alone
+   * cannot say so once the drain has aborted it first (a controller keeps its first reason), and
+   * the drain's reason writes through `put` (`writesThrough`): this is what still refuses a step
+   * left in flight by a body that returned without awaiting it. `executeJob` passes it.
+   */
+  readonly ended?: AbortSignal;
 }
 
 export interface StepRunner {
@@ -179,7 +175,7 @@ function trace(into: string[], name: string): void {
   if (into.length > MAX_TRACE_NAMES) into.shift();
 }
 
-export function createStepRunner(options: StepRunnerOptions): StepRunner {
+export function stepRunner(options: StepRunnerOptions): StepRunner {
   const { runId, jobName, store } = options;
   /** Every name this attempt has claimed. Membership only — the trace is `used`. */
   const claimed = new Set<string>();
@@ -187,6 +183,13 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
   const replayed: string[] = [];
   const clock = options.clock;
   const pollMs = finiteOption('step.waitForEvent', 'eventPollMs', options.eventPollMs ?? 30_000);
+  // `job()` refuses a non-finite `stepTimeout` at declaration (`job.ts`); screened again here, where
+  // it is read, because this runner is also built directly and `withStepTimeout` reads a `NaN`
+  // ceiling as none at all — the backstop `pollMs` above already is.
+  const stepTimeoutMs =
+    options.stepTimeoutMs === undefined
+      ? undefined
+      : finiteOption('step.run', 'stepTimeoutMs', options.stepTimeoutMs);
   const runSignal = options.signal ?? NEVER_ABORTED;
 
   /**
@@ -244,6 +247,19 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
 
   const cancelled = (): boolean => runSignal.aborted;
 
+  /**
+   * A cancelled run whose step result is still this attempt's to record: the worker's drain cut
+   * it short (`X_DRAINING`, `worker-drain-cutoff.ts`), the write is fenced on the claim, and the
+   * attempt has not ended. The side effect behind the record HAPPENED — refused, the next worker
+   * runs it again. The fence is what makes this safe and not the in-process signal: a row the
+   * drain has already handed back answers the write `X_JOB_LEASE_LOST` in the store.
+   */
+  const writesThrough = (): boolean =>
+    options.fence !== undefined &&
+    options.ended?.aborted !== true &&
+    isUltimateError(runSignal.reason) &&
+    runSignal.reason.code === 'X_DRAINING';
+
   /**
    * The statement itself, declared deliberate to the N+1 detector. One write per step IS the
    * design this file's header states — each step completes at its own instant and its output has
@@ -256,7 +272,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
   const persist = (record: StepRecord): Promise<void> =>
     expectedQueryLoop(
       'a durable step is written the instant it completes, one statement per step by design',
-      () => store.put(record),
+      () => store.put(record, options.fence),
     );
 
   /**
@@ -266,7 +282,9 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
    * never ran — or overwrite one it did. The write is refused, and refusing it unwinds the body.
    */
   const put = async (record: StepRecord): Promise<void> => {
-    if (cancelled()) throw new JobAbortedError({ job: jobName, step: record.name });
+    if (cancelled() && !writesThrough()) {
+      throw new JobAbortedError({ job: jobName, step: record.name });
+    }
     await persist(record);
     remember(record);
   };
@@ -294,15 +312,14 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     // to compose, and DISPOSED in the `finally` below, which is the whole reason `run-signal.ts`
     // exists — `worker-run.ts` disposes the run's own the same way.
     const composed =
-      options.stepTimeoutMs === undefined ? null : createRunSignal([runSignal, deadline.signal]);
+      stepTimeoutMs === undefined ? null : createRunSignal([runSignal, deadline.signal]);
     const signal = composed?.signal ?? runSignal;
     try {
       const output = await withStepTimeout(
         fn(signal),
-        options.stepTimeoutMs,
+        stepTimeoutMs,
         deadline,
-        () =>
-          new JobTimeoutError({ job: jobName, step: name, timeoutMs: options.stepTimeoutMs ?? 0 }),
+        () => new JobTimeoutError({ job: jobName, step: name, timeoutMs: stepTimeoutMs ?? 0 }),
       );
       // Persist BEFORE returning: a crash one line later must not re-run this step.
       await put({
@@ -331,8 +348,16 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
           attempts,
           error: renderThrowable(error),
         };
-        await persist(failure);
-        remember(failure);
+        try {
+          await persist(failure);
+          remember(failure);
+        } catch (unwritten) {
+          logger.warn('jobs.step.failure-unrecorded', {
+            job: jobName,
+            step: name,
+            error: renderThrowable(unwritten),
+          });
+        }
       }
       throw error;
     } finally {
@@ -389,7 +414,9 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     }
 
     const at = now();
-    const startedAt = existing?.startedAt ?? at;
+    // A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
+    // with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
+    const startedAt = existing?.startedAt ?? (await options.events?.now()) ?? at;
     const deadline =
       startedAt +
       finiteDurationMs(waitOptions.timeout ?? 86_400_000, 'step.waitForEvent', 'timeout');
@@ -411,7 +438,13 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
       return hit.payload as T;
     }
 
-    if (at >= deadline) {
+    // `deadline` is on the BUS's clock, so the bus calls the timeout: asked only once this runner
+    // believes the time is up, which costs a wait one statement at its end. A runner AHEAD of the
+    // bus used to give up early by the skew — on an answer still inside its window. (A runner
+    // BEHIND waits the skew longer; knowing that sooner would cost a round trip on every poll.)
+    let remaining = deadline - at;
+    if (remaining <= 0) remaining = deadline - ((await options.events?.now()) ?? at);
+    if (remaining <= 0) {
       if (waitOptions.required === true) {
         throw new JobTimeoutError({
           job: jobName,
@@ -437,7 +470,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
       return undefined;
     }
 
-    const resumeAt = Math.min(deadline, at + pollMs);
+    const resumeAt = at + Math.min(remaining, pollMs);
     await put({
       runId,
       name,
@@ -463,34 +496,3 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     replayedNames: () => [...replayed],
   };
 }
-
-/**
- * A step's own ceiling. It ABORTS before it rejects, in that order: `run()` fails the step on this
- * rejection and the job retries, so a body still holding a socket open past the deadline would be
- * racing the attempt that replaced it. Cancelling first is the only thing that can stop it.
- */
-function withStepTimeout<T>(
-  work: Promise<T> | T,
-  timeoutMs: number | undefined,
-  deadline: AbortController,
-  error: () => Error,
-): Promise<T> {
-  if (timeoutMs === undefined || timeoutMs <= 0) return Promise.resolve(work);
-  return new Promise<T>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      const failure = error();
-      deadline.abort(failure);
-      reject(failure);
-    }, timeoutMs);
-    Promise.resolve(work).then(
-      (value) => {
-        clearTimeout(timer);
-        resolve(value);
-      },
-      (cause) => {
-        clearTimeout(timer);
-        reject(cause);
-      },
-    );
-  });
-}

package/src/task.ts

@@ -141,7 +141,7 @@ export function task(definition: TaskDefinition): TaskHandle {
   // `for (let i = 0; i < handle.maxCatchUp; i += 1)`, so zero (and any negative, and any fraction
   // below one) returns an empty list on every round and the task NEVER fires: no error, no log
   // line, no queue row, forever. Refused where it is written, exactly as `job()` refuses
-  // `concurrency: 0` and `createPacer` refuses `rate: 0`. `Number.isInteger` covers `NaN` and
+  // `concurrency: 0` and `backfillPacer` refuses `rate: 0`. `Number.isInteger` covers `NaN` and
   // `Infinity` in the same predicate — an unbounded catch-up is a burst nobody declared.
   assert(
     definition.maxCatchUp === undefined ||