@ultimat3/jobs 23.0.0 → 24.0.0

package/src/steps.ts

@@ -13,6 +13,15 @@ import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
 import { JobAbortedError, JobTimeoutError, StepDuplicateError } from './errors';
 import { createRunSignal } from './run-signal';
+import {
+  isStepSuspension,
+  isWaitTimedOut,
+  StepSuspension,
+  WAIT_TIMED_OUT,
+} from './steps-suspension';
+import { withStepTimeout } from './steps-timeout';
+
+export { isStepSuspension, StepSuspension } from './steps-suspension';
 
 /**
  * The runtime list is the declaration and `StepStatus` is derived from it, the shape
@@ -45,9 +54,23 @@ export interface StepRecord {
   readonly error?: string;
 }
 
+/** The claim a step write is made under: `claimOf(claimed)`, plus the row it names. */
+export interface StepFence {
+  readonly job: string;
+  readonly jobId: string;
+  readonly workerId: string;
+  readonly claim: number;
+}
+
 export interface StepStore {
   get(runId: string, name: string): Promise<StepRecord | undefined>;
-  put(record: StepRecord): Promise<void>;
+  /**
+   * With `by`, the write lands only while that claim still holds the row (`running`, same worker,
+   * same claim ordinal — `SQL_ACK`'s fence) and is `X_JOB_LEASE_LOST` otherwise: the in-process
+   * signal cannot stop a body in ANOTHER process whose lease lapsed. Without it the write is
+   * unfenced — a transfer (`x jobs drain`) or a runner driven by hand, with no row to hold.
+   */
+  put(record: StepRecord, by?: StepFence): Promise<void>;
   list(runId: string): Promise<readonly StepRecord[]>;
   del(runId: string, name: string): Promise<void>;
   clear(runId: string): Promise<void>;
@@ -55,6 +78,13 @@ export interface StepStore {
 
 /** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
 export interface EventLookup {
+  /**
+   * The bus's OWN clock, in epoch ms: what a publish issued now would be stamped with. A consumer
+   * that wants "published after I asked" takes its "asked at" from here and never from its own
+   * process — `publishedAt` is this clock's, and two clocks compared is a skew between pods
+   * deciding which answers count. `waitForEvent` stamps a NEW wait with it.
+   */
+  now(): Promise<number>;
   find(
     event: string,
     correlationKey: string | undefined,
@@ -90,53 +120,12 @@ export interface StepApi {
   ): Promise<T | undefined>;
 }
 
-/**
- * Control flow, not a failure: it must never be logged as an error or counted as an attempt.
- * Branded by string rather than `instanceof` so two copies of this module still agree.
- */
-export class StepSuspension extends Error {
-  static readonly brand = 'ultimate.jobs.suspension';
-  readonly brand: string = StepSuspension.brand;
-  readonly step: string;
-  readonly resumeAt: number;
-  readonly reason: 'sleep' | 'event';
-
-  constructor(input: { step: string; resumeAt: number; reason: 'sleep' | 'event' }) {
-    super(`step "${input.step}" suspended until ${new Date(input.resumeAt).toISOString()}`);
-    this.name = 'StepSuspension';
-    this.step = input.step;
-    this.resumeAt = input.resumeAt;
-    this.reason = input.reason;
-  }
-}
-
-/**
- * What a timed-out `waitForEvent` persists, and maps back to `undefined` on replay. `undefined`
- * itself cannot survive a store — the pg driver writes `JSON.stringify(output ?? null)` — so the
- * timeout replayed as `null` and `evt === undefined` read as "an event arrived". A namespaced key
- * rather than a bare `{ timedOut: true }`, so no event payload can be mistaken for the marker.
- */
-const WAIT_TIMED_OUT = Object.freeze<Record<string, true>>({
-  '~ultimate.waitTimedOut': true,
-});
-
-function isWaitTimedOut(output: unknown): boolean {
-  return (
-    typeof output === 'object' &&
-    output !== null &&
-    Object.keys(output).length === 1 &&
-    (output as Record<string, unknown>)['~ultimate.waitTimedOut'] === true
-  );
-}
-
-export function isStepSuspension(error: unknown): error is StepSuspension {
-  return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
-}
-
 export interface StepRunnerOptions {
   readonly runId: string;
   readonly jobName: string;
   readonly store: StepStore;
+  /** The claim this attempt runs under — every write is fenced on it. `executeJob` passes it. */
+  readonly fence?: StepFence;
   readonly clock?: Clock;
   readonly events?: EventLookup;
   /** How long a waiting step stays parked between event polls. Default 30s. */
@@ -256,7 +245,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
   const persist = (record: StepRecord): Promise<void> =>
     expectedQueryLoop(
       'a durable step is written the instant it completes, one statement per step by design',
-      () => store.put(record),
+      () => store.put(record, options.fence),
     );
 
   /**
@@ -331,8 +320,16 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
           attempts,
           error: renderThrowable(error),
         };
-        await persist(failure);
-        remember(failure);
+        try {
+          await persist(failure);
+          remember(failure);
+        } catch (unwritten) {
+          logger.warn('jobs.step.failure-unrecorded', {
+            job: jobName,
+            step: name,
+            error: renderThrowable(unwritten),
+          });
+        }
       }
       throw error;
     } finally {
@@ -389,7 +386,9 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     }
 
     const at = now();
-    const startedAt = existing?.startedAt ?? at;
+    // A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
+    // with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
+    const startedAt = existing?.startedAt ?? (await options.events?.now()) ?? at;
     const deadline =
       startedAt +
       finiteDurationMs(waitOptions.timeout ?? 86_400_000, 'step.waitForEvent', 'timeout');
@@ -411,7 +410,13 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
       return hit.payload as T;
     }
 
-    if (at >= deadline) {
+    // `deadline` is on the BUS's clock, so the bus calls the timeout: asked only once this runner
+    // believes the time is up, which costs a wait one statement at its end. A runner AHEAD of the
+    // bus used to give up early by the skew — on an answer still inside its window. (A runner
+    // BEHIND waits the skew longer; knowing that sooner would cost a round trip on every poll.)
+    let remaining = deadline - at;
+    if (remaining <= 0) remaining = deadline - ((await options.events?.now()) ?? at);
+    if (remaining <= 0) {
       if (waitOptions.required === true) {
         throw new JobTimeoutError({
           job: jobName,
@@ -437,7 +442,7 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
       return undefined;
     }
 
-    const resumeAt = Math.min(deadline, at + pollMs);
+    const resumeAt = at + Math.min(remaining, pollMs);
     await put({
       runId,
       name,
@@ -463,34 +468,3 @@ export function createStepRunner(options: StepRunnerOptions): StepRunner {
     replayedNames: () => [...replayed],
   };
 }
-
-/**
- * A step's own ceiling. It ABORTS before it rejects, in that order: `run()` fails the step on this
- * rejection and the job retries, so a body still holding a socket open past the deadline would be
- * racing the attempt that replaced it. Cancelling first is the only thing that can stop it.
- */
-function withStepTimeout<T>(
-  work: Promise<T> | T,
-  timeoutMs: number | undefined,
-  deadline: AbortController,
-  error: () => Error,
-): Promise<T> {
-  if (timeoutMs === undefined || timeoutMs <= 0) return Promise.resolve(work);
-  return new Promise<T>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      const failure = error();
-      deadline.abort(failure);
-      reject(failure);
-    }, timeoutMs);
-    Promise.resolve(work).then(
-      (value) => {
-        clearTimeout(timer);
-        resolve(value);
-      },
-      (cause) => {
-        clearTimeout(timer);
-        reject(cause);
-      },
-    );
-  });
-}

package/src/worker-admit.ts

@@ -95,7 +95,8 @@ export function createAdmission(options: AdmissionOptions): Admit {
     try {
       grant = await fleetSlots.acquire(job);
     } catch (error) {
-      lease.release();
+      // Never started: the slot AND the rate stamp go back (`Lease.abandon`).
+      lease.abandon();
       // This job and every one behind it go BACK, unburned — rethrowing alone stranded them in
       // `running` with an attempt spent on work that never started.
       await handBack(driver, [job, ...behind], back);
@@ -106,7 +107,9 @@ export function createAdmission(options: AdmissionOptions): Admit {
     // refusal before the body: it fails as an attempt, never as a round.
     if (grant.outcome === 'undecidable') return { kind: 'run', lease, refusal: grant.error };
 
-    lease.release();
+    // Over the fleet cap, so this run does not start: `abandon`, never `release`, or a tenant
+    // waiting behind a full `job.concurrency` pays one `ratePerTenant` start per poll interval.
+    lease.abandon();
     try {
       // The two `whenBusy` answers. `'fail'` settles the run here, its body never started.
       if (grant.outcome === 'fail') {
@@ -119,7 +122,8 @@ export function createAdmission(options: AdmissionOptions): Admit {
           context: options.context,
           ...(options.clock === undefined ? {} : { clock: options.clock }),
         });
-        return { kind: 'refused', execution };
+        // No execution: the nack did not land, so the row is not this worker's to have refused.
+        return execution === undefined ? { kind: 'waiting' } : { kind: 'refused', execution };
       }
       const keyed = grant.key === undefined ? '' : `, key ${grant.key}`;
       await shed(job, queue, `job concurrency (${grant.limit}${keyed})`);

package/src/worker-fleet-slots.ts

@@ -3,7 +3,9 @@
 // Apart from `worker.ts` because the claim loop's question is "may I start this one?" — which job
 // holds which slot, and who gives it back, is bookkeeping of its own.
 
+import type { Clock } from '@ultimat3/core';
 import { logger, renderThrowable } from '@ultimat3/core';
+import { nowMs } from './clock';
 import type { ClaimedJob, JobDriver } from './driver';
 import { ConcurrencyUnenforceableError } from './errors-concurrency';
 import { getJob, registeredJobs } from './job';
@@ -11,12 +13,6 @@ import type { HeldLease, LeaseStore } from './leases';
 import { jobLeaseKey } from './leases';
 import { type IntervalScheduler, startRenewalTimer } from './renewal-timer';
 
-/**
- * A renewal that REJECTED is not a lost slot: there is a TTL behind it and the interval gets
- * several tries inside it, exactly as `heartbeat.ts` treats a failed `driver.heartbeat`. The
- * heartbeat cannot cover for this one either way — it renews `x_jobs.visible_at`, a different row
- * on a different clock, and knows nothing about `x_job_leases`.
- */
 const noop = (): void => undefined;
 
 export interface FleetSlotOptions {
@@ -33,6 +29,8 @@ export interface FleetSlotOptions {
   readonly renewIntervalMs: number;
   /** What every renewal runs on (`renewal-timer.ts`). Default: a real, unrefed interval. */
   readonly schedule?: IntervalScheduler;
+  /** What "a whole TTL without a renewal landing" is measured on. Default: the system clock. */
+  readonly clock?: Clock;
 }
 
 /**
@@ -97,8 +95,14 @@ export interface FleetSlots {
 }
 
 export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
-  /** The fleet slot each in-flight job holds, so the renewal finds it and the drain frees it. */
-  const held = new Map<string, HeldLease>();
+  /**
+   * The fleet slots each in-flight job holds, OLDEST FIRST, so the renewal finds one and the
+   * settle frees one. A list, because one worker can hold the same job twice: a claim that lapsed
+   * while its body kept running, and this worker's own re-claim of the row. One entry per job id
+   * let the second overwrite the first — the first run's settle then released the second's slot,
+   * and the first's was never released at all.
+   */
+  const held = new Map<string, HeldLease[]>();
 
   return {
     async acquire(claimed) {
@@ -127,7 +131,7 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
         slotHolder(options.workerId, claimed.id),
       );
       if (slot !== undefined) {
-        held.set(claimed.id, slot);
+        held.set(claimed.id, [...(held.get(claimed.id) ?? []), slot]);
         return GRANTED;
       }
       if (handle.whenBusy !== 'fail' || key === undefined) return { outcome: 'wait', limit, key };
@@ -144,47 +148,77 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
     },
 
     startRenewal(jobId, onLost) {
-      const slot = held.get(jobId);
+      // The NEWEST: a run arms its renewal right behind its own acquire.
+      const slot = held.get(jobId)?.at(-1);
       if (slot === undefined) return noop;
-      // Renewed on the lease heartbeat's own interval and released in the same `finally`: one
-      // clock for "this worker still owns the job" and "this worker still owns the slot" is one
-      // fewer way for them to disagree — and `timer.stopped()` is the same latch `heartbeat.ts`
-      // reads, for the same reason.
-      const timer = startRenewalTimer(
-        options.renewIntervalMs,
-        () =>
-          options.leases
-            ?.renew(slot, options.ttlMs)
-            .then((renewed) => {
-              // `=== false`, never `!renewed`, for the reason `heartbeat.ts` reads `held` that way:
-              // a store written before this return value existed resolves `undefined`, and treating
-              // that as a loss would cancel every job on every renewal. Only an explicit no is one.
-              //
-              // `stopped()` re-read AFTER the await for the other half: the run settles, this timer
-              // is stopped and `worker.ts` releases the slot — so the renewal already on the wire
-              // finds the row gone and answers `false` for a job that FINISHED. Reported, that is
-              // `jobs.worker.slot-lost` at error and an abort on a controller `runSignal.dispose()`
-              // has already torn down: noise about a run nobody lost.
-              if (renewed !== false || timer.stopped()) return;
-              timer.stop();
-              logger.error('jobs.worker.slot-lost', {
-                workerId: options.workerId,
-                jobId,
-                leaseKey: slot.key,
-                slot: slot.slot,
-              });
-              onLost?.(slot);
-            })
-            .catch(noop),
-        options.schedule,
-      );
+      // Renewed on the lease heartbeat's own interval and released in the same `finally` — and
+      // LOST by the same two facts `heartbeat.ts` loses a job's lease by: the store saying "not
+      // yours", or a whole TTL on this process's clock with no renewal landing. The heartbeat
+      // cannot cover for this one: it renews `x_jobs.visible_at`, a different row.
+      const now = (): number => nowMs(options.clock);
+      let renewedAt = now();
+      let renewing = false;
+      let lost = false;
+      const lapsed = (): boolean => now() - renewedAt >= options.ttlMs;
+
+      /** Once, then renewal stops: extending a slot this worker no longer holds pushes out another's. */
+      const reportLost = (reason: 'expired' | 'not-ours', error?: unknown): void => {
+        // `stopped()` for the run that FINISHED: its release deletes the row, so the renewal
+        // already on the wire answers `false` for a slot nobody lost.
+        if (lost || timer.stopped()) return;
+        lost = true;
+        timer.stop();
+        logger.error('jobs.worker.slot-lost', {
+          workerId: options.workerId,
+          jobId,
+          leaseKey: slot.key,
+          slot: slot.slot,
+          reason,
+          ...(error === undefined ? {} : { error: renderThrowable(error) }),
+        });
+        onLost?.(slot);
+      };
+
+      const renew = async (): Promise<void> => {
+        if (lost || timer.stopped() || options.leases === undefined) return;
+        // Decided BEFORE the store is asked: a renewal hung on a dead connection never rejects.
+        if (lapsed()) return reportLost('expired');
+        // One in flight at a time, on the connection that is already the thing failing.
+        if (renewing) return;
+        renewing = true;
+        try {
+          const renewed = await options.leases.renew(slot, options.ttlMs);
+          if (timer.stopped()) return;
+          // `=== false`, never `!renewed`: a store written before this answer existed resolves
+          // `undefined`, and reading that as a loss would cancel every job on every renewal.
+          if (renewed === false) return reportLost('not-ours');
+          // A renewal that lands LATE is still late: the slot may already be another worker's.
+          if (lapsed()) return reportLost('expired');
+          renewedAt = now();
+        } catch (error) {
+          // One failed renewal is not a lost slot: the TTL gives the interval several tries.
+          logger.warn('jobs.worker.slot-renewal-failed', {
+            workerId: options.workerId,
+            jobId,
+            leaseKey: slot.key,
+            error: renderThrowable(error),
+          });
+          if (lapsed()) reportLost('expired', error);
+        } finally {
+          renewing = false;
+        }
+      };
+      const timer = startRenewalTimer(options.renewIntervalMs, renew, options.schedule);
       return () => timer.stop();
     },
 
     async release(jobId) {
-      const slot = held.get(jobId);
+      // The OLDEST: the run that settles first is, bar a race nobody loses by, the superseded one
+      // — and if it is not, the superseded run's next renewal answers `false` and cancels it.
+      const [slot, ...rest] = held.get(jobId) ?? [];
       if (slot === undefined) return;
-      held.delete(jobId);
+      if (rest.length === 0) held.delete(jobId);
+      else held.set(jobId, rest);
       // Never lets a settle fail over bookkeeping: an unreleased slot expires on its own TTL.
       await options.leases?.release(slot).catch((error: unknown) => {
         logger.warn('jobs.worker.lease-release-failed', {

package/src/worker-key-busy.ts

@@ -4,7 +4,7 @@
 // would be counted, reported and parked as a fault the author declared was an answer.
 
 import type { Clock, Ctx } from '@ultimat3/core';
-import { logger } from '@ultimat3/core';
+import { logger, renderThrowable } from '@ultimat3/core';
 import { nowMs } from './clock';
 import type { ClaimedJob, JobDriver } from './driver';
 import { claimOf } from './driver';
@@ -33,7 +33,9 @@ export interface RefuseKeyBusyOptions {
  * Not `reportError`ed: the error tracker is for failures somebody has to look at, and this one
  * was asked for by the declaration. The row and this log line carry it.
  */
-export async function refuseKeyBusy(options: RefuseKeyBusyOptions): Promise<JobExecution> {
+export async function refuseKeyBusy(
+  options: RefuseKeyBusyOptions,
+): Promise<JobExecution | undefined> {
   const { driver, claimed, key, limit } = options;
   const startedAt = nowMs(options.clock);
   const refusal = new JobKeyBusyError({ job: claimed.name, key, limit });
@@ -47,6 +49,17 @@ export async function refuseKeyBusy(options: RefuseKeyBusyOptions): Promise<JobE
     countsAsAttempt: false,
     fail: true,
   });
+  if (!settled) {
+    // The claim lapsed and the row is another worker's now, or it was cancelled: nothing was
+    // refused HERE, so there is no execution to count — `executeJob`'s own line for the same miss.
+    logger.warn('jobs.settle.unowned', {
+      job: claimed.name,
+      jobId: claimed.id,
+      workerId: options.workerId,
+      settling: 'refused',
+    });
+    return undefined;
+  }
   logger.info('jobs.attempt.refused', {
     workerId: options.workerId,
     job: claimed.name,
@@ -55,15 +68,30 @@ export async function refuseKeyBusy(options: RefuseKeyBusyOptions): Promise<JobE
     limit,
   });
   // The one ending no body is there to record: "a second run was refused" exists for the app only
-  // if the job is told. After the settle and only when it landed, like every other announcement.
+  // if the job is told. After the settle, like every other announcement.
   const handle = getJob(claimed.name);
-  if (settled && handle !== undefined) {
-    await announceSettled({
-      handle,
-      claimed,
-      ctx: options.context(),
-      settlement: { outcome: 'refused', error, code: refusal.code },
-    });
+  if (handle !== undefined) {
+    let ctx: Ctx | undefined;
+    try {
+      ctx = options.context();
+    } catch (raised) {
+      // `context()` is the app's own function. The row is already settled `failed`: a throw from
+      // here failed the whole claim ROUND over a refusal that had landed.
+      logger.error('jobs.attempt.refused-unannounced', {
+        workerId: options.workerId,
+        job: claimed.name,
+        jobId: claimed.id,
+        error: renderThrowable(raised),
+      });
+    }
+    if (ctx !== undefined) {
+      await announceSettled({
+        handle,
+        claimed,
+        ctx,
+        settlement: { outcome: 'refused', error, code: refusal.code },
+      });
+    }
   }
   return {
     outcome: 'refused',

package/src/worker-loop.ts

@@ -15,8 +15,28 @@ export interface ClaimLoopOptions {
   /** One pass. It reports what it found through `passed()`, so a manual `tick()` counts too. */
   readonly round: () => Promise<unknown>;
   readonly onError: (error: unknown) => void;
+  /** What every wait is armed on. Default: real, unrefed timeouts; a test hands in its own. */
+  readonly timers?: LoopTimers;
 }
 
+/** The two calls the loop makes on a timer. The handle is whatever `set` answered. */
+export interface LoopTimers {
+  set(run: () => void, ms: number): unknown;
+  clear(handle: unknown): void;
+}
+
+/** Unrefed: never the thing keeping a drained process alive — the hooks stop this loop. */
+const REAL_TIMERS: LoopTimers = {
+  set(run, ms) {
+    const handle = setTimeout(run, ms);
+    handle.unref?.();
+    return handle;
+  },
+  clear(handle) {
+    clearTimeout(handle as ReturnType<typeof setTimeout>);
+  },
+};
+
 export interface ClaimLoop {
   start(): void;
   stop(): void;
@@ -47,23 +67,30 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
     floorMs: floor,
     ...(options.ceilingMs === undefined ? {} : { ceilingMs: options.ceilingMs }),
   });
+  const timers = options.timers ?? REAL_TIMERS;
   let delayMs = floor;
   let active = false;
-  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timer: unknown;
   /** The timer's own pass is in flight. A wake landing now asks for ONE more, never a second chain. */
   let passing = false;
   let again = false;
   let unsubscribe: (() => void) | undefined;
   /** One timer per floor-wide bucket of due times, so a burst of retries holds a bounded few. */
-  const due = new Map<number, ReturnType<typeof setTimeout>>();
+  const due = new Map<number, unknown>();
 
   const arm = (): void => {
-    timer = setTimeout(() => {
+    timer = timers.set(() => {
       timer = undefined;
       passing = true;
       void options
         .round()
-        .catch(options.onError)
+        .catch((error: unknown) => {
+          // A failed pass never reached `passed()`, so the wait in hand is whatever cut the last
+          // one short — 0 after a wake — and re-arming on it is a loop that spins for as long as
+          // the database is down. Not an idle pass either: try again at the floor.
+          delayMs = backoff.next(true);
+          options.onError(error);
+        })
         .finally(() => {
           passing = false;
           if (!active) return;
@@ -72,8 +99,6 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
           arm();
         });
     }, delayMs);
-    // Never the thing keeping a drained process alive: the hooks stop this loop.
-    timer.unref?.();
   };
 
   /** The next pass, now — or right behind the one in flight, whose snapshot may predate the news. */
@@ -84,7 +109,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
       return;
     }
     if (timer === undefined) return;
-    clearTimeout(timer);
+    timers.clear(timer);
     delayMs = 0;
     arm();
   };
@@ -115,9 +140,9 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
     stop() {
       active = false;
       again = false;
-      if (timer !== undefined) clearTimeout(timer);
+      if (timer !== undefined) timers.clear(timer);
       timer = undefined;
-      for (const hint of due.values()) clearTimeout(hint);
+      for (const hint of due.values()) timers.clear(hint);
       due.clear();
       unsubscribe?.();
       unsubscribe = undefined;
@@ -134,11 +159,10 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
       const step = Math.max(floor, DUE_MARGIN_MS);
       const bucket = Math.ceil((performance.now() + wait) / step);
       if (due.has(bucket)) return;
-      const hint = setTimeout(() => {
+      const hint = timers.set(() => {
         due.delete(bucket);
         kick();
       }, wait);
-      hint.unref?.();
       due.set(bucket, hint);
     },
   };

package/src/worker-registry.ts

@@ -41,7 +41,13 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
   const startedAt = nowMs(options.clock);
   const host = options.host ?? Bun.env['HOSTNAME'] ?? 'unknown';
 
-  const announce = (): Promise<void> =>
+  /**
+   * Every announce still on the wire. `stop()` waits them out before it forgets: the row an
+   * announce writes after the forget is a stopped worker listed as serving for a whole TTL.
+   */
+  const announcing = new Set<Promise<void>>();
+
+  const send = (): Promise<void> =>
     registry
       .announceWorker(
         {
@@ -63,6 +69,13 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
         });
       });
 
+  const announce = (): Promise<void> => {
+    const sent = send().finally(() => announcing.delete(sent));
+    announcing.add(sent);
+    return sent;
+  };
+
+  // Not awaited — a worker serves whether or not its row has landed — and held in `announcing`.
   void announce();
   const timer = startRenewalTimer(
     options.intervalMs,
@@ -75,6 +88,8 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
   return {
     async stop() {
       timer.stop();
+      // `send` never rejects (it logs), so neither does this.
+      await Promise.all(announcing);
       await registry.forgetWorker(options.workerId).catch((error: unknown) => {
         logger.warn('jobs.worker.forget-failed', {
           workerId: options.workerId,