@ultimat3/jobs 23.0.0 → 25.0.0

package/src/worker-fleet-slots.ts

@@ -3,7 +3,9 @@
 // Apart from `worker.ts` because the claim loop's question is "may I start this one?" — which job
 // holds which slot, and who gives it back, is bookkeeping of its own.
 
+import type { Clock } from '@ultimat3/core';
 import { logger, renderThrowable } from '@ultimat3/core';
+import { nowMs } from './clock';
 import type { ClaimedJob, JobDriver } from './driver';
 import { ConcurrencyUnenforceableError } from './errors-concurrency';
 import { getJob, registeredJobs } from './job';
@@ -11,12 +13,6 @@ import type { HeldLease, LeaseStore } from './leases';
 import { jobLeaseKey } from './leases';
 import { type IntervalScheduler, startRenewalTimer } from './renewal-timer';
 
-/**
- * A renewal that REJECTED is not a lost slot: there is a TTL behind it and the interval gets
- * several tries inside it, exactly as `heartbeat.ts` treats a failed `driver.heartbeat`. The
- * heartbeat cannot cover for this one either way — it renews `x_jobs.visible_at`, a different row
- * on a different clock, and knows nothing about `x_job_leases`.
- */
 const noop = (): void => undefined;
 
 export interface FleetSlotOptions {
@@ -33,6 +29,8 @@ export interface FleetSlotOptions {
   readonly renewIntervalMs: number;
   /** What every renewal runs on (`renewal-timer.ts`). Default: a real, unrefed interval. */
   readonly schedule?: IntervalScheduler;
+  /** What "a whole TTL without a renewal landing" is measured on. Default: the system clock. */
+  readonly clock?: Clock;
 }
 
 /**
@@ -75,7 +73,7 @@ export interface FleetSlots {
    * to do without one. `granted` is also the answer for "no cap declared": a job with no
    * `concurrency` never touches the lease table.
    *
-   * A driver with no lease store cannot reach here: `createWorker().start()` refuses to boot when
+   * A driver with no lease store cannot reach here: `jobWorker().start()` refuses to boot when
    * a registered job declares `concurrency` and the driver has none, because a cap that silently
    * holds per process is the documented-guarantee-that-does-nothing axiom 3 exists to make
    * impossible.
@@ -97,8 +95,14 @@ export interface FleetSlots {
 }
 
 export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
-  /** The fleet slot each in-flight job holds, so the renewal finds it and the drain frees it. */
-  const held = new Map<string, HeldLease>();
+  /**
+   * The fleet slots each in-flight job holds, OLDEST FIRST, so the renewal finds one and the
+   * settle frees one. A list, because one worker can hold the same job twice: a claim that lapsed
+   * while its body kept running, and this worker's own re-claim of the row. One entry per job id
+   * let the second overwrite the first — the first run's settle then released the second's slot,
+   * and the first's was never released at all.
+   */
+  const held = new Map<string, HeldLease[]>();
 
   return {
     async acquire(claimed) {
@@ -127,7 +131,7 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
         slotHolder(options.workerId, claimed.id),
       );
       if (slot !== undefined) {
-        held.set(claimed.id, slot);
+        held.set(claimed.id, [...(held.get(claimed.id) ?? []), slot]);
         return GRANTED;
       }
       if (handle.whenBusy !== 'fail' || key === undefined) return { outcome: 'wait', limit, key };
@@ -144,47 +148,80 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
     },
 
     startRenewal(jobId, onLost) {
-      const slot = held.get(jobId);
+      // The NEWEST: a run arms its renewal right behind its own acquire.
+      const slot = held.get(jobId)?.at(-1);
       if (slot === undefined) return noop;
-      // Renewed on the lease heartbeat's own interval and released in the same `finally`: one
-      // clock for "this worker still owns the job" and "this worker still owns the slot" is one
-      // fewer way for them to disagree — and `timer.stopped()` is the same latch `heartbeat.ts`
-      // reads, for the same reason.
-      const timer = startRenewalTimer(
-        options.renewIntervalMs,
-        () =>
-          options.leases
-            ?.renew(slot, options.ttlMs)
-            .then((renewed) => {
-              // `=== false`, never `!renewed`, for the reason `heartbeat.ts` reads `held` that way:
-              // a store written before this return value existed resolves `undefined`, and treating
-              // that as a loss would cancel every job on every renewal. Only an explicit no is one.
-              //
-              // `stopped()` re-read AFTER the await for the other half: the run settles, this timer
-              // is stopped and `worker.ts` releases the slot — so the renewal already on the wire
-              // finds the row gone and answers `false` for a job that FINISHED. Reported, that is
-              // `jobs.worker.slot-lost` at error and an abort on a controller `runSignal.dispose()`
-              // has already torn down: noise about a run nobody lost.
-              if (renewed !== false || timer.stopped()) return;
-              timer.stop();
-              logger.error('jobs.worker.slot-lost', {
-                workerId: options.workerId,
-                jobId,
-                leaseKey: slot.key,
-                slot: slot.slot,
-              });
-              onLost?.(slot);
-            })
-            .catch(noop),
-        options.schedule,
-      );
+      // Renewed on the lease heartbeat's own interval and released in the same `finally` — and
+      // LOST by the same two facts `heartbeat.ts` loses a job's lease by: the store saying "not
+      // yours", or a whole TTL on this process's clock with no renewal landing. The heartbeat
+      // cannot cover for this one: it renews `x_jobs.visible_at`, a different row.
+      const now = (): number => nowMs(options.clock);
+      let renewedAt = now();
+      let renewing = false;
+      let lost = false;
+      const lapsed = (): boolean => now() - renewedAt >= options.ttlMs;
+
+      /** Once, then renewal stops: extending a slot this worker no longer holds pushes out another's. */
+      const reportLost = (reason: 'expired' | 'not-ours', error?: unknown): void => {
+        // `stopped()` for the run that FINISHED: its release deletes the row, so the renewal
+        // already on the wire answers `false` for a slot nobody lost.
+        if (lost || timer.stopped()) return;
+        lost = true;
+        timer.stop();
+        logger.error('jobs.worker.slot-lost', {
+          workerId: options.workerId,
+          jobId,
+          leaseKey: slot.key,
+          slot: slot.slot,
+          reason,
+          ...(error === undefined ? {} : { error: renderThrowable(error) }),
+        });
+        onLost?.(slot);
+      };
+
+      const renew = async (): Promise<void> => {
+        if (lost || timer.stopped() || options.leases === undefined) return;
+        // Decided BEFORE the store is asked: a renewal hung on a dead connection never rejects.
+        if (lapsed()) return reportLost('expired');
+        // One in flight at a time, on the connection that is already the thing failing.
+        if (renewing) return;
+        renewing = true;
+        // Stamped BEFORE the call, as `heartbeat.ts` does: the store dates the new expiry at the
+        // statement, so a slow reply must not stretch this worker's view past it.
+        const sentAt = now();
+        try {
+          const renewed = await options.leases.renew(slot, options.ttlMs);
+          if (timer.stopped()) return;
+          // `=== false`, never `!renewed`: a store written before this answer existed resolves
+          // `undefined`, and reading that as a loss would cancel every job on every renewal.
+          if (renewed === false) return reportLost('not-ours');
+          // A renewal that lands LATE is still late: the slot may already be another worker's.
+          if (lapsed()) return reportLost('expired');
+          renewedAt = sentAt;
+        } catch (error) {
+          // One failed renewal is not a lost slot: the TTL gives the interval several tries.
+          logger.warn('jobs.worker.slot-renewal-failed', {
+            workerId: options.workerId,
+            jobId,
+            leaseKey: slot.key,
+            error: renderThrowable(error),
+          });
+          if (lapsed()) reportLost('expired', error);
+        } finally {
+          renewing = false;
+        }
+      };
+      const timer = startRenewalTimer(options.renewIntervalMs, renew, options.schedule);
       return () => timer.stop();
     },
 
     async release(jobId) {
-      const slot = held.get(jobId);
+      // The OLDEST: the run that settles first is, bar a race nobody loses by, the superseded one
+      // — and if it is not, the superseded run's next renewal answers `false` and cancels it.
+      const [slot, ...rest] = held.get(jobId) ?? [];
       if (slot === undefined) return;
-      held.delete(jobId);
+      if (rest.length === 0) held.delete(jobId);
+      else held.set(jobId, rest);
       // Never lets a settle fail over bookkeeping: an unreleased slot expires on its own TTL.
       await options.leases?.release(slot).catch((error: unknown) => {
         logger.warn('jobs.worker.lease-release-failed', {

package/src/worker-held.ts

@@ -0,0 +1,100 @@
+// The runs one worker holds right now, each with the claim it runs under — so a drain that reaches
+// its cut-off can hand back every row still held instead of leaving it under a lease another
+// worker waits a whole visibility timeout to see lapse. Split from `worker.ts`, whose job is the
+// claim loop; this one is "what am I holding, and give it back".
+
+import type { ClaimedJob, JobDriver } from './driver';
+import { handBack } from './worker-hand-back';
+
+export interface HeldRuns {
+  /** How many runs are held — what `stats().inFlight` reports. */
+  readonly size: number;
+  /**
+   * The job ids held, each once — what the registry row reports. Per RUN underneath, because one
+   * worker can hold a job twice (a claim that lapsed under a body still running, then its own
+   * re-claim): keyed by id, the first run to end dropped an id the second still held.
+   */
+  ids(): string[];
+  /**
+   * A run started. `release` is what the run's own `finally` would do for the process — the
+   * in-flight count core's drain waits on, the fleet slot — run early when the run is handed back.
+   */
+  add(run: Promise<unknown>, claimed: ClaimedJob, release: () => Promise<void>): void;
+  /** The run settled on its own: no longer held. */
+  settle(run: Promise<unknown>): void;
+  /** Resolves the moment nothing is held — settled or handed back. */
+  empty(): Promise<void>;
+  /**
+   * Every run still held goes back to the queue — `countsAsAttempt: false`, no delay — and stops
+   * being this worker's. Answers how many. The bodies themselves cannot be stopped; what they
+   * write after this is fenced out by the claim that just moved (`StepFence`, `SQL_ACK`).
+   */
+  handBackAll(): Promise<number>;
+}
+
+interface Holding {
+  readonly claimed: ClaimedJob;
+  readonly release: () => Promise<void>;
+}
+
+export function createHeldRuns(options: {
+  readonly driver: JobDriver;
+  readonly workerId: string;
+}): HeldRuns {
+  const held = new Map<Promise<unknown>, Holding>();
+  let waiters: (() => void)[] = [];
+
+  /** Pending hand-backs: `empty()` answers only once their nacks have LANDED, never mid-wire. */
+  let handingBack = 0;
+  const wakeIfEmpty = (): void => {
+    if (held.size > 0 || handingBack > 0 || waiters.length === 0) return;
+    const woken = waiters;
+    waiters = [];
+    for (const wake of woken) wake();
+  };
+
+  return {
+    get size() {
+      return held.size;
+    },
+    ids() {
+      return [...new Set([...held.values()].map((holding) => holding.claimed.id))];
+    },
+    add(run, claimed, release) {
+      held.set(run, { claimed, release });
+    },
+    settle(run) {
+      held.delete(run);
+      wakeIfEmpty();
+    },
+    empty() {
+      if (held.size === 0 && handingBack === 0) return Promise.resolve();
+      return new Promise<void>((resolve) => {
+        waiters.push(resolve);
+      });
+    },
+    async handBackAll() {
+      // Taken out of the set BEFORE the first await: a run settling while the nacks are on the
+      // wire is a run the hand-back already owns, and must not be counted twice. And `empty()`
+      // stays unanswered until the nacks land — the teardown waiting on it closes the driver next.
+      const stranded = [...held.values()];
+      held.clear();
+      if (stranded.length === 0) return 0;
+      handingBack += 1;
+      try {
+        await handBack(
+          options.driver,
+          stranded.map((holding) => holding.claimed),
+          { delayMs: 0, workerId: options.workerId },
+        );
+        // After the nacks, never before: the slot is what lets the replacement run a
+        // `concurrency: 1` job, and freed first it could be taken while the row was still ours.
+        for (const holding of stranded) await holding.release();
+      } finally {
+        handingBack -= 1;
+        wakeIfEmpty();
+      }
+      return stranded.length;
+    },
+  };
+}

package/src/worker-key-busy.ts

@@ -4,7 +4,7 @@
 // would be counted, reported and parked as a fault the author declared was an answer.
 
 import type { Clock, Ctx } from '@ultimat3/core';
-import { logger } from '@ultimat3/core';
+import { logger, renderThrowable } from '@ultimat3/core';
 import { nowMs } from './clock';
 import type { ClaimedJob, JobDriver } from './driver';
 import { claimOf } from './driver';
@@ -33,7 +33,9 @@ export interface RefuseKeyBusyOptions {
  * Not `reportError`ed: the error tracker is for failures somebody has to look at, and this one
  * was asked for by the declaration. The row and this log line carry it.
  */
-export async function refuseKeyBusy(options: RefuseKeyBusyOptions): Promise<JobExecution> {
+export async function refuseKeyBusy(
+  options: RefuseKeyBusyOptions,
+): Promise<JobExecution | undefined> {
   const { driver, claimed, key, limit } = options;
   const startedAt = nowMs(options.clock);
   const refusal = new JobKeyBusyError({ job: claimed.name, key, limit });
@@ -47,6 +49,17 @@ export async function refuseKeyBusy(options: RefuseKeyBusyOptions): Promise<JobE
     countsAsAttempt: false,
     fail: true,
   });
+  if (!settled) {
+    // The claim lapsed and the row is another worker's now, or it was cancelled: nothing was
+    // refused HERE, so there is no execution to count — `executeJob`'s own line for the same miss.
+    logger.warn('jobs.settle.unowned', {
+      job: claimed.name,
+      jobId: claimed.id,
+      workerId: options.workerId,
+      settling: 'refused',
+    });
+    return undefined;
+  }
   logger.info('jobs.attempt.refused', {
     workerId: options.workerId,
     job: claimed.name,
@@ -55,15 +68,30 @@ export async function refuseKeyBusy(options: RefuseKeyBusyOptions): Promise<JobE
     limit,
   });
   // The one ending no body is there to record: "a second run was refused" exists for the app only
-  // if the job is told. After the settle and only when it landed, like every other announcement.
+  // if the job is told. After the settle, like every other announcement.
   const handle = getJob(claimed.name);
-  if (settled && handle !== undefined) {
-    await announceSettled({
-      handle,
-      claimed,
-      ctx: options.context(),
-      settlement: { outcome: 'refused', error, code: refusal.code },
-    });
+  if (handle !== undefined) {
+    let ctx: Ctx | undefined;
+    try {
+      ctx = options.context();
+    } catch (raised) {
+      // `context()` is the app's own function. The row is already settled `failed`: a throw from
+      // here failed the whole claim ROUND over a refusal that had landed.
+      logger.error('jobs.attempt.refused-unannounced', {
+        workerId: options.workerId,
+        job: claimed.name,
+        jobId: claimed.id,
+        error: renderThrowable(raised),
+      });
+    }
+    if (ctx !== undefined) {
+      await announceSettled({
+        handle,
+        claimed,
+        ctx,
+        settlement: { outcome: 'refused', error, code: refusal.code },
+      });
+    }
   }
   return {
     outcome: 'refused',

package/src/worker-loop.ts

@@ -15,8 +15,28 @@ export interface ClaimLoopOptions {
   /** One pass. It reports what it found through `passed()`, so a manual `tick()` counts too. */
   readonly round: () => Promise<unknown>;
   readonly onError: (error: unknown) => void;
+  /** What every wait is armed on. Default: real, unrefed timeouts; a test hands in its own. */
+  readonly timers?: LoopTimers;
 }
 
+/** The two calls the loop makes on a timer. The handle is whatever `set` answered. */
+export interface LoopTimers {
+  set(run: () => void, ms: number): unknown;
+  clear(handle: unknown): void;
+}
+
+/** Unrefed: never the thing keeping a drained process alive — the hooks stop this loop. */
+const REAL_TIMERS: LoopTimers = {
+  set(run, ms) {
+    const handle = setTimeout(run, ms);
+    handle.unref?.();
+    return handle;
+  },
+  clear(handle) {
+    clearTimeout(handle as ReturnType<typeof setTimeout>);
+  },
+};
+
 export interface ClaimLoop {
   start(): void;
   stop(): void;
@@ -47,23 +67,30 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
     floorMs: floor,
     ...(options.ceilingMs === undefined ? {} : { ceilingMs: options.ceilingMs }),
   });
+  const timers = options.timers ?? REAL_TIMERS;
   let delayMs = floor;
   let active = false;
-  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timer: unknown;
   /** The timer's own pass is in flight. A wake landing now asks for ONE more, never a second chain. */
   let passing = false;
   let again = false;
   let unsubscribe: (() => void) | undefined;
   /** One timer per floor-wide bucket of due times, so a burst of retries holds a bounded few. */
-  const due = new Map<number, ReturnType<typeof setTimeout>>();
+  const due = new Map<number, unknown>();
 
   const arm = (): void => {
-    timer = setTimeout(() => {
+    timer = timers.set(() => {
       timer = undefined;
       passing = true;
       void options
         .round()
-        .catch(options.onError)
+        .catch((error: unknown) => {
+          // A failed pass never reached `passed()`, so the wait in hand is whatever cut the last
+          // one short — 0 after a wake — and re-arming on it is a loop that spins for as long as
+          // the database is down. Not an idle pass either: try again at the floor.
+          delayMs = backoff.next(true);
+          options.onError(error);
+        })
         .finally(() => {
           passing = false;
           if (!active) return;
@@ -72,8 +99,6 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
           arm();
         });
     }, delayMs);
-    // Never the thing keeping a drained process alive: the hooks stop this loop.
-    timer.unref?.();
   };
 
   /** The next pass, now — or right behind the one in flight, whose snapshot may predate the news. */
@@ -84,7 +109,7 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
       return;
     }
     if (timer === undefined) return;
-    clearTimeout(timer);
+    timers.clear(timer);
     delayMs = 0;
     arm();
   };
@@ -115,9 +140,9 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
     stop() {
       active = false;
       again = false;
-      if (timer !== undefined) clearTimeout(timer);
+      if (timer !== undefined) timers.clear(timer);
       timer = undefined;
-      for (const hint of due.values()) clearTimeout(hint);
+      for (const hint of due.values()) timers.clear(hint);
       due.clear();
       unsubscribe?.();
       unsubscribe = undefined;
@@ -134,11 +159,10 @@ export function createClaimLoop(options: ClaimLoopOptions): ClaimLoop {
       const step = Math.max(floor, DUE_MARGIN_MS);
       const bucket = Math.ceil((performance.now() + wait) / step);
       if (due.has(bucket)) return;
-      const hint = setTimeout(() => {
+      const hint = timers.set(() => {
         due.delete(bucket);
         kick();
       }, wait);
-      hint.unref?.();
       due.set(bucket, hint);
     },
   };

package/src/worker-options.ts

@@ -1,22 +1,19 @@
-// Every numeric knob `createWorker` accepts, read and REFUSED in one place — the slot table
+// Every numeric knob `jobWorker` accepts, read and REFUSED in one place — the slot table
 // included, because a queue name is data and a slot count is a bound, and both arrive from the
 // same deployment config.
 //
 // WHY A REFUSAL AND NOT A CLAMP. `Number(process.env.JOB_VISIBILITY_MS)` on an unset variable is
 // `NaN`; `??` guards only nullish, and `Math.max`/`Math.min`/`Math.floor` PROPAGATE it. So the
 // value arrives at a lease deadline, a claim limit and a timer interval intact, and every
-// comparison against it reads FALSE — measured on `createMemoryDriver`: `visibleAt = at + NaN`,
+// comparison against it reads FALSE — measured on `memoryJobDriver`: `visibleAt = at + NaN`,
 // the reclaim scan asks `visibleAt <= at`, and a job whose worker died is never claimable again.
 // At-least-once becomes never, with no error and a row `x jobs ls` still prints as `running`.
 // `slice(0, NaN)` is `[]`, so a `concurrency: NaN` worker claims nothing and reports healthy.
-// Same shape as `createLimiter`'s `maxTenants` and `backfill()`'s `batch`, refused the same way.
+// Same shape as `concurrencyLimiter`'s `maxTenants` and `backfill()`'s `batch`, refused the same way.
 
-import { finiteOption } from '@ultimat3/core';
+import { finiteOption, JOBS_CONCURRENCY_DEFAULT } from '@ultimat3/core';
 import { DEFAULT_VISIBILITY_TIMEOUT_MS } from './driver';
 
-/** Slots a queue gets when `concurrency` names no number for it. */
-const DEFAULT_SLOTS = 5;
-
 /** The subset of `WorkerOptions` this module reads. Structural, so `WorkerOptions` satisfies it. */
 export interface WorkerNumericOptions {
   readonly concurrency?: number | Readonly<Record<string, number>> | undefined;
@@ -34,7 +31,10 @@ export interface WorkerTimings {
 }
 
 /**
- * Slots for one queue. A queue NAME is deployment data, so the table is read by OWN keys:
+ * Slots for one queue — `JOBS_CONCURRENCY_DEFAULT` (core's, the same number `app.config.ts`'s
+ * `jobs.concurrency` defaults to) for a queue `concurrency` names no number for. This package kept
+ * its own 5 until 25.0.0, so a table `{ banks: 4 }` left `default` at 5 beside an unset config's 8.
+ * A queue NAME is deployment data, so the table is read by OWN keys:
  * `concurrency['constructor']` answers `Object.prototype.constructor`, and
  * `Math.max(0, <function> - inFlight)` is `NaN`, which the `free === 0` guard does not catch.
  * `bun run proto-index` cannot see this one — the table is a parameter, not a literal in a file.
@@ -43,25 +43,29 @@ const slotTable =
   (declared: number | Readonly<Record<string, number>> | undefined) =>
   (queue: string): number => {
     if (typeof declared === 'number') return declared;
-    if (declared === undefined || !Object.hasOwn(declared, queue)) return DEFAULT_SLOTS;
-    return declared[queue] ?? DEFAULT_SLOTS;
+    if (declared === undefined || !Object.hasOwn(declared, queue)) return JOBS_CONCURRENCY_DEFAULT;
+    return declared[queue] ?? JOBS_CONCURRENCY_DEFAULT;
   };
 
 export function resolveWorkerTimings(options: WorkerNumericOptions): WorkerTimings {
   const visibilityTimeoutMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS;
-  finiteOption('createWorker', 'visibilityTimeoutMs', visibilityTimeoutMs);
+  finiteOption('jobWorker', 'visibilityTimeoutMs', visibilityTimeoutMs);
   const pollIntervalMs = options.pollIntervalMs ?? 250;
   // `setTimeout(fn, NaN)` coerces the delay to 0, so the claim loop stops being a poll and becomes
   // a spin: one round trip to Postgres per event-loop turn, from every worker replica.
-  finiteOption('createWorker', 'pollIntervalMs', pollIntervalMs);
+  finiteOption('jobWorker', 'pollIntervalMs', pollIntervalMs);
   const heartbeatIntervalMs = options.heartbeatIntervalMs ?? Math.floor(visibilityTimeoutMs / 3);
-  finiteOption('createWorker', 'heartbeatIntervalMs', heartbeatIntervalMs);
+  finiteOption('jobWorker', 'heartbeatIntervalMs', heartbeatIntervalMs);
   const declared = options.concurrency;
-  if (typeof declared === 'number') finiteOption('createWorker', 'concurrency', declared);
+  if (typeof declared === 'number') finiteOption('jobWorker', 'concurrency', declared);
   else if (declared !== undefined) {
     // Per queue, by own key: an inherited member is not this table's to answer with either.
     for (const queue of Object.keys(declared)) {
-      finiteOption('createWorker', `concurrency.${queue}`, declared[queue] ?? DEFAULT_SLOTS);
+      finiteOption(
+        'jobWorker',
+        `concurrency.${queue}`,
+        declared[queue] ?? JOBS_CONCURRENCY_DEFAULT,
+      );
     }
   }
   return {

package/src/worker-registry.ts

@@ -41,7 +41,13 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
   const startedAt = nowMs(options.clock);
   const host = options.host ?? Bun.env['HOSTNAME'] ?? 'unknown';
 
-  const announce = (): Promise<void> =>
+  /**
+   * Every announce still on the wire. `stop()` waits them out before it forgets: the row an
+   * announce writes after the forget is a stopped worker listed as serving for a whole TTL.
+   */
+  const announcing = new Set<Promise<void>>();
+
+  const send = (): Promise<void> =>
     registry
       .announceWorker(
         {
@@ -63,6 +69,13 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
         });
       });
 
+  const announce = (): Promise<void> => {
+    const sent = send().finally(() => announcing.delete(sent));
+    announcing.add(sent);
+    return sent;
+  };
+
+  // Not awaited — a worker serves whether or not its row has landed — and held in `announcing`.
   void announce();
   const timer = startRenewalTimer(
     options.intervalMs,
@@ -75,6 +88,8 @@ export function startWorkerRegistry(options: WorkerRegistryOptions): WorkerRegis
   return {
     async stop() {
       timer.stop();
+      // `send` never rejects (it logs), so neither does this.
+      await Promise.all(announcing);
       await registry.forgetWorker(options.workerId).catch((error: unknown) => {
         logger.warn('jobs.worker.forget-failed', {
           workerId: options.workerId,