@ultimat3/jobs 19.2.0 → 19.3.2

package/src/worker.ts

@@ -3,7 +3,7 @@
 // bug here — the visibility timeout re-delivers it — but a worker that exits mid-job on EVERY
 // deploy turns "at least once" into "always twice", so draining is on by default.
 
-import type { Clock, Ctx } from '@ultimat3/core';
+import type { ShutdownReason } from '@ultimat3/core';
 import {
   beginWork,
   logger,
@@ -14,19 +14,18 @@ import {
   uuid,
 } from '@ultimat3/core';
 import { nowMs } from './clock';
-import { settleAllBy } from './drain-wait';
-import type { ClaimedJob, JobDriver, QueueStats } from './driver';
+import { createDrainBudget, settleAllBy } from './drain-wait';
+import type { ClaimedJob } from './driver';
 import { DEFAULT_QUEUE } from './driver';
-import { ConcurrencyUnenforceableError } from './errors';
-import type { JobExecution, JobOutcome } from './execute';
+import { ConcurrencyUnenforceableError, JobDrainedError } from './errors';
+import type { JobExecution } from './execute';
 import { getJob, registeredJobs } from './job';
-import type { Limiter } from './limits';
 import { createLimiter } from './limits';
-import { recordQueueDeadJobs, recordQueueOldestReady } from './metrics';
-import type { EventLookup } from './steps';
+import { JOB_OUTCOME_LABELS, recordQueueDeadJobs, recordQueueOldestReady } from './metrics';
 import { createFleetSlots } from './worker-fleet-slots';
 import { resolveWorkerTimings } from './worker-options';
 import { runClaimedJob } from './worker-run';
+import type { Worker, WorkerOptions, WorkerStats } from './worker-types';
 
 /**
  * How often the claim loop republishes `queue_depth`. Its own interval, not `pollIntervalMs`:
@@ -36,56 +35,7 @@ import { runClaimedJob } from './worker-run';
  */
 const QUEUE_DEPTH_INTERVAL_MS = 15_000;
 
-/**
- * `JobOutcome` -> the `jobs_total` label, and `null` for the outcome that is not one. `suspended`
- * is deliberately unmapped: parking a run is control flow, so counting it would make every
- * `step.sleep` read as a finished job and make the failure ratio meaningless.
- */
-const JOB_OUTCOME_LABELS = Object.freeze<Record<JobOutcome, 'ok' | 'failed' | 'dead' | null>>({
-  completed: 'ok',
-  suspended: null,
-  retried: 'failed',
-  'dead-lettered': 'dead',
-});
-
-export interface WorkerOptions {
-  readonly driver: JobDriver;
-  /** Queues this process serves. Default `['default']`. */
-  readonly queues?: readonly string[];
-  /** Slots per queue. A number applies to every queue. */
-  readonly concurrency?: number | Readonly<Record<string, number>>;
-  readonly limiter?: Limiter;
-  readonly clock?: Clock;
-  readonly events?: EventLookup;
-  /** Supplies the ambient Ctx for a job run; the app wires ALS + tenant here. */
-  readonly context: () => Ctx;
-  readonly visibilityTimeoutMs?: number;
-  readonly pollIntervalMs?: number;
-  readonly heartbeatIntervalMs?: number;
-  readonly workerId?: string;
-  /** Default true. Registers a SIGTERM drain via `onShutdown`. */
-  readonly drainOnShutdown?: boolean;
-}
-
-export interface WorkerStats {
-  readonly workerId: string;
-  readonly queues: readonly string[];
-  readonly state: 'idle' | 'running' | 'draining' | 'stopped';
-  readonly inFlight: number;
-  readonly processed: number;
-  readonly failed: number;
-  readonly suspended: number;
-  readonly deadLettered: number;
-  readonly queueDepth: readonly QueueStats[];
-}
-
-export interface Worker {
-  start(): void;
-  /** One claim+run round. Returns jobs processed. Tests drive this instead of the timer. */
-  tick(): Promise<readonly JobExecution[]>;
-  stop(reason?: string): Promise<void>;
-  stats(): Promise<WorkerStats>;
-}
+export type { Worker, WorkerOptions, WorkerStats } from './worker-types';
 
 export function createWorker(options: WorkerOptions): Worker {
   const workerId = options.workerId ?? `worker-${uuid()}`;
@@ -111,6 +61,23 @@ export function createWorker(options: WorkerOptions): Worker {
    * `inFlight` waited on a set the round it was racing had not finished filling.
    */
   const rounds = new Set<Promise<unknown>>();
+  /**
+   * The drain, as every run this worker starts hears it: composed into each run's `ctx.signal`
+   * (`worker-run.ts`), aborted by the `accept` hook with a `JobDrainedError`. ONE controller and
+   * not one per run, because the fact it carries — "this process is going away" — is one fact.
+   * Replaced with a fresh one when a teardown ends: a controller aborted once stays aborted, and
+   * a restarted worker would otherwise hand every job it claimed a signal born cancelled.
+   */
+  let drainSignal = new AbortController();
+  /**
+   * The deadline the teardown waits under. `undefined` for a manual `stop()`, bound the moment a
+   * shutdown lands — and bound LATE when that shutdown lands on a teardown already in flight: the
+   * `close` hook joins the memoised `stopping` rather than starting a second, and until 2026-09-07
+   * the teardown it joined kept the `undefined` it was started with. Core abandoned the hook at
+   * the deadline; the worker sat on a body ignoring `ctx.signal` with its driver open. Fresh per
+   * teardown, for the controller's reason: a restarted worker's manual stop is unbounded again.
+   */
+  let budget = createDrainBudget();
   let state: WorkerStats['state'] = 'idle';
   let loop: ReturnType<typeof setTimeout> | undefined;
   /**
@@ -125,6 +92,7 @@ export function createWorker(options: WorkerOptions): Worker {
   let failed = 0;
   let suspended = 0;
   let deadLettered = 0;
+  let interrupted = 0;
   let depthPublishedAt = Number.NEGATIVE_INFINITY;
 
   /**
@@ -167,6 +135,7 @@ export function createWorker(options: WorkerOptions): Worker {
       workerId,
       visibilityTimeoutMs,
       heartbeatIntervalMs,
+      drain: drainSignal.signal,
       ...(options.clock === undefined ? {} : { clock: options.clock }),
       ...(options.events === undefined ? {} : { events: options.events }),
     });
@@ -275,6 +244,7 @@ export function createWorker(options: WorkerOptions): Worker {
             if (execution.outcome === 'completed') processed += 1;
             else if (execution.outcome === 'suspended') suspended += 1;
             else if (execution.outcome === 'retried') failed += 1;
+            else if (execution.outcome === 'interrupted') interrupted += 1;
             else deadLettered += 1;
             // The other half of this package's metrics contract: `queue_depth` says how much work
             // is waiting, `jobs_total` says whether any of it is succeeding. Depth alone cannot
@@ -368,19 +338,39 @@ export function createWorker(options: WorkerOptions): Worker {
   };
 
   /**
-   * The whole of the `accept` phase: stop taking work, and nothing else. Synchronous on purpose —
-   * a phase whose job is to be over before the load balancer's next health check must not contain
-   * a wait, and the hook behind this one is somebody else's "stop listening".
+   * The whole of the `accept` phase: stop taking work, tell the work already held, and nothing
+   * else. Synchronous on purpose — a phase whose job is to be over before the load balancer's next
+   * health check must not contain a wait, and the hook behind this one is somebody else's "stop
+   * listening". An abort is synchronous and costs nothing, which is why it belongs HERE and not in
+   * the teardown: core runs `accept`, then waits out in-flight work (every claimed job is
+   * `beginWork()`ed) under the same budget, then `close`. Told in `close`, a body that reads
+   * `ctx.signal` would hear it after the in-flight wait had already spent the whole budget on it —
+   * which is what happened until 2026-09-07: a job that would have stopped in a second was waited
+   * on for the full deadline and abandoned there, exactly like one that ignores the signal.
+   *
+   * Only a SHUTDOWN aborts, and only a shutdown binds the budget. A manual `stop()` passes
+   * nothing: a caller that asked has no budget to spend and wants its work finished, the same
+   * line `settleAllBy` draws for the wait. The bind comes BEFORE the abort's once-guard and on
+   * every call, because the second shutdown to reach a worker is the one that finds the abort
+   * already fired and the teardown already waiting — with no deadline, if the first was manual.
    */
-  const stopAccepting = (): void => {
+  const stopAccepting = (shutdown?: ShutdownReason): void => {
     if (state === 'stopped') return;
     state = 'draining';
     if (loop !== undefined) clearTimeout(loop);
     loop = undefined;
+    if (shutdown === undefined) return;
+    budget.bind(shutdown.deadlineAt);
+    if (drainSignal.signal.aborted) return;
+    logger.info('jobs.worker.drain-signalled', {
+      workerId,
+      signal: shutdown.signal,
+      inFlight: inFlight.size,
+    });
+    drainSignal.abort(new JobDrainedError({ workerId, signal: shutdown.signal }));
   };
 
-  const teardown = async (reason: string, deadlineAt?: number): Promise<void> => {
-    stopAccepting();
+  const teardown = async (reason: string): Promise<void> => {
     logger.info('jobs.worker.draining', { workerId, reason, inFlight: inFlight.size });
     try {
       // Stop claiming, finish what we hold, then close. Anything else re-runs work on deploy.
@@ -388,13 +378,14 @@ export function createWorker(options: WorkerOptions): Worker {
       // `claim()`, and the jobs it starts join `inFlight` after any snapshot taken here — so a
       // drain that waited on `inFlight` alone closed the driver under a job that had just begun.
       //
-      // Both waits share ONE deadline on the SIGTERM path (`undefined` on a manual stop, which
-      // waits as long as its jobs take). Nothing can kill a body that ignores `ctx.signal`, so an
-      // unbounded wait here is a teardown that never ends: driver never closed, state never past
-      // 'draining', and the memoized `stopping` every later `stop()` joins never settling.
-      // Abandoning costs a lapsed lease and a redelivered job — at-least-once, as promised.
-      const rounded = await settleAllBy([...rounds], deadlineAt);
-      const drained = (await settleAllBy([...inFlight], deadlineAt)) && rounded;
+      // Both waits share ONE budget: unbound on a manual stop, which waits as long as its jobs
+      // take, and bound by the SIGTERM — whether it arrived before this teardown or lands in the
+      // middle of it. Nothing can kill a body that ignores `ctx.signal`, so an unbounded wait
+      // here is a teardown that never ends: driver never closed, state never past 'draining', and
+      // the memoized `stopping` every later `stop()` joins never settling. Abandoning costs a
+      // lapsed lease and a redelivered job — at-least-once, as promised.
+      const rounded = await settleAllBy([...rounds], budget);
+      const drained = (await settleAllBy([...inFlight], budget)) && rounded;
       if (!drained) {
         logger.warn('jobs.worker.drain-abandoned', {
           workerId,
@@ -414,20 +405,28 @@ export function createWorker(options: WorkerOptions): Worker {
       state = 'stopped';
       for (const release of releaseShutdownHooks) release();
       releaseShutdownHooks = [];
+      // A run this drain abandoned still follows the old controller through its own composition;
+      // the next start's jobs must not. Fresh here, in the one place a teardown always reaches —
+      // and the budget with it, or the next manual stop would inherit a deadline already spent.
+      drainSignal = new AbortController();
+      budget = createDrainBudget();
     }
   };
 
-  const stop = async (reason = 'stop', deadlineAt?: number): Promise<void> => {
+  const stop = async (reason = 'stop', shutdown?: ShutdownReason): Promise<void> => {
     // Answered immediately once this worker is done: the teardown always REACHES 'stopped' (its
     // waits are bounded and the state is set in a `finally`), so a caller landing after an
     // abandoned drain gets an answer rather than joining a promise that never settles.
     if (state === 'stopped') return;
-    // One teardown, joined rather than repeated: a SIGTERM landing on a manual stop must wait out
-    // the same in-flight work, not close the driver a second time underneath it. Cleared as it
-    // settles, so a worker that started again tears down again instead of joining a promise that
-    // settled a lifetime ago. A close that threw still stopped this worker — the failure is the
-    // caller's to see on the promise it awaited, not a teardown to run twice.
-    stopping ??= teardown(reason, deadlineAt).finally(() => {
+    // Before the join, every time: a SIGTERM landing on a manual stop still aborts every held
+    // run's `ctx.signal` and binds the teardown already waiting to the shutdown's deadline.
+    stopAccepting(shutdown);
+    // One teardown, joined rather than repeated: that SIGTERM must wait out the same in-flight
+    // work, not close the driver a second time underneath it. Cleared as it settles, so a worker
+    // that started again tears down again instead of joining a promise that settled a lifetime
+    // ago. A close that threw still stopped this worker — the failure is the caller's to see on
+    // the promise it awaited, not a teardown to run twice.
+    stopping ??= teardown(reason).finally(() => {
       stopping = undefined;
     });
     await stopping;
@@ -453,19 +452,22 @@ export function createWorker(options: WorkerOptions): Worker {
       }
       state = 'running';
       logger.info('jobs.worker.started', { workerId, queues });
-      // TWO hooks, for the two phases that answer two questions. `accept` stops claiming and
-      // returns, so every hook behind it — the HTTP server's "stop listening", the sync node's
-      // "stop upgrading" — runs while the budget is still whole; one hook doing both spent all of
-      // it in the phase whose whole purpose is to be quick. `close` waits out what this worker
-      // holds and closes the driver, bounded by the deadline the hook is handed.
+      // TWO hooks, for the two phases that answer two questions. `accept` stops claiming, aborts
+      // every held run's `ctx.signal` and returns, so every hook behind it — the HTTP server's
+      // "stop listening", the sync node's "stop upgrading" — runs while the budget is still whole;
+      // one hook doing both spent all of it in the phase whose whole purpose is to be quick.
+      // `close` waits out what this worker holds and closes the driver, bounded by the deadline
+      // the hook is handed.
       //
       // Both unregisters are kept, never discarded: `stop()` hands them back, so
       // start -> stop -> start holds one pair rather than one per start, each retaining the
       // driver of a worker that is already gone.
       if (options.drainOnShutdown !== false) {
         releaseShutdownHooks = [
-          onShutdown(`jobs.worker.${workerId}.accept`, stopAccepting, { phase: 'accept' }),
-          onShutdown(`jobs.worker.${workerId}`, (reason) => stop('SIGTERM', reason.deadlineAt), {
+          onShutdown(`jobs.worker.${workerId}.accept`, (reason) => stopAccepting(reason), {
+            phase: 'accept',
+          }),
+          onShutdown(`jobs.worker.${workerId}`, (reason) => stop(reason.signal, reason), {
             phase: 'close',
           }),
         ];
@@ -484,6 +486,7 @@ export function createWorker(options: WorkerOptions): Worker {
         failed,
         suspended,
         deadLettered,
+        interrupted,
         queueDepth: [...(await options.driver.stats())],
       };
     },