@ultimat3/jobs 23.0.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CLAUDE.md +72 -75
  2. package/README.md +83 -38
  3. package/package.json +7 -7
  4. package/src/backfill-errors.ts +3 -2
  5. package/src/backfill-ledger.ts +25 -3
  6. package/src/backfill-pass.ts +22 -5
  7. package/src/backfill-rate.ts +2 -2
  8. package/src/backfill.ts +2 -2
  9. package/src/claim-exhausted.ts +76 -0
  10. package/src/counters.ts +1 -1
  11. package/src/driver-memory-operator.ts +34 -12
  12. package/src/driver-memory.ts +80 -38
  13. package/src/driver-pg-jobs-sql.ts +27 -19
  14. package/src/driver-pg-operator-sql.ts +32 -12
  15. package/src/driver-pg-operator.ts +19 -3
  16. package/src/driver-pg-rows.ts +3 -3
  17. package/src/driver-pg-settle-sql.ts +2 -3
  18. package/src/driver-pg-sql.ts +99 -30
  19. package/src/driver-pg.ts +103 -98
  20. package/src/driver.ts +46 -16
  21. package/src/errors-concurrency.ts +1 -1
  22. package/src/errors-operator.ts +12 -4
  23. package/src/errors-requeue.ts +21 -3
  24. package/src/errors.ts +9 -17
  25. package/src/events-pg.ts +11 -21
  26. package/src/events.ts +26 -15
  27. package/src/execute.ts +47 -5
  28. package/src/export.ts +2 -2
  29. package/src/heartbeat.ts +7 -2
  30. package/src/index.ts +45 -99
  31. package/src/inspect-operator.ts +4 -4
  32. package/src/inspect.ts +5 -4
  33. package/src/introspection.ts +23 -11
  34. package/src/job.ts +6 -7
  35. package/src/leases.ts +1 -1
  36. package/src/limits.ts +46 -15
  37. package/src/outbox-lease.ts +1 -1
  38. package/src/outbox-pg.ts +6 -7
  39. package/src/outbox-relay.ts +6 -6
  40. package/src/outbox.ts +47 -19
  41. package/src/purge.ts +1 -1
  42. package/src/queue-wake.ts +1 -2
  43. package/src/renewal-timer.ts +1 -1
  44. package/src/retry-classification.ts +52 -15
  45. package/src/scheduler-leader.ts +2 -2
  46. package/src/scheduler-pg.ts +19 -16
  47. package/src/scheduler-state.ts +10 -3
  48. package/src/scheduler.ts +14 -15
  49. package/src/steps-memory.ts +2 -2
  50. package/src/steps-suspension.ts +46 -0
  51. package/src/steps-timeout.ts +33 -0
  52. package/src/steps.ts +90 -88
  53. package/src/task.ts +1 -1
  54. package/src/webhook-attempt.ts +206 -0
  55. package/src/webhook-ledger.ts +15 -4
  56. package/src/webhook.ts +64 -118
  57. package/src/worker-admit.ts +14 -6
  58. package/src/worker-drain-cutoff.ts +110 -0
  59. package/src/worker-fleet-slots.ts +82 -45
  60. package/src/worker-held.ts +100 -0
  61. package/src/worker-key-busy.ts +38 -10
  62. package/src/worker-loop.ts +35 -11
  63. package/src/worker-options.ts +19 -15
  64. package/src/worker-registry.ts +16 -1
  65. package/src/worker-run.ts +38 -8
  66. package/src/worker-tally.ts +53 -0
  67. package/src/worker-types.ts +7 -4
  68. package/src/worker.ts +124 -73
  69. package/src/driver-nats.ts +0 -86
  70. package/src/driver-redis.ts +0 -89
package/src/worker-run.ts CHANGED
@@ -17,6 +17,7 @@ import type { RunSignal } from './run-signal';
17
17
  import { createRunSignal } from './run-signal';
18
18
  import type { EventLookup } from './steps';
19
19
  import type { FleetSlots } from './worker-fleet-slots';
20
+ import { handBack } from './worker-hand-back';
20
21
 
21
22
  export interface RunClaimedOptions {
22
23
  readonly driver: JobDriver;
@@ -27,14 +28,16 @@ export interface RunClaimedOptions {
27
28
  readonly workerId: string;
28
29
  readonly visibilityTimeoutMs: number;
29
30
  readonly heartbeatIntervalMs: number;
31
+ /** How soon a run whose wiring threw before its body started is claimable again — a shed's. */
32
+ readonly pollIntervalMs: number;
30
33
  readonly clock?: Clock;
31
34
  /** What every renewal runs on (`renewal-timer.ts`). Default: a real, unrefed interval. */
32
35
  readonly schedule?: IntervalScheduler;
33
36
  readonly events?: EventLookup;
34
37
  /**
35
- * The worker's drain, composed into every run it starts: aborted with a `JobDrainedError` when
36
- * the process is going away, so the body hears it on `ctx.signal` — the one seam it already
37
- * reads — before core's in-flight wait starts spending the budget on it.
38
+ * The worker's drain, composed into every run it starts: aborted with a `JobDrainedError` at
39
+ * the drain's cut-off — near the deadline, never at SIGTERM (`worker-drain-cutoff.ts`) — so a
40
+ * body still running then hears it on `ctx.signal`, the one seam it already reads.
38
41
  */
39
42
  readonly drain?: AbortSignal;
40
43
  /** Why this run may not start — handed to `executeJob`, which fails the attempt with it. */
@@ -73,10 +76,28 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
73
76
  return unknownJob(claimed);
74
77
  }
75
78
 
79
+ // A throw from the wiring below means the body never started, so the row goes BACK exactly as a
80
+ // shed returns one — uncounted, no `lastError` — before the throw reaches the round. Rethrowing
81
+ // alone left it `running` until the lease lapsed, an attempt spent on nothing; on the final
82
+ // attempt the claim then dead-letters a job whose body never ran. Once `executeJob` is entered
83
+ // the attempt is real and settles itself, so `started` is set at that door and nowhere else.
84
+ let started = false;
85
+ const handedBack = async (error: unknown): Promise<never> => {
86
+ if (!started) {
87
+ await handBack(driver, [claimed], { delayMs: options.pollIntervalMs, workerId });
88
+ }
89
+ throw error;
90
+ };
91
+
76
92
  // Read BEFORE the timers start. `context()` is the app's own function and it can throw; started
77
93
  // first, a heartbeat interval and a slot renewal were left running for a job that never ran,
78
94
  // with nothing left holding a reference to stop them.
79
- const base = options.context();
95
+ let base: Ctx;
96
+ try {
97
+ base = options.context();
98
+ } catch (error) {
99
+ return handedBack(error);
100
+ }
80
101
 
81
102
  // The lease, kept alive and NOT kept quiet: a renewal that stops landing means the queue hands
82
103
  // this job to another worker while this one is still running it, and `.catch(() => undefined)`
@@ -110,7 +131,7 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
110
131
  // when the run settles (an app whose `context()` carries a process-lifetime signal was
111
132
  // accumulating one composite per job), and the worker can abort it itself — which is the only
112
133
  // way a fleet slot taken by somebody else reaches the body running under it. The worker's
113
- // drain is the third source: SIGTERM reaches the body through the same signal, carrying the
134
+ // drain is the third source: its cut-off reaches the body through the same signal, with the
114
135
  // `X_DRAINING` reason `executeJob` reads to hand the attempt back uncounted.
115
136
  runSignal = createRunSignal([base.signal, heartbeat.signal, options.drain]);
116
137
  const signal = runSignal;
@@ -132,8 +153,9 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
132
153
 
133
154
  return await withSpan(
134
155
  `job.${handle.name}`,
135
- () =>
136
- executeJob({
156
+ () => {
157
+ started = true;
158
+ return executeJob({
137
159
  driver,
138
160
  claimed,
139
161
  handle,
@@ -141,7 +163,8 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
141
163
  ...(options.clock === undefined ? {} : { clock: options.clock }),
142
164
  ...(options.events === undefined ? {} : { events: options.events }),
143
165
  ...(options.refusal === undefined ? {} : { refusal: options.refusal }),
144
- }),
166
+ });
167
+ },
145
168
  {
146
169
  ...(parent === undefined ? {} : { parent }),
147
170
  attributes: {
@@ -152,6 +175,13 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
152
175
  },
153
176
  },
154
177
  );
178
+ } catch (error) {
179
+ // Stopped BEFORE the row goes back, never after: a renewal landing on a handed-back row
180
+ // answers "not yours", which `heartbeat.ts` reports as a lease lost under a running job.
181
+ // Both are idempotent, so the `finally` repeating them costs nothing.
182
+ stopSlotRenewal?.();
183
+ heartbeat.stop();
184
+ return await handedBack(error);
155
185
  } finally {
156
186
  stopSlotRenewal?.();
157
187
  heartbeat.stop();
@@ -0,0 +1,53 @@
1
+ // What one worker has settled, by outcome — the counters `stats()` reports. Split from `worker.ts`
2
+ // at the file-size ceiling; the claim loop counts, this file says what each count means.
3
+
4
+ import type { JobExecution } from './execute';
5
+ import type { WorkerStats } from './worker-types';
6
+
7
+ export type WorkerTallies = Pick<
8
+ WorkerStats,
9
+ 'processed' | 'failed' | 'suspended' | 'deadLettered' | 'interrupted' | 'refused' | 'dropped'
10
+ >;
11
+
12
+ export interface WorkerTally {
13
+ /** One run's execution, counted under its outcome. */
14
+ count(execution: JobExecution): void;
15
+ /** Rows the claim itself buried (`claim-exhausted.ts`), never handed out as work. */
16
+ buried(ended: { readonly deadLettered: number; readonly dropped: number }): void;
17
+ /** A run refused by its concurrency key, body never started (`worker-key-busy.ts`). */
18
+ refused(): void;
19
+ snapshot(): WorkerTallies;
20
+ }
21
+
22
+ export function createWorkerTally(): WorkerTally {
23
+ const tallies = {
24
+ processed: 0,
25
+ failed: 0,
26
+ suspended: 0,
27
+ deadLettered: 0,
28
+ interrupted: 0,
29
+ refused: 0,
30
+ dropped: 0,
31
+ };
32
+ return {
33
+ count(execution) {
34
+ // `retried` is counted as `failed`: the attempt failed, the job did not.
35
+ if (execution.outcome === 'completed') tallies.processed += 1;
36
+ else if (execution.outcome === 'suspended') tallies.suspended += 1;
37
+ else if (execution.outcome === 'retried') tallies.failed += 1;
38
+ else if (execution.outcome === 'interrupted') tallies.interrupted += 1;
39
+ else if (execution.outcome === 'dead-lettered') tallies.deadLettered += 1;
40
+ else if (execution.outcome === 'dropped') tallies.dropped += 1;
41
+ },
42
+ buried(ended) {
43
+ tallies.deadLettered += ended.deadLettered;
44
+ tallies.dropped += ended.dropped;
45
+ },
46
+ refused() {
47
+ tallies.refused += 1;
48
+ },
49
+ snapshot() {
50
+ return { ...tallies };
51
+ },
52
+ };
53
+ }
@@ -1,4 +1,4 @@
1
- // The `worker` role's public contract: what `createWorker` takes, what it hands back, and what
1
+ // The `worker` role's public contract: what `jobWorker` takes, what it hands back, and what
2
2
  // `stats()` reports. Apart from `worker.ts` because that file's job is the claim loop and the
3
3
  // drain, and a contract three files import should not sit under 500 lines of loop.
4
4
 
@@ -13,7 +13,10 @@ export interface WorkerOptions {
13
13
  readonly driver: JobDriver;
14
14
  /** Queues this process serves. Default `['default']`. */
15
15
  readonly queues?: readonly string[];
16
- /** Slots per queue. A number applies to every queue. */
16
+ /**
17
+ * Slots per queue. A number applies to every queue; a table names some, and a queue it leaves out
18
+ * (or no `concurrency` at all) runs at core's `JOBS_CONCURRENCY_DEFAULT`, 8 — 5 before 25.0.0.
19
+ */
17
20
  readonly concurrency?: number | Readonly<Record<string, number>>;
18
21
  readonly limiter?: Limiter;
19
22
  readonly clock?: Clock;
@@ -71,8 +74,8 @@ export interface Worker {
71
74
  /**
72
75
  * Stop claiming, wait for every job this worker holds, close the driver. Unbounded, and it
73
76
  * aborts nothing: a caller that asked wants its work finished. SIGTERM takes the other path —
74
- * the shutdown hooks `start()` registers abort every held run's `ctx.signal` and wait under
75
- * the lifecycle's deadline.
77
+ * it stops claiming and lets held runs finish under the lifecycle's deadline; only at the cut-off
78
+ * (`deadlineAt − margin`) is a still-running `ctx.signal` aborted, and its claim handed back.
76
79
  */
77
80
  stop(reason?: string): Promise<void>;
78
81
  stats(): Promise<WorkerStats>;
package/src/worker.ts CHANGED
@@ -1,20 +1,31 @@
1
1
  // The `worker` role: per-queue pools, a claim loop, lease heartbeats, and a graceful drain on
2
- // SIGTERM (stop claiming -> finish in-flight -> close). A worker that exits mid-job is not a
3
- // bug here — the visibility timeout re-delivers it — but a worker that exits mid-job on EVERY
4
- // deploy turns "at least once" into "always twice", so draining is on by default.
2
+ // SIGTERM (stop claiming -> finish in-flight -> cut off near the deadline -> hand back -> close).
3
+ // A worker that exits mid-job on EVERY deploy turns "at least once" into "always twice", so
4
+ // draining is on by default, and a drain never cancels work it still has budget to finish.
5
5
 
6
6
  import type { ShutdownReason } from '@ultimat3/core';
7
- import { beginWork, logger, onShutdown, recordJob, renderThrowable, uuid } from '@ultimat3/core';
7
+ import {
8
+ beginWork,
9
+ isDraining,
10
+ logger,
11
+ onShutdown,
12
+ recordJob,
13
+ renderThrowable,
14
+ uuidV7,
15
+ } from '@ultimat3/core';
16
+ import { announceExhausted } from './claim-exhausted';
8
17
  import { nowMs } from './clock';
9
18
  import { createDrainBudget, settleAllBy } from './drain-wait';
10
- import type { ClaimedJob } from './driver';
19
+ import type { ClaimedJob, JobRecord } from './driver';
11
20
  import { DEFAULT_QUEUE } from './driver';
12
- import { JobDrainedError } from './errors';
13
21
  import type { JobExecution } from './execute';
14
- import { createLimiter } from './limits';
22
+ import { registeredJobs } from './job';
23
+ import { concurrencyLimiter } from './limits';
15
24
  import { JOB_OUTCOME_LABELS } from './metrics';
16
25
  import { claimAsks, createAdmission } from './worker-admit';
26
+ import { armWorkerCutoff, type DrainCutoff } from './worker-drain-cutoff';
17
27
  import { assertConcurrencyEnforceable, createFleetSlots } from './worker-fleet-slots';
28
+ import { createHeldRuns } from './worker-held';
18
29
  import type { ClaimLoop } from './worker-loop';
19
30
  import { createClaimLoop } from './worker-loop';
20
31
  import { resolveWorkerTimings } from './worker-options';
@@ -22,18 +33,19 @@ import { createQueueDepthPublisher } from './worker-queue-depth';
22
33
  import type { WorkerRegistration } from './worker-registry';
23
34
  import { reportUnregisteredQueues, startWorkerRegistry } from './worker-registry';
24
35
  import { runClaimedJob } from './worker-run';
36
+ import { createWorkerTally } from './worker-tally';
25
37
  import type { Worker, WorkerOptions, WorkerStats } from './worker-types';
26
38
 
27
39
  export type { Worker, WorkerOptions, WorkerStats } from './worker-types';
28
40
 
29
- export function createWorker(options: WorkerOptions): Worker {
30
- const workerId = options.workerId ?? `worker-${uuid()}`;
41
+ export function jobWorker(options: WorkerOptions): Worker {
42
+ const workerId = options.workerId ?? `worker-${uuidV7()}`;
31
43
  const queues = options.queues ?? [DEFAULT_QUEUE];
32
44
  // Every numeric knob, read and refused in one place — `worker-options.ts` says why a non-finite
33
45
  // one is a refusal rather than a clamp, and carries the slot table's own-key read with it.
34
46
  const { visibilityTimeoutMs, pollIntervalMs, heartbeatIntervalMs, slotsFor } =
35
47
  resolveWorkerTimings(options);
36
- const limiter = options.limiter ?? createLimiter({});
48
+ const limiter = options.limiter ?? concurrencyLimiter({});
37
49
  const driverLeases = options.driver.leases;
38
50
  // `job.concurrency`, held as a row every replica sees. The TTL is the visibility timeout and the
39
51
  // renewal rides the lease heartbeat's interval — `worker-fleet-slots.ts` says why both.
@@ -43,6 +55,7 @@ export function createWorker(options: WorkerOptions): Worker {
43
55
  ttlMs: visibilityTimeoutMs,
44
56
  renewIntervalMs: heartbeatIntervalMs,
45
57
  ...(options.schedule === undefined ? {} : { schedule: options.schedule }),
58
+ ...(options.clock === undefined ? {} : { clock: options.clock }),
46
59
  });
47
60
 
48
61
  const admit = createAdmission({
@@ -55,20 +68,21 @@ export function createWorker(options: WorkerOptions): Worker {
55
68
  ...(options.clock === undefined ? {} : { clock: options.clock }),
56
69
  });
57
70
 
58
- const inFlight = new Set<Promise<unknown>>();
71
+ /** The runs this worker holds, each with its claim — what a drain's cut-off hands back. */
72
+ const inFlight = createHeldRuns({ driver: options.driver, workerId });
59
73
  /**
60
74
  * Claim rounds in flight. Jobs land in `inFlight` mid-round, so a drain that waited only on
61
75
  * `inFlight` waited on a set the round it was racing had not finished filling.
62
76
  */
63
77
  const rounds = new Set<Promise<unknown>>();
64
78
  /**
65
- * The drain, as every run this worker starts hears it: composed into each run's `ctx.signal`
66
- * (`worker-run.ts`), aborted by the `accept` hook with a `JobDrainedError`. ONE controller and
67
- * not one per run, because the fact it carries — "this process is going away" — is one fact.
68
- * Replaced with a fresh one when a teardown ends: a controller aborted once stays aborted, and
69
- * a restarted worker would otherwise hand every job it claimed a signal born cancelled.
79
+ * The drain, as every run hears it: composed into each `ctx.signal` (`worker-run.ts`), aborted
80
+ * with a `JobDrainedError` at the CUT-OFF (`worker-drain-cutoff.ts`), never at SIGTERM itself.
81
+ * ONE controller, because "this process is going away" is one fact. Fresh after each teardown:
82
+ * aborted once it stays aborted, and a restarted worker's jobs would be born cancelled.
70
83
  */
71
84
  let drainSignal = new AbortController();
85
+ let cutoff: DrainCutoff | undefined; // armed by the first shutdown to reach this worker
72
86
  /**
73
87
  * The deadline the teardown waits under. `undefined` for a manual `stop()`, bound the moment a
74
88
  * shutdown lands — and bound LATE when that shutdown lands on a teardown already in flight: the
@@ -87,15 +101,7 @@ export function createWorker(options: WorkerOptions): Worker {
87
101
  let releaseShutdownHooks: (() => void)[] = [];
88
102
  /** The teardown in flight, so a second `stop()` joins it instead of running a second one. */
89
103
  let stopping: Promise<void> | undefined;
90
- let processed = 0;
91
- let failed = 0;
92
- let suspended = 0;
93
- let deadLettered = 0;
94
- let interrupted = 0;
95
- let refused = 0;
96
- let dropped = 0;
97
- /** The ids this worker holds right now — what its registry row reports. */
98
- const holding = new Set<string>();
104
+ const tally = createWorkerTally();
99
105
  let registration: WorkerRegistration | undefined;
100
106
  /** `queue_depth` and its two siblings, republished on their own interval (`worker-queue-depth.ts`). */
101
107
  const publishQueueDepth = createQueueDepthPublisher({
@@ -115,6 +121,7 @@ export function createWorker(options: WorkerOptions): Worker {
115
121
  workerId,
116
122
  visibilityTimeoutMs,
117
123
  heartbeatIntervalMs,
124
+ pollIntervalMs,
118
125
  drain: drainSignal.signal,
119
126
  ...(options.clock === undefined ? {} : { clock: options.clock }),
120
127
  ...(options.schedule === undefined ? {} : { schedule: options.schedule }),
@@ -147,8 +154,35 @@ export function createWorker(options: WorkerOptions): Worker {
147
154
  // claiming" now, not at the next tick. What this round already holds still runs to the end
148
155
  // — that is the drain, and `stop()` waits for it.
149
156
  if (!claiming()) break;
150
- const claimed = await options.driver.claim({ ...ask, visibilityTimeoutMs, workerId });
157
+ // A lease that lapsed on a row's final attempt is settled by the claim itself and handed to
158
+ // `onExhausted`, never out as work: this round is the only thing that can count it, log it
159
+ // and tell the job. `dropExhausted` is the policy the row does not carry.
160
+ const buried: JobRecord[] = [];
161
+ const claimed = await options.driver.claim({
162
+ ...ask,
163
+ visibilityTimeoutMs,
164
+ workerId,
165
+ dropExhausted: registeredJobs()
166
+ .filter((handle) => handle.retry.deadLetter === false)
167
+ .map((handle) => handle.name),
168
+ onExhausted: (dead) => {
169
+ buried.push(...dead);
170
+ },
171
+ });
151
172
  if (claimed.length > 0) found = true;
173
+ if (buried.length > 0) {
174
+ found = true;
175
+ const ended = await announceExhausted({
176
+ exhausted: buried,
177
+ workerId,
178
+ context: options.context,
179
+ });
180
+ tally.buried(ended);
181
+ for (const row of buried) {
182
+ const label = JOB_OUTCOME_LABELS[row.state === 'failed' ? 'dropped' : 'dead-lettered'];
183
+ if (label !== null) recordJob(row.queue, label);
184
+ }
185
+ }
152
186
 
153
187
  for (const [index, job] of claimed.entries()) {
154
188
  const queue = job.queue;
@@ -158,7 +192,7 @@ export function createWorker(options: WorkerOptions): Worker {
158
192
  if (admission.kind === 'refused') {
159
193
  // Settled without a body: counted and returned with the pass, the one job this loop
160
194
  // itself finishes.
161
- refused += 1;
195
+ tally.refused();
162
196
  const label = JOB_OUTCOME_LABELS[admission.execution.outcome];
163
197
  if (label !== null) recordJob(queue, label);
164
198
  started.push(Promise.resolve(admission.execution));
@@ -170,15 +204,17 @@ export function createWorker(options: WorkerOptions): Worker {
170
204
  // core's own in-flight wait sits between `accept` and `inflight` and exists for exactly
171
205
  // this. Counted nowhere, the worker had to wait for its own jobs inside a hook.
172
206
  const finishWork = beginWork();
173
- holding.add(job.id);
207
+ // Once per run, whichever comes first: the run's own `finally`, or the drain handing its
208
+ // row back while the body is still running (`worker-held.ts`).
209
+ let slotReleased = false;
210
+ const releaseSlot = async (): Promise<void> => {
211
+ if (slotReleased) return;
212
+ slotReleased = true;
213
+ await fleetSlots.release(job.id);
214
+ };
174
215
  const running = runClaimed(job, admission.refusal)
175
216
  .then((execution) => {
176
- if (execution.outcome === 'completed') processed += 1;
177
- else if (execution.outcome === 'suspended') suspended += 1;
178
- else if (execution.outcome === 'retried') failed += 1;
179
- else if (execution.outcome === 'interrupted') interrupted += 1;
180
- else if (execution.outcome === 'dead-lettered') deadLettered += 1;
181
- else if (execution.outcome === 'dropped') dropped += 1;
217
+ tally.count(execution);
182
218
  // The other half of this package's metrics contract: `queue_depth` says how much work
183
219
  // is waiting, `jobs_total` says whether any of it is succeeding. Depth alone cannot
184
220
  // tell a drained queue from a queue nothing ever claimed. Labelled by QUEUE and
@@ -193,7 +229,6 @@ export function createWorker(options: WorkerOptions): Worker {
193
229
  return execution;
194
230
  })
195
231
  .finally(async () => {
196
- holding.delete(job.id);
197
232
  lease.release();
198
233
  // The slot is free NOW. A full worker asks the queue for nothing, so its loop backs
199
234
  // off like an idle one — and waiting that out with a backlog behind the job that
@@ -207,24 +242,27 @@ export function createWorker(options: WorkerOptions): Worker {
207
242
  // `finally` is for the one that could, because a lost `finishWork()` is an in-flight
208
243
  // count that never returns to zero and a drain that waits out its whole budget.
209
244
  try {
210
- await fleetSlots.release(job.id);
245
+ await releaseSlot();
211
246
  } finally {
212
247
  finishWork();
213
248
  }
214
249
  });
215
250
 
216
251
  started.push(running);
217
- inFlight.add(running);
252
+ inFlight.add(running, job, async () => {
253
+ await releaseSlot();
254
+ finishWork();
255
+ });
218
256
  // The claim loop no longer awaits these, so this is the one place a rejection is observed:
219
257
  // unobserved it is an unhandled rejection, which on Bun's default is the whole process.
220
258
  // `executeJob` settles the job itself, so reaching here means the driver could not be
221
259
  // told how it ended — the lease will lapse and the queue will deliver it again.
222
260
  void running.then(
223
261
  () => {
224
- inFlight.delete(running);
262
+ inFlight.settle(running);
225
263
  },
226
264
  (error: unknown) => {
227
- inFlight.delete(running);
265
+ inFlight.settle(running);
228
266
  logger.error('jobs.worker.settle-failed', {
229
267
  workerId,
230
268
  job: job.name,
@@ -269,7 +307,7 @@ export function createWorker(options: WorkerOptions): Worker {
269
307
  * whole batch was done. `worker-loop.ts` owns the timer and everything that cuts its wait short.
270
308
  */
271
309
  const loop: ClaimLoop = createClaimLoop({
272
- subject: 'createWorker',
310
+ subject: 'jobWorker',
273
311
  floorMs: pollIntervalMs,
274
312
  ...(options.idlePollMaxMs === undefined ? {} : { ceilingMs: options.idlePollMaxMs }),
275
313
  queues,
@@ -280,21 +318,18 @@ export function createWorker(options: WorkerOptions): Worker {
280
318
  });
281
319
 
282
320
  /**
283
- * The whole of the `accept` phase: stop taking work, tell the work already held, and nothing
284
- * else. Synchronous on purpose — a phase whose job is to be over before the load balancer's next
285
- * health check must not contain a wait, and the hook behind this one is somebody else's "stop
286
- * listening". An abort is synchronous and costs nothing, which is why it belongs HERE and not in
287
- * the teardown: core runs `accept`, then waits out in-flight work (every claimed job is
288
- * `beginWork()`ed) under the same budget, then `close`. Told in `close`, a body that reads
289
- * `ctx.signal` would hear it after the in-flight wait had already spent the whole budget on it —
290
- * which is what happened until 2026-09-07: a job that would have stopped in a second was waited
291
- * on for the full deadline and abandoned there, exactly like one that ignores the signal.
321
+ * The whole of the `accept` phase: stop taking work and ARM the cut-off — nothing else, and no
322
+ * wait: the hook behind this one is somebody else's "stop listening". Armed here and not in
323
+ * `close`, which runs only after core's in-flight wait has spent the budget.
324
+ *
325
+ * The jobs held are NOT told. Until 2026-10-05 this aborted every run at SIGTERM, and a step
326
+ * whose side effect completed during the drain was refused and run again by the next worker.
327
+ * The cut-off (`worker-drain-cutoff.ts`) tells them at `deadlineAt − margin`, then hands back
328
+ * what is still held so the replacement claims it at once, not after the visibility timeout.
292
329
  *
293
- * Only a SHUTDOWN aborts, and only a shutdown binds the budget. A manual `stop()` passes
294
- * nothing: a caller that asked has no budget to spend and wants its work finished, the same
295
- * line `settleAllBy` draws for the wait. The bind comes BEFORE the abort's once-guard and on
296
- * every call, because the second shutdown to reach a worker is the one that finds the abort
297
- * already fired and the teardown already waiting — with no deadline, if the first was manual.
330
+ * Only a SHUTDOWN binds the budget and arms it: a manual `stop()` has no budget and wants its
331
+ * work finished. The bind runs on every call — a SIGTERM landing on a manual stop is what binds
332
+ * the teardown already waiting — and a tighter deadline re-arms through the budget's watch.
298
333
  */
299
334
  const stopAccepting = (shutdown?: ShutdownReason): void => {
300
335
  if (state === 'stopped') return;
@@ -302,13 +337,19 @@ export function createWorker(options: WorkerOptions): Worker {
302
337
  loop.stop();
303
338
  if (shutdown === undefined) return;
304
339
  budget.bind(shutdown.deadlineAt);
305
- if (drainSignal.signal.aborted) return;
340
+ if (cutoff !== undefined) return;
306
341
  logger.info('jobs.worker.drain-signalled', {
307
342
  workerId,
308
343
  signal: shutdown.signal,
309
344
  inFlight: inFlight.size,
310
345
  });
311
- drainSignal.abort(new JobDrainedError({ workerId, signal: shutdown.signal }));
346
+ cutoff = armWorkerCutoff({
347
+ budget,
348
+ workerId,
349
+ signal: shutdown.signal,
350
+ drain: drainSignal,
351
+ held: inFlight,
352
+ });
312
353
  };
313
354
 
314
355
  const teardown = async (reason: string): Promise<void> => {
@@ -323,10 +364,11 @@ export function createWorker(options: WorkerOptions): Worker {
323
364
  // take, and bound by the SIGTERM — whether it arrived before this teardown or lands in the
324
365
  // middle of it. Nothing can kill a body that ignores `ctx.signal`, so an unbounded wait
325
366
  // here is a teardown that never ends: driver never closed, state never past 'draining', and
326
- // the memoized `stopping` every later `stop()` joins never settling. Abandoning costs a
327
- // lapsed lease and a redelivered job — at-least-once, as promised.
367
+ // the memoized `stopping` every later `stop()` joins never settling. `empty()`, not a
368
+ // snapshot: the cut-off's hand-back empties the set under bodies still running, and that
369
+ // ends this wait. What a late round started is handed back below, before the close.
328
370
  const rounded = await settleAllBy([...rounds], budget);
329
- const drained = (await settleAllBy([...inFlight], budget)) && rounded;
371
+ const drained = (await settleAllBy([inFlight.empty()], budget)) && rounded;
330
372
  if (!drained) {
331
373
  logger.warn('jobs.worker.drain-abandoned', {
332
374
  workerId,
@@ -335,9 +377,16 @@ export function createWorker(options: WorkerOptions): Worker {
335
377
  fix: 'raise the drain budget past the slowest job — configureLifecycle({ deadlineMs: 600_000 }) — and set terminationGracePeriodSeconds to at least as many seconds',
336
378
  });
337
379
  }
338
- // Before the close, and awaited: the row is deleted through the driver being closed.
339
- await registration?.stop();
380
+ if (inFlight.size > 0 && budget.deadlineAt !== undefined) await inFlight.handBackAll();
381
+ // Before the close, and awaited: the row is deleted through the driver being closed. Under
382
+ // the SAME budget as the jobs: `forgetWorker` is a statement on the pool that may be the
383
+ // thing failing, and unbounded it held a SIGTERM teardown open with the driver never closed.
384
+ // Abandoned, the row expires on its own TTL — what a killed worker's does.
385
+ const registered = registration;
340
386
  registration = undefined;
387
+ if (registered !== undefined && !(await settleAllBy([registered.stop()], budget))) {
388
+ logger.warn('jobs.worker.registry-abandoned', { workerId, reason });
389
+ }
341
390
  await options.driver.close?.();
342
391
  } finally {
343
392
  // Whatever the close did, this worker is done: a state left at 'draining' is a drain that
@@ -352,6 +401,8 @@ export function createWorker(options: WorkerOptions): Worker {
352
401
  // A run this drain abandoned still follows the old controller through its own composition;
353
402
  // the next start's jobs must not. Fresh here, in the one place a teardown always reaches —
354
403
  // and the budget with it, or the next manual stop would inherit a deadline already spent.
404
+ cutoff?.dispose();
405
+ cutoff = undefined;
355
406
  drainSignal = new AbortController();
356
407
  budget = createDrainBudget();
357
408
  }
@@ -362,8 +413,8 @@ export function createWorker(options: WorkerOptions): Worker {
362
413
  // waits are bounded and the state is set in a `finally`), so a caller landing after an
363
414
  // abandoned drain gets an answer rather than joining a promise that never settles.
364
415
  if (state === 'stopped') return;
365
- // Before the join, every time: a SIGTERM landing on a manual stop still aborts every held
366
- // run's `ctx.signal` and binds the teardown already waiting to the shutdown's deadline.
416
+ // Before the join, every time: a SIGTERM landing on a manual stop still arms the cut-off and
417
+ // binds the teardown already waiting to the shutdown's deadline.
367
418
  stopAccepting(shutdown);
368
419
  // One teardown, joined rather than repeated: that SIGTERM must wait out the same in-flight
369
420
  // work, not close the driver a second time underneath it. Cleared as it settles, so a worker
@@ -381,6 +432,12 @@ export function createWorker(options: WorkerOptions): Worker {
381
432
  // Only from a standstill. A start mid-drain would put the claim loop back on a driver the
382
433
  // drain is about to close, and stack a second shutdown hook on the one still running.
383
434
  if (state !== 'idle' && state !== 'stopped') return;
435
+ // Nor inside the PROCESS's drain: a boot that reached here after SIGTERM registers hooks the
436
+ // drain is already past, so a worker that claimed now held jobs nothing would ever drain.
437
+ if (isDraining()) {
438
+ logger.warn('jobs.worker.start-refused-draining', { workerId });
439
+ return;
440
+ }
384
441
  // Refused HERE, at the earliest decidable point, and refused rather than logged: an agent
385
442
  // reads "max in-flight runs of THIS job across the fleet", writes `concurrency: 1` on
386
443
  // `rebuildSearchIndex`, ships, and two workers run it on the first deploy — while
@@ -395,7 +452,7 @@ export function createWorker(options: WorkerOptions): Worker {
395
452
  host: options.host,
396
453
  queues,
397
454
  concurrency: queues.reduce((slots, queue) => slots + slotsFor(queue), 0),
398
- inFlight: () => [...holding],
455
+ inFlight: () => inFlight.ids(),
399
456
  // The visibility timeout, renewed on the heartbeat interval: a killed worker leaves the
400
457
  // registry on exactly the schedule the queue takes its jobs back.
401
458
  ttlMs: visibilityTimeoutMs,
@@ -403,8 +460,8 @@ export function createWorker(options: WorkerOptions): Worker {
403
460
  ...(options.clock === undefined ? {} : { clock: options.clock }),
404
461
  ...(options.schedule === undefined ? {} : { schedule: options.schedule }),
405
462
  });
406
- // TWO hooks, for the two phases that answer two questions. `accept` stops claiming, aborts
407
- // every held run's `ctx.signal` and returns, so every hook behind it — the HTTP server's
463
+ // TWO hooks, for the two phases that answer two questions. `accept` stops claiming, arms
464
+ // the cut-off that tells the held runs near the deadline, and returns, so every hook behind it — the HTTP server's
408
465
  // "stop listening", the sync node's "stop upgrading" — runs while the budget is still whole;
409
466
  // one hook doing both spent all of it in the phase whose whole purpose is to be quick.
410
467
  // `close` waits out what this worker holds and closes the driver, bounded by the deadline
@@ -433,13 +490,7 @@ export function createWorker(options: WorkerOptions): Worker {
433
490
  queues,
434
491
  state,
435
492
  inFlight: inFlight.size,
436
- processed,
437
- failed,
438
- suspended,
439
- deadLettered,
440
- interrupted,
441
- refused,
442
- dropped,
493
+ ...tally.snapshot(),
443
494
  pollDelayMs: loop.delayMs(),
444
495
  queueDepth: [...(await options.driver.stats())],
445
496
  };