@ultimat3/jobs 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +72 -75
- package/README.md +83 -38
- package/package.json +7 -7
- package/src/backfill-errors.ts +3 -2
- package/src/backfill-ledger.ts +25 -3
- package/src/backfill-pass.ts +22 -5
- package/src/backfill-rate.ts +2 -2
- package/src/backfill.ts +2 -2
- package/src/claim-exhausted.ts +76 -0
- package/src/counters.ts +1 -1
- package/src/driver-memory-operator.ts +34 -12
- package/src/driver-memory.ts +80 -38
- package/src/driver-pg-jobs-sql.ts +27 -19
- package/src/driver-pg-operator-sql.ts +32 -12
- package/src/driver-pg-operator.ts +19 -3
- package/src/driver-pg-rows.ts +3 -3
- package/src/driver-pg-settle-sql.ts +2 -3
- package/src/driver-pg-sql.ts +99 -30
- package/src/driver-pg.ts +103 -98
- package/src/driver.ts +46 -16
- package/src/errors-concurrency.ts +1 -1
- package/src/errors-operator.ts +12 -4
- package/src/errors-requeue.ts +21 -3
- package/src/errors.ts +9 -17
- package/src/events-pg.ts +11 -21
- package/src/events.ts +26 -15
- package/src/execute.ts +47 -5
- package/src/export.ts +2 -2
- package/src/heartbeat.ts +7 -2
- package/src/index.ts +45 -99
- package/src/inspect-operator.ts +4 -4
- package/src/inspect.ts +5 -4
- package/src/introspection.ts +23 -11
- package/src/job.ts +6 -7
- package/src/leases.ts +1 -1
- package/src/limits.ts +46 -15
- package/src/outbox-lease.ts +1 -1
- package/src/outbox-pg.ts +6 -7
- package/src/outbox-relay.ts +6 -6
- package/src/outbox.ts +47 -19
- package/src/purge.ts +1 -1
- package/src/queue-wake.ts +1 -2
- package/src/renewal-timer.ts +1 -1
- package/src/retry-classification.ts +52 -15
- package/src/scheduler-leader.ts +2 -2
- package/src/scheduler-pg.ts +19 -16
- package/src/scheduler-state.ts +10 -3
- package/src/scheduler.ts +14 -15
- package/src/steps-memory.ts +2 -2
- package/src/steps-suspension.ts +46 -0
- package/src/steps-timeout.ts +33 -0
- package/src/steps.ts +90 -88
- package/src/task.ts +1 -1
- package/src/webhook-attempt.ts +206 -0
- package/src/webhook-ledger.ts +15 -4
- package/src/webhook.ts +64 -118
- package/src/worker-admit.ts +14 -6
- package/src/worker-drain-cutoff.ts +110 -0
- package/src/worker-fleet-slots.ts +82 -45
- package/src/worker-held.ts +100 -0
- package/src/worker-key-busy.ts +38 -10
- package/src/worker-loop.ts +35 -11
- package/src/worker-options.ts +19 -15
- package/src/worker-registry.ts +16 -1
- package/src/worker-run.ts +38 -8
- package/src/worker-tally.ts +53 -0
- package/src/worker-types.ts +7 -4
- package/src/worker.ts +124 -73
- package/src/driver-nats.ts +0 -86
- package/src/driver-redis.ts +0 -89
package/src/worker-run.ts
CHANGED
|
@@ -17,6 +17,7 @@ import type { RunSignal } from './run-signal';
|
|
|
17
17
|
import { createRunSignal } from './run-signal';
|
|
18
18
|
import type { EventLookup } from './steps';
|
|
19
19
|
import type { FleetSlots } from './worker-fleet-slots';
|
|
20
|
+
import { handBack } from './worker-hand-back';
|
|
20
21
|
|
|
21
22
|
export interface RunClaimedOptions {
|
|
22
23
|
readonly driver: JobDriver;
|
|
@@ -27,14 +28,16 @@ export interface RunClaimedOptions {
|
|
|
27
28
|
readonly workerId: string;
|
|
28
29
|
readonly visibilityTimeoutMs: number;
|
|
29
30
|
readonly heartbeatIntervalMs: number;
|
|
31
|
+
/** How soon a run whose wiring threw before its body started is claimable again — a shed's. */
|
|
32
|
+
readonly pollIntervalMs: number;
|
|
30
33
|
readonly clock?: Clock;
|
|
31
34
|
/** What every renewal runs on (`renewal-timer.ts`). Default: a real, unrefed interval. */
|
|
32
35
|
readonly schedule?: IntervalScheduler;
|
|
33
36
|
readonly events?: EventLookup;
|
|
34
37
|
/**
|
|
35
|
-
* The worker's drain, composed into every run it starts: aborted with a `JobDrainedError`
|
|
36
|
-
* the
|
|
37
|
-
*
|
|
38
|
+
* The worker's drain, composed into every run it starts: aborted with a `JobDrainedError` at
|
|
39
|
+
* the drain's cut-off — near the deadline, never at SIGTERM (`worker-drain-cutoff.ts`) — so a
|
|
40
|
+
* body still running then hears it on `ctx.signal`, the one seam it already reads.
|
|
38
41
|
*/
|
|
39
42
|
readonly drain?: AbortSignal;
|
|
40
43
|
/** Why this run may not start — handed to `executeJob`, which fails the attempt with it. */
|
|
@@ -73,10 +76,28 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
|
|
|
73
76
|
return unknownJob(claimed);
|
|
74
77
|
}
|
|
75
78
|
|
|
79
|
+
// A throw from the wiring below means the body never started, so the row goes BACK exactly as a
|
|
80
|
+
// shed returns one — uncounted, no `lastError` — before the throw reaches the round. Rethrowing
|
|
81
|
+
// alone left it `running` until the lease lapsed, an attempt spent on nothing; on the final
|
|
82
|
+
// attempt the claim then dead-letters a job whose body never ran. Once `executeJob` is entered
|
|
83
|
+
// the attempt is real and settles itself, so `started` is set at that door and nowhere else.
|
|
84
|
+
let started = false;
|
|
85
|
+
const handedBack = async (error: unknown): Promise<never> => {
|
|
86
|
+
if (!started) {
|
|
87
|
+
await handBack(driver, [claimed], { delayMs: options.pollIntervalMs, workerId });
|
|
88
|
+
}
|
|
89
|
+
throw error;
|
|
90
|
+
};
|
|
91
|
+
|
|
76
92
|
// Read BEFORE the timers start. `context()` is the app's own function and it can throw; started
|
|
77
93
|
// first, a heartbeat interval and a slot renewal were left running for a job that never ran,
|
|
78
94
|
// with nothing left holding a reference to stop them.
|
|
79
|
-
|
|
95
|
+
let base: Ctx;
|
|
96
|
+
try {
|
|
97
|
+
base = options.context();
|
|
98
|
+
} catch (error) {
|
|
99
|
+
return handedBack(error);
|
|
100
|
+
}
|
|
80
101
|
|
|
81
102
|
// The lease, kept alive and NOT kept quiet: a renewal that stops landing means the queue hands
|
|
82
103
|
// this job to another worker while this one is still running it, and `.catch(() => undefined)`
|
|
@@ -110,7 +131,7 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
|
|
|
110
131
|
// when the run settles (an app whose `context()` carries a process-lifetime signal was
|
|
111
132
|
// accumulating one composite per job), and the worker can abort it itself — which is the only
|
|
112
133
|
// way a fleet slot taken by somebody else reaches the body running under it. The worker's
|
|
113
|
-
// drain is the third source:
|
|
134
|
+
// drain is the third source: its cut-off reaches the body through the same signal, with the
|
|
114
135
|
// `X_DRAINING` reason `executeJob` reads to hand the attempt back uncounted.
|
|
115
136
|
runSignal = createRunSignal([base.signal, heartbeat.signal, options.drain]);
|
|
116
137
|
const signal = runSignal;
|
|
@@ -132,8 +153,9 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
|
|
|
132
153
|
|
|
133
154
|
return await withSpan(
|
|
134
155
|
`job.${handle.name}`,
|
|
135
|
-
() =>
|
|
136
|
-
|
|
156
|
+
() => {
|
|
157
|
+
started = true;
|
|
158
|
+
return executeJob({
|
|
137
159
|
driver,
|
|
138
160
|
claimed,
|
|
139
161
|
handle,
|
|
@@ -141,7 +163,8 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
|
|
|
141
163
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
142
164
|
...(options.events === undefined ? {} : { events: options.events }),
|
|
143
165
|
...(options.refusal === undefined ? {} : { refusal: options.refusal }),
|
|
144
|
-
})
|
|
166
|
+
});
|
|
167
|
+
},
|
|
145
168
|
{
|
|
146
169
|
...(parent === undefined ? {} : { parent }),
|
|
147
170
|
attributes: {
|
|
@@ -152,6 +175,13 @@ export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExec
|
|
|
152
175
|
},
|
|
153
176
|
},
|
|
154
177
|
);
|
|
178
|
+
} catch (error) {
|
|
179
|
+
// Stopped BEFORE the row goes back, never after: a renewal landing on a handed-back row
|
|
180
|
+
// answers "not yours", which `heartbeat.ts` reports as a lease lost under a running job.
|
|
181
|
+
// Both are idempotent, so the `finally` repeating them costs nothing.
|
|
182
|
+
stopSlotRenewal?.();
|
|
183
|
+
heartbeat.stop();
|
|
184
|
+
return await handedBack(error);
|
|
155
185
|
} finally {
|
|
156
186
|
stopSlotRenewal?.();
|
|
157
187
|
heartbeat.stop();
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// What one worker has settled, by outcome — the counters `stats()` reports. Split from `worker.ts`
|
|
2
|
+
// at the file-size ceiling; the claim loop counts, this file says what each count means.
|
|
3
|
+
|
|
4
|
+
import type { JobExecution } from './execute';
|
|
5
|
+
import type { WorkerStats } from './worker-types';
|
|
6
|
+
|
|
7
|
+
export type WorkerTallies = Pick<
|
|
8
|
+
WorkerStats,
|
|
9
|
+
'processed' | 'failed' | 'suspended' | 'deadLettered' | 'interrupted' | 'refused' | 'dropped'
|
|
10
|
+
>;
|
|
11
|
+
|
|
12
|
+
export interface WorkerTally {
|
|
13
|
+
/** One run's execution, counted under its outcome. */
|
|
14
|
+
count(execution: JobExecution): void;
|
|
15
|
+
/** Rows the claim itself buried (`claim-exhausted.ts`), never handed out as work. */
|
|
16
|
+
buried(ended: { readonly deadLettered: number; readonly dropped: number }): void;
|
|
17
|
+
/** A run refused by its concurrency key, body never started (`worker-key-busy.ts`). */
|
|
18
|
+
refused(): void;
|
|
19
|
+
snapshot(): WorkerTallies;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function createWorkerTally(): WorkerTally {
|
|
23
|
+
const tallies = {
|
|
24
|
+
processed: 0,
|
|
25
|
+
failed: 0,
|
|
26
|
+
suspended: 0,
|
|
27
|
+
deadLettered: 0,
|
|
28
|
+
interrupted: 0,
|
|
29
|
+
refused: 0,
|
|
30
|
+
dropped: 0,
|
|
31
|
+
};
|
|
32
|
+
return {
|
|
33
|
+
count(execution) {
|
|
34
|
+
// `retried` is counted as `failed`: the attempt failed, the job did not.
|
|
35
|
+
if (execution.outcome === 'completed') tallies.processed += 1;
|
|
36
|
+
else if (execution.outcome === 'suspended') tallies.suspended += 1;
|
|
37
|
+
else if (execution.outcome === 'retried') tallies.failed += 1;
|
|
38
|
+
else if (execution.outcome === 'interrupted') tallies.interrupted += 1;
|
|
39
|
+
else if (execution.outcome === 'dead-lettered') tallies.deadLettered += 1;
|
|
40
|
+
else if (execution.outcome === 'dropped') tallies.dropped += 1;
|
|
41
|
+
},
|
|
42
|
+
buried(ended) {
|
|
43
|
+
tallies.deadLettered += ended.deadLettered;
|
|
44
|
+
tallies.dropped += ended.dropped;
|
|
45
|
+
},
|
|
46
|
+
refused() {
|
|
47
|
+
tallies.refused += 1;
|
|
48
|
+
},
|
|
49
|
+
snapshot() {
|
|
50
|
+
return { ...tallies };
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
package/src/worker-types.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// The `worker` role's public contract: what `
|
|
1
|
+
// The `worker` role's public contract: what `jobWorker` takes, what it hands back, and what
|
|
2
2
|
// `stats()` reports. Apart from `worker.ts` because that file's job is the claim loop and the
|
|
3
3
|
// drain, and a contract three files import should not sit under 500 lines of loop.
|
|
4
4
|
|
|
@@ -13,7 +13,10 @@ export interface WorkerOptions {
|
|
|
13
13
|
readonly driver: JobDriver;
|
|
14
14
|
/** Queues this process serves. Default `['default']`. */
|
|
15
15
|
readonly queues?: readonly string[];
|
|
16
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* Slots per queue. A number applies to every queue; a table names some, and a queue it leaves out
|
|
18
|
+
* (or no `concurrency` at all) runs at core's `JOBS_CONCURRENCY_DEFAULT`, 8 — 5 before 25.0.0.
|
|
19
|
+
*/
|
|
17
20
|
readonly concurrency?: number | Readonly<Record<string, number>>;
|
|
18
21
|
readonly limiter?: Limiter;
|
|
19
22
|
readonly clock?: Clock;
|
|
@@ -71,8 +74,8 @@ export interface Worker {
|
|
|
71
74
|
/**
|
|
72
75
|
* Stop claiming, wait for every job this worker holds, close the driver. Unbounded, and it
|
|
73
76
|
* aborts nothing: a caller that asked wants its work finished. SIGTERM takes the other path —
|
|
74
|
-
*
|
|
75
|
-
*
|
|
77
|
+
* it stops claiming and lets held runs finish under the lifecycle's deadline; only at the cut-off
|
|
78
|
+
* (`deadlineAt − margin`) is a still-running `ctx.signal` aborted, and its claim handed back.
|
|
76
79
|
*/
|
|
77
80
|
stop(reason?: string): Promise<void>;
|
|
78
81
|
stats(): Promise<WorkerStats>;
|
package/src/worker.ts
CHANGED
|
@@ -1,20 +1,31 @@
|
|
|
1
1
|
// The `worker` role: per-queue pools, a claim loop, lease heartbeats, and a graceful drain on
|
|
2
|
-
// SIGTERM (stop claiming -> finish in-flight ->
|
|
3
|
-
//
|
|
4
|
-
//
|
|
2
|
+
// SIGTERM (stop claiming -> finish in-flight -> cut off near the deadline -> hand back -> close).
|
|
3
|
+
// A worker that exits mid-job on EVERY deploy turns "at least once" into "always twice", so
|
|
4
|
+
// draining is on by default, and a drain never cancels work it still has budget to finish.
|
|
5
5
|
|
|
6
6
|
import type { ShutdownReason } from '@ultimat3/core';
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
beginWork,
|
|
9
|
+
isDraining,
|
|
10
|
+
logger,
|
|
11
|
+
onShutdown,
|
|
12
|
+
recordJob,
|
|
13
|
+
renderThrowable,
|
|
14
|
+
uuidV7,
|
|
15
|
+
} from '@ultimat3/core';
|
|
16
|
+
import { announceExhausted } from './claim-exhausted';
|
|
8
17
|
import { nowMs } from './clock';
|
|
9
18
|
import { createDrainBudget, settleAllBy } from './drain-wait';
|
|
10
|
-
import type { ClaimedJob } from './driver';
|
|
19
|
+
import type { ClaimedJob, JobRecord } from './driver';
|
|
11
20
|
import { DEFAULT_QUEUE } from './driver';
|
|
12
|
-
import { JobDrainedError } from './errors';
|
|
13
21
|
import type { JobExecution } from './execute';
|
|
14
|
-
import {
|
|
22
|
+
import { registeredJobs } from './job';
|
|
23
|
+
import { concurrencyLimiter } from './limits';
|
|
15
24
|
import { JOB_OUTCOME_LABELS } from './metrics';
|
|
16
25
|
import { claimAsks, createAdmission } from './worker-admit';
|
|
26
|
+
import { armWorkerCutoff, type DrainCutoff } from './worker-drain-cutoff';
|
|
17
27
|
import { assertConcurrencyEnforceable, createFleetSlots } from './worker-fleet-slots';
|
|
28
|
+
import { createHeldRuns } from './worker-held';
|
|
18
29
|
import type { ClaimLoop } from './worker-loop';
|
|
19
30
|
import { createClaimLoop } from './worker-loop';
|
|
20
31
|
import { resolveWorkerTimings } from './worker-options';
|
|
@@ -22,18 +33,19 @@ import { createQueueDepthPublisher } from './worker-queue-depth';
|
|
|
22
33
|
import type { WorkerRegistration } from './worker-registry';
|
|
23
34
|
import { reportUnregisteredQueues, startWorkerRegistry } from './worker-registry';
|
|
24
35
|
import { runClaimedJob } from './worker-run';
|
|
36
|
+
import { createWorkerTally } from './worker-tally';
|
|
25
37
|
import type { Worker, WorkerOptions, WorkerStats } from './worker-types';
|
|
26
38
|
|
|
27
39
|
export type { Worker, WorkerOptions, WorkerStats } from './worker-types';
|
|
28
40
|
|
|
29
|
-
export function
|
|
30
|
-
const workerId = options.workerId ?? `worker-${
|
|
41
|
+
export function jobWorker(options: WorkerOptions): Worker {
|
|
42
|
+
const workerId = options.workerId ?? `worker-${uuidV7()}`;
|
|
31
43
|
const queues = options.queues ?? [DEFAULT_QUEUE];
|
|
32
44
|
// Every numeric knob, read and refused in one place — `worker-options.ts` says why a non-finite
|
|
33
45
|
// one is a refusal rather than a clamp, and carries the slot table's own-key read with it.
|
|
34
46
|
const { visibilityTimeoutMs, pollIntervalMs, heartbeatIntervalMs, slotsFor } =
|
|
35
47
|
resolveWorkerTimings(options);
|
|
36
|
-
const limiter = options.limiter ??
|
|
48
|
+
const limiter = options.limiter ?? concurrencyLimiter({});
|
|
37
49
|
const driverLeases = options.driver.leases;
|
|
38
50
|
// `job.concurrency`, held as a row every replica sees. The TTL is the visibility timeout and the
|
|
39
51
|
// renewal rides the lease heartbeat's interval — `worker-fleet-slots.ts` says why both.
|
|
@@ -43,6 +55,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
43
55
|
ttlMs: visibilityTimeoutMs,
|
|
44
56
|
renewIntervalMs: heartbeatIntervalMs,
|
|
45
57
|
...(options.schedule === undefined ? {} : { schedule: options.schedule }),
|
|
58
|
+
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
46
59
|
});
|
|
47
60
|
|
|
48
61
|
const admit = createAdmission({
|
|
@@ -55,20 +68,21 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
55
68
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
56
69
|
});
|
|
57
70
|
|
|
58
|
-
|
|
71
|
+
/** The runs this worker holds, each with its claim — what a drain's cut-off hands back. */
|
|
72
|
+
const inFlight = createHeldRuns({ driver: options.driver, workerId });
|
|
59
73
|
/**
|
|
60
74
|
* Claim rounds in flight. Jobs land in `inFlight` mid-round, so a drain that waited only on
|
|
61
75
|
* `inFlight` waited on a set the round it was racing had not finished filling.
|
|
62
76
|
*/
|
|
63
77
|
const rounds = new Set<Promise<unknown>>();
|
|
64
78
|
/**
|
|
65
|
-
* The drain, as every run
|
|
66
|
-
* (`worker-
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* a restarted worker would otherwise hand every job it claimed a signal born cancelled.
|
|
79
|
+
* The drain, as every run hears it: composed into each `ctx.signal` (`worker-run.ts`), aborted
|
|
80
|
+
* with a `JobDrainedError` at the CUT-OFF (`worker-drain-cutoff.ts`), never at SIGTERM itself.
|
|
81
|
+
* ONE controller, because "this process is going away" is one fact. Fresh after each teardown:
|
|
82
|
+
* aborted once it stays aborted, and a restarted worker's jobs would be born cancelled.
|
|
70
83
|
*/
|
|
71
84
|
let drainSignal = new AbortController();
|
|
85
|
+
let cutoff: DrainCutoff | undefined; // armed by the first shutdown to reach this worker
|
|
72
86
|
/**
|
|
73
87
|
* The deadline the teardown waits under. `undefined` for a manual `stop()`, bound the moment a
|
|
74
88
|
* shutdown lands — and bound LATE when that shutdown lands on a teardown already in flight: the
|
|
@@ -87,15 +101,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
87
101
|
let releaseShutdownHooks: (() => void)[] = [];
|
|
88
102
|
/** The teardown in flight, so a second `stop()` joins it instead of running a second one. */
|
|
89
103
|
let stopping: Promise<void> | undefined;
|
|
90
|
-
|
|
91
|
-
let failed = 0;
|
|
92
|
-
let suspended = 0;
|
|
93
|
-
let deadLettered = 0;
|
|
94
|
-
let interrupted = 0;
|
|
95
|
-
let refused = 0;
|
|
96
|
-
let dropped = 0;
|
|
97
|
-
/** The ids this worker holds right now — what its registry row reports. */
|
|
98
|
-
const holding = new Set<string>();
|
|
104
|
+
const tally = createWorkerTally();
|
|
99
105
|
let registration: WorkerRegistration | undefined;
|
|
100
106
|
/** `queue_depth` and its two siblings, republished on their own interval (`worker-queue-depth.ts`). */
|
|
101
107
|
const publishQueueDepth = createQueueDepthPublisher({
|
|
@@ -115,6 +121,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
115
121
|
workerId,
|
|
116
122
|
visibilityTimeoutMs,
|
|
117
123
|
heartbeatIntervalMs,
|
|
124
|
+
pollIntervalMs,
|
|
118
125
|
drain: drainSignal.signal,
|
|
119
126
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
120
127
|
...(options.schedule === undefined ? {} : { schedule: options.schedule }),
|
|
@@ -147,8 +154,35 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
147
154
|
// claiming" now, not at the next tick. What this round already holds still runs to the end
|
|
148
155
|
// — that is the drain, and `stop()` waits for it.
|
|
149
156
|
if (!claiming()) break;
|
|
150
|
-
|
|
157
|
+
// A lease that lapsed on a row's final attempt is settled by the claim itself and handed to
|
|
158
|
+
// `onExhausted`, never out as work: this round is the only thing that can count it, log it
|
|
159
|
+
// and tell the job. `dropExhausted` is the policy the row does not carry.
|
|
160
|
+
const buried: JobRecord[] = [];
|
|
161
|
+
const claimed = await options.driver.claim({
|
|
162
|
+
...ask,
|
|
163
|
+
visibilityTimeoutMs,
|
|
164
|
+
workerId,
|
|
165
|
+
dropExhausted: registeredJobs()
|
|
166
|
+
.filter((handle) => handle.retry.deadLetter === false)
|
|
167
|
+
.map((handle) => handle.name),
|
|
168
|
+
onExhausted: (dead) => {
|
|
169
|
+
buried.push(...dead);
|
|
170
|
+
},
|
|
171
|
+
});
|
|
151
172
|
if (claimed.length > 0) found = true;
|
|
173
|
+
if (buried.length > 0) {
|
|
174
|
+
found = true;
|
|
175
|
+
const ended = await announceExhausted({
|
|
176
|
+
exhausted: buried,
|
|
177
|
+
workerId,
|
|
178
|
+
context: options.context,
|
|
179
|
+
});
|
|
180
|
+
tally.buried(ended);
|
|
181
|
+
for (const row of buried) {
|
|
182
|
+
const label = JOB_OUTCOME_LABELS[row.state === 'failed' ? 'dropped' : 'dead-lettered'];
|
|
183
|
+
if (label !== null) recordJob(row.queue, label);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
152
186
|
|
|
153
187
|
for (const [index, job] of claimed.entries()) {
|
|
154
188
|
const queue = job.queue;
|
|
@@ -158,7 +192,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
158
192
|
if (admission.kind === 'refused') {
|
|
159
193
|
// Settled without a body: counted and returned with the pass, the one job this loop
|
|
160
194
|
// itself finishes.
|
|
161
|
-
refused
|
|
195
|
+
tally.refused();
|
|
162
196
|
const label = JOB_OUTCOME_LABELS[admission.execution.outcome];
|
|
163
197
|
if (label !== null) recordJob(queue, label);
|
|
164
198
|
started.push(Promise.resolve(admission.execution));
|
|
@@ -170,15 +204,17 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
170
204
|
// core's own in-flight wait sits between `accept` and `inflight` and exists for exactly
|
|
171
205
|
// this. Counted nowhere, the worker had to wait for its own jobs inside a hook.
|
|
172
206
|
const finishWork = beginWork();
|
|
173
|
-
|
|
207
|
+
// Once per run, whichever comes first: the run's own `finally`, or the drain handing its
|
|
208
|
+
// row back while the body is still running (`worker-held.ts`).
|
|
209
|
+
let slotReleased = false;
|
|
210
|
+
const releaseSlot = async (): Promise<void> => {
|
|
211
|
+
if (slotReleased) return;
|
|
212
|
+
slotReleased = true;
|
|
213
|
+
await fleetSlots.release(job.id);
|
|
214
|
+
};
|
|
174
215
|
const running = runClaimed(job, admission.refusal)
|
|
175
216
|
.then((execution) => {
|
|
176
|
-
|
|
177
|
-
else if (execution.outcome === 'suspended') suspended += 1;
|
|
178
|
-
else if (execution.outcome === 'retried') failed += 1;
|
|
179
|
-
else if (execution.outcome === 'interrupted') interrupted += 1;
|
|
180
|
-
else if (execution.outcome === 'dead-lettered') deadLettered += 1;
|
|
181
|
-
else if (execution.outcome === 'dropped') dropped += 1;
|
|
217
|
+
tally.count(execution);
|
|
182
218
|
// The other half of this package's metrics contract: `queue_depth` says how much work
|
|
183
219
|
// is waiting, `jobs_total` says whether any of it is succeeding. Depth alone cannot
|
|
184
220
|
// tell a drained queue from a queue nothing ever claimed. Labelled by QUEUE and
|
|
@@ -193,7 +229,6 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
193
229
|
return execution;
|
|
194
230
|
})
|
|
195
231
|
.finally(async () => {
|
|
196
|
-
holding.delete(job.id);
|
|
197
232
|
lease.release();
|
|
198
233
|
// The slot is free NOW. A full worker asks the queue for nothing, so its loop backs
|
|
199
234
|
// off like an idle one — and waiting that out with a backlog behind the job that
|
|
@@ -207,24 +242,27 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
207
242
|
// `finally` is for the one that could, because a lost `finishWork()` is an in-flight
|
|
208
243
|
// count that never returns to zero and a drain that waits out its whole budget.
|
|
209
244
|
try {
|
|
210
|
-
await
|
|
245
|
+
await releaseSlot();
|
|
211
246
|
} finally {
|
|
212
247
|
finishWork();
|
|
213
248
|
}
|
|
214
249
|
});
|
|
215
250
|
|
|
216
251
|
started.push(running);
|
|
217
|
-
inFlight.add(running)
|
|
252
|
+
inFlight.add(running, job, async () => {
|
|
253
|
+
await releaseSlot();
|
|
254
|
+
finishWork();
|
|
255
|
+
});
|
|
218
256
|
// The claim loop no longer awaits these, so this is the one place a rejection is observed:
|
|
219
257
|
// unobserved it is an unhandled rejection, which on Bun's default is the whole process.
|
|
220
258
|
// `executeJob` settles the job itself, so reaching here means the driver could not be
|
|
221
259
|
// told how it ended — the lease will lapse and the queue will deliver it again.
|
|
222
260
|
void running.then(
|
|
223
261
|
() => {
|
|
224
|
-
inFlight.
|
|
262
|
+
inFlight.settle(running);
|
|
225
263
|
},
|
|
226
264
|
(error: unknown) => {
|
|
227
|
-
inFlight.
|
|
265
|
+
inFlight.settle(running);
|
|
228
266
|
logger.error('jobs.worker.settle-failed', {
|
|
229
267
|
workerId,
|
|
230
268
|
job: job.name,
|
|
@@ -269,7 +307,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
269
307
|
* whole batch was done. `worker-loop.ts` owns the timer and everything that cuts its wait short.
|
|
270
308
|
*/
|
|
271
309
|
const loop: ClaimLoop = createClaimLoop({
|
|
272
|
-
subject: '
|
|
310
|
+
subject: 'jobWorker',
|
|
273
311
|
floorMs: pollIntervalMs,
|
|
274
312
|
...(options.idlePollMaxMs === undefined ? {} : { ceilingMs: options.idlePollMaxMs }),
|
|
275
313
|
queues,
|
|
@@ -280,21 +318,18 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
280
318
|
});
|
|
281
319
|
|
|
282
320
|
/**
|
|
283
|
-
* The whole of the `accept` phase: stop taking work
|
|
284
|
-
*
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
* `
|
|
290
|
-
*
|
|
291
|
-
* on for the full deadline and abandoned there, exactly like one that ignores the signal.
|
|
321
|
+
* The whole of the `accept` phase: stop taking work and ARM the cut-off — nothing else, and no
|
|
322
|
+
* wait: the hook behind this one is somebody else's "stop listening". Armed here and not in
|
|
323
|
+
* `close`, which runs only after core's in-flight wait has spent the budget.
|
|
324
|
+
*
|
|
325
|
+
* The jobs held are NOT told. Until 2026-10-05 this aborted every run at SIGTERM, and a step
|
|
326
|
+
* whose side effect completed during the drain was refused and run again by the next worker.
|
|
327
|
+
* The cut-off (`worker-drain-cutoff.ts`) tells them at `deadlineAt − margin`, then hands back
|
|
328
|
+
* what is still held so the replacement claims it at once, not after the visibility timeout.
|
|
292
329
|
*
|
|
293
|
-
* Only a SHUTDOWN
|
|
294
|
-
*
|
|
295
|
-
*
|
|
296
|
-
* every call, because the second shutdown to reach a worker is the one that finds the abort
|
|
297
|
-
* already fired and the teardown already waiting — with no deadline, if the first was manual.
|
|
330
|
+
* Only a SHUTDOWN binds the budget and arms it: a manual `stop()` has no budget and wants its
|
|
331
|
+
* work finished. The bind runs on every call — a SIGTERM landing on a manual stop is what binds
|
|
332
|
+
* the teardown already waiting — and a tighter deadline re-arms through the budget's watch.
|
|
298
333
|
*/
|
|
299
334
|
const stopAccepting = (shutdown?: ShutdownReason): void => {
|
|
300
335
|
if (state === 'stopped') return;
|
|
@@ -302,13 +337,19 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
302
337
|
loop.stop();
|
|
303
338
|
if (shutdown === undefined) return;
|
|
304
339
|
budget.bind(shutdown.deadlineAt);
|
|
305
|
-
if (
|
|
340
|
+
if (cutoff !== undefined) return;
|
|
306
341
|
logger.info('jobs.worker.drain-signalled', {
|
|
307
342
|
workerId,
|
|
308
343
|
signal: shutdown.signal,
|
|
309
344
|
inFlight: inFlight.size,
|
|
310
345
|
});
|
|
311
|
-
|
|
346
|
+
cutoff = armWorkerCutoff({
|
|
347
|
+
budget,
|
|
348
|
+
workerId,
|
|
349
|
+
signal: shutdown.signal,
|
|
350
|
+
drain: drainSignal,
|
|
351
|
+
held: inFlight,
|
|
352
|
+
});
|
|
312
353
|
};
|
|
313
354
|
|
|
314
355
|
const teardown = async (reason: string): Promise<void> => {
|
|
@@ -323,10 +364,11 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
323
364
|
// take, and bound by the SIGTERM — whether it arrived before this teardown or lands in the
|
|
324
365
|
// middle of it. Nothing can kill a body that ignores `ctx.signal`, so an unbounded wait
|
|
325
366
|
// here is a teardown that never ends: driver never closed, state never past 'draining', and
|
|
326
|
-
// the memoized `stopping` every later `stop()` joins never settling.
|
|
327
|
-
//
|
|
367
|
+
// the memoized `stopping` every later `stop()` joins never settling. `empty()`, not a
|
|
368
|
+
// snapshot: the cut-off's hand-back empties the set under bodies still running, and that
|
|
369
|
+
// ends this wait. What a late round started is handed back below, before the close.
|
|
328
370
|
const rounded = await settleAllBy([...rounds], budget);
|
|
329
|
-
const drained = (await settleAllBy([
|
|
371
|
+
const drained = (await settleAllBy([inFlight.empty()], budget)) && rounded;
|
|
330
372
|
if (!drained) {
|
|
331
373
|
logger.warn('jobs.worker.drain-abandoned', {
|
|
332
374
|
workerId,
|
|
@@ -335,9 +377,16 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
335
377
|
fix: 'raise the drain budget past the slowest job — configureLifecycle({ deadlineMs: 600_000 }) — and set terminationGracePeriodSeconds to at least as many seconds',
|
|
336
378
|
});
|
|
337
379
|
}
|
|
338
|
-
|
|
339
|
-
|
|
380
|
+
if (inFlight.size > 0 && budget.deadlineAt !== undefined) await inFlight.handBackAll();
|
|
381
|
+
// Before the close, and awaited: the row is deleted through the driver being closed. Under
|
|
382
|
+
// the SAME budget as the jobs: `forgetWorker` is a statement on the pool that may be the
|
|
383
|
+
// thing failing, and unbounded it held a SIGTERM teardown open with the driver never closed.
|
|
384
|
+
// Abandoned, the row expires on its own TTL — what a killed worker's does.
|
|
385
|
+
const registered = registration;
|
|
340
386
|
registration = undefined;
|
|
387
|
+
if (registered !== undefined && !(await settleAllBy([registered.stop()], budget))) {
|
|
388
|
+
logger.warn('jobs.worker.registry-abandoned', { workerId, reason });
|
|
389
|
+
}
|
|
341
390
|
await options.driver.close?.();
|
|
342
391
|
} finally {
|
|
343
392
|
// Whatever the close did, this worker is done: a state left at 'draining' is a drain that
|
|
@@ -352,6 +401,8 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
352
401
|
// A run this drain abandoned still follows the old controller through its own composition;
|
|
353
402
|
// the next start's jobs must not. Fresh here, in the one place a teardown always reaches —
|
|
354
403
|
// and the budget with it, or the next manual stop would inherit a deadline already spent.
|
|
404
|
+
cutoff?.dispose();
|
|
405
|
+
cutoff = undefined;
|
|
355
406
|
drainSignal = new AbortController();
|
|
356
407
|
budget = createDrainBudget();
|
|
357
408
|
}
|
|
@@ -362,8 +413,8 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
362
413
|
// waits are bounded and the state is set in a `finally`), so a caller landing after an
|
|
363
414
|
// abandoned drain gets an answer rather than joining a promise that never settles.
|
|
364
415
|
if (state === 'stopped') return;
|
|
365
|
-
// Before the join, every time: a SIGTERM landing on a manual stop still
|
|
366
|
-
//
|
|
416
|
+
// Before the join, every time: a SIGTERM landing on a manual stop still arms the cut-off and
|
|
417
|
+
// binds the teardown already waiting to the shutdown's deadline.
|
|
367
418
|
stopAccepting(shutdown);
|
|
368
419
|
// One teardown, joined rather than repeated: that SIGTERM must wait out the same in-flight
|
|
369
420
|
// work, not close the driver a second time underneath it. Cleared as it settles, so a worker
|
|
@@ -381,6 +432,12 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
381
432
|
// Only from a standstill. A start mid-drain would put the claim loop back on a driver the
|
|
382
433
|
// drain is about to close, and stack a second shutdown hook on the one still running.
|
|
383
434
|
if (state !== 'idle' && state !== 'stopped') return;
|
|
435
|
+
// Nor inside the PROCESS's drain: a boot that reached here after SIGTERM registers hooks the
|
|
436
|
+
// drain is already past, so a worker that claimed now held jobs nothing would ever drain.
|
|
437
|
+
if (isDraining()) {
|
|
438
|
+
logger.warn('jobs.worker.start-refused-draining', { workerId });
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
384
441
|
// Refused HERE, at the earliest decidable point, and refused rather than logged: an agent
|
|
385
442
|
// reads "max in-flight runs of THIS job across the fleet", writes `concurrency: 1` on
|
|
386
443
|
// `rebuildSearchIndex`, ships, and two workers run it on the first deploy — while
|
|
@@ -395,7 +452,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
395
452
|
host: options.host,
|
|
396
453
|
queues,
|
|
397
454
|
concurrency: queues.reduce((slots, queue) => slots + slotsFor(queue), 0),
|
|
398
|
-
inFlight: () =>
|
|
455
|
+
inFlight: () => inFlight.ids(),
|
|
399
456
|
// The visibility timeout, renewed on the heartbeat interval: a killed worker leaves the
|
|
400
457
|
// registry on exactly the schedule the queue takes its jobs back.
|
|
401
458
|
ttlMs: visibilityTimeoutMs,
|
|
@@ -403,8 +460,8 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
403
460
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
404
461
|
...(options.schedule === undefined ? {} : { schedule: options.schedule }),
|
|
405
462
|
});
|
|
406
|
-
// TWO hooks, for the two phases that answer two questions. `accept` stops claiming,
|
|
407
|
-
//
|
|
463
|
+
// TWO hooks, for the two phases that answer two questions. `accept` stops claiming, arms
|
|
464
|
+
// the cut-off that tells the held runs near the deadline, and returns, so every hook behind it — the HTTP server's
|
|
408
465
|
// "stop listening", the sync node's "stop upgrading" — runs while the budget is still whole;
|
|
409
466
|
// one hook doing both spent all of it in the phase whose whole purpose is to be quick.
|
|
410
467
|
// `close` waits out what this worker holds and closes the driver, bounded by the deadline
|
|
@@ -433,13 +490,7 @@ export function createWorker(options: WorkerOptions): Worker {
|
|
|
433
490
|
queues,
|
|
434
491
|
state,
|
|
435
492
|
inFlight: inFlight.size,
|
|
436
|
-
|
|
437
|
-
failed,
|
|
438
|
-
suspended,
|
|
439
|
-
deadLettered,
|
|
440
|
-
interrupted,
|
|
441
|
-
refused,
|
|
442
|
-
dropped,
|
|
493
|
+
...tally.snapshot(),
|
|
443
494
|
pollDelayMs: loop.delayMs(),
|
|
444
495
|
queueDepth: [...(await options.driver.stats())],
|
|
445
496
|
};
|