@ultimat3/jobs 28.0.0 → 29.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +8 -10
- package/README.md +23 -1
- package/package.json +7 -7
- package/src/claim-exhausted.ts +6 -3
- package/src/metrics.ts +24 -0
- package/src/worker-claim-sweep.ts +113 -0
- package/src/worker-process-cap.ts +83 -0
- package/src/worker-types.ts +9 -0
- package/src/worker.ts +40 -40
package/CLAUDE.md
CHANGED
|
@@ -76,7 +76,8 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
|
|
|
76
76
|
`finiteCount()` (a count, caller's minimum); `worker-options.ts` is where `jobWorker` reads them.
|
|
77
77
|
**`bun run finite-bounds` is a floor, never proof**: `concurrencyLimiter`'s five numbers are screened
|
|
78
78
|
with min 0 (zero is a HARD STOP). **A row count is `finiteCount` (min 0)**; `retry.attempts` is one with min 1.
|
|
79
|
-
- **`WorkerOptions.concurrency` is read by OWN key** (`worker-slots.test.ts`)
|
|
79
|
+
- **`WorkerOptions.concurrency` is read by OWN key** (`worker-slots.test.ts`); `maxInFlight` BEFORE
|
|
80
|
+
the claim, its count back with the lease, before the `kick` (history).
|
|
80
81
|
- **`job.concurrency` is enforced by `JobDriver.leases`** (one row per held slot in `x_job_leases`);
|
|
81
82
|
`limits.ts` is the per-process fast path. No lease store + a declared `concurrency` makes `start()`
|
|
82
83
|
throw `X_JOB_CONCURRENCY_UNENFORCEABLE`.
|
|
@@ -129,9 +130,8 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
|
|
|
129
130
|
out before `leader.release()`. Same two-hook rule; an ABANDONED round does not release
|
|
130
131
|
(`jobs.scheduler.drain-abandoned`). `stillLeading()` is asked every round and before every task,
|
|
131
132
|
and answers from memory inside `renewEveryMs`.
|
|
132
|
-
- **`run-once` fires ONE catch-up**: the watermark rides the fire (`ScheduledFire.watermarkMs
|
|
133
|
-
|
|
134
|
-
occurrence (`latestOccurrenceBy`, bisection).
|
|
133
|
+
- **`run-once` fires ONE catch-up**: the watermark rides the fire (`ScheduledFire.watermarkMs`),
|
|
134
|
+
never a `markFired` behind it; `skip` fires the latest missed occurrence (`latestOccurrenceBy`).
|
|
135
135
|
- **A lease for a run that never STARTED is `abandon()`ed, never `release()`d** — the rate stamp
|
|
136
136
|
goes with it (`worker-admit.ts`). **A failed pass re-arms at the floor** (`worker-loop.ts`
|
|
137
137
|
catch); `timers` is the test seam.
|
|
@@ -229,8 +229,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
|
|
|
229
229
|
- **`webhook()`**: ONE event to ONE endpoint, no steps; each attempt but a CANCELLED one recorded
|
|
230
230
|
before the throw; `ledger.isDisabled` gates the socket. **The wire format is core's.** An org
|
|
231
231
|
tenant reads `orgId` OFF THE INPUT; none, or not the run's org: TERMINAL `X_JOB_TENANT_MISMATCH`.
|
|
232
|
-
- A `-fixture.ts` file does not ship; `backfill-pass-fixture.ts` raises a plain `Error`
|
|
233
|
-
purpose (it stands in for app code).
|
|
232
|
+
- A `-fixture.ts` file does not ship; `backfill-pass-fixture.ts` raises a plain `Error` on purpose.
|
|
234
233
|
|
|
235
234
|
## Files
|
|
236
235
|
|
|
@@ -283,6 +282,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
|
|
|
283
282
|
| `worker-run.ts` | one claimed job, wired: heartbeat, slot renewal, run signal, span |
|
|
284
283
|
| `run-signal.ts` | the signal ONE run is cancelled by |
|
|
285
284
|
| `worker-fleet-slots.ts` | the fleet slot an in-flight job holds — take, renew, hand back — and the boot refusal |
|
|
285
|
+
| `worker-process-cap.ts` / `worker-claim-sweep.ts` | `maxInFlight`'s count; the pass's asks, rotated |
|
|
286
286
|
| `purge.ts` | `purge()` — a factory over `job()`: the retention sweep and its target seam |
|
|
287
287
|
| `task.ts` | the `task()` primitive + registry + the handle's surface + `registerTask` |
|
|
288
288
|
| `scheduler.ts` | `scheduler` role: the dispatch round, catch-up, leader election, the drain |
|
|
@@ -304,9 +304,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
|
|
|
304
304
|
| `errors-operator.ts` / `errors-requeue.ts` | the operator verbs' refusals; `requeue`'s |
|
|
305
305
|
| `errors-tenant.ts` | `JobTenantMismatchError` |
|
|
306
306
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
Commands: `bun test packages/jobs` · `bun run --filter @ultimat3/jobs typecheck`.
|
|
307
|
+
Commands: `bun test packages/jobs` · `bun run --filter @ultimat3/jobs typecheck`. Test harnesses:
|
|
308
|
+
history, "Moved 2026-10-01 — test harnesses".
|
|
311
309
|
|
|
312
310
|
Why each rule above is shaped the way it is: [`docs/history/jobs.md`](../../docs/history/jobs.md).
|
package/README.md
CHANGED
|
@@ -816,7 +816,7 @@ or `ready` — so a new branch cannot land in one driver and not another. Its le
|
|
|
816
816
|
|
|
817
817
|
| Role | Entry | Behaviour |
|
|
818
818
|
|---|---|---|
|
|
819
|
-
| `worker` | `jobWorker({ driver, context, queues, concurrency })` | per-queue pools, lease heartbeat, SIGTERM drain: stop claiming → finish in-flight → close. `concurrency` is a number for every queue or a table per queue; a queue it does not name gets core's `JOBS_CONCURRENCY_DEFAULT` (8 — this package's own default was 5 before 25.0.0) |
|
|
819
|
+
| `worker` | `jobWorker({ driver, context, queues, concurrency, maxInFlight })` | per-queue pools under one optional process cap, lease heartbeat, SIGTERM drain: stop claiming → finish in-flight → close. `concurrency` is a number for every queue or a table per queue; a queue it does not name gets core's `JOBS_CONCURRENCY_DEFAULT` (8 — this package's own default was 5 before 25.0.0). `maxInFlight` caps the runs held across EVERY queue (app config `jobs.maxInFlight`; unset: no cap) — [Limits](#the-process-cap-maxinflight) |
|
|
820
820
|
| `scheduler` | `jobScheduler({ driver, leader, state })` | one dispatch round at a time, catch-up policy, SIGTERM drain: stop dispatching → finish the round → release the lock |
|
|
821
821
|
|
|
822
822
|
Both roles register the same **pair** of hooks and bound the `close` half the same way. The
|
|
@@ -1047,6 +1047,28 @@ cannot hold the cap, so `jobWorker().start()` **refuses to boot**
|
|
|
1047
1047
|
Over any cap the claim is handed straight back without burning an attempt — one org's 50k-row
|
|
1048
1048
|
import cannot starve the fleet.
|
|
1049
1049
|
|
|
1050
|
+
### The process cap: `maxInFlight`
|
|
1051
|
+
|
|
1052
|
+
`jobWorker({ maxInFlight })` — `jobs.maxInFlight` in `app.config.ts` — is the most runs ONE worker
|
|
1053
|
+
process holds at once across every queue it serves: Sidekiq's `concurrency`. The per-queue slots
|
|
1054
|
+
(`concurrency`) still bound each queue under it. Unset, there is no process cap: the sum of the
|
|
1055
|
+
queue slots is the bound, as before.
|
|
1056
|
+
|
|
1057
|
+
| Rule | Detail |
|
|
1058
|
+
|---|---|
|
|
1059
|
+
| read BEFORE the claim | a worker at the cap asks the queue for nothing; no row is claimed only to be shed. `LimitConfig.global` is the other kind: it sheds what was already claimed |
|
|
1060
|
+
| a finish frees one claim at once | the run's `finally` returns its count beside its limiter lease, then kicks a pass — no poll wait |
|
|
1061
|
+
| no idle slot while a queue has work | a pass splits the room it has over the queues it has not asked yet (`ceil(room / queues left)`); a queue with less work than its share leaves the rest to the queues after it, and a queue that filled its share is asked again while room is left |
|
|
1062
|
+
| no queue starves | queues are asked in the worker's order **rotated to start after the last queue given a job**, so on a worker at the cap each freed slot goes to the next queue in turn that has work. Three backlogged queues on `maxInFlight: 1` run `a, b, c, a, b, c` — never `a` until it drains. This holds while the loop IDLES at the cap too: once the cap has filled, an idle pass takes the rotated asks instead of the one-statement claim over every queue (which hands out the oldest row of any queue), until a pass finds no work at all. A first fill from empty is that one statement's, in the driver's order |
|
|
1063
|
+
| drain, heartbeat, lease | unchanged: SIGTERM stops claiming, held runs finish under the deadline, the cut-off hands back what is left |
|
|
1064
|
+
| refused | `0`, negative, fractional, `NaN`, `Infinity`: `X_INVARIANT` at `jobWorker()`, `X_CONFIG_INVALID` naming `jobs.maxInFlight` at boot |
|
|
1065
|
+
|
|
1066
|
+
What a scaler reads, per process: `worker.stats()` → `inFlight` and `maxInFlight` (`null` when
|
|
1067
|
+
uncapped); the registry row's `concurrency` is the tighter of the cap and the slot sum; and two
|
|
1068
|
+
unlabelled gauges, `job_worker_in_flight` and `job_worker_capacity`, each the SUM over the live workers in the process (a stopped worker leaves both). Pods needed is
|
|
1069
|
+
`ceil((queue_depth + Σ job_worker_in_flight) / job_worker_capacity)`. A fleet-wide ceiling is a
|
|
1070
|
+
different number: `maxInFlight` × replicas, or `job.concurrency` for one job.
|
|
1071
|
+
|
|
1050
1072
|
### One run per key
|
|
1051
1073
|
|
|
1052
1074
|
A number caps the whole job. `{ key, limit, whenBusy }` caps it **per key** — "at most one run
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/jobs",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "29.1.0",
|
|
4
4
|
"description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -26,17 +26,17 @@
|
|
|
26
26
|
"LICENSE"
|
|
27
27
|
],
|
|
28
28
|
"engines": {
|
|
29
|
-
"bun": ">=1.4.
|
|
29
|
+
"bun": ">=1.4.4"
|
|
30
30
|
},
|
|
31
31
|
"scripts": {
|
|
32
32
|
"typecheck": "tsc --noEmit -p tsconfig.json",
|
|
33
33
|
"test": "bun test"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@ultimat3/core": "
|
|
37
|
-
"@ultimat3/db": "
|
|
38
|
-
"@ultimat3/entity": "
|
|
39
|
-
"@ultimat3/schema": "
|
|
40
|
-
"@ultimat3/time": "
|
|
36
|
+
"@ultimat3/core": "29.1.0",
|
|
37
|
+
"@ultimat3/db": "29.1.0",
|
|
38
|
+
"@ultimat3/entity": "29.1.0",
|
|
39
|
+
"@ultimat3/schema": "29.1.0",
|
|
40
|
+
"@ultimat3/time": "29.1.0"
|
|
41
41
|
}
|
|
42
42
|
}
|
package/src/claim-exhausted.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// The claim round's half of a burial: a row the claim settled `dead` (`ClaimOptions.onExhausted`)
|
|
2
2
|
// has no body to end it and no settle to announce it, so the pass that buried it says so — one log
|
|
3
|
-
// line an operator can alert on, and the job's `onSettled`, exactly as a
|
|
4
|
-
// settled itself would get.
|
|
3
|
+
// line an operator can alert on, its `jobs_total` count, and the job's `onSettled`, exactly as a
|
|
4
|
+
// dead letter a worker settled itself would get.
|
|
5
5
|
|
|
6
6
|
import type { Ctx } from '@ultimat3/core';
|
|
7
|
-
import { logger, renderThrowable } from '@ultimat3/core';
|
|
7
|
+
import { logger, recordJob, renderThrowable } from '@ultimat3/core';
|
|
8
8
|
import type { JobRecord } from './driver';
|
|
9
9
|
import { getJob } from './job';
|
|
10
|
+
import { JOB_OUTCOME_LABELS } from './metrics';
|
|
10
11
|
import { announceSettled } from './settled';
|
|
11
12
|
|
|
12
13
|
export interface AnnounceExhaustedOptions {
|
|
@@ -38,6 +39,8 @@ export async function announceExhausted(
|
|
|
38
39
|
const error = record.lastError ?? '';
|
|
39
40
|
const outcome = record.state === 'failed' ? 'dropped' : 'dead-lettered';
|
|
40
41
|
if (outcome === 'dropped') dropped += 1;
|
|
42
|
+
const label = JOB_OUTCOME_LABELS[outcome];
|
|
43
|
+
if (label !== null) recordJob(record.queue, label);
|
|
41
44
|
// `error`, not `warn`: nobody recovered from this one, and a job that outlives every worker
|
|
42
45
|
// that takes it is a crash loop somebody has to look at.
|
|
43
46
|
logger.error('jobs.claim.exhausted', {
|
package/src/metrics.ts
CHANGED
|
@@ -56,3 +56,27 @@ export function recordQueueOldestReady(queue: string, oldestReadyMs: number): vo
|
|
|
56
56
|
export function recordQueueDeadJobs(queue: string, dead: number): void {
|
|
57
57
|
queueDeadJobs.record(dead, { queue });
|
|
58
58
|
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* What an autoscaler sizes a worker fleet with: `ceil((queue_depth + Σ in_flight) / capacity)`
|
|
62
|
+
* pods. Unlabelled — one series per PROCESS, which is what `maxInFlight` caps and what one worker
|
|
63
|
+
* pod is; the scrape's own `instance` label tells pods apart, and `sum` across them is the fleet.
|
|
64
|
+
*/
|
|
65
|
+
export const workerInFlight: Gauge = gauge('job_worker_in_flight', {
|
|
66
|
+
unit: '{job}',
|
|
67
|
+
description: 'Jobs this worker process is running right now, across every queue it serves',
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
export const workerCapacity: Gauge = gauge('job_worker_capacity', {
|
|
71
|
+
unit: '{job}',
|
|
72
|
+
description:
|
|
73
|
+
'The most jobs this worker process runs at once: maxInFlight, or the sum of its queue slots when tighter or unset',
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
export function recordWorkerInFlight(held: number): void {
|
|
77
|
+
workerInFlight.record(held);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function recordWorkerCapacity(capacity: number): void {
|
|
81
|
+
workerCapacity.record(capacity);
|
|
82
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// Single responsibility: which queue one claim pass asks, in what order and for how many — under
|
|
2
|
+
// each queue's own slots AND the process cap (`maxInFlight`, `worker-process-cap.ts`). The worker
|
|
3
|
+
// owns what a claim DOES; this owns the plan, so the fairness rule is one file a test can drive.
|
|
4
|
+
//
|
|
5
|
+
// THE FAIRNESS RULE. Queues are asked in the worker's order ROTATED to start after the last queue
|
|
6
|
+
// that was given a job, and a pass splits the room it has left evenly over the queues it has not
|
|
7
|
+
// asked yet (`ceil(room / queues left)`); a queue that filled its share is asked again, in the same order, while room is left. So a free slot goes to some queue
|
|
8
|
+
// with work whenever any has it, and a backlogged queue cannot monopolise a capped worker: each
|
|
9
|
+
// slot a finish frees goes to the next queue in turn — at the cap too, where the loop idles between
|
|
10
|
+
// finishes. An idle worker whose cap has not filled asks once over every queue (one statement, the
|
|
11
|
+
// driver's order), so a first fill from empty is the driver's; the turn holds from the cap on.
|
|
12
|
+
|
|
13
|
+
import { type ClaimAsk, claimAsks } from './worker-admit';
|
|
14
|
+
|
|
15
|
+
/** What one claim answered: runs ADMITTED (a shed or a refusal holds nothing), per queue. */
|
|
16
|
+
export type ClaimAnswer = ReadonlyMap<string, number>;
|
|
17
|
+
|
|
18
|
+
export interface ClaimSweepInput {
|
|
19
|
+
/** The worker's queues, in its own order. */
|
|
20
|
+
readonly queues: readonly string[];
|
|
21
|
+
/** A queue's free slots, re-read before every ask — a run admitted or finished moves it. */
|
|
22
|
+
readonly freeSlots: (queue: string) => number;
|
|
23
|
+
/** Room under the process cap, re-read likewise; `Infinity` with none. */
|
|
24
|
+
readonly processFree: () => number;
|
|
25
|
+
/** The loop is idling: one statement over every open queue (`claimAsks`), unless contended. */
|
|
26
|
+
readonly idle: boolean;
|
|
27
|
+
/** May the pass still claim? Re-read per ask — a `stop()` between two queues is a stop now. */
|
|
28
|
+
readonly claiming: () => boolean;
|
|
29
|
+
readonly claim: (ask: ClaimAsk) => Promise<ClaimAnswer>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface ClaimSweep {
|
|
33
|
+
/** One pass. Resolves once every ask it made has been answered. */
|
|
34
|
+
pass(input: ClaimSweepInput): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** `queues` from `start`, wrapping. */
|
|
38
|
+
const rotated = (queues: readonly string[], start: number): readonly string[] =>
|
|
39
|
+
queues.length === 0
|
|
40
|
+
? queues
|
|
41
|
+
: [...queues.slice(start % queues.length), ...queues.slice(0, start % queues.length)];
|
|
42
|
+
|
|
43
|
+
/** The cursor lives across passes: that is what makes the rotation a turn, not a shuffle. */
|
|
44
|
+
export function createClaimSweep(): ClaimSweep {
|
|
45
|
+
let cursor = 0;
|
|
46
|
+
/**
|
|
47
|
+
* The cap has been FULL since the last pass that found the queues empty. A worker at the cap
|
|
48
|
+
* finds nothing, so its loop idles, and an idle pass is ONE statement over every queue — rows in
|
|
49
|
+
* the DRIVER's order, the oldest of any queue, never by turn. So while contended, an idle pass
|
|
50
|
+
* takes the rotated sweep; once a rotated idle pass finds no work at all, the worker is idle for
|
|
51
|
+
* real and goes back to its one-statement pass. Set by the cap filling, not by runs held: at
|
|
52
|
+
* `maxInFlight: 1` a finish leaves none held at the very pass that must pick the next queue.
|
|
53
|
+
*/
|
|
54
|
+
let contended = false;
|
|
55
|
+
|
|
56
|
+
const served = (queues: readonly string[], answer: ClaimAnswer): number => {
|
|
57
|
+
let admitted = 0;
|
|
58
|
+
for (const [queue, count] of answer) {
|
|
59
|
+
if (count <= 0) continue;
|
|
60
|
+
admitted += count;
|
|
61
|
+
const index = queues.indexOf(queue);
|
|
62
|
+
if (index !== -1) cursor = index + 1;
|
|
63
|
+
}
|
|
64
|
+
return admitted;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
async pass(input) {
|
|
69
|
+
const order = rotated(input.queues, cursor);
|
|
70
|
+
if (input.processFree() <= 0) {
|
|
71
|
+
contended = true;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (input.idle && !contended) {
|
|
75
|
+
const room = input.processFree();
|
|
76
|
+
for (const ask of claimAsks(order, input.freeSlots, true)) {
|
|
77
|
+
if (!input.claiming()) return;
|
|
78
|
+
served(input.queues, await input.claim({ ...ask, limit: Math.min(ask.limit, room) }));
|
|
79
|
+
}
|
|
80
|
+
if (input.processFree() <= 0) contended = true;
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
let admitted = 0;
|
|
84
|
+
let asking = order;
|
|
85
|
+
while (asking.length > 0) {
|
|
86
|
+
const again: string[] = [];
|
|
87
|
+
for (const [index, queue] of asking.entries()) {
|
|
88
|
+
if (!input.claiming()) return;
|
|
89
|
+
const room = input.processFree();
|
|
90
|
+
if (room <= 0) {
|
|
91
|
+
contended = true;
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const share = Math.ceil(room / (asking.length - index));
|
|
95
|
+
const free = Math.max(0, input.freeSlots(queue));
|
|
96
|
+
const limit = Math.min(free, share);
|
|
97
|
+
if (limit <= 0) continue;
|
|
98
|
+
const answer = await input.claim({ queues: [queue], limit });
|
|
99
|
+
const got = served(input.queues, answer);
|
|
100
|
+
admitted += got;
|
|
101
|
+
// Its SHARE bound it, not its slots, and it filled it with runs that STARTED: it may
|
|
102
|
+
// hold more, for room a quieter queue left. A shed is not "more" — asking again would
|
|
103
|
+
// claim the same capped rows back and forth. With no cap the share never binds, so an
|
|
104
|
+
// uncapped pass is one ask per queue, as it always was.
|
|
105
|
+
if (limit < free && got >= limit) again.push(queue);
|
|
106
|
+
}
|
|
107
|
+
asking = again;
|
|
108
|
+
}
|
|
109
|
+
if (input.processFree() <= 0) contended = true;
|
|
110
|
+
else if (input.idle && admitted === 0) contended = false;
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Single responsibility: `WorkerOptions.maxInFlight` — the one count of runs this worker holds
|
|
2
|
+
// across EVERY queue, and the room it leaves. Sidekiq's `concurrency`: the per-queue slots bound
|
|
3
|
+
// each pool, this bounds the process. Read BEFORE a claim, so a worker at the cap asks the queue
|
|
4
|
+
// for nothing and no row is claimed only to be shed.
|
|
5
|
+
|
|
6
|
+
import { finiteCount } from '@ultimat3/core';
|
|
7
|
+
import { recordWorkerCapacity, recordWorkerInFlight } from './metrics';
|
|
8
|
+
|
|
9
|
+
export interface ProcessCap {
|
|
10
|
+
/** The cap as declared, `undefined` when the worker has none. */
|
|
11
|
+
readonly max: number | undefined;
|
|
12
|
+
/** Room left under the cap: `Infinity` with no cap, never below 0. */
|
|
13
|
+
free(): number;
|
|
14
|
+
/** Runs counted against the cap right now. */
|
|
15
|
+
held(): number;
|
|
16
|
+
/**
|
|
17
|
+
* One run admitted. The answer hands its count back, ONCE however often it is called — the
|
|
18
|
+
* run's `finally` and nothing else, at the same instant its limiter lease goes back, so the
|
|
19
|
+
* `kick` that follows sees the room it made.
|
|
20
|
+
*/
|
|
21
|
+
take(): () => void;
|
|
22
|
+
/** The worker serves again (`start()`): counted in the process gauges. */
|
|
23
|
+
revive(): void;
|
|
24
|
+
/** The worker stopped: out of the process gauges, its capacity with it. */
|
|
25
|
+
retire(): void;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface Counted {
|
|
29
|
+
held: number;
|
|
30
|
+
readonly capacity: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Every worker this PROCESS has live. The two gauges are unlabelled, one series per process, so
|
|
35
|
+
* they carry the SUM: a second `jobWorker` in one process used to reset the first's in-flight to 0
|
|
36
|
+
* and overwrite its capacity, and each later update published one worker's count as the whole.
|
|
37
|
+
*/
|
|
38
|
+
const live = new Set<Counted>();
|
|
39
|
+
|
|
40
|
+
const publish = (): void => {
|
|
41
|
+
let held = 0;
|
|
42
|
+
let capacity = 0;
|
|
43
|
+
for (const worker of live) {
|
|
44
|
+
held += worker.held;
|
|
45
|
+
capacity += worker.capacity;
|
|
46
|
+
}
|
|
47
|
+
recordWorkerInFlight(held);
|
|
48
|
+
recordWorkerCapacity(capacity);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/** `maxInFlight`, refused when it is not a whole number ≥ 1 — `jobWorker`'s other counts' rule. */
|
|
52
|
+
export function createProcessCap(max: number | undefined, slots: number): ProcessCap {
|
|
53
|
+
if (max !== undefined) finiteCount('jobWorker', 'maxInFlight', max, 1);
|
|
54
|
+
// What a scaler divides by: the most this worker can run, whichever bound is tighter.
|
|
55
|
+
const counted: Counted = { held: 0, capacity: max === undefined ? slots : Math.min(max, slots) };
|
|
56
|
+
live.add(counted);
|
|
57
|
+
publish();
|
|
58
|
+
return {
|
|
59
|
+
max,
|
|
60
|
+
free: () => (max === undefined ? Number.POSITIVE_INFINITY : Math.max(0, max - counted.held)),
|
|
61
|
+
held: () => counted.held,
|
|
62
|
+
take() {
|
|
63
|
+
counted.held += 1;
|
|
64
|
+
publish();
|
|
65
|
+
let returned = false;
|
|
66
|
+
return () => {
|
|
67
|
+
if (returned) return;
|
|
68
|
+
returned = true;
|
|
69
|
+
counted.held -= 1;
|
|
70
|
+
publish();
|
|
71
|
+
};
|
|
72
|
+
},
|
|
73
|
+
revive() {
|
|
74
|
+
if (live.has(counted)) return;
|
|
75
|
+
live.add(counted);
|
|
76
|
+
publish();
|
|
77
|
+
},
|
|
78
|
+
retire() {
|
|
79
|
+
if (!live.delete(counted)) return;
|
|
80
|
+
publish();
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
package/src/worker-types.ts
CHANGED
|
@@ -18,6 +18,13 @@ export interface WorkerOptions {
|
|
|
18
18
|
* (or no `concurrency` at all) runs at core's `JOBS_CONCURRENCY_DEFAULT`, 8 — 5 before 25.0.0.
|
|
19
19
|
*/
|
|
20
20
|
readonly concurrency?: number | Readonly<Record<string, number>>;
|
|
21
|
+
/**
|
|
22
|
+
* The most runs this process holds at once, across EVERY queue — Sidekiq's `concurrency`. The
|
|
23
|
+
* per-queue slots still bound each queue under it. A worker at the cap claims nothing until a
|
|
24
|
+
* run ends; a freed slot goes to the next queue in turn that has work (`worker-claim-sweep.ts`).
|
|
25
|
+
* Unset: no process cap, the sum of the queue slots. A whole number ≥ 1, else `X_INVARIANT`.
|
|
26
|
+
*/
|
|
27
|
+
readonly maxInFlight?: number;
|
|
21
28
|
readonly limiter?: Limiter;
|
|
22
29
|
readonly clock?: Clock;
|
|
23
30
|
readonly events?: EventLookup;
|
|
@@ -52,6 +59,8 @@ export interface WorkerStats {
|
|
|
52
59
|
readonly queues: readonly string[];
|
|
53
60
|
readonly state: 'idle' | 'running' | 'draining' | 'stopped';
|
|
54
61
|
readonly inFlight: number;
|
|
62
|
+
/** `WorkerOptions.maxInFlight`, `null` when uncapped — what a scaler divides the backlog by. */
|
|
63
|
+
readonly maxInFlight: number | null;
|
|
55
64
|
readonly processed: number;
|
|
56
65
|
readonly failed: number;
|
|
57
66
|
readonly suspended: number;
|
package/src/worker.ts
CHANGED
|
@@ -22,13 +22,15 @@ import type { JobExecution } from './execute';
|
|
|
22
22
|
import { registeredJobs } from './job';
|
|
23
23
|
import { concurrencyLimiter } from './limits';
|
|
24
24
|
import { JOB_OUTCOME_LABELS } from './metrics';
|
|
25
|
-
import {
|
|
25
|
+
import { type ClaimAsk, createAdmission } from './worker-admit';
|
|
26
|
+
import { type ClaimAnswer, createClaimSweep } from './worker-claim-sweep';
|
|
26
27
|
import { armWorkerCutoff, type DrainCutoff } from './worker-drain-cutoff';
|
|
27
28
|
import { assertConcurrencyEnforceable, createFleetSlots } from './worker-fleet-slots';
|
|
28
29
|
import { createHeldRuns } from './worker-held';
|
|
29
30
|
import type { ClaimLoop } from './worker-loop';
|
|
30
31
|
import { createClaimLoop } from './worker-loop';
|
|
31
32
|
import { resolveWorkerTimings } from './worker-options';
|
|
33
|
+
import { createProcessCap } from './worker-process-cap';
|
|
32
34
|
import { createQueueDepthPublisher } from './worker-queue-depth';
|
|
33
35
|
import type { WorkerRegistration } from './worker-registry';
|
|
34
36
|
import { reportUnregisteredQueues, startWorkerRegistry } from './worker-registry';
|
|
@@ -46,6 +48,9 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
46
48
|
const { visibilityTimeoutMs, pollIntervalMs, heartbeatIntervalMs, slotsFor } =
|
|
47
49
|
resolveWorkerTimings(options);
|
|
48
50
|
const limiter = options.limiter ?? concurrencyLimiter({});
|
|
51
|
+
const slots = queues.reduce((total, queue) => total + slotsFor(queue), 0);
|
|
52
|
+
const cap = createProcessCap(options.maxInFlight, slots); // read before each claim
|
|
53
|
+
const sweep = createClaimSweep();
|
|
49
54
|
const driverLeases = options.driver.leases;
|
|
50
55
|
// `job.concurrency`, held as a row every replica sees. The TTL is the visibility timeout and the
|
|
51
56
|
// renewal rides the lease heartbeat's interval — `worker-fleet-slots.ts` says why both.
|
|
@@ -132,28 +137,19 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
132
137
|
const claiming = (): boolean => state !== 'draining' && state !== 'stopped';
|
|
133
138
|
|
|
134
139
|
/**
|
|
135
|
-
* One claim pass:
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
* as slow as its slowest member and, because the pass walks every queue before it waits, left
|
|
139
|
-
* every OTHER queue idle behind one long-running job too.
|
|
140
|
+
* One claim pass: everything it hands back STARTED, never waited for — a slot is free again the
|
|
141
|
+
* moment its own job settles, so the next pass refills exactly that slot. Waiting for the batch
|
|
142
|
+
* made a pool as slow as its slowest member, and every OTHER queue idle behind it.
|
|
140
143
|
*/
|
|
141
144
|
const claimRound = async (): Promise<readonly Promise<JobExecution>[]> => {
|
|
142
145
|
await publishQueueDepth();
|
|
143
146
|
const started: Promise<JobExecution>[] = [];
|
|
144
147
|
let found = false;
|
|
145
148
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
);
|
|
151
|
-
|
|
152
|
-
for (const ask of asks) {
|
|
153
|
-
// Re-read per claim, not once at the top: a `stop()` between two queues means "stop
|
|
154
|
-
// claiming" now, not at the next tick. What this round already holds still runs to the end
|
|
155
|
-
// — that is the drain, and `stop()` waits for it.
|
|
156
|
-
if (!claiming()) break;
|
|
149
|
+
// One ask; WHICH queue, how many and in what order is `worker-claim-sweep.ts`'s (`maxInFlight`
|
|
150
|
+
// and the turn). It re-reads `claiming()` per ask: a `stop()` mid-pass stops claiming now.
|
|
151
|
+
const claimOne = async (ask: ClaimAsk): Promise<ClaimAnswer> => {
|
|
152
|
+
const admitted = new Map<string, number>();
|
|
157
153
|
// A lease that lapsed on a row's final attempt is settled by the claim itself and handed to
|
|
158
154
|
// `onExhausted`, never out as work: this round is the only thing that can count it, log it
|
|
159
155
|
// and tell the job. `dropExhausted` is the policy the row does not carry.
|
|
@@ -172,16 +168,9 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
172
168
|
if (claimed.length > 0) found = true;
|
|
173
169
|
if (buried.length > 0) {
|
|
174
170
|
found = true;
|
|
175
|
-
|
|
176
|
-
exhausted: buried,
|
|
177
|
-
|
|
178
|
-
context: options.context,
|
|
179
|
-
});
|
|
180
|
-
tally.buried(ended);
|
|
181
|
-
for (const row of buried) {
|
|
182
|
-
const label = JOB_OUTCOME_LABELS[row.state === 'failed' ? 'dropped' : 'dead-lettered'];
|
|
183
|
-
if (label !== null) recordJob(row.queue, label);
|
|
184
|
-
}
|
|
171
|
+
tally.buried(
|
|
172
|
+
await announceExhausted({ exhausted: buried, workerId, context: options.context }),
|
|
173
|
+
);
|
|
185
174
|
}
|
|
186
175
|
|
|
187
176
|
for (const [index, job] of claimed.entries()) {
|
|
@@ -199,6 +188,8 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
199
188
|
continue;
|
|
200
189
|
}
|
|
201
190
|
const { lease } = admission;
|
|
191
|
+
admitted.set(queue, (admitted.get(queue) ?? 0) + 1);
|
|
192
|
+
const returnCap = cap.take(); // `maxInFlight`'s count, handed back beside the lease
|
|
202
193
|
|
|
203
194
|
// The claimed job is the process's in-flight work, counted where the DRAIN can see it:
|
|
204
195
|
// core's own in-flight wait sits between `accept` and `inflight` and exists for exactly
|
|
@@ -215,10 +206,8 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
215
206
|
const running = runClaimed(job, admission.refusal)
|
|
216
207
|
.then((execution) => {
|
|
217
208
|
tally.count(execution);
|
|
218
|
-
//
|
|
219
|
-
//
|
|
220
|
-
// tell a drained queue from a queue nothing ever claimed. Labelled by QUEUE and
|
|
221
|
-
// OUTCOME only — a label per job name is unbounded in an app's own vocabulary.
|
|
209
|
+
// `jobs_total`, beside `queue_depth`: depth cannot tell a drained queue from one nothing
|
|
210
|
+
// claimed. QUEUE and OUTCOME only — a job-name label is unbounded in an app's vocabulary.
|
|
222
211
|
const label = JOB_OUTCOME_LABELS[execution.outcome];
|
|
223
212
|
if (label !== null) recordJob(queue, label);
|
|
224
213
|
// Handed back with a time on it — a retry's backoff, a sleep's wake: this worker
|
|
@@ -230,17 +219,15 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
230
219
|
})
|
|
231
220
|
.finally(async () => {
|
|
232
221
|
lease.release();
|
|
222
|
+
returnCap();
|
|
233
223
|
// The slot is free NOW. A full worker asks the queue for nothing, so its loop backs
|
|
234
224
|
// off like an idle one — and waiting that out with a backlog behind the job that
|
|
235
225
|
// just finished is throughput lost to a timer.
|
|
236
226
|
loop.kick();
|
|
237
|
-
// AWAITED, never `void`: the slot is a row in `x_job_leases`,
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
//
|
|
241
|
-
// swallows its own failures, so awaiting it cannot reject a job that finished; the
|
|
242
|
-
// `finally` is for the one that could, because a lost `finishWork()` is an in-flight
|
|
243
|
-
// count that never returns to zero and a drain that waits out its whole budget.
|
|
227
|
+
// AWAITED, never `void`: the slot is a row in `x_job_leases`, and a DELETE still on the
|
|
228
|
+
// wire at `driver.close()` left a `concurrency: 1` job unclaimable by the replacement
|
|
229
|
+
// pod for a visibility window. `release` swallows its own failures; the `finally` is
|
|
230
|
+
// for the one that could not, as a lost `finishWork()` holds the drain its whole budget.
|
|
244
231
|
try {
|
|
245
232
|
await releaseSlot();
|
|
246
233
|
} finally {
|
|
@@ -272,7 +259,17 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
272
259
|
},
|
|
273
260
|
);
|
|
274
261
|
}
|
|
275
|
-
|
|
262
|
+
return admitted;
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
await sweep.pass({
|
|
266
|
+
queues,
|
|
267
|
+
freeSlots: (queue) => slotsFor(queue) - limiter.inFlight({ queue }),
|
|
268
|
+
processFree: cap.free,
|
|
269
|
+
idle: loop.idle(),
|
|
270
|
+
claiming,
|
|
271
|
+
claim: claimOne,
|
|
272
|
+
});
|
|
276
273
|
|
|
277
274
|
// The loop's next wait: the floor while passes find work, doubling to the ceiling while they
|
|
278
275
|
// do not (`idle-backoff.ts`). Reported by the pass itself, so `tick()` and the timer agree.
|
|
@@ -403,6 +400,7 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
403
400
|
// and the budget with it, or the next manual stop would inherit a deadline already spent.
|
|
404
401
|
cutoff?.dispose();
|
|
405
402
|
cutoff = undefined;
|
|
403
|
+
cap.retire(); // out of the process gauges until a `start()` serves again
|
|
406
404
|
drainSignal = new AbortController();
|
|
407
405
|
budget = createDrainBudget();
|
|
408
406
|
}
|
|
@@ -444,6 +442,7 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
444
442
|
// `x jobs show` and the manifest both confirm a guarantee that does not exist.
|
|
445
443
|
assertConcurrencyEnforceable(options.driver);
|
|
446
444
|
state = 'running';
|
|
445
|
+
cap.revive();
|
|
447
446
|
logger.info('jobs.worker.started', { workerId, queues });
|
|
448
447
|
reportUnregisteredQueues(workerId, queues);
|
|
449
448
|
registration = startWorkerRegistry({
|
|
@@ -451,7 +450,7 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
451
450
|
workerId,
|
|
452
451
|
host: options.host,
|
|
453
452
|
queues,
|
|
454
|
-
concurrency:
|
|
453
|
+
concurrency: Math.min(slots, cap.max ?? slots),
|
|
455
454
|
inFlight: () => inFlight.ids(),
|
|
456
455
|
// The visibility timeout, renewed on the heartbeat interval: a killed worker leaves the
|
|
457
456
|
// registry on exactly the schedule the queue takes its jobs back.
|
|
@@ -490,6 +489,7 @@ export function jobWorker(options: WorkerOptions): Worker {
|
|
|
490
489
|
queues,
|
|
491
490
|
state,
|
|
492
491
|
inFlight: inFlight.size,
|
|
492
|
+
maxInFlight: cap.max ?? null,
|
|
493
493
|
...tally.snapshot(),
|
|
494
494
|
pollDelayMs: loop.delayMs(),
|
|
495
495
|
queueDepth: [...(await options.driver.stats())],
|