@ultimat3/jobs 28.0.0 → 29.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CLAUDE.md CHANGED
@@ -76,7 +76,8 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
76
76
  `finiteCount()` (a count, caller's minimum); `worker-options.ts` is where `jobWorker` reads them.
77
77
  **`bun run finite-bounds` is a floor, never proof**: `concurrencyLimiter`'s five numbers are screened
78
78
  with min 0 (zero is a HARD STOP). **A row count is `finiteCount` (min 0)**; `retry.attempts` is one with min 1.
79
- - **`WorkerOptions.concurrency` is read by OWN key** (`worker-slots.test.ts`).
79
+ - **`WorkerOptions.concurrency` is read by OWN key** (`worker-slots.test.ts`); `maxInFlight` BEFORE
80
+ the claim, its count back with the lease, before the `kick` (history).
80
81
  - **`job.concurrency` is enforced by `JobDriver.leases`** (one row per held slot in `x_job_leases`);
81
82
  `limits.ts` is the per-process fast path. No lease store + a declared `concurrency` makes `start()`
82
83
  throw `X_JOB_CONCURRENCY_UNENFORCEABLE`.
@@ -129,9 +130,8 @@ Tier 3. The `job` + `task` primitives, durable steps, transactional outbox, queu
129
130
  out before `leader.release()`. Same two-hook rule; an ABANDONED round does not release
130
131
  (`jobs.scheduler.drain-abandoned`). `stillLeading()` is asked every round and before every task,
131
132
  and answers from memory inside `renewEveryMs`.
132
- - **`run-once` fires ONE catch-up**: the watermark rides the fire (`ScheduledFire.watermarkMs`,
133
- `SQL_SCHEDULER_FIRE` `$4`), never a `markFired` behind it; `skip` fires the true latest missed
134
- occurrence (`latestOccurrenceBy`, bisection).
133
+ - **`run-once` fires ONE catch-up**: the watermark rides the fire (`ScheduledFire.watermarkMs`),
134
+ never a `markFired` behind it; `skip` fires the latest missed occurrence (`latestOccurrenceBy`).
135
135
  - **A lease for a run that never STARTED is `abandon()`ed, never `release()`d** — the rate stamp
136
136
  goes with it (`worker-admit.ts`). **A failed pass re-arms at the floor** (`worker-loop.ts`
137
137
  catch); `timers` is the test seam.
@@ -229,8 +229,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
229
229
  - **`webhook()`**: ONE event to ONE endpoint, no steps; each attempt but a CANCELLED one recorded
230
230
  before the throw; `ledger.isDisabled` gates the socket. **The wire format is core's.** An org
231
231
  tenant reads `orgId` OFF THE INPUT; none, or not the run's org: TERMINAL `X_JOB_TENANT_MISMATCH`.
232
- - A `-fixture.ts` file does not ship; `backfill-pass-fixture.ts` raises a plain `Error` subclass on
233
- purpose (it stands in for app code).
232
+ - A `-fixture.ts` file does not ship; `backfill-pass-fixture.ts` raises a plain `Error` on purpose.
234
233
 
235
234
  ## Files
236
235
 
@@ -283,6 +282,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
283
282
  | `worker-run.ts` | one claimed job, wired: heartbeat, slot renewal, run signal, span |
284
283
  | `run-signal.ts` | the signal ONE run is cancelled by |
285
284
  | `worker-fleet-slots.ts` | the fleet slot an in-flight job holds — take, renew, hand back — and the boot refusal |
285
+ | `worker-process-cap.ts` / `worker-claim-sweep.ts` | `maxInFlight`'s count; the pass's asks, rotated |
286
286
  | `purge.ts` | `purge()` — a factory over `job()`: the retention sweep and its target seam |
287
287
  | `task.ts` | the `task()` primitive + registry + the handle's surface + `registerTask` |
288
288
  | `scheduler.ts` | `scheduler` role: the dispatch round, catch-up, leader election, the drain |
@@ -304,9 +304,7 @@ Long form: [`docs/history/jobs.md`](../../docs/history/jobs.md), "Moved 2026-10-
304
304
  | `errors-operator.ts` / `errors-requeue.ts` | the operator verbs' refusals; `requeue`'s |
305
305
  | `errors-tenant.ts` | `JobTenantMismatchError` |
306
306
 
307
- Which `-fixture.ts` harness each suite shares, and what each `.job.` suite proves:
308
- `docs/history/jobs.md`, "Moved 2026-10-01 — test harnesses".
309
-
310
- Commands: `bun test packages/jobs` · `bun run --filter @ultimat3/jobs typecheck`.
307
+ Commands: `bun test packages/jobs` · `bun run --filter @ultimat3/jobs typecheck`. Test harnesses:
308
+ history, "Moved 2026-10-01 — test harnesses".
311
309
 
312
310
  Why each rule above is shaped the way it is: [`docs/history/jobs.md`](../../docs/history/jobs.md).
package/README.md CHANGED
@@ -816,7 +816,7 @@ or `ready` — so a new branch cannot land in one driver and not another. Its le
816
816
 
817
817
  | Role | Entry | Behaviour |
818
818
  |---|---|---|
819
- | `worker` | `jobWorker({ driver, context, queues, concurrency })` | per-queue pools, lease heartbeat, SIGTERM drain: stop claiming → finish in-flight → close. `concurrency` is a number for every queue or a table per queue; a queue it does not name gets core's `JOBS_CONCURRENCY_DEFAULT` (8 — this package's own default was 5 before 25.0.0) |
819
+ | `worker` | `jobWorker({ driver, context, queues, concurrency, maxInFlight })` | per-queue pools under one optional process cap, lease heartbeat, SIGTERM drain: stop claiming → finish in-flight → close. `concurrency` is a number for every queue or a table per queue; a queue it does not name gets core's `JOBS_CONCURRENCY_DEFAULT` (8 — this package's own default was 5 before 25.0.0). `maxInFlight` caps the runs held across EVERY queue (app config `jobs.maxInFlight`; unset: no cap) — [Limits](#the-process-cap-maxinflight) |
820
820
  | `scheduler` | `jobScheduler({ driver, leader, state })` | one dispatch round at a time, catch-up policy, SIGTERM drain: stop dispatching → finish the round → release the lock |
821
821
 
822
822
  Both roles register the same **pair** of hooks and bound the `close` half the same way. The
@@ -1047,6 +1047,28 @@ cannot hold the cap, so `jobWorker().start()` **refuses to boot**
1047
1047
  Over any cap the claim is handed straight back without burning an attempt — one org's 50k-row
1048
1048
  import cannot starve the fleet.
1049
1049
 
1050
+ ### The process cap: `maxInFlight`
1051
+
1052
+ `jobWorker({ maxInFlight })` — `jobs.maxInFlight` in `app.config.ts` — is the most runs ONE worker
1053
+ process holds at once across every queue it serves: Sidekiq's `concurrency`. The per-queue slots
1054
+ (`concurrency`) still bound each queue under it. Unset, there is no process cap: the sum of the
1055
+ queue slots is the bound, as before.
1056
+
1057
+ | Rule | Detail |
1058
+ |---|---|
1059
+ | read BEFORE the claim | a worker at the cap asks the queue for nothing; no row is claimed only to be shed. `LimitConfig.global` is the other kind: it sheds what was already claimed |
1060
+ | a finish frees one claim at once | the run's `finally` returns its count beside its limiter lease, then kicks a pass — no poll wait |
1061
+ | no idle slot while a queue has work | a pass splits the room it has over the queues it has not asked yet (`ceil(room / queues left)`); a queue with less work than its share leaves the rest to the queues after it, and a queue that filled its share is asked again while room is left |
1062
+ | no queue starves | queues are asked in the worker's order **rotated to start after the last queue given a job**, so on a worker at the cap each freed slot goes to the next queue in turn that has work. Three backlogged queues on `maxInFlight: 1` run `a, b, c, a, b, c` — never `a` until it drains. This holds while the loop IDLES at the cap too: once the cap has filled, an idle pass takes the rotated asks instead of the one-statement claim over every queue (which hands out the oldest row of any queue), until a pass finds no work at all. A first fill from empty is that one statement's, in the driver's order |
1063
+ | drain, heartbeat, lease | unchanged: SIGTERM stops claiming, held runs finish under the deadline, the cut-off hands back what is left |
1064
+ | refused | `0`, negative, fractional, `NaN`, `Infinity`: `X_INVARIANT` at `jobWorker()`, `X_CONFIG_INVALID` naming `jobs.maxInFlight` at boot |
1065
+
1066
+ What a scaler reads, per process: `worker.stats()` → `inFlight` and `maxInFlight` (`null` when
1067
+ uncapped); the registry row's `concurrency` is the tighter of the cap and the slot sum; and two
1068
+ unlabelled gauges, `job_worker_in_flight` and `job_worker_capacity`, each the SUM over the live workers in the process (a stopped worker leaves both). Pods needed is
1069
+ `ceil((queue_depth + Σ job_worker_in_flight) / job_worker_capacity)`. A fleet-wide ceiling is a
1070
+ different number: `maxInFlight` × replicas, or `job.concurrency` for one job.
1071
+
1050
1072
  ### One run per key
1051
1073
 
1052
1074
  A number caps the whole job. `{ key, limit, whenBusy }` caps it **per key** — "at most one run
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/jobs",
3
- "version": "28.0.0",
3
+ "version": "29.0.0",
4
4
  "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -26,17 +26,17 @@
26
26
  "LICENSE"
27
27
  ],
28
28
  "engines": {
29
- "bun": ">=1.4.3"
29
+ "bun": ">=1.4.4"
30
30
  },
31
31
  "scripts": {
32
32
  "typecheck": "tsc --noEmit -p tsconfig.json",
33
33
  "test": "bun test"
34
34
  },
35
35
  "dependencies": {
36
- "@ultimat3/core": "28.0.0",
37
- "@ultimat3/db": "28.0.0",
38
- "@ultimat3/entity": "28.0.0",
39
- "@ultimat3/schema": "28.0.0",
40
- "@ultimat3/time": "28.0.0"
36
+ "@ultimat3/core": "29.0.0",
37
+ "@ultimat3/db": "29.0.0",
38
+ "@ultimat3/entity": "29.0.0",
39
+ "@ultimat3/schema": "29.0.0",
40
+ "@ultimat3/time": "29.0.0"
41
41
  }
42
42
  }
@@ -1,12 +1,13 @@
1
1
  // The claim round's half of a burial: a row the claim settled `dead` (`ClaimOptions.onExhausted`)
2
2
  // has no body to end it and no settle to announce it, so the pass that buried it says so — one log
3
- // line an operator can alert on, and the job's `onSettled`, exactly as a dead letter a worker
4
- // settled itself would get.
3
+ // line an operator can alert on, its `jobs_total` count, and the job's `onSettled`, exactly as a
4
+ // dead letter a worker settled itself would get.
5
5
 
6
6
  import type { Ctx } from '@ultimat3/core';
7
- import { logger, renderThrowable } from '@ultimat3/core';
7
+ import { logger, recordJob, renderThrowable } from '@ultimat3/core';
8
8
  import type { JobRecord } from './driver';
9
9
  import { getJob } from './job';
10
+ import { JOB_OUTCOME_LABELS } from './metrics';
10
11
  import { announceSettled } from './settled';
11
12
 
12
13
  export interface AnnounceExhaustedOptions {
@@ -38,6 +39,8 @@ export async function announceExhausted(
38
39
  const error = record.lastError ?? '';
39
40
  const outcome = record.state === 'failed' ? 'dropped' : 'dead-lettered';
40
41
  if (outcome === 'dropped') dropped += 1;
42
+ const label = JOB_OUTCOME_LABELS[outcome];
43
+ if (label !== null) recordJob(record.queue, label);
41
44
  // `error`, not `warn`: nobody recovered from this one, and a job that outlives every worker
42
45
  // that takes it is a crash loop somebody has to look at.
43
46
  logger.error('jobs.claim.exhausted', {
package/src/metrics.ts CHANGED
@@ -56,3 +56,27 @@ export function recordQueueOldestReady(queue: string, oldestReadyMs: number): vo
56
56
  export function recordQueueDeadJobs(queue: string, dead: number): void {
57
57
  queueDeadJobs.record(dead, { queue });
58
58
  }
59
+
60
+ /**
61
+ * What an autoscaler sizes a worker fleet with: `ceil((queue_depth + Σ in_flight) / capacity)`
62
+ * pods. Unlabelled — one series per PROCESS, which is what `maxInFlight` caps and what one worker
63
+ * pod is; the scrape's own `instance` label tells pods apart, and `sum` across them is the fleet.
64
+ */
65
+ export const workerInFlight: Gauge = gauge('job_worker_in_flight', {
66
+ unit: '{job}',
67
+ description: 'Jobs this worker process is running right now, across every queue it serves',
68
+ });
69
+
70
+ export const workerCapacity: Gauge = gauge('job_worker_capacity', {
71
+ unit: '{job}',
72
+ description:
73
+ 'The most jobs this worker process runs at once: maxInFlight, or the sum of its queue slots when tighter or unset',
74
+ });
75
+
76
+ export function recordWorkerInFlight(held: number): void {
77
+ workerInFlight.record(held);
78
+ }
79
+
80
+ export function recordWorkerCapacity(capacity: number): void {
81
+ workerCapacity.record(capacity);
82
+ }
@@ -0,0 +1,113 @@
1
+ // Single responsibility: which queue one claim pass asks, in what order and for how many — under
2
+ // each queue's own slots AND the process cap (`maxInFlight`, `worker-process-cap.ts`). The worker
3
+ // owns what a claim DOES; this owns the plan, so the fairness rule is one file a test can drive.
4
+ //
5
+ // THE FAIRNESS RULE. Queues are asked in the worker's order ROTATED to start after the last queue
6
+ // that was given a job, and a pass splits the room it has left evenly over the queues it has not
7
+ // asked yet (`ceil(room / queues left)`); a queue that filled its share is asked again, in the same order, while room is left. So a free slot goes to some queue
8
+ // with work whenever any has it, and a backlogged queue cannot monopolise a capped worker: each
9
+ // slot a finish frees goes to the next queue in turn — at the cap too, where the loop idles between
10
+ // finishes. An idle worker whose cap has not filled asks once over every queue (one statement, the
11
+ // driver's order), so a first fill from empty is the driver's; the turn holds from the cap on.
12
+
13
+ import { type ClaimAsk, claimAsks } from './worker-admit';
14
+
15
+ /** What one claim answered: runs ADMITTED (a shed or a refusal holds nothing), per queue. */
16
+ export type ClaimAnswer = ReadonlyMap<string, number>;
17
+
18
+ export interface ClaimSweepInput {
19
+ /** The worker's queues, in its own order. */
20
+ readonly queues: readonly string[];
21
+ /** A queue's free slots, re-read before every ask — a run admitted or finished moves it. */
22
+ readonly freeSlots: (queue: string) => number;
23
+ /** Room under the process cap, re-read likewise; `Infinity` with none. */
24
+ readonly processFree: () => number;
25
+ /** The loop is idling: one statement over every open queue (`claimAsks`), unless contended. */
26
+ readonly idle: boolean;
27
+ /** May the pass still claim? Re-read per ask — a `stop()` between two queues is a stop now. */
28
+ readonly claiming: () => boolean;
29
+ readonly claim: (ask: ClaimAsk) => Promise<ClaimAnswer>;
30
+ }
31
+
32
+ export interface ClaimSweep {
33
+ /** One pass. Resolves once every ask it made has been answered. */
34
+ pass(input: ClaimSweepInput): Promise<void>;
35
+ }
36
+
37
+ /** `queues` from `start`, wrapping. */
38
+ const rotated = (queues: readonly string[], start: number): readonly string[] =>
39
+ queues.length === 0
40
+ ? queues
41
+ : [...queues.slice(start % queues.length), ...queues.slice(0, start % queues.length)];
42
+
43
+ /** The cursor lives across passes: that is what makes the rotation a turn, not a shuffle. */
44
+ export function createClaimSweep(): ClaimSweep {
45
+ let cursor = 0;
46
+ /**
47
+ * The cap has been FULL since the last pass that found the queues empty. A worker at the cap
48
+ * finds nothing, so its loop idles, and an idle pass is ONE statement over every queue — rows in
49
+ * the DRIVER's order, the oldest of any queue, never by turn. So while contended, an idle pass
50
+ * takes the rotated sweep; once a rotated idle pass finds no work at all, the worker is idle for
51
+ * real and goes back to its one-statement pass. Set by the cap filling, not by runs held: at
52
+ * `maxInFlight: 1` a finish leaves none held at the very pass that must pick the next queue.
53
+ */
54
+ let contended = false;
55
+
56
+ const served = (queues: readonly string[], answer: ClaimAnswer): number => {
57
+ let admitted = 0;
58
+ for (const [queue, count] of answer) {
59
+ if (count <= 0) continue;
60
+ admitted += count;
61
+ const index = queues.indexOf(queue);
62
+ if (index !== -1) cursor = index + 1;
63
+ }
64
+ return admitted;
65
+ };
66
+
67
+ return {
68
+ async pass(input) {
69
+ const order = rotated(input.queues, cursor);
70
+ if (input.processFree() <= 0) {
71
+ contended = true;
72
+ return;
73
+ }
74
+ if (input.idle && !contended) {
75
+ const room = input.processFree();
76
+ for (const ask of claimAsks(order, input.freeSlots, true)) {
77
+ if (!input.claiming()) return;
78
+ served(input.queues, await input.claim({ ...ask, limit: Math.min(ask.limit, room) }));
79
+ }
80
+ if (input.processFree() <= 0) contended = true;
81
+ return;
82
+ }
83
+ let admitted = 0;
84
+ let asking = order;
85
+ while (asking.length > 0) {
86
+ const again: string[] = [];
87
+ for (const [index, queue] of asking.entries()) {
88
+ if (!input.claiming()) return;
89
+ const room = input.processFree();
90
+ if (room <= 0) {
91
+ contended = true;
92
+ return;
93
+ }
94
+ const share = Math.ceil(room / (asking.length - index));
95
+ const free = Math.max(0, input.freeSlots(queue));
96
+ const limit = Math.min(free, share);
97
+ if (limit <= 0) continue;
98
+ const answer = await input.claim({ queues: [queue], limit });
99
+ const got = served(input.queues, answer);
100
+ admitted += got;
101
+ // Its SHARE bound it, not its slots, and it filled it with runs that STARTED: it may
102
+ // hold more, for room a quieter queue left. A shed is not "more" — asking again would
103
+ // claim the same capped rows back and forth. With no cap the share never binds, so an
104
+ // uncapped pass is one ask per queue, as it always was.
105
+ if (limit < free && got >= limit) again.push(queue);
106
+ }
107
+ asking = again;
108
+ }
109
+ if (input.processFree() <= 0) contended = true;
110
+ else if (input.idle && admitted === 0) contended = false;
111
+ },
112
+ };
113
+ }
@@ -0,0 +1,83 @@
1
+ // Single responsibility: `WorkerOptions.maxInFlight` — the one count of runs this worker holds
2
+ // across EVERY queue, and the room it leaves. Sidekiq's `concurrency`: the per-queue slots bound
3
+ // each pool, this bounds the process. Read BEFORE a claim, so a worker at the cap asks the queue
4
+ // for nothing and no row is claimed only to be shed.
5
+
6
+ import { finiteCount } from '@ultimat3/core';
7
+ import { recordWorkerCapacity, recordWorkerInFlight } from './metrics';
8
+
9
+ export interface ProcessCap {
10
+ /** The cap as declared, `undefined` when the worker has none. */
11
+ readonly max: number | undefined;
12
+ /** Room left under the cap: `Infinity` with no cap, never below 0. */
13
+ free(): number;
14
+ /** Runs counted against the cap right now. */
15
+ held(): number;
16
+ /**
17
+ * One run admitted. The answer hands its count back, ONCE however often it is called — the
18
+ * run's `finally` and nothing else, at the same instant its limiter lease goes back, so the
19
+ * `kick` that follows sees the room it made.
20
+ */
21
+ take(): () => void;
22
+ /** The worker serves again (`start()`): counted in the process gauges. */
23
+ revive(): void;
24
+ /** The worker stopped: out of the process gauges, its capacity with it. */
25
+ retire(): void;
26
+ }
27
+
28
+ interface Counted {
29
+ held: number;
30
+ readonly capacity: number;
31
+ }
32
+
33
+ /**
34
+ * Every worker this PROCESS has live. The two gauges are unlabelled, one series per process, so
35
+ * they carry the SUM: a second `jobWorker` in one process used to reset the first's in-flight to 0
36
+ * and overwrite its capacity, and each later update published one worker's count as the whole.
37
+ */
38
+ const live = new Set<Counted>();
39
+
40
+ const publish = (): void => {
41
+ let held = 0;
42
+ let capacity = 0;
43
+ for (const worker of live) {
44
+ held += worker.held;
45
+ capacity += worker.capacity;
46
+ }
47
+ recordWorkerInFlight(held);
48
+ recordWorkerCapacity(capacity);
49
+ };
50
+
51
+ /** `maxInFlight`, refused when it is not a whole number ≥ 1 — `jobWorker`'s other counts' rule. */
52
+ export function createProcessCap(max: number | undefined, slots: number): ProcessCap {
53
+ if (max !== undefined) finiteCount('jobWorker', 'maxInFlight', max, 1);
54
+ // What a scaler divides by: the most this worker can run, whichever bound is tighter.
55
+ const counted: Counted = { held: 0, capacity: max === undefined ? slots : Math.min(max, slots) };
56
+ live.add(counted);
57
+ publish();
58
+ return {
59
+ max,
60
+ free: () => (max === undefined ? Number.POSITIVE_INFINITY : Math.max(0, max - counted.held)),
61
+ held: () => counted.held,
62
+ take() {
63
+ counted.held += 1;
64
+ publish();
65
+ let returned = false;
66
+ return () => {
67
+ if (returned) return;
68
+ returned = true;
69
+ counted.held -= 1;
70
+ publish();
71
+ };
72
+ },
73
+ revive() {
74
+ if (live.has(counted)) return;
75
+ live.add(counted);
76
+ publish();
77
+ },
78
+ retire() {
79
+ if (!live.delete(counted)) return;
80
+ publish();
81
+ },
82
+ };
83
+ }
@@ -18,6 +18,13 @@ export interface WorkerOptions {
18
18
  * (or no `concurrency` at all) runs at core's `JOBS_CONCURRENCY_DEFAULT`, 8 — 5 before 25.0.0.
19
19
  */
20
20
  readonly concurrency?: number | Readonly<Record<string, number>>;
21
+ /**
22
+ * The most runs this process holds at once, across EVERY queue — Sidekiq's `concurrency`. The
23
+ * per-queue slots still bound each queue under it. A worker at the cap claims nothing until a
24
+ * run ends; a freed slot goes to the next queue in turn that has work (`worker-claim-sweep.ts`).
25
+ * Unset: no process cap, the sum of the queue slots. A whole number ≥ 1, else `X_INVARIANT`.
26
+ */
27
+ readonly maxInFlight?: number;
21
28
  readonly limiter?: Limiter;
22
29
  readonly clock?: Clock;
23
30
  readonly events?: EventLookup;
@@ -52,6 +59,8 @@ export interface WorkerStats {
52
59
  readonly queues: readonly string[];
53
60
  readonly state: 'idle' | 'running' | 'draining' | 'stopped';
54
61
  readonly inFlight: number;
62
+ /** `WorkerOptions.maxInFlight`, `null` when uncapped — what a scaler divides the backlog by. */
63
+ readonly maxInFlight: number | null;
55
64
  readonly processed: number;
56
65
  readonly failed: number;
57
66
  readonly suspended: number;
package/src/worker.ts CHANGED
@@ -22,13 +22,15 @@ import type { JobExecution } from './execute';
22
22
  import { registeredJobs } from './job';
23
23
  import { concurrencyLimiter } from './limits';
24
24
  import { JOB_OUTCOME_LABELS } from './metrics';
25
- import { claimAsks, createAdmission } from './worker-admit';
25
+ import { type ClaimAsk, createAdmission } from './worker-admit';
26
+ import { type ClaimAnswer, createClaimSweep } from './worker-claim-sweep';
26
27
  import { armWorkerCutoff, type DrainCutoff } from './worker-drain-cutoff';
27
28
  import { assertConcurrencyEnforceable, createFleetSlots } from './worker-fleet-slots';
28
29
  import { createHeldRuns } from './worker-held';
29
30
  import type { ClaimLoop } from './worker-loop';
30
31
  import { createClaimLoop } from './worker-loop';
31
32
  import { resolveWorkerTimings } from './worker-options';
33
+ import { createProcessCap } from './worker-process-cap';
32
34
  import { createQueueDepthPublisher } from './worker-queue-depth';
33
35
  import type { WorkerRegistration } from './worker-registry';
34
36
  import { reportUnregisteredQueues, startWorkerRegistry } from './worker-registry';
@@ -46,6 +48,9 @@ export function jobWorker(options: WorkerOptions): Worker {
46
48
  const { visibilityTimeoutMs, pollIntervalMs, heartbeatIntervalMs, slotsFor } =
47
49
  resolveWorkerTimings(options);
48
50
  const limiter = options.limiter ?? concurrencyLimiter({});
51
+ const slots = queues.reduce((total, queue) => total + slotsFor(queue), 0);
52
+ const cap = createProcessCap(options.maxInFlight, slots); // read before each claim
53
+ const sweep = createClaimSweep();
49
54
  const driverLeases = options.driver.leases;
50
55
  // `job.concurrency`, held as a row every replica sees. The TTL is the visibility timeout and the
51
56
  // renewal rides the lease heartbeat's interval — `worker-fleet-slots.ts` says why both.
@@ -132,28 +137,19 @@ export function jobWorker(options: WorkerOptions): Worker {
132
137
  const claiming = (): boolean => state !== 'draining' && state !== 'stopped';
133
138
 
134
139
  /**
135
- * One claim pass: every queue asked once, and everything it hands back STARTED. It does not wait
136
- * for the jobs — a slot is free again the moment its own job settles and the limiter releases
137
- * the lease, so the next pass refills exactly that slot. Waiting for the whole batch made a pool
138
- * as slow as its slowest member and, because the pass walks every queue before it waits, left
139
- * every OTHER queue idle behind one long-running job too.
140
+ * One claim pass: everything it hands back STARTED, never waited for — a slot is free again the
141
+ * moment its own job settles, so the next pass refills exactly that slot. Waiting for the batch
142
+ * made a pool as slow as its slowest member, and every OTHER queue idle behind it.
140
143
  */
141
144
  const claimRound = async (): Promise<readonly Promise<JobExecution>[]> => {
142
145
  await publishQueueDepth();
143
146
  const started: Promise<JobExecution>[] = [];
144
147
  let found = false;
145
148
 
146
- const asks = claimAsks(
147
- queues,
148
- (queue) => slotsFor(queue) - limiter.inFlight({ queue }),
149
- loop.idle(),
150
- );
151
-
152
- for (const ask of asks) {
153
- // Re-read per claim, not once at the top: a `stop()` between two queues means "stop
154
- // claiming" now, not at the next tick. What this round already holds still runs to the end
155
- // — that is the drain, and `stop()` waits for it.
156
- if (!claiming()) break;
149
+ // One ask; WHICH queue, how many and in what order is `worker-claim-sweep.ts`'s (`maxInFlight`
150
+ // and the turn). It re-reads `claiming()` per ask: a `stop()` mid-pass stops claiming now.
151
+ const claimOne = async (ask: ClaimAsk): Promise<ClaimAnswer> => {
152
+ const admitted = new Map<string, number>();
157
153
  // A lease that lapsed on a row's final attempt is settled by the claim itself and handed to
158
154
  // `onExhausted`, never out as work: this round is the only thing that can count it, log it
159
155
  // and tell the job. `dropExhausted` is the policy the row does not carry.
@@ -172,16 +168,9 @@ export function jobWorker(options: WorkerOptions): Worker {
172
168
  if (claimed.length > 0) found = true;
173
169
  if (buried.length > 0) {
174
170
  found = true;
175
- const ended = await announceExhausted({
176
- exhausted: buried,
177
- workerId,
178
- context: options.context,
179
- });
180
- tally.buried(ended);
181
- for (const row of buried) {
182
- const label = JOB_OUTCOME_LABELS[row.state === 'failed' ? 'dropped' : 'dead-lettered'];
183
- if (label !== null) recordJob(row.queue, label);
184
- }
171
+ tally.buried(
172
+ await announceExhausted({ exhausted: buried, workerId, context: options.context }),
173
+ );
185
174
  }
186
175
 
187
176
  for (const [index, job] of claimed.entries()) {
@@ -199,6 +188,8 @@ export function jobWorker(options: WorkerOptions): Worker {
199
188
  continue;
200
189
  }
201
190
  const { lease } = admission;
191
+ admitted.set(queue, (admitted.get(queue) ?? 0) + 1);
192
+ const returnCap = cap.take(); // `maxInFlight`'s count, handed back beside the lease
202
193
 
203
194
  // The claimed job is the process's in-flight work, counted where the DRAIN can see it:
204
195
  // core's own in-flight wait sits between `accept` and `inflight` and exists for exactly
@@ -215,10 +206,8 @@ export function jobWorker(options: WorkerOptions): Worker {
215
206
  const running = runClaimed(job, admission.refusal)
216
207
  .then((execution) => {
217
208
  tally.count(execution);
218
- // The other half of this package's metrics contract: `queue_depth` says how much work
219
- // is waiting, `jobs_total` says whether any of it is succeeding. Depth alone cannot
220
- // tell a drained queue from a queue nothing ever claimed. Labelled by QUEUE and
221
- // OUTCOME only — a label per job name is unbounded in an app's own vocabulary.
209
+ // `jobs_total`, beside `queue_depth`: depth cannot tell a drained queue from one nothing
210
+ // claimed. QUEUE and OUTCOME only — a job-name label is unbounded in an app's vocabulary.
222
211
  const label = JOB_OUTCOME_LABELS[execution.outcome];
223
212
  if (label !== null) recordJob(queue, label);
224
213
  // Handed back with a time on it — a retry's backoff, a sleep's wake: this worker
@@ -230,17 +219,15 @@ export function jobWorker(options: WorkerOptions): Worker {
230
219
  })
231
220
  .finally(async () => {
232
221
  lease.release();
222
+ returnCap();
233
223
  // The slot is free NOW. A full worker asks the queue for nothing, so its loop backs
234
224
  // off like an idle one — and waiting that out with a backlog behind the job that
235
225
  // just finished is throughput lost to a timer.
236
226
  loop.kick();
237
- // AWAITED, never `void`: the slot is a row in `x_job_leases`, so the DELETE was still
238
- // on the wire when the teardown's `allSettled` returned and `driver.close()` took the
239
- // connection out from under it — a `concurrency: 1` job unclaimable by the pod
240
- // replacing this one for a whole visibility window after every deploy. `release`
241
- // swallows its own failures, so awaiting it cannot reject a job that finished; the
242
- // `finally` is for the one that could, because a lost `finishWork()` is an in-flight
243
- // count that never returns to zero and a drain that waits out its whole budget.
227
+ // AWAITED, never `void`: the slot is a row in `x_job_leases`, and a DELETE still on the
228
+ // wire at `driver.close()` left a `concurrency: 1` job unclaimable by the replacement
229
+ // pod for a visibility window. `release` swallows its own failures; the `finally` is
230
+ // for the one that could not, as a lost `finishWork()` holds the drain its whole budget.
244
231
  try {
245
232
  await releaseSlot();
246
233
  } finally {
@@ -272,7 +259,17 @@ export function jobWorker(options: WorkerOptions): Worker {
272
259
  },
273
260
  );
274
261
  }
275
- }
262
+ return admitted;
263
+ };
264
+
265
+ await sweep.pass({
266
+ queues,
267
+ freeSlots: (queue) => slotsFor(queue) - limiter.inFlight({ queue }),
268
+ processFree: cap.free,
269
+ idle: loop.idle(),
270
+ claiming,
271
+ claim: claimOne,
272
+ });
276
273
 
277
274
  // The loop's next wait: the floor while passes find work, doubling to the ceiling while they
278
275
  // do not (`idle-backoff.ts`). Reported by the pass itself, so `tick()` and the timer agree.
@@ -403,6 +400,7 @@ export function jobWorker(options: WorkerOptions): Worker {
403
400
  // and the budget with it, or the next manual stop would inherit a deadline already spent.
404
401
  cutoff?.dispose();
405
402
  cutoff = undefined;
403
+ cap.retire(); // out of the process gauges until a `start()` serves again
406
404
  drainSignal = new AbortController();
407
405
  budget = createDrainBudget();
408
406
  }
@@ -444,6 +442,7 @@ export function jobWorker(options: WorkerOptions): Worker {
444
442
  // `x jobs show` and the manifest both confirm a guarantee that does not exist.
445
443
  assertConcurrencyEnforceable(options.driver);
446
444
  state = 'running';
445
+ cap.revive();
447
446
  logger.info('jobs.worker.started', { workerId, queues });
448
447
  reportUnregisteredQueues(workerId, queues);
449
448
  registration = startWorkerRegistry({
@@ -451,7 +450,7 @@ export function jobWorker(options: WorkerOptions): Worker {
451
450
  workerId,
452
451
  host: options.host,
453
452
  queues,
454
- concurrency: queues.reduce((slots, queue) => slots + slotsFor(queue), 0),
453
+ concurrency: Math.min(slots, cap.max ?? slots),
455
454
  inFlight: () => inFlight.ids(),
456
455
  // The visibility timeout, renewed on the heartbeat interval: a killed worker leaves the
457
456
  // registry on exactly the schedule the queue takes its jobs back.
@@ -490,6 +489,7 @@ export function jobWorker(options: WorkerOptions): Worker {
490
489
  queues,
491
490
  state,
492
491
  inFlight: inFlight.size,
492
+ maxInFlight: cap.max ?? null,
493
493
  ...tally.snapshot(),
494
494
  pollDelayMs: loop.delayMs(),
495
495
  queueDepth: [...(await options.driver.stats())],