@ultimat3/jobs 24.0.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CLAUDE.md +25 -27
  2. package/README.md +50 -34
  3. package/package.json +7 -7
  4. package/src/backfill-errors.ts +3 -2
  5. package/src/backfill-ledger.ts +1 -1
  6. package/src/backfill-rate.ts +2 -2
  7. package/src/backfill.ts +2 -2
  8. package/src/counters.ts +1 -1
  9. package/src/driver-memory-operator.ts +3 -3
  10. package/src/driver-memory.ts +14 -17
  11. package/src/driver-pg-settle-sql.ts +2 -3
  12. package/src/driver-pg-sql.ts +1 -1
  13. package/src/driver-pg.ts +18 -18
  14. package/src/driver.ts +9 -15
  15. package/src/errors-concurrency.ts +1 -1
  16. package/src/errors.ts +5 -5
  17. package/src/events-pg.ts +4 -4
  18. package/src/events.ts +6 -6
  19. package/src/execute.ts +46 -5
  20. package/src/export.ts +2 -2
  21. package/src/heartbeat.ts +7 -2
  22. package/src/index.ts +37 -97
  23. package/src/inspect-operator.ts +1 -1
  24. package/src/inspect.ts +1 -1
  25. package/src/job.ts +1 -1
  26. package/src/leases.ts +1 -1
  27. package/src/limits.ts +13 -8
  28. package/src/outbox-lease.ts +1 -1
  29. package/src/outbox-pg.ts +5 -5
  30. package/src/outbox-relay.ts +6 -6
  31. package/src/outbox.ts +13 -13
  32. package/src/renewal-timer.ts +1 -1
  33. package/src/retry-classification.ts +52 -15
  34. package/src/scheduler-leader.ts +2 -2
  35. package/src/scheduler-pg.ts +16 -14
  36. package/src/scheduler-state.ts +2 -2
  37. package/src/scheduler.ts +5 -5
  38. package/src/steps-memory.ts +2 -2
  39. package/src/steps.ts +35 -7
  40. package/src/task.ts +1 -1
  41. package/src/webhook-attempt.ts +206 -0
  42. package/src/webhook-ledger.ts +15 -4
  43. package/src/webhook.ts +64 -118
  44. package/src/worker-admit.ts +7 -3
  45. package/src/worker-drain-cutoff.ts +110 -0
  46. package/src/worker-fleet-slots.ts +5 -2
  47. package/src/worker-held.ts +100 -0
  48. package/src/worker-options.ts +19 -15
  49. package/src/worker-run.ts +38 -8
  50. package/src/worker-tally.ts +53 -0
  51. package/src/worker-types.ts +7 -4
  52. package/src/worker.ts +85 -77
  53. package/src/driver-nats.ts +0 -89
  54. package/src/driver-redis.ts +0 -92
package/src/webhook.ts CHANGED
@@ -19,14 +19,10 @@ import type { Clock, Ctx } from '@ultimat3/core';
19
19
  import {
20
20
  finiteOption,
21
21
  isCanonicalWebhookField,
22
- // Core's ONE table, never a fifth copy of it. The copy that lived here omitted 409, so a
23
- // receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
24
- // no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
25
- isRetryableStatus,
26
- renderThrowable,
22
+ isUltimateError,
27
23
  systemClock,
24
+ throwIfAborted,
28
25
  WEBHOOK_FIELD_MAX,
29
- webhookHeaders,
30
26
  } from '@ultimat3/core';
31
27
  import { t } from '@ultimat3/schema';
32
28
  import type { DurationInput } from './clock';
@@ -36,10 +32,9 @@ import { job } from './job';
36
32
  import type { RetryPolicy } from './retry';
37
33
  import { DEFAULT_RETRY } from './retry';
38
34
  import type { JobTenant } from './tenant';
35
+ import type { Outcome, WebhookFetch } from './webhook-attempt';
36
+ import { attemptDelivery, deliveryError, deliverySignal } from './webhook-attempt';
39
37
  import {
40
- WebhookDeliveryFailedError,
41
- WebhookDeliveryRejectedError,
42
- WebhookDeliveryThrottledError,
43
38
  WebhookEndpointDisabledError,
44
39
  WebhookEndpointInvalidError,
45
40
  WebhookEndpointUnknownError,
@@ -47,19 +42,9 @@ import {
47
42
  WebhookEventUnknownError,
48
43
  } from './webhook-errors';
49
44
  import type { WebhookLedger } from './webhook-ledger';
50
- import type { WebhookResolve, WebhookTarget } from './webhook-target';
45
+ import type { WebhookResolve } from './webhook-target';
51
46
  import { resolveWebhookHost, webhookTarget } from './webhook-target';
52
47
 
53
- /** Just the call. `typeof fetch` also carries `preconnect`, which no test double should have to. */
54
- /**
55
- * The transport. `init.tls.serverName` carries the hostname a PINNED connection proves, because
56
- * the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
57
- */
58
- export type WebhookFetch = (
59
- url: string,
60
- init: RequestInit & { readonly tls?: { readonly serverName?: string } },
61
- ) => Promise<Response>;
62
-
63
48
  /**
64
49
  * Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
65
50
  * retrying queue against a dead receiver, which is long enough for an outage and short enough that
@@ -67,12 +52,6 @@ export type WebhookFetch = (
67
52
  */
68
53
  export const DEFAULT_WEBHOOK_DISABLE_AFTER = 10;
69
54
 
70
- /** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
71
- export const WEBHOOK_CONTENT_TYPE = 'application/json';
72
-
73
- /** `Retry-After` in seconds. The HTTP-date form is ignored on purpose — see `retryAfterSeconds`. */
74
- const RETRY_AFTER_SECONDS = /^\d{1,7}$/;
75
-
76
55
  export interface WebhookEndpoint {
77
56
  readonly id: string;
78
57
  /**
@@ -184,13 +163,6 @@ export interface WebhookDefinition {
184
163
  readonly timeout?: DurationInput;
185
164
  }
186
165
 
187
- /** `Retry-After: 120`. The HTTP-date form needs the receiver's clock, which is what we do not trust. */
188
- const retryAfterSeconds = (response: Response): number | undefined => {
189
- const header = response.headers.get('retry-after');
190
- if (header === null || !RETRY_AFTER_SECONDS.test(header.trim())) return undefined;
191
- return Number(header.trim());
192
- };
193
-
194
166
  export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
195
167
  const clock = definition.clock ?? systemClock;
196
168
  const disableAfter = finiteOption(
@@ -200,7 +172,8 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
200
172
  );
201
173
  const send = definition.fetch ?? ((url, init) => fetch(url, init));
202
174
 
203
- return job<WebhookDeliveryInput>({
175
+ // Named so `run` can read the deadline `job()` resolved — one parse of `timeout`, never two.
176
+ const handle: JobHandle<WebhookDeliveryInput> = job<WebhookDeliveryInput>({
204
177
  name: definition.name,
205
178
  input: t.object({ endpointId: t.string, eventId: t.string }),
206
179
  // Endpoint AND event: the same event fans out to every subscribed endpoint, so a key on the
@@ -211,6 +184,16 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
211
184
  ...(definition.queue === undefined ? {} : { queue: definition.queue }),
212
185
  ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
213
186
  async run({ input, ctx, attempt }): Promise<WebhookReport> {
187
+ // On the clock `executeJob`'s deadline timer runs on, read as the body starts — which is
188
+ // when that timer is armed.
189
+ const deadline =
190
+ handle.timeoutMs === undefined
191
+ ? undefined
192
+ : {
193
+ atMs: performance.now() + handle.timeoutMs,
194
+ job: definition.name,
195
+ timeoutMs: handle.timeoutMs,
196
+ };
214
197
  const endpoint = await definition.endpoint({ endpointId: input.endpointId, ctx });
215
198
  if (endpoint === null) {
216
199
  throw new WebhookEndpointUnknownError({
@@ -220,7 +203,9 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
220
203
  }
221
204
  // Before the ledger and before the socket: a disabled endpoint costs nothing, which is the
222
205
  // whole point of disabling one.
223
- if (endpoint.disabled === true) {
206
+ // Asked of BOTH sources, because `disableAfter` writes only the ledger: an endpoint the
207
+ // mechanism switched off must stop here whether or not the app mirrored it onto its row.
208
+ if (endpoint.disabled === true || (await definition.ledger.isDisabled(endpoint.id))) {
224
209
  throw new WebhookEndpointDisabledError({
225
210
  webhook: definition.name,
226
211
  endpointId: endpoint.id,
@@ -260,17 +245,41 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
260
245
  // again now, so a receiver's freshness window measures the request in front of it rather
261
246
  // than the age of the fact behind it.
262
247
  const timestampSeconds = Math.floor(nowMs(clock) / 1_000);
248
+ // The last moment a cancelled attempt can stop without the receiver hearing anything — the
249
+ // drain included: a POST not yet started is handed back to the next pod at no cost.
250
+ stopIfCancelled(ctx);
263
251
  const startedAt = clock.monotonic();
264
- const outcome: Outcome = await ('unresolved' in target
265
- ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
266
- : attemptDelivery(send, endpoint, target, {
267
- secret: endpoint.secret,
268
- timestampSeconds,
269
- eventId: input.eventId,
270
- topic: event.topic,
271
- body: event.body,
272
- }));
252
+ // Past this line the drain no longer cancels: see `deliverySignal`.
253
+ const request = deliverySignal(ctx.signal, deadline);
254
+ let outcome: Outcome;
255
+ try {
256
+ outcome = await ('unresolved' in target
257
+ ? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
258
+ : attemptDelivery(
259
+ send,
260
+ endpoint,
261
+ target,
262
+ {
263
+ secret: endpoint.secret,
264
+ timestampSeconds,
265
+ eventId: input.eventId,
266
+ topic: event.topic,
267
+ body: event.body,
268
+ },
269
+ request.signal,
270
+ ));
271
+ } finally {
272
+ request.dispose();
273
+ }
273
274
  const durationMs = Math.max(0, clock.monotonic() - startedAt);
275
+ // A failure the REQUEST's cancellation caused is not the receiver's, and is not this
276
+ // attempt's to record: the queue has already handed the delivery to another worker, whose
277
+ // outcome is the one the consecutive count must hear. Recorded, it was a failure nobody saw
278
+ // counting toward `disableAfter` — an abandoned attempt switching off a healthy endpoint.
279
+ // Read off the request's signal, never the attempt's: a receiver that really failed while
280
+ // the worker drained is a failure like any other, and the count hears it. A delivery that
281
+ // LANDED is a fact, and is always recorded.
282
+ if (!outcome.ok && request.signal.aborted) stopIfCancelled(ctx);
274
283
 
275
284
  // Recorded whatever happened, and BEFORE the throw: a failure that is not on the ledger is a
276
285
  // failure the consecutive count cannot see, which is an endpoint that never gets disabled.
@@ -311,6 +320,7 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
311
320
  throw deliveryError(definition.name, endpoint, outcome);
312
321
  },
313
322
  });
323
+ return handle;
314
324
  }
315
325
 
316
326
  /** A URL no delivery may open, or a secret that would make the POST unsigned. The parsed url. */
@@ -346,78 +356,14 @@ function assertDeliverable(name: string, endpoint: WebhookEndpoint): URL {
346
356
  return parsed;
347
357
  }
348
358
 
349
- type Outcome =
350
- | { readonly ok: true; readonly status: number }
351
- | {
352
- readonly ok: false;
353
- readonly status: number | null;
354
- readonly detail: string;
355
- readonly retryAfterSeconds?: number;
356
- };
357
-
358
- async function attemptDelivery(
359
- send: WebhookFetch,
360
- endpoint: WebhookEndpoint,
361
- target: WebhookTarget,
362
- signing: Parameters<typeof webhookHeaders>[0],
363
- ): Promise<Outcome> {
364
- let response: Response;
365
- try {
366
- response = await send(target.url, {
367
- method: 'POST',
368
- headers: {
369
- ...endpoint.headers,
370
- // The name the pinned address is reached AS — after the row's own headers, so an endpoint
371
- // cannot redirect the request to a different virtual host than the one it registered.
372
- host: target.host,
373
- 'content-type': WEBHOOK_CONTENT_TYPE,
374
- // LAST, so an endpoint row's own headers can never overwrite the signature it is proved by.
375
- ...webhookHeaders(signing),
376
- },
377
- ...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
378
- body: signing.body,
379
- // Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
380
- // named, and the signature would travel with it.
381
- redirect: 'manual',
382
- });
383
- } catch (error) {
384
- // `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
385
- // a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
386
- return { ok: false, status: null, detail: renderThrowable(error) };
387
- }
388
- if (response.ok) return { ok: true, status: response.status };
389
- const stated = retryAfterSeconds(response);
390
- return {
391
- ok: false,
392
- status: response.status,
393
- detail: `status ${response.status}`,
394
- ...(stated === undefined ? {} : { retryAfterSeconds: stated }),
395
- };
396
- }
397
-
398
- /** Which of the three failure codes this outcome is. The split is "can the same request land?". */
399
- function deliveryError(
400
- name: string,
401
- endpoint: WebhookEndpoint,
402
- outcome: Extract<Outcome, { ok: false }>,
403
- ): Error {
404
- const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
405
- if (outcome.status === null) {
406
- return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
407
- }
408
- if (!isRetryableStatus(outcome.status)) {
409
- return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
410
- }
411
- if (outcome.retryAfterSeconds !== undefined) {
412
- return new WebhookDeliveryThrottledError({
413
- ...base,
414
- status: outcome.status,
415
- retryAfterSeconds: outcome.retryAfterSeconds,
416
- });
417
- }
418
- return new WebhookDeliveryFailedError({
419
- ...base,
420
- status: outcome.status,
421
- detail: outcome.detail,
422
- });
359
+ /**
360
+ * Throw the attempt's cancellation as itself — the deadline's `X_JOB_TIMEOUT`, the drain's
361
+ * `X_DRAINING`, a lost lease — so `executeJob` reads the stop it caused, never a delivery failure
362
+ * naming a receiver that did nothing wrong. Core's `X_ABORTED` when the reason carries no code.
363
+ */
364
+ function stopIfCancelled(ctx: Ctx): void {
365
+ if (!ctx.signal.aborted) return;
366
+ const reason: unknown = ctx.signal.reason;
367
+ if (isUltimateError(reason)) throw reason;
368
+ throwIfAborted(ctx);
423
369
  }
@@ -76,8 +76,10 @@ export function createAdmission(options: AdmissionOptions): Admit {
76
76
  try {
77
77
  await shed(job, queue, limiter.blockedBy(key) ?? 'unknown');
78
78
  } catch (error) {
79
- // The nack itself failed: the jobs BEHIND it go back before the round reports.
80
- await handBack(driver, behind, back);
79
+ // The nack itself failed, so THIS job went nowhere either: it goes back with the jobs
80
+ // behind it, one more best-effort try, before the round reports. Left out, it sat
81
+ // `running` until its lease lapsed — an attempt spent on a body that never started.
82
+ await handBack(driver, [job, ...behind], back);
81
83
  throw error;
82
84
  }
83
85
  return { kind: 'waiting' };
@@ -128,7 +130,9 @@ export function createAdmission(options: AdmissionOptions): Admit {
128
130
  const keyed = grant.key === undefined ? '' : `, key ${grant.key}`;
129
131
  await shed(job, queue, `job concurrency (${grant.limit}${keyed})`);
130
132
  } catch (error) {
131
- await handBack(driver, behind, back);
133
+ // This job too, for the reason above. A refusal that DID settle before throwing is safe to
134
+ // repeat: every nack is fenced on `running` and the claim, so a settled row ignores it.
135
+ await handBack(driver, [job, ...behind], back);
132
136
  throw error;
133
137
  }
134
138
  return { kind: 'waiting' };
@@ -0,0 +1,110 @@
1
+ // When a draining worker stops waiting for the jobs it holds. SIGTERM stops the CLAIMING; the jobs
2
+ // already running keep going, because a deploy that cancels work in flight re-runs every side
3
+ // effect it interrupted. Only near the end of the drain budget is what is still running told to
4
+ // stop (`cancel`), and a beat later what is still held goes back to the queue (`handBack`).
5
+
6
+ import { logger, renderThrowable, systemClock } from '@ultimat3/core';
7
+ import type { DrainBudget } from './drain-wait';
8
+ import { JobDrainedError } from './errors';
9
+ import type { HeldRuns } from './worker-held';
10
+
11
+ /**
12
+ * The most of the budget's tail kept for unwinding: the cancel lands this far before the deadline,
13
+ * the hand-back half as far, and the teardown — the registry row, the driver — gets the rest. Two
14
+ * seconds of a 25 s default, and never more than HALF of a small budget, so a budget of 300 ms
15
+ * still spends 150 ms letting its jobs finish before anything is cut.
16
+ */
17
+ export const DRAIN_CANCEL_MARGIN_MS = 2_000;
18
+
19
+ export interface CutoffTimes {
20
+ /** Real monotonic ms: the held runs' `ctx.signal` aborts with `X_DRAINING`. */
21
+ readonly cancelAt: number;
22
+ /** Real monotonic ms: every claim still held is handed back uncounted. */
23
+ readonly handBackAt: number;
24
+ }
25
+
26
+ /** The cut-off for a deadline, read against the instant it was bound. */
27
+ export function cutoffTimes(deadlineAt: number, now: number): CutoffTimes {
28
+ const remaining = Math.max(0, deadlineAt - now);
29
+ const margin = Math.min(DRAIN_CANCEL_MARGIN_MS, remaining / 2);
30
+ return { cancelAt: deadlineAt - margin, handBackAt: deadlineAt - margin / 2 };
31
+ }
32
+
33
+ export interface DrainCutoff {
34
+ /** Stop listening to the budget and clear both timers — the teardown is over. */
35
+ dispose(): void;
36
+ }
37
+
38
+ /**
39
+ * Arm `cancel` and `handBack` on `budget`, re-armed each time it tightens (the earliest deadline
40
+ * wins — `drain-wait.ts`). Each fires at most once. The timers are `unref`ed: a drained process is
41
+ * never kept alive by its own cut-off.
42
+ */
43
+ export function armDrainCutoff(
44
+ budget: DrainBudget,
45
+ steps: { readonly cancel: () => void; readonly handBack: () => Promise<void> },
46
+ ): DrainCutoff {
47
+ const timers: ReturnType<typeof setTimeout>[] = [];
48
+ let cancelled = false;
49
+ let handedBack = false;
50
+ const clear = (): void => {
51
+ for (const timer of timers.splice(0)) clearTimeout(timer);
52
+ };
53
+ const at = (instant: number, fire: () => void): void => {
54
+ const timer = setTimeout(fire, Math.max(0, instant - systemClock.monotonic()));
55
+ timer.unref?.();
56
+ timers.push(timer);
57
+ };
58
+ const unwatch = budget.watch((deadlineAt) => {
59
+ clear();
60
+ const times = cutoffTimes(deadlineAt, systemClock.monotonic());
61
+ if (!cancelled) {
62
+ at(times.cancelAt, () => {
63
+ cancelled = true;
64
+ steps.cancel();
65
+ });
66
+ }
67
+ if (!handedBack) {
68
+ at(times.handBackAt, () => {
69
+ handedBack = true;
70
+ // Each nack's failure is logged where it happens (`worker-hand-back.ts`); this is the one
71
+ // place anything else could still escape a timer, so it is observed here.
72
+ void steps.handBack().catch((error: unknown) => {
73
+ logger.error('jobs.worker.drain-hand-back-failed', { error: renderThrowable(error) });
74
+ });
75
+ });
76
+ }
77
+ });
78
+ return {
79
+ dispose() {
80
+ unwatch();
81
+ clear();
82
+ },
83
+ };
84
+ }
85
+
86
+ /**
87
+ * The worker's cut-off: `drain` aborted with `X_DRAINING` at `cancelAt`, every run `held` still
88
+ * holds handed back at `handBackAt`. Aborted even with nothing held: a round still inside its
89
+ * `claim()` starts its runs under this controller, and they must hear the cut-off too.
90
+ */
91
+ export function armWorkerCutoff(input: {
92
+ readonly budget: DrainBudget;
93
+ readonly workerId: string;
94
+ /** The shutdown's signal name, carried on the reason the bodies read. */
95
+ readonly signal: string;
96
+ readonly drain: AbortController;
97
+ readonly held: HeldRuns;
98
+ }): DrainCutoff {
99
+ const { workerId, held } = input;
100
+ return armDrainCutoff(input.budget, {
101
+ cancel: () => {
102
+ if (held.size > 0) logger.warn('jobs.worker.drain-cancelling', { workerId, held: held.size });
103
+ input.drain.abort(new JobDrainedError({ workerId, signal: input.signal }));
104
+ },
105
+ handBack: async () => {
106
+ const handed = await held.handBackAll();
107
+ if (handed > 0) logger.warn('jobs.worker.drain-handed-back', { workerId, handed });
108
+ },
109
+ });
110
+ }
@@ -73,7 +73,7 @@ export interface FleetSlots {
73
73
  * to do without one. `granted` is also the answer for "no cap declared": a job with no
74
74
  * `concurrency` never touches the lease table.
75
75
  *
76
- * A driver with no lease store cannot reach here: `createWorker().start()` refuses to boot when
76
+ * A driver with no lease store cannot reach here: `jobWorker().start()` refuses to boot when
77
77
  * a registered job declares `concurrency` and the driver has none, because a cap that silently
78
78
  * holds per process is the documented-guarantee-that-does-nothing axiom 3 exists to make
79
79
  * impossible.
@@ -186,6 +186,9 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
186
186
  // One in flight at a time, on the connection that is already the thing failing.
187
187
  if (renewing) return;
188
188
  renewing = true;
189
+ // Stamped BEFORE the call, as `heartbeat.ts` does: the store dates the new expiry at the
190
+ // statement, so a slow reply must not stretch this worker's view past it.
191
+ const sentAt = now();
189
192
  try {
190
193
  const renewed = await options.leases.renew(slot, options.ttlMs);
191
194
  if (timer.stopped()) return;
@@ -194,7 +197,7 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
194
197
  if (renewed === false) return reportLost('not-ours');
195
198
  // A renewal that lands LATE is still late: the slot may already be another worker's.
196
199
  if (lapsed()) return reportLost('expired');
197
- renewedAt = now();
200
+ renewedAt = sentAt;
198
201
  } catch (error) {
199
202
  // One failed renewal is not a lost slot: the TTL gives the interval several tries.
200
203
  logger.warn('jobs.worker.slot-renewal-failed', {
@@ -0,0 +1,100 @@
1
+ // The runs one worker holds right now, each with the claim it runs under — so a drain that reaches
2
+ // its cut-off can hand back every row still held instead of leaving it under a lease another
3
+ // worker waits a whole visibility timeout to see lapse. Split from `worker.ts`, whose job is the
4
+ // claim loop; this one is "what am I holding, and give it back".
5
+
6
+ import type { ClaimedJob, JobDriver } from './driver';
7
+ import { handBack } from './worker-hand-back';
8
+
9
+ export interface HeldRuns {
10
+ /** How many runs are held — what `stats().inFlight` reports. */
11
+ readonly size: number;
12
+ /**
13
+ * The job ids held, each once — what the registry row reports. Per RUN underneath, because one
14
+ * worker can hold a job twice (a claim that lapsed under a body still running, then its own
15
+ * re-claim): keyed by id, the first run to end dropped an id the second still held.
16
+ */
17
+ ids(): string[];
18
+ /**
19
+ * A run started. `release` is what the run's own `finally` would do for the process — the
20
+ * in-flight count core's drain waits on, the fleet slot — run early when the run is handed back.
21
+ */
22
+ add(run: Promise<unknown>, claimed: ClaimedJob, release: () => Promise<void>): void;
23
+ /** The run settled on its own: no longer held. */
24
+ settle(run: Promise<unknown>): void;
25
+ /** Resolves the moment nothing is held — settled or handed back. */
26
+ empty(): Promise<void>;
27
+ /**
28
+ * Every run still held goes back to the queue — `countsAsAttempt: false`, no delay — and stops
29
+ * being this worker's. Answers how many. The bodies themselves cannot be stopped; what they
30
+ * write after this is fenced out by the claim that just moved (`StepFence`, `SQL_ACK`).
31
+ */
32
+ handBackAll(): Promise<number>;
33
+ }
34
+
35
+ interface Holding {
36
+ readonly claimed: ClaimedJob;
37
+ readonly release: () => Promise<void>;
38
+ }
39
+
40
+ export function createHeldRuns(options: {
41
+ readonly driver: JobDriver;
42
+ readonly workerId: string;
43
+ }): HeldRuns {
44
+ const held = new Map<Promise<unknown>, Holding>();
45
+ let waiters: (() => void)[] = [];
46
+
47
+ /** Pending hand-backs: `empty()` answers only once their nacks have LANDED, never mid-wire. */
48
+ let handingBack = 0;
49
+ const wakeIfEmpty = (): void => {
50
+ if (held.size > 0 || handingBack > 0 || waiters.length === 0) return;
51
+ const woken = waiters;
52
+ waiters = [];
53
+ for (const wake of woken) wake();
54
+ };
55
+
56
+ return {
57
+ get size() {
58
+ return held.size;
59
+ },
60
+ ids() {
61
+ return [...new Set([...held.values()].map((holding) => holding.claimed.id))];
62
+ },
63
+ add(run, claimed, release) {
64
+ held.set(run, { claimed, release });
65
+ },
66
+ settle(run) {
67
+ held.delete(run);
68
+ wakeIfEmpty();
69
+ },
70
+ empty() {
71
+ if (held.size === 0 && handingBack === 0) return Promise.resolve();
72
+ return new Promise<void>((resolve) => {
73
+ waiters.push(resolve);
74
+ });
75
+ },
76
+ async handBackAll() {
77
+ // Taken out of the set BEFORE the first await: a run settling while the nacks are on the
78
+ // wire is a run the hand-back already owns, and must not be counted twice. And `empty()`
79
+ // stays unanswered until the nacks land — the teardown waiting on it closes the driver next.
80
+ const stranded = [...held.values()];
81
+ held.clear();
82
+ if (stranded.length === 0) return 0;
83
+ handingBack += 1;
84
+ try {
85
+ await handBack(
86
+ options.driver,
87
+ stranded.map((holding) => holding.claimed),
88
+ { delayMs: 0, workerId: options.workerId },
89
+ );
90
+ // After the nacks, never before: the slot is what lets the replacement run a
91
+ // `concurrency: 1` job, and freed first it could be taken while the row was still ours.
92
+ for (const holding of stranded) await holding.release();
93
+ } finally {
94
+ handingBack -= 1;
95
+ wakeIfEmpty();
96
+ }
97
+ return stranded.length;
98
+ },
99
+ };
100
+ }
@@ -1,22 +1,19 @@
1
- // Every numeric knob `createWorker` accepts, read and REFUSED in one place — the slot table
1
+ // Every numeric knob `jobWorker` accepts, read and REFUSED in one place — the slot table
2
2
  // included, because a queue name is data and a slot count is a bound, and both arrive from the
3
3
  // same deployment config.
4
4
  //
5
5
  // WHY A REFUSAL AND NOT A CLAMP. `Number(process.env.JOB_VISIBILITY_MS)` on an unset variable is
6
6
  // `NaN`; `??` guards only nullish, and `Math.max`/`Math.min`/`Math.floor` PROPAGATE it. So the
7
7
  // value arrives at a lease deadline, a claim limit and a timer interval intact, and every
8
- // comparison against it reads FALSE — measured on `createMemoryDriver`: `visibleAt = at + NaN`,
8
+ // comparison against it reads FALSE — measured on `memoryJobDriver`: `visibleAt = at + NaN`,
9
9
  // the reclaim scan asks `visibleAt <= at`, and a job whose worker died is never claimable again.
10
10
  // At-least-once becomes never, with no error and a row `x jobs ls` still prints as `running`.
11
11
  // `slice(0, NaN)` is `[]`, so a `concurrency: NaN` worker claims nothing and reports healthy.
12
- // Same shape as `createLimiter`'s `maxTenants` and `backfill()`'s `batch`, refused the same way.
12
+ // Same shape as `concurrencyLimiter`'s `maxTenants` and `backfill()`'s `batch`, refused the same way.
13
13
 
14
- import { finiteOption } from '@ultimat3/core';
14
+ import { finiteOption, JOBS_CONCURRENCY_DEFAULT } from '@ultimat3/core';
15
15
  import { DEFAULT_VISIBILITY_TIMEOUT_MS } from './driver';
16
16
 
17
- /** Slots a queue gets when `concurrency` names no number for it. */
18
- const DEFAULT_SLOTS = 5;
19
-
20
17
  /** The subset of `WorkerOptions` this module reads. Structural, so `WorkerOptions` satisfies it. */
21
18
  export interface WorkerNumericOptions {
22
19
  readonly concurrency?: number | Readonly<Record<string, number>> | undefined;
@@ -34,7 +31,10 @@ export interface WorkerTimings {
34
31
  }
35
32
 
36
33
  /**
37
- * Slots for one queue. A queue NAME is deployment data, so the table is read by OWN keys:
34
+ * Slots for one queue — `JOBS_CONCURRENCY_DEFAULT` (core's, the same number `app.config.ts`'s
35
+ * `jobs.concurrency` defaults to) for a queue `concurrency` names no number for. This package kept
36
+ * its own 5 until 25.0.0, so a table `{ banks: 4 }` left `default` at 5 beside an unset config's 8.
37
+ * A queue NAME is deployment data, so the table is read by OWN keys:
38
38
  * `concurrency['constructor']` answers `Object.prototype.constructor`, and
39
39
  * `Math.max(0, <function> - inFlight)` is `NaN`, which the `free === 0` guard does not catch.
40
40
  * `bun run proto-index` cannot see this one — the table is a parameter, not a literal in a file.
@@ -43,25 +43,29 @@ const slotTable =
43
43
  (declared: number | Readonly<Record<string, number>> | undefined) =>
44
44
  (queue: string): number => {
45
45
  if (typeof declared === 'number') return declared;
46
- if (declared === undefined || !Object.hasOwn(declared, queue)) return DEFAULT_SLOTS;
47
- return declared[queue] ?? DEFAULT_SLOTS;
46
+ if (declared === undefined || !Object.hasOwn(declared, queue)) return JOBS_CONCURRENCY_DEFAULT;
47
+ return declared[queue] ?? JOBS_CONCURRENCY_DEFAULT;
48
48
  };
49
49
 
50
50
  export function resolveWorkerTimings(options: WorkerNumericOptions): WorkerTimings {
51
51
  const visibilityTimeoutMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS;
52
- finiteOption('createWorker', 'visibilityTimeoutMs', visibilityTimeoutMs);
52
+ finiteOption('jobWorker', 'visibilityTimeoutMs', visibilityTimeoutMs);
53
53
  const pollIntervalMs = options.pollIntervalMs ?? 250;
54
54
  // `setTimeout(fn, NaN)` coerces the delay to 0, so the claim loop stops being a poll and becomes
55
55
  // a spin: one round trip to Postgres per event-loop turn, from every worker replica.
56
- finiteOption('createWorker', 'pollIntervalMs', pollIntervalMs);
56
+ finiteOption('jobWorker', 'pollIntervalMs', pollIntervalMs);
57
57
  const heartbeatIntervalMs = options.heartbeatIntervalMs ?? Math.floor(visibilityTimeoutMs / 3);
58
- finiteOption('createWorker', 'heartbeatIntervalMs', heartbeatIntervalMs);
58
+ finiteOption('jobWorker', 'heartbeatIntervalMs', heartbeatIntervalMs);
59
59
  const declared = options.concurrency;
60
- if (typeof declared === 'number') finiteOption('createWorker', 'concurrency', declared);
60
+ if (typeof declared === 'number') finiteOption('jobWorker', 'concurrency', declared);
61
61
  else if (declared !== undefined) {
62
62
  // Per queue, by own key: an inherited member is not this table's to answer with either.
63
63
  for (const queue of Object.keys(declared)) {
64
- finiteOption('createWorker', `concurrency.${queue}`, declared[queue] ?? DEFAULT_SLOTS);
64
+ finiteOption(
65
+ 'jobWorker',
66
+ `concurrency.${queue}`,
67
+ declared[queue] ?? JOBS_CONCURRENCY_DEFAULT,
68
+ );
65
69
  }
66
70
  }
67
71
  return {