@ultimat3/jobs 24.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +25 -27
- package/README.md +50 -34
- package/package.json +7 -7
- package/src/backfill-errors.ts +3 -2
- package/src/backfill-ledger.ts +1 -1
- package/src/backfill-rate.ts +2 -2
- package/src/backfill.ts +2 -2
- package/src/counters.ts +1 -1
- package/src/driver-memory-operator.ts +3 -3
- package/src/driver-memory.ts +14 -17
- package/src/driver-pg-settle-sql.ts +2 -3
- package/src/driver-pg-sql.ts +1 -1
- package/src/driver-pg.ts +18 -18
- package/src/driver.ts +9 -15
- package/src/errors-concurrency.ts +1 -1
- package/src/errors.ts +5 -5
- package/src/events-pg.ts +4 -4
- package/src/events.ts +6 -6
- package/src/execute.ts +46 -5
- package/src/export.ts +2 -2
- package/src/heartbeat.ts +7 -2
- package/src/index.ts +37 -97
- package/src/inspect-operator.ts +1 -1
- package/src/inspect.ts +1 -1
- package/src/job.ts +1 -1
- package/src/leases.ts +1 -1
- package/src/limits.ts +13 -8
- package/src/outbox-lease.ts +1 -1
- package/src/outbox-pg.ts +5 -5
- package/src/outbox-relay.ts +6 -6
- package/src/outbox.ts +13 -13
- package/src/renewal-timer.ts +1 -1
- package/src/retry-classification.ts +52 -15
- package/src/scheduler-leader.ts +2 -2
- package/src/scheduler-pg.ts +16 -14
- package/src/scheduler-state.ts +2 -2
- package/src/scheduler.ts +5 -5
- package/src/steps-memory.ts +2 -2
- package/src/steps.ts +35 -7
- package/src/task.ts +1 -1
- package/src/webhook-attempt.ts +206 -0
- package/src/webhook-ledger.ts +15 -4
- package/src/webhook.ts +64 -118
- package/src/worker-admit.ts +7 -3
- package/src/worker-drain-cutoff.ts +110 -0
- package/src/worker-fleet-slots.ts +5 -2
- package/src/worker-held.ts +100 -0
- package/src/worker-options.ts +19 -15
- package/src/worker-run.ts +38 -8
- package/src/worker-tally.ts +53 -0
- package/src/worker-types.ts +7 -4
- package/src/worker.ts +85 -77
- package/src/driver-nats.ts +0 -89
- package/src/driver-redis.ts +0 -92
package/src/webhook.ts
CHANGED
|
@@ -19,14 +19,10 @@ import type { Clock, Ctx } from '@ultimat3/core';
|
|
|
19
19
|
import {
|
|
20
20
|
finiteOption,
|
|
21
21
|
isCanonicalWebhookField,
|
|
22
|
-
|
|
23
|
-
// receiver saying "a concurrent writer won this round" dead-lettered on attempt 1 as a refusal
|
|
24
|
-
// no retry could change — the exact divergence `retryable-status.ts` was extracted to end.
|
|
25
|
-
isRetryableStatus,
|
|
26
|
-
renderThrowable,
|
|
22
|
+
isUltimateError,
|
|
27
23
|
systemClock,
|
|
24
|
+
throwIfAborted,
|
|
28
25
|
WEBHOOK_FIELD_MAX,
|
|
29
|
-
webhookHeaders,
|
|
30
26
|
} from '@ultimat3/core';
|
|
31
27
|
import { t } from '@ultimat3/schema';
|
|
32
28
|
import type { DurationInput } from './clock';
|
|
@@ -36,10 +32,9 @@ import { job } from './job';
|
|
|
36
32
|
import type { RetryPolicy } from './retry';
|
|
37
33
|
import { DEFAULT_RETRY } from './retry';
|
|
38
34
|
import type { JobTenant } from './tenant';
|
|
35
|
+
import type { Outcome, WebhookFetch } from './webhook-attempt';
|
|
36
|
+
import { attemptDelivery, deliveryError, deliverySignal } from './webhook-attempt';
|
|
39
37
|
import {
|
|
40
|
-
WebhookDeliveryFailedError,
|
|
41
|
-
WebhookDeliveryRejectedError,
|
|
42
|
-
WebhookDeliveryThrottledError,
|
|
43
38
|
WebhookEndpointDisabledError,
|
|
44
39
|
WebhookEndpointInvalidError,
|
|
45
40
|
WebhookEndpointUnknownError,
|
|
@@ -47,19 +42,9 @@ import {
|
|
|
47
42
|
WebhookEventUnknownError,
|
|
48
43
|
} from './webhook-errors';
|
|
49
44
|
import type { WebhookLedger } from './webhook-ledger';
|
|
50
|
-
import type { WebhookResolve
|
|
45
|
+
import type { WebhookResolve } from './webhook-target';
|
|
51
46
|
import { resolveWebhookHost, webhookTarget } from './webhook-target';
|
|
52
47
|
|
|
53
|
-
/** Just the call. `typeof fetch` also carries `preconnect`, which no test double should have to. */
|
|
54
|
-
/**
|
|
55
|
-
* The transport. `init.tls.serverName` carries the hostname a PINNED connection proves, because
|
|
56
|
-
* the url it is handed names the approved address (`webhook-target.ts`); Bun's `fetch` honours it.
|
|
57
|
-
*/
|
|
58
|
-
export type WebhookFetch = (
|
|
59
|
-
url: string,
|
|
60
|
-
init: RequestInit & { readonly tls?: { readonly serverName?: string } },
|
|
61
|
-
) => Promise<Response>;
|
|
62
|
-
|
|
63
48
|
/**
|
|
64
49
|
* Consecutive failures before an endpoint stops taking deliveries. Ten is roughly a day of a
|
|
65
50
|
* retrying queue against a dead receiver, which is long enough for an outage and short enough that
|
|
@@ -67,12 +52,6 @@ export type WebhookFetch = (
|
|
|
67
52
|
*/
|
|
68
53
|
export const DEFAULT_WEBHOOK_DISABLE_AFTER = 10;
|
|
69
54
|
|
|
70
|
-
/** The one content type a delivery announces. The BODY is the app's; how it is framed is not. */
|
|
71
|
-
export const WEBHOOK_CONTENT_TYPE = 'application/json';
|
|
72
|
-
|
|
73
|
-
/** `Retry-After` in seconds. The HTTP-date form is ignored on purpose — see `retryAfterSeconds`. */
|
|
74
|
-
const RETRY_AFTER_SECONDS = /^\d{1,7}$/;
|
|
75
|
-
|
|
76
55
|
export interface WebhookEndpoint {
|
|
77
56
|
readonly id: string;
|
|
78
57
|
/**
|
|
@@ -184,13 +163,6 @@ export interface WebhookDefinition {
|
|
|
184
163
|
readonly timeout?: DurationInput;
|
|
185
164
|
}
|
|
186
165
|
|
|
187
|
-
/** `Retry-After: 120`. The HTTP-date form needs the receiver's clock, which is what we do not trust. */
|
|
188
|
-
const retryAfterSeconds = (response: Response): number | undefined => {
|
|
189
|
-
const header = response.headers.get('retry-after');
|
|
190
|
-
if (header === null || !RETRY_AFTER_SECONDS.test(header.trim())) return undefined;
|
|
191
|
-
return Number(header.trim());
|
|
192
|
-
};
|
|
193
|
-
|
|
194
166
|
export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliveryInput> {
|
|
195
167
|
const clock = definition.clock ?? systemClock;
|
|
196
168
|
const disableAfter = finiteOption(
|
|
@@ -200,7 +172,8 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
|
|
|
200
172
|
);
|
|
201
173
|
const send = definition.fetch ?? ((url, init) => fetch(url, init));
|
|
202
174
|
|
|
203
|
-
|
|
175
|
+
// Named so `run` can read the deadline `job()` resolved — one parse of `timeout`, never two.
|
|
176
|
+
const handle: JobHandle<WebhookDeliveryInput> = job<WebhookDeliveryInput>({
|
|
204
177
|
name: definition.name,
|
|
205
178
|
input: t.object({ endpointId: t.string, eventId: t.string }),
|
|
206
179
|
// Endpoint AND event: the same event fans out to every subscribed endpoint, so a key on the
|
|
@@ -211,6 +184,16 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
|
|
|
211
184
|
...(definition.queue === undefined ? {} : { queue: definition.queue }),
|
|
212
185
|
...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
|
|
213
186
|
async run({ input, ctx, attempt }): Promise<WebhookReport> {
|
|
187
|
+
// On the clock `executeJob`'s deadline timer runs on, read as the body starts — which is
|
|
188
|
+
// when that timer is armed.
|
|
189
|
+
const deadline =
|
|
190
|
+
handle.timeoutMs === undefined
|
|
191
|
+
? undefined
|
|
192
|
+
: {
|
|
193
|
+
atMs: performance.now() + handle.timeoutMs,
|
|
194
|
+
job: definition.name,
|
|
195
|
+
timeoutMs: handle.timeoutMs,
|
|
196
|
+
};
|
|
214
197
|
const endpoint = await definition.endpoint({ endpointId: input.endpointId, ctx });
|
|
215
198
|
if (endpoint === null) {
|
|
216
199
|
throw new WebhookEndpointUnknownError({
|
|
@@ -220,7 +203,9 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
|
|
|
220
203
|
}
|
|
221
204
|
// Before the ledger and before the socket: a disabled endpoint costs nothing, which is the
|
|
222
205
|
// whole point of disabling one.
|
|
223
|
-
|
|
206
|
+
// Asked of BOTH sources, because `disableAfter` writes only the ledger: an endpoint the
|
|
207
|
+
// mechanism switched off must stop here whether or not the app mirrored it onto its row.
|
|
208
|
+
if (endpoint.disabled === true || (await definition.ledger.isDisabled(endpoint.id))) {
|
|
224
209
|
throw new WebhookEndpointDisabledError({
|
|
225
210
|
webhook: definition.name,
|
|
226
211
|
endpointId: endpoint.id,
|
|
@@ -260,17 +245,41 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
|
|
|
260
245
|
// again now, so a receiver's freshness window measures the request in front of it rather
|
|
261
246
|
// than the age of the fact behind it.
|
|
262
247
|
const timestampSeconds = Math.floor(nowMs(clock) / 1_000);
|
|
248
|
+
// The last moment a cancelled attempt can stop without the receiver hearing anything — the
|
|
249
|
+
// drain included: a POST not yet started is handed back to the next pod at no cost.
|
|
250
|
+
stopIfCancelled(ctx);
|
|
263
251
|
const startedAt = clock.monotonic();
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
252
|
+
// Past this line the drain no longer cancels: see `deliverySignal`.
|
|
253
|
+
const request = deliverySignal(ctx.signal, deadline);
|
|
254
|
+
let outcome: Outcome;
|
|
255
|
+
try {
|
|
256
|
+
outcome = await ('unresolved' in target
|
|
257
|
+
? Promise.resolve({ ok: false as const, status: null, detail: target.unresolved })
|
|
258
|
+
: attemptDelivery(
|
|
259
|
+
send,
|
|
260
|
+
endpoint,
|
|
261
|
+
target,
|
|
262
|
+
{
|
|
263
|
+
secret: endpoint.secret,
|
|
264
|
+
timestampSeconds,
|
|
265
|
+
eventId: input.eventId,
|
|
266
|
+
topic: event.topic,
|
|
267
|
+
body: event.body,
|
|
268
|
+
},
|
|
269
|
+
request.signal,
|
|
270
|
+
));
|
|
271
|
+
} finally {
|
|
272
|
+
request.dispose();
|
|
273
|
+
}
|
|
273
274
|
const durationMs = Math.max(0, clock.monotonic() - startedAt);
|
|
275
|
+
// A failure the REQUEST's cancellation caused is not the receiver's, and is not this
|
|
276
|
+
// attempt's to record: the queue has already handed the delivery to another worker, whose
|
|
277
|
+
// outcome is the one the consecutive count must hear. Recorded, it was a failure nobody saw
|
|
278
|
+
// counting toward `disableAfter` — an abandoned attempt switching off a healthy endpoint.
|
|
279
|
+
// Read off the request's signal, never the attempt's: a receiver that really failed while
|
|
280
|
+
// the worker drained is a failure like any other, and the count hears it. A delivery that
|
|
281
|
+
// LANDED is a fact, and is always recorded.
|
|
282
|
+
if (!outcome.ok && request.signal.aborted) stopIfCancelled(ctx);
|
|
274
283
|
|
|
275
284
|
// Recorded whatever happened, and BEFORE the throw: a failure that is not on the ledger is a
|
|
276
285
|
// failure the consecutive count cannot see, which is an endpoint that never gets disabled.
|
|
@@ -311,6 +320,7 @@ export function webhook(definition: WebhookDefinition): JobHandle<WebhookDeliver
|
|
|
311
320
|
throw deliveryError(definition.name, endpoint, outcome);
|
|
312
321
|
},
|
|
313
322
|
});
|
|
323
|
+
return handle;
|
|
314
324
|
}
|
|
315
325
|
|
|
316
326
|
/** A URL no delivery may open, or a secret that would make the POST unsigned. The parsed url. */
|
|
@@ -346,78 +356,14 @@ function assertDeliverable(name: string, endpoint: WebhookEndpoint): URL {
|
|
|
346
356
|
return parsed;
|
|
347
357
|
}
|
|
348
358
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
send: WebhookFetch,
|
|
360
|
-
endpoint: WebhookEndpoint,
|
|
361
|
-
target: WebhookTarget,
|
|
362
|
-
signing: Parameters<typeof webhookHeaders>[0],
|
|
363
|
-
): Promise<Outcome> {
|
|
364
|
-
let response: Response;
|
|
365
|
-
try {
|
|
366
|
-
response = await send(target.url, {
|
|
367
|
-
method: 'POST',
|
|
368
|
-
headers: {
|
|
369
|
-
...endpoint.headers,
|
|
370
|
-
// The name the pinned address is reached AS — after the row's own headers, so an endpoint
|
|
371
|
-
// cannot redirect the request to a different virtual host than the one it registered.
|
|
372
|
-
host: target.host,
|
|
373
|
-
'content-type': WEBHOOK_CONTENT_TYPE,
|
|
374
|
-
// LAST, so an endpoint row's own headers can never overwrite the signature it is proved by.
|
|
375
|
-
...webhookHeaders(signing),
|
|
376
|
-
},
|
|
377
|
-
...(target.serverName === undefined ? {} : { tls: { serverName: target.serverName } }),
|
|
378
|
-
body: signing.body,
|
|
379
|
-
// Never followed: a 3xx would re-POST a body signed for one host to whatever the receiver
|
|
380
|
-
// named, and the signature would travel with it.
|
|
381
|
-
redirect: 'manual',
|
|
382
|
-
});
|
|
383
|
-
} catch (error) {
|
|
384
|
-
// `renderThrowable`, never `String(error)` or `${error}`: this string lands in a `cause` and on
|
|
385
|
-
// a durable ledger row, and a null-prototype throwable makes both of those a `TypeError`.
|
|
386
|
-
return { ok: false, status: null, detail: renderThrowable(error) };
|
|
387
|
-
}
|
|
388
|
-
if (response.ok) return { ok: true, status: response.status };
|
|
389
|
-
const stated = retryAfterSeconds(response);
|
|
390
|
-
return {
|
|
391
|
-
ok: false,
|
|
392
|
-
status: response.status,
|
|
393
|
-
detail: `status ${response.status}`,
|
|
394
|
-
...(stated === undefined ? {} : { retryAfterSeconds: stated }),
|
|
395
|
-
};
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
/** Which of the three failure codes this outcome is. The split is "can the same request land?". */
|
|
399
|
-
function deliveryError(
|
|
400
|
-
name: string,
|
|
401
|
-
endpoint: WebhookEndpoint,
|
|
402
|
-
outcome: Extract<Outcome, { ok: false }>,
|
|
403
|
-
): Error {
|
|
404
|
-
const base = { webhook: name, endpointId: endpoint.id, url: endpoint.url };
|
|
405
|
-
if (outcome.status === null) {
|
|
406
|
-
return new WebhookDeliveryFailedError({ ...base, status: null, detail: outcome.detail });
|
|
407
|
-
}
|
|
408
|
-
if (!isRetryableStatus(outcome.status)) {
|
|
409
|
-
return new WebhookDeliveryRejectedError({ ...base, status: outcome.status });
|
|
410
|
-
}
|
|
411
|
-
if (outcome.retryAfterSeconds !== undefined) {
|
|
412
|
-
return new WebhookDeliveryThrottledError({
|
|
413
|
-
...base,
|
|
414
|
-
status: outcome.status,
|
|
415
|
-
retryAfterSeconds: outcome.retryAfterSeconds,
|
|
416
|
-
});
|
|
417
|
-
}
|
|
418
|
-
return new WebhookDeliveryFailedError({
|
|
419
|
-
...base,
|
|
420
|
-
status: outcome.status,
|
|
421
|
-
detail: outcome.detail,
|
|
422
|
-
});
|
|
359
|
+
/**
|
|
360
|
+
* Throw the attempt's cancellation as itself — the deadline's `X_JOB_TIMEOUT`, the drain's
|
|
361
|
+
* `X_DRAINING`, a lost lease — so `executeJob` reads the stop it caused, never a delivery failure
|
|
362
|
+
* naming a receiver that did nothing wrong. Core's `X_ABORTED` when the reason carries no code.
|
|
363
|
+
*/
|
|
364
|
+
function stopIfCancelled(ctx: Ctx): void {
|
|
365
|
+
if (!ctx.signal.aborted) return;
|
|
366
|
+
const reason: unknown = ctx.signal.reason;
|
|
367
|
+
if (isUltimateError(reason)) throw reason;
|
|
368
|
+
throwIfAborted(ctx);
|
|
423
369
|
}
|
package/src/worker-admit.ts
CHANGED
|
@@ -76,8 +76,10 @@ export function createAdmission(options: AdmissionOptions): Admit {
|
|
|
76
76
|
try {
|
|
77
77
|
await shed(job, queue, limiter.blockedBy(key) ?? 'unknown');
|
|
78
78
|
} catch (error) {
|
|
79
|
-
// The nack itself failed
|
|
80
|
-
|
|
79
|
+
// The nack itself failed, so THIS job went nowhere either: it goes back with the jobs
|
|
80
|
+
// behind it, one more best-effort try, before the round reports. Left out, it sat
|
|
81
|
+
// `running` until its lease lapsed — an attempt spent on a body that never started.
|
|
82
|
+
await handBack(driver, [job, ...behind], back);
|
|
81
83
|
throw error;
|
|
82
84
|
}
|
|
83
85
|
return { kind: 'waiting' };
|
|
@@ -128,7 +130,9 @@ export function createAdmission(options: AdmissionOptions): Admit {
|
|
|
128
130
|
const keyed = grant.key === undefined ? '' : `, key ${grant.key}`;
|
|
129
131
|
await shed(job, queue, `job concurrency (${grant.limit}${keyed})`);
|
|
130
132
|
} catch (error) {
|
|
131
|
-
|
|
133
|
+
// This job too, for the reason above. A refusal that DID settle before throwing is safe to
|
|
134
|
+
// repeat: every nack is fenced on `running` and the claim, so a settled row ignores it.
|
|
135
|
+
await handBack(driver, [job, ...behind], back);
|
|
132
136
|
throw error;
|
|
133
137
|
}
|
|
134
138
|
return { kind: 'waiting' };
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// When a draining worker stops waiting for the jobs it holds. SIGTERM stops the CLAIMING; the jobs
|
|
2
|
+
// already running keep going, because a deploy that cancels work in flight re-runs every side
|
|
3
|
+
// effect it interrupted. Only near the end of the drain budget is what is still running told to
|
|
4
|
+
// stop (`cancel`), and a beat later what is still held goes back to the queue (`handBack`).
|
|
5
|
+
|
|
6
|
+
import { logger, renderThrowable, systemClock } from '@ultimat3/core';
|
|
7
|
+
import type { DrainBudget } from './drain-wait';
|
|
8
|
+
import { JobDrainedError } from './errors';
|
|
9
|
+
import type { HeldRuns } from './worker-held';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The most of the budget's tail kept for unwinding: the cancel lands this far before the deadline,
|
|
13
|
+
* the hand-back half as far, and the teardown — the registry row, the driver — gets the rest. Two
|
|
14
|
+
* seconds of a 25 s default, and never more than HALF of a small budget, so a budget of 300 ms
|
|
15
|
+
* still spends 150 ms letting its jobs finish before anything is cut.
|
|
16
|
+
*/
|
|
17
|
+
export const DRAIN_CANCEL_MARGIN_MS = 2_000;
|
|
18
|
+
|
|
19
|
+
export interface CutoffTimes {
|
|
20
|
+
/** Real monotonic ms: the held runs' `ctx.signal` aborts with `X_DRAINING`. */
|
|
21
|
+
readonly cancelAt: number;
|
|
22
|
+
/** Real monotonic ms: every claim still held is handed back uncounted. */
|
|
23
|
+
readonly handBackAt: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The cut-off for a deadline, read against the instant it was bound. */
|
|
27
|
+
export function cutoffTimes(deadlineAt: number, now: number): CutoffTimes {
|
|
28
|
+
const remaining = Math.max(0, deadlineAt - now);
|
|
29
|
+
const margin = Math.min(DRAIN_CANCEL_MARGIN_MS, remaining / 2);
|
|
30
|
+
return { cancelAt: deadlineAt - margin, handBackAt: deadlineAt - margin / 2 };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface DrainCutoff {
|
|
34
|
+
/** Stop listening to the budget and clear both timers — the teardown is over. */
|
|
35
|
+
dispose(): void;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Arm `cancel` and `handBack` on `budget`, re-armed each time it tightens (the earliest deadline
|
|
40
|
+
* wins — `drain-wait.ts`). Each fires at most once. The timers are `unref`ed: a drained process is
|
|
41
|
+
* never kept alive by its own cut-off.
|
|
42
|
+
*/
|
|
43
|
+
export function armDrainCutoff(
|
|
44
|
+
budget: DrainBudget,
|
|
45
|
+
steps: { readonly cancel: () => void; readonly handBack: () => Promise<void> },
|
|
46
|
+
): DrainCutoff {
|
|
47
|
+
const timers: ReturnType<typeof setTimeout>[] = [];
|
|
48
|
+
let cancelled = false;
|
|
49
|
+
let handedBack = false;
|
|
50
|
+
const clear = (): void => {
|
|
51
|
+
for (const timer of timers.splice(0)) clearTimeout(timer);
|
|
52
|
+
};
|
|
53
|
+
const at = (instant: number, fire: () => void): void => {
|
|
54
|
+
const timer = setTimeout(fire, Math.max(0, instant - systemClock.monotonic()));
|
|
55
|
+
timer.unref?.();
|
|
56
|
+
timers.push(timer);
|
|
57
|
+
};
|
|
58
|
+
const unwatch = budget.watch((deadlineAt) => {
|
|
59
|
+
clear();
|
|
60
|
+
const times = cutoffTimes(deadlineAt, systemClock.monotonic());
|
|
61
|
+
if (!cancelled) {
|
|
62
|
+
at(times.cancelAt, () => {
|
|
63
|
+
cancelled = true;
|
|
64
|
+
steps.cancel();
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
if (!handedBack) {
|
|
68
|
+
at(times.handBackAt, () => {
|
|
69
|
+
handedBack = true;
|
|
70
|
+
// Each nack's failure is logged where it happens (`worker-hand-back.ts`); this is the one
|
|
71
|
+
// place anything else could still escape a timer, so it is observed here.
|
|
72
|
+
void steps.handBack().catch((error: unknown) => {
|
|
73
|
+
logger.error('jobs.worker.drain-hand-back-failed', { error: renderThrowable(error) });
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
return {
|
|
79
|
+
dispose() {
|
|
80
|
+
unwatch();
|
|
81
|
+
clear();
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The worker's cut-off: `drain` aborted with `X_DRAINING` at `cancelAt`, every run `held` still
|
|
88
|
+
* holds handed back at `handBackAt`. Aborted even with nothing held: a round still inside its
|
|
89
|
+
* `claim()` starts its runs under this controller, and they must hear the cut-off too.
|
|
90
|
+
*/
|
|
91
|
+
export function armWorkerCutoff(input: {
|
|
92
|
+
readonly budget: DrainBudget;
|
|
93
|
+
readonly workerId: string;
|
|
94
|
+
/** The shutdown's signal name, carried on the reason the bodies read. */
|
|
95
|
+
readonly signal: string;
|
|
96
|
+
readonly drain: AbortController;
|
|
97
|
+
readonly held: HeldRuns;
|
|
98
|
+
}): DrainCutoff {
|
|
99
|
+
const { workerId, held } = input;
|
|
100
|
+
return armDrainCutoff(input.budget, {
|
|
101
|
+
cancel: () => {
|
|
102
|
+
if (held.size > 0) logger.warn('jobs.worker.drain-cancelling', { workerId, held: held.size });
|
|
103
|
+
input.drain.abort(new JobDrainedError({ workerId, signal: input.signal }));
|
|
104
|
+
},
|
|
105
|
+
handBack: async () => {
|
|
106
|
+
const handed = await held.handBackAll();
|
|
107
|
+
if (handed > 0) logger.warn('jobs.worker.drain-handed-back', { workerId, handed });
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
}
|
|
@@ -73,7 +73,7 @@ export interface FleetSlots {
|
|
|
73
73
|
* to do without one. `granted` is also the answer for "no cap declared": a job with no
|
|
74
74
|
* `concurrency` never touches the lease table.
|
|
75
75
|
*
|
|
76
|
-
* A driver with no lease store cannot reach here: `
|
|
76
|
+
* A driver with no lease store cannot reach here: `jobWorker().start()` refuses to boot when
|
|
77
77
|
* a registered job declares `concurrency` and the driver has none, because a cap that silently
|
|
78
78
|
* holds per process is the documented-guarantee-that-does-nothing axiom 3 exists to make
|
|
79
79
|
* impossible.
|
|
@@ -186,6 +186,9 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
|
|
|
186
186
|
// One in flight at a time, on the connection that is already the thing failing.
|
|
187
187
|
if (renewing) return;
|
|
188
188
|
renewing = true;
|
|
189
|
+
// Stamped BEFORE the call, as `heartbeat.ts` does: the store dates the new expiry at the
|
|
190
|
+
// statement, so a slow reply must not stretch this worker's view past it.
|
|
191
|
+
const sentAt = now();
|
|
189
192
|
try {
|
|
190
193
|
const renewed = await options.leases.renew(slot, options.ttlMs);
|
|
191
194
|
if (timer.stopped()) return;
|
|
@@ -194,7 +197,7 @@ export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
|
|
|
194
197
|
if (renewed === false) return reportLost('not-ours');
|
|
195
198
|
// A renewal that lands LATE is still late: the slot may already be another worker's.
|
|
196
199
|
if (lapsed()) return reportLost('expired');
|
|
197
|
-
renewedAt =
|
|
200
|
+
renewedAt = sentAt;
|
|
198
201
|
} catch (error) {
|
|
199
202
|
// One failed renewal is not a lost slot: the TTL gives the interval several tries.
|
|
200
203
|
logger.warn('jobs.worker.slot-renewal-failed', {
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// The runs one worker holds right now, each with the claim it runs under — so a drain that reaches
|
|
2
|
+
// its cut-off can hand back every row still held instead of leaving it under a lease another
|
|
3
|
+
// worker waits a whole visibility timeout to see lapse. Split from `worker.ts`, whose job is the
|
|
4
|
+
// claim loop; this one is "what am I holding, and give it back".
|
|
5
|
+
|
|
6
|
+
import type { ClaimedJob, JobDriver } from './driver';
|
|
7
|
+
import { handBack } from './worker-hand-back';
|
|
8
|
+
|
|
9
|
+
export interface HeldRuns {
|
|
10
|
+
/** How many runs are held — what `stats().inFlight` reports. */
|
|
11
|
+
readonly size: number;
|
|
12
|
+
/**
|
|
13
|
+
* The job ids held, each once — what the registry row reports. Per RUN underneath, because one
|
|
14
|
+
* worker can hold a job twice (a claim that lapsed under a body still running, then its own
|
|
15
|
+
* re-claim): keyed by id, the first run to end dropped an id the second still held.
|
|
16
|
+
*/
|
|
17
|
+
ids(): string[];
|
|
18
|
+
/**
|
|
19
|
+
* A run started. `release` is what the run's own `finally` would do for the process — the
|
|
20
|
+
* in-flight count core's drain waits on, the fleet slot — run early when the run is handed back.
|
|
21
|
+
*/
|
|
22
|
+
add(run: Promise<unknown>, claimed: ClaimedJob, release: () => Promise<void>): void;
|
|
23
|
+
/** The run settled on its own: no longer held. */
|
|
24
|
+
settle(run: Promise<unknown>): void;
|
|
25
|
+
/** Resolves the moment nothing is held — settled or handed back. */
|
|
26
|
+
empty(): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Every run still held goes back to the queue — `countsAsAttempt: false`, no delay — and stops
|
|
29
|
+
* being this worker's. Answers how many. The bodies themselves cannot be stopped; what they
|
|
30
|
+
* write after this is fenced out by the claim that just moved (`StepFence`, `SQL_ACK`).
|
|
31
|
+
*/
|
|
32
|
+
handBackAll(): Promise<number>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface Holding {
|
|
36
|
+
readonly claimed: ClaimedJob;
|
|
37
|
+
readonly release: () => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function createHeldRuns(options: {
|
|
41
|
+
readonly driver: JobDriver;
|
|
42
|
+
readonly workerId: string;
|
|
43
|
+
}): HeldRuns {
|
|
44
|
+
const held = new Map<Promise<unknown>, Holding>();
|
|
45
|
+
let waiters: (() => void)[] = [];
|
|
46
|
+
|
|
47
|
+
/** Pending hand-backs: `empty()` answers only once their nacks have LANDED, never mid-wire. */
|
|
48
|
+
let handingBack = 0;
|
|
49
|
+
const wakeIfEmpty = (): void => {
|
|
50
|
+
if (held.size > 0 || handingBack > 0 || waiters.length === 0) return;
|
|
51
|
+
const woken = waiters;
|
|
52
|
+
waiters = [];
|
|
53
|
+
for (const wake of woken) wake();
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
return {
|
|
57
|
+
get size() {
|
|
58
|
+
return held.size;
|
|
59
|
+
},
|
|
60
|
+
ids() {
|
|
61
|
+
return [...new Set([...held.values()].map((holding) => holding.claimed.id))];
|
|
62
|
+
},
|
|
63
|
+
add(run, claimed, release) {
|
|
64
|
+
held.set(run, { claimed, release });
|
|
65
|
+
},
|
|
66
|
+
settle(run) {
|
|
67
|
+
held.delete(run);
|
|
68
|
+
wakeIfEmpty();
|
|
69
|
+
},
|
|
70
|
+
empty() {
|
|
71
|
+
if (held.size === 0 && handingBack === 0) return Promise.resolve();
|
|
72
|
+
return new Promise<void>((resolve) => {
|
|
73
|
+
waiters.push(resolve);
|
|
74
|
+
});
|
|
75
|
+
},
|
|
76
|
+
async handBackAll() {
|
|
77
|
+
// Taken out of the set BEFORE the first await: a run settling while the nacks are on the
|
|
78
|
+
// wire is a run the hand-back already owns, and must not be counted twice. And `empty()`
|
|
79
|
+
// stays unanswered until the nacks land — the teardown waiting on it closes the driver next.
|
|
80
|
+
const stranded = [...held.values()];
|
|
81
|
+
held.clear();
|
|
82
|
+
if (stranded.length === 0) return 0;
|
|
83
|
+
handingBack += 1;
|
|
84
|
+
try {
|
|
85
|
+
await handBack(
|
|
86
|
+
options.driver,
|
|
87
|
+
stranded.map((holding) => holding.claimed),
|
|
88
|
+
{ delayMs: 0, workerId: options.workerId },
|
|
89
|
+
);
|
|
90
|
+
// After the nacks, never before: the slot is what lets the replacement run a
|
|
91
|
+
// `concurrency: 1` job, and freed first it could be taken while the row was still ours.
|
|
92
|
+
for (const holding of stranded) await holding.release();
|
|
93
|
+
} finally {
|
|
94
|
+
handingBack -= 1;
|
|
95
|
+
wakeIfEmpty();
|
|
96
|
+
}
|
|
97
|
+
return stranded.length;
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
}
|
package/src/worker-options.ts
CHANGED
|
@@ -1,22 +1,19 @@
|
|
|
1
|
-
// Every numeric knob `
|
|
1
|
+
// Every numeric knob `jobWorker` accepts, read and REFUSED in one place — the slot table
|
|
2
2
|
// included, because a queue name is data and a slot count is a bound, and both arrive from the
|
|
3
3
|
// same deployment config.
|
|
4
4
|
//
|
|
5
5
|
// WHY A REFUSAL AND NOT A CLAMP. `Number(process.env.JOB_VISIBILITY_MS)` on an unset variable is
|
|
6
6
|
// `NaN`; `??` guards only nullish, and `Math.max`/`Math.min`/`Math.floor` PROPAGATE it. So the
|
|
7
7
|
// value arrives at a lease deadline, a claim limit and a timer interval intact, and every
|
|
8
|
-
// comparison against it reads FALSE — measured on `
|
|
8
|
+
// comparison against it reads FALSE — measured on `memoryJobDriver`: `visibleAt = at + NaN`,
|
|
9
9
|
// the reclaim scan asks `visibleAt <= at`, and a job whose worker died is never claimable again.
|
|
10
10
|
// At-least-once becomes never, with no error and a row `x jobs ls` still prints as `running`.
|
|
11
11
|
// `slice(0, NaN)` is `[]`, so a `concurrency: NaN` worker claims nothing and reports healthy.
|
|
12
|
-
// Same shape as `
|
|
12
|
+
// Same shape as `concurrencyLimiter`'s `maxTenants` and `backfill()`'s `batch`, refused the same way.
|
|
13
13
|
|
|
14
|
-
import { finiteOption } from '@ultimat3/core';
|
|
14
|
+
import { finiteOption, JOBS_CONCURRENCY_DEFAULT } from '@ultimat3/core';
|
|
15
15
|
import { DEFAULT_VISIBILITY_TIMEOUT_MS } from './driver';
|
|
16
16
|
|
|
17
|
-
/** Slots a queue gets when `concurrency` names no number for it. */
|
|
18
|
-
const DEFAULT_SLOTS = 5;
|
|
19
|
-
|
|
20
17
|
/** The subset of `WorkerOptions` this module reads. Structural, so `WorkerOptions` satisfies it. */
|
|
21
18
|
export interface WorkerNumericOptions {
|
|
22
19
|
readonly concurrency?: number | Readonly<Record<string, number>> | undefined;
|
|
@@ -34,7 +31,10 @@ export interface WorkerTimings {
|
|
|
34
31
|
}
|
|
35
32
|
|
|
36
33
|
/**
|
|
37
|
-
* Slots for one queue
|
|
34
|
+
* Slots for one queue — `JOBS_CONCURRENCY_DEFAULT` (core's, the same number `app.config.ts`'s
|
|
35
|
+
* `jobs.concurrency` defaults to) for a queue `concurrency` names no number for. This package kept
|
|
36
|
+
* its own 5 until 25.0.0, so a table `{ banks: 4 }` left `default` at 5 beside an unset config's 8.
|
|
37
|
+
* A queue NAME is deployment data, so the table is read by OWN keys:
|
|
38
38
|
* `concurrency['constructor']` answers `Object.prototype.constructor`, and
|
|
39
39
|
* `Math.max(0, <function> - inFlight)` is `NaN`, which the `free === 0` guard does not catch.
|
|
40
40
|
* `bun run proto-index` cannot see this one — the table is a parameter, not a literal in a file.
|
|
@@ -43,25 +43,29 @@ const slotTable =
|
|
|
43
43
|
(declared: number | Readonly<Record<string, number>> | undefined) =>
|
|
44
44
|
(queue: string): number => {
|
|
45
45
|
if (typeof declared === 'number') return declared;
|
|
46
|
-
if (declared === undefined || !Object.hasOwn(declared, queue)) return
|
|
47
|
-
return declared[queue] ??
|
|
46
|
+
if (declared === undefined || !Object.hasOwn(declared, queue)) return JOBS_CONCURRENCY_DEFAULT;
|
|
47
|
+
return declared[queue] ?? JOBS_CONCURRENCY_DEFAULT;
|
|
48
48
|
};
|
|
49
49
|
|
|
50
50
|
export function resolveWorkerTimings(options: WorkerNumericOptions): WorkerTimings {
|
|
51
51
|
const visibilityTimeoutMs = options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS;
|
|
52
|
-
finiteOption('
|
|
52
|
+
finiteOption('jobWorker', 'visibilityTimeoutMs', visibilityTimeoutMs);
|
|
53
53
|
const pollIntervalMs = options.pollIntervalMs ?? 250;
|
|
54
54
|
// `setTimeout(fn, NaN)` coerces the delay to 0, so the claim loop stops being a poll and becomes
|
|
55
55
|
// a spin: one round trip to Postgres per event-loop turn, from every worker replica.
|
|
56
|
-
finiteOption('
|
|
56
|
+
finiteOption('jobWorker', 'pollIntervalMs', pollIntervalMs);
|
|
57
57
|
const heartbeatIntervalMs = options.heartbeatIntervalMs ?? Math.floor(visibilityTimeoutMs / 3);
|
|
58
|
-
finiteOption('
|
|
58
|
+
finiteOption('jobWorker', 'heartbeatIntervalMs', heartbeatIntervalMs);
|
|
59
59
|
const declared = options.concurrency;
|
|
60
|
-
if (typeof declared === 'number') finiteOption('
|
|
60
|
+
if (typeof declared === 'number') finiteOption('jobWorker', 'concurrency', declared);
|
|
61
61
|
else if (declared !== undefined) {
|
|
62
62
|
// Per queue, by own key: an inherited member is not this table's to answer with either.
|
|
63
63
|
for (const queue of Object.keys(declared)) {
|
|
64
|
-
finiteOption(
|
|
64
|
+
finiteOption(
|
|
65
|
+
'jobWorker',
|
|
66
|
+
`concurrency.${queue}`,
|
|
67
|
+
declared[queue] ?? JOBS_CONCURRENCY_DEFAULT,
|
|
68
|
+
);
|
|
65
69
|
}
|
|
66
70
|
}
|
|
67
71
|
return {
|