@gobing-ai/ts-infra 0.4.59 → 0.4.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -4
- package/dist/application/index.d.ts.map +1 -1
- package/dist/application/index.js +3 -0
- package/dist/application/types.d.ts +11 -2
- package/dist/application/types.d.ts.map +1 -1
- package/dist/application-node.d.ts.map +1 -1
- package/dist/application-node.js +34 -4
- package/dist/execution-policy.d.ts +76 -0
- package/dist/execution-policy.d.ts.map +1 -0
- package/dist/execution-policy.js +123 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/job-queue/db-job-queue.d.ts +14 -1
- package/dist/job-queue/db-job-queue.d.ts.map +1 -1
- package/dist/job-queue/db-job-queue.js +165 -26
- package/dist/job-queue/types.d.ts +47 -7
- package/dist/job-queue/types.d.ts.map +1 -1
- package/dist/scheduler/cloudflare.d.ts +4 -0
- package/dist/scheduler/cloudflare.d.ts.map +1 -1
- package/dist/scheduler/cloudflare.js +2 -1
- package/dist/scheduler/index.d.ts +1 -1
- package/dist/scheduler/index.d.ts.map +1 -1
- package/dist/scheduler/node.d.ts +10 -1
- package/dist/scheduler/node.d.ts.map +1 -1
- package/dist/scheduler/node.js +28 -6
- package/dist/scheduler/types.d.ts +20 -3
- package/dist/scheduler/types.d.ts.map +1 -1
- package/dist/scheduler/wrap-handler.d.ts.map +1 -1
- package/dist/scheduler/wrap-handler.js +2 -2
- package/package.json +6 -6
- package/src/application/index.ts +3 -0
- package/src/application/types.ts +16 -3
- package/src/application-node.ts +40 -4
- package/src/execution-policy.ts +168 -0
- package/src/index.ts +3 -1
- package/src/job-queue/db-job-queue.ts +184 -25
- package/src/job-queue/types.ts +47 -7
- package/src/scheduler/cloudflare.ts +3 -1
- package/src/scheduler/index.ts +6 -1
- package/src/scheduler/node.ts +39 -6
- package/src/scheduler/types.ts +34 -5
- package/src/scheduler/wrap-handler.ts +3 -2
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
QueueJobFailedDetail,
|
|
8
8
|
QueueJobRetryingDetail,
|
|
9
9
|
} from '../events';
|
|
10
|
+
import { resolveExecutionTimeoutMs, runWithExecutionDeadline } from '../execution-policy';
|
|
10
11
|
import { settleWithin } from '../internals/drain';
|
|
11
12
|
import { getLogger, type Logger } from '../logger';
|
|
12
13
|
import {
|
|
@@ -67,7 +68,7 @@ function enqueuedDetail(jobId: string, type: string, options: EnqueueOptions | u
|
|
|
67
68
|
return detail;
|
|
68
69
|
}
|
|
69
70
|
|
|
70
|
-
/** DB-backed queue consumer with polling, retry, and
|
|
71
|
+
/** DB-backed queue consumer with polling, retry, lease ownership, and execution-deadline handling. */
|
|
71
72
|
export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
72
73
|
private readonly handlers = new Map<string, JobHandler<T>>();
|
|
73
74
|
private readonly pollInterval: number;
|
|
@@ -77,6 +78,11 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
77
78
|
private readonly baseDelay: number;
|
|
78
79
|
private readonly maxDelay: number;
|
|
79
80
|
private readonly drainTimeoutMs: number;
|
|
81
|
+
private readonly drainPolicy: 'bounded' | 'drain-to-completion';
|
|
82
|
+
/** Default execution policy for jobs without their own persisted decision; `null` = unlimited. */
|
|
83
|
+
private readonly defaultTimeoutMs: number | null;
|
|
84
|
+
/** Live attempts owned by this consumer, keyed by job id — the target of `cancel()`. */
|
|
85
|
+
private readonly activeAttempts = new Map<string, { controller: AbortController; state: AttemptState }>();
|
|
80
86
|
/**
|
|
81
87
|
* Validated event sink: the bus plus the non-empty queue name its lifecycle rows
|
|
82
88
|
* require (ADR-068). `undefined` for silent consumers — no `events`, no identity.
|
|
@@ -98,6 +104,19 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
98
104
|
this.baseDelay = nonNegativeFiniteConfig('baseDelay', config.baseDelay ?? 1_000);
|
|
99
105
|
this.maxDelay = nonNegativeFiniteConfig('maxDelay', config.maxDelay ?? 60_000);
|
|
100
106
|
this.drainTimeoutMs = nonNegativeFiniteConfig('drainTimeoutMs', config.drainTimeoutMs ?? 30_000);
|
|
107
|
+
this.drainPolicy = config.drainPolicy ?? 'bounded';
|
|
108
|
+
if (
|
|
109
|
+
config.defaultTimeoutMs !== undefined &&
|
|
110
|
+
config.defaultTimeoutMs !== null &&
|
|
111
|
+
(typeof config.defaultTimeoutMs !== 'number' ||
|
|
112
|
+
!Number.isInteger(config.defaultTimeoutMs) ||
|
|
113
|
+
config.defaultTimeoutMs <= 0)
|
|
114
|
+
) {
|
|
115
|
+
throw new RangeError(
|
|
116
|
+
`Queue consumer defaultTimeoutMs must be a positive integer or null; received ${String(config.defaultTimeoutMs)}`,
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
this.defaultTimeoutMs = config.defaultTimeoutMs ?? null;
|
|
101
120
|
this.eventSink = resolveEventSink(config);
|
|
102
121
|
}
|
|
103
122
|
|
|
@@ -131,6 +150,7 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
131
150
|
this.timer = null;
|
|
132
151
|
}
|
|
133
152
|
|
|
153
|
+
const drainToCompletion = this.drainPolicy === 'drain-to-completion';
|
|
134
154
|
const deadline = Date.now() + this.drainTimeoutMs;
|
|
135
155
|
|
|
136
156
|
// Wait for an already-running poll cycle before consulting `inFlight`.
|
|
@@ -140,11 +160,25 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
140
160
|
// to claim and process jobs after stop() resolved.
|
|
141
161
|
const pending = this.pollPromise;
|
|
142
162
|
if (pending !== null) {
|
|
143
|
-
|
|
163
|
+
if (drainToCompletion) {
|
|
164
|
+
await pending;
|
|
165
|
+
} else {
|
|
166
|
+
await settleWithin(pending, deadline);
|
|
167
|
+
}
|
|
144
168
|
}
|
|
145
169
|
|
|
146
|
-
|
|
147
|
-
|
|
170
|
+
if (drainToCompletion) {
|
|
171
|
+
// Shutdown policy (A21): wait for every in-flight attempt to settle,
|
|
172
|
+
// however long that takes. Leases keep renewing while we wait.
|
|
173
|
+
while (this.inFlight > 0) {
|
|
174
|
+
await sleep(10);
|
|
175
|
+
}
|
|
176
|
+
} else {
|
|
177
|
+
while (this.inFlight > 0 && Date.now() < deadline) {
|
|
178
|
+
await sleep(10);
|
|
179
|
+
}
|
|
180
|
+
// Bounded drain expiry (A21): attempts still running keep ownership —
|
|
181
|
+
// their leases keep renewing and they are acknowledged when they settle.
|
|
148
182
|
}
|
|
149
183
|
if (wasRunning) {
|
|
150
184
|
const drained = this.inFlight === 0;
|
|
@@ -162,6 +196,14 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
162
196
|
}
|
|
163
197
|
}
|
|
164
198
|
|
|
199
|
+
async cancel(jobId: string): Promise<boolean> {
|
|
200
|
+
const attempt = this.activeAttempts.get(jobId);
|
|
201
|
+
if (attempt === undefined) return false;
|
|
202
|
+
attempt.state.cancelled = true;
|
|
203
|
+
attempt.controller.abort();
|
|
204
|
+
return true;
|
|
205
|
+
}
|
|
206
|
+
|
|
165
207
|
async stats(): Promise<QueueStats> {
|
|
166
208
|
return this.dao.getStats();
|
|
167
209
|
}
|
|
@@ -172,7 +214,7 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
172
214
|
await this.dao.resetStuckJobs(this.visibilityTimeout);
|
|
173
215
|
await this.dao.failExpiredJobs();
|
|
174
216
|
|
|
175
|
-
const jobs = await this.dao.claimReady(this.batchSize);
|
|
217
|
+
const jobs = await this.dao.claimReady(this.batchSize, { leaseMs: this.visibilityTimeout });
|
|
176
218
|
let processed = 0;
|
|
177
219
|
|
|
178
220
|
for (let index = 0; index < jobs.length; index += this.maxConcurrency) {
|
|
@@ -229,7 +271,7 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
229
271
|
} catch (error) {
|
|
230
272
|
// Corrupt payload — the job can never parse; route it through the
|
|
231
273
|
// retry/fail path instead of rejecting the whole batch.
|
|
232
|
-
await this.failOrRetry(record, error, 0);
|
|
274
|
+
await this.failOrRetry(record, error, 0, record.attemptToken ?? undefined);
|
|
233
275
|
return;
|
|
234
276
|
}
|
|
235
277
|
return traceAsync('queue.job.process', async () => {
|
|
@@ -240,43 +282,146 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
240
282
|
});
|
|
241
283
|
|
|
242
284
|
const handler = this.handlers.get(job.type);
|
|
285
|
+
const token = record.attemptToken ?? undefined;
|
|
243
286
|
if (handler === undefined) {
|
|
244
|
-
await this.failOrRetry(job, new Error(`No handler registered for job type "${job.type}"`), 0);
|
|
287
|
+
await this.failOrRetry(job, new Error(`No handler registered for job type "${job.type}"`), 0, token);
|
|
245
288
|
return;
|
|
246
289
|
}
|
|
247
290
|
|
|
248
|
-
|
|
291
|
+
// Per-attempt ownership (A21): the fresh claim token fences every
|
|
292
|
+
// acknowledgement, and the external controller carries lease-loss and
|
|
293
|
+
// manual cancellation into the shared deadline clock.
|
|
294
|
+
let attemptWake: (() => void) | undefined;
|
|
295
|
+
const attempt: AttemptState = { lost: false, cancelled: false, settled: false };
|
|
296
|
+
const attemptAbort = new AbortController();
|
|
297
|
+
if (token !== undefined) {
|
|
298
|
+
const { promise: settledSignal, resolve: resolveSettled } = Promise.withResolvers<void>();
|
|
299
|
+
attempt.settledSignal = settledSignal;
|
|
300
|
+
attemptWake = resolveSettled;
|
|
301
|
+
// Shares the `attempt` object reference so cancel() mutations are
|
|
302
|
+
// visible in the outcome dispatch below.
|
|
303
|
+
this.activeAttempts.set(record.id, { controller: attemptAbort, state: attempt });
|
|
304
|
+
}
|
|
305
|
+
const renewal =
|
|
306
|
+
token === undefined
|
|
307
|
+
? undefined
|
|
308
|
+
: this.renewUntilSettled(record.id, token, attempt, attemptAbort).catch(() => {});
|
|
249
309
|
try {
|
|
250
|
-
|
|
251
|
-
|
|
310
|
+
const resolvedTimeout = resolveExecutionTimeoutMs(
|
|
311
|
+
`job "${job.type}"`,
|
|
312
|
+
record.timeoutUnlimited === 1 ? null : (record.timeoutMs ?? undefined),
|
|
313
|
+
this.defaultTimeoutMs,
|
|
314
|
+
);
|
|
315
|
+
const startMs = performance.now();
|
|
316
|
+
const outcome = await runWithExecutionDeadline((context) => handler(job, context), {
|
|
317
|
+
timeoutMs: resolvedTimeout,
|
|
318
|
+
signal: attemptAbort.signal,
|
|
319
|
+
});
|
|
320
|
+
attempt.settled = true;
|
|
321
|
+
attemptWake?.();
|
|
252
322
|
const durationMs = performance.now() - startMs;
|
|
253
|
-
|
|
254
|
-
getQueueJobProcessingDuration().record(durationMs, { type: job.type });
|
|
255
|
-
const completed: QueueJobCompletedDetail = {
|
|
256
|
-
jobId: job.id,
|
|
323
|
+
getQueueJobProcessingDuration().record(Number.isFinite(durationMs) ? durationMs : 0, {
|
|
257
324
|
type: job.type,
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
if (attempt.lost) {
|
|
328
|
+
// Ownership was lost: the row now belongs to a replacement attempt
|
|
329
|
+
// claimed through the expired lease. Fence every acknowledgement —
|
|
330
|
+
// a stale ack must never mutate the replacement's work (R4).
|
|
331
|
+
queueLogger().warn('queue attempt lost lease ownership; acknowledgement fenced', {
|
|
332
|
+
jobId: record.id,
|
|
333
|
+
type: job.type,
|
|
334
|
+
outcome: outcome.outcome,
|
|
335
|
+
});
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
if (outcome.outcome === 'error') {
|
|
340
|
+
await this.failOrRetry(job, outcome.error, Number.isFinite(durationMs) ? durationMs : 0, token);
|
|
341
|
+
} else if (outcome.timedOut) {
|
|
342
|
+
// Deadline expiry is a failure like any other, reported only after
|
|
343
|
+
// the handler has settled — even if it ignored the abort.
|
|
344
|
+
await this.failOrRetry(job, outcome.error, Number.isFinite(durationMs) ? durationMs : 0, token);
|
|
345
|
+
} else if (attempt.cancelled) {
|
|
346
|
+
// Manual cancellation is terminal: settle, then fail without retry.
|
|
347
|
+
const message = 'job cancelled';
|
|
348
|
+
const applied = await this.dao.markFailed(record.id, job.attempts + 1, message, token);
|
|
349
|
+
if (applied) {
|
|
350
|
+
getQueueJobFailedTotal().add(1, { type: job.type });
|
|
351
|
+
await this.eventSink?.bus.emit('queue.job.failed', {
|
|
352
|
+
jobId: job.id,
|
|
353
|
+
type: job.type,
|
|
354
|
+
error: message,
|
|
355
|
+
attempt: job.attempts + 1,
|
|
356
|
+
maxRetries: job.maxRetries,
|
|
357
|
+
durationMs: Number.isFinite(durationMs) ? durationMs : 0,
|
|
358
|
+
severity: 'warning',
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
} else {
|
|
362
|
+
await this.dao.markCompleted(record.id, token);
|
|
363
|
+
getQueueJobCompletedTotal().add(1, { type: job.type });
|
|
364
|
+
const completed: QueueJobCompletedDetail = {
|
|
365
|
+
jobId: job.id,
|
|
366
|
+
type: job.type,
|
|
367
|
+
durationMs: Number.isFinite(durationMs) ? durationMs : 0,
|
|
368
|
+
attempt: job.attempts,
|
|
369
|
+
severity: 'info',
|
|
370
|
+
};
|
|
371
|
+
await this.eventSink?.bus.emit('queue.job.completed', completed);
|
|
372
|
+
}
|
|
373
|
+
} finally {
|
|
374
|
+
attempt.settled = true;
|
|
375
|
+
attemptWake?.();
|
|
376
|
+
if (renewal !== undefined) await renewal;
|
|
377
|
+
if (token !== undefined) this.activeAttempts.delete(record.id);
|
|
267
378
|
}
|
|
268
379
|
});
|
|
269
380
|
}
|
|
270
381
|
|
|
382
|
+
/**
|
|
383
|
+
* Extend the attempt's lease on an interval derived from the visibility
|
|
384
|
+
* timeout, proving this consumer is still alive and working. A failed or
|
|
385
|
+
* erroring renewal means ownership was lost: abort the attempt so the
|
|
386
|
+
* handler can stop, and mark the attempt fenced.
|
|
387
|
+
*/
|
|
388
|
+
private async renewUntilSettled(
|
|
389
|
+
id: string,
|
|
390
|
+
token: string,
|
|
391
|
+
attempt: AttemptState,
|
|
392
|
+
attemptAbort: AbortController,
|
|
393
|
+
): Promise<void> {
|
|
394
|
+
const intervalMs = Math.max(1, Math.floor(this.visibilityTimeout / 3));
|
|
395
|
+
while (!attempt.settled) {
|
|
396
|
+
// Settle-aware wait: wakes early when the attempt finishes so a long
|
|
397
|
+
// visibility timeout never delays acknowledgement past the handler.
|
|
398
|
+
await Promise.race([sleep(intervalMs), attempt.settledSignal]);
|
|
399
|
+
if (attempt.settled) return;
|
|
400
|
+
let renewed = false;
|
|
401
|
+
try {
|
|
402
|
+
renewed = await this.dao.renewLease(id, token, this.visibilityTimeout);
|
|
403
|
+
} catch {
|
|
404
|
+
renewed = false; // DB unavailability is treated like ownership loss
|
|
405
|
+
}
|
|
406
|
+
if (attempt.settled) return;
|
|
407
|
+
if (!renewed) {
|
|
408
|
+
attempt.lost = true;
|
|
409
|
+
attemptAbort.abort();
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
271
415
|
private async failOrRetry(
|
|
272
416
|
job: Pick<Job<T>, 'id' | 'type' | 'attempts' | 'maxRetries'>,
|
|
273
417
|
error: unknown,
|
|
274
418
|
durationMs: number,
|
|
419
|
+
attemptToken?: string,
|
|
275
420
|
): Promise<void> {
|
|
276
421
|
const attempts = job.attempts + 1;
|
|
277
422
|
const message = error instanceof Error ? error.message : String(error);
|
|
278
423
|
if (attempts >= job.maxRetries) {
|
|
279
|
-
await this.dao.markFailed(job.id, attempts, message);
|
|
424
|
+
await this.dao.markFailed(job.id, attempts, message, attemptToken);
|
|
280
425
|
getQueueJobFailedTotal().add(1, { type: job.type });
|
|
281
426
|
const failed: QueueJobFailedDetail = {
|
|
282
427
|
jobId: job.id,
|
|
@@ -293,7 +438,7 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
293
438
|
|
|
294
439
|
const delay = Math.min(this.maxDelay, this.baseDelay * 2 ** Math.max(0, attempts - 1));
|
|
295
440
|
const nextRetryAt = Date.now() + delay;
|
|
296
|
-
await this.dao.markForRetry(job.id, attempts, message, nextRetryAt);
|
|
441
|
+
await this.dao.markForRetry(job.id, attempts, message, nextRetryAt, attemptToken);
|
|
297
442
|
const retrying: QueueJobRetryingDetail = {
|
|
298
443
|
jobId: job.id,
|
|
299
444
|
type: job.type,
|
|
@@ -307,6 +452,18 @@ export class DBQueueConsumer<T = unknown> implements QueueConsumer<T> {
|
|
|
307
452
|
}
|
|
308
453
|
}
|
|
309
454
|
|
|
455
|
+
/** Live per-attempt state shared between `processJob`, the renewal loop, and `cancel()`. */
|
|
456
|
+
type AttemptState = {
|
|
457
|
+
/** Set when the lease was lost — the attempt's acknowledgements are fenced. */
|
|
458
|
+
lost: boolean;
|
|
459
|
+
/** Set by a manual `cancel()` — the attempt is failed without retry after settling. */
|
|
460
|
+
cancelled: boolean;
|
|
461
|
+
/** Set once the handler has settled — stops the renewal loop. */
|
|
462
|
+
settled: boolean;
|
|
463
|
+
/** Resolved when the attempt settles so the renewal wait wakes early. */
|
|
464
|
+
settledSignal?: Promise<void>;
|
|
465
|
+
};
|
|
466
|
+
|
|
310
467
|
function toJob<T>(record: QueueJobRecord): Job<T> {
|
|
311
468
|
return {
|
|
312
469
|
id: record.id,
|
|
@@ -320,6 +477,8 @@ function toJob<T>(record: QueueJobRecord): Job<T> {
|
|
|
320
477
|
nextRetryAt: record.nextRetryAt,
|
|
321
478
|
lastError: record.lastError,
|
|
322
479
|
processingAt: record.processingAt,
|
|
480
|
+
// Persisted job policy: explicit unlimited stays `null`, absence stays undefined.
|
|
481
|
+
timeoutMs: record.timeoutUnlimited === 1 ? null : (record.timeoutMs ?? undefined),
|
|
323
482
|
};
|
|
324
483
|
}
|
|
325
484
|
|
package/src/job-queue/types.ts
CHANGED
|
@@ -7,8 +7,9 @@
|
|
|
7
7
|
|
|
8
8
|
import type { EventBus } from '../event-bus/event-bus';
|
|
9
9
|
import type { QueueEvents } from '../events';
|
|
10
|
+
import type { ExecutionContext } from '../execution-policy';
|
|
10
11
|
|
|
11
|
-
/** A queued job with status tracking, retry metadata, and
|
|
12
|
+
/** A queued job with status tracking, retry metadata, timestamps, and its persisted execution policy. */
|
|
12
13
|
export interface Job<T = unknown> {
|
|
13
14
|
id: string;
|
|
14
15
|
type: string;
|
|
@@ -22,14 +23,27 @@ export interface Job<T = unknown> {
|
|
|
22
23
|
nextRetryAt: number | null;
|
|
23
24
|
lastError: string | null;
|
|
24
25
|
processingAt: number | null;
|
|
26
|
+
/**
|
|
27
|
+
* Job-level execution policy as persisted: a positive integer ms deadline,
|
|
28
|
+
* explicit `null` = unlimited, or `undefined` = no job-level decision (the
|
|
29
|
+
* consumer default applies). Never the resolved per-attempt value.
|
|
30
|
+
*/
|
|
31
|
+
timeoutMs?: number | null;
|
|
25
32
|
}
|
|
26
33
|
|
|
27
|
-
/** Options for enqueuing a job: retry policy, delay, and
|
|
34
|
+
/** Options for enqueuing a job: retry policy, delay, TTL, and execution policy. */
|
|
28
35
|
export interface EnqueueOptions {
|
|
29
36
|
/** Total attempts allowed (default 3) — `maxRetries: 1` runs the job once with no retry. */
|
|
30
37
|
maxRetries?: number;
|
|
31
38
|
delay?: number;
|
|
32
39
|
ttlMs?: number;
|
|
40
|
+
/**
|
|
41
|
+
* Job execution policy persisted with the row: a positive integer ms
|
|
42
|
+
* deadline, explicit `null` = unlimited (disables this job's deadline),
|
|
43
|
+
* omitted = no job-level decision (consumers apply their own default).
|
|
44
|
+
* Invalid explicit values are rejected before the row is created.
|
|
45
|
+
*/
|
|
46
|
+
timeoutMs?: number | null;
|
|
33
47
|
}
|
|
34
48
|
|
|
35
49
|
/** Producer interface for the job queue — enqueue single jobs or batches. */
|
|
@@ -39,8 +53,12 @@ export interface JobQueue<T = unknown> {
|
|
|
39
53
|
stats(): Promise<QueueStats>;
|
|
40
54
|
}
|
|
41
55
|
|
|
42
|
-
/**
|
|
43
|
-
|
|
56
|
+
/**
|
|
57
|
+
* Async handler that processes a single job. The context carries the shared
|
|
58
|
+
* execution deadline clock for this attempt; single-argument handlers stay
|
|
59
|
+
* assignable.
|
|
60
|
+
*/
|
|
61
|
+
export type JobHandler<T = unknown> = (job: Job<T>, context: ExecutionContext) => Promise<void>;
|
|
44
62
|
|
|
45
63
|
/** Aggregate statistics for a job queue: counts by status. */
|
|
46
64
|
export interface QueueStats {
|
|
@@ -50,7 +68,7 @@ export interface QueueStats {
|
|
|
50
68
|
failed: number;
|
|
51
69
|
}
|
|
52
70
|
|
|
53
|
-
/** Configuration for a queue consumer: polling, concurrency, and
|
|
71
|
+
/** Configuration for a queue consumer: polling, concurrency, backoff, and the default execution policy. */
|
|
54
72
|
export interface QueueConsumerConfig {
|
|
55
73
|
pollInterval?: number;
|
|
56
74
|
batchSize?: number;
|
|
@@ -60,6 +78,20 @@ export interface QueueConsumerConfig {
|
|
|
60
78
|
maxDelay?: number;
|
|
61
79
|
/** Upper bound (ms, default 30_000) on how long `stop()` waits for in-flight work to drain. */
|
|
62
80
|
drainTimeoutMs?: number;
|
|
81
|
+
/**
|
|
82
|
+
* Shutdown drain policy (A21). `'bounded'` (default): `stop()` gives up after
|
|
83
|
+
* `drainTimeoutMs`; attempts still running keep their leases, keep renewing, and
|
|
84
|
+
* are acknowledged when they settle. `'drain-to-completion'`: `stop()` waits for
|
|
85
|
+
* all in-flight attempts to settle, however long that takes.
|
|
86
|
+
*/
|
|
87
|
+
drainPolicy?: 'bounded' | 'drain-to-completion';
|
|
88
|
+
/**
|
|
89
|
+
* Default execution policy for jobs without their own persisted decision
|
|
90
|
+
* (A21): a positive integer ms deadline or explicit `null` = unlimited.
|
|
91
|
+
* First-match-wins: an explicit job option — including job `null` — beats
|
|
92
|
+
* this default; absence inherits it.
|
|
93
|
+
*/
|
|
94
|
+
defaultTimeoutMs?: number | null;
|
|
63
95
|
/**
|
|
64
96
|
* Identity of the queue this consumer polls. Runtime-required whenever `events` is
|
|
65
97
|
* configured (ADR-068): an observable consumer must never emit an anonymous lifecycle
|
|
@@ -81,10 +113,18 @@ export interface QueueConsumerConfig {
|
|
|
81
113
|
export interface QueueConsumer<T = unknown> {
|
|
82
114
|
register(type: string, handler: JobHandler<T>): void;
|
|
83
115
|
start(): Promise<void>;
|
|
116
|
+
/**
|
|
117
|
+
* Request cooperative cancellation of a claimed attempt (A21). The context
|
|
118
|
+
* signal aborts with reason `'cancelled'`; the attempt is allowed to settle
|
|
119
|
+
* and is then failed without retry. Resolves `false` when the job is not
|
|
120
|
+
* currently owned by this consumer.
|
|
121
|
+
*/
|
|
122
|
+
cancel(jobId: string): Promise<boolean>;
|
|
84
123
|
/**
|
|
85
124
|
* Stop polling and drain work already in flight, including a poll cycle that has
|
|
86
|
-
* claimed nothing yet.
|
|
87
|
-
*
|
|
125
|
+
* claimed nothing yet. Under the default `'bounded'` policy this resolves once the
|
|
126
|
+
* drain completes or `drainTimeoutMs` elapses; `'drain-to-completion'` ignores the
|
|
127
|
+
* bound and waits for every in-flight attempt.
|
|
88
128
|
*/
|
|
89
129
|
stop(): Promise<void>;
|
|
90
130
|
stats(): Promise<QueueStats>;
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Cloudflare Workers scheduler adapter using Cron Triggers.
|
|
3
3
|
* Uses minimal local type declarations — no @cloudflare/workers-types dependency.
|
|
4
4
|
*/
|
|
5
|
+
|
|
6
|
+
import { unlimitedExecutionContext } from '../execution-policy';
|
|
5
7
|
import {
|
|
6
8
|
getSchedulerJobDuration,
|
|
7
9
|
getSchedulerJobExecutedTotal,
|
|
@@ -60,7 +62,7 @@ export class CloudflareSchedulerAdapter implements SchedulerAdapter {
|
|
|
60
62
|
const startMs = performance.now();
|
|
61
63
|
getSchedulerJobExecutedTotal().add(1, { cron: event.cron });
|
|
62
64
|
ctx.waitUntil(
|
|
63
|
-
action()
|
|
65
|
+
action(unlimitedExecutionContext())
|
|
64
66
|
.catch((error: unknown) => {
|
|
65
67
|
getSchedulerJobFailedTotal().add(1, { cron: event.cron });
|
|
66
68
|
throw error;
|
package/src/scheduler/index.ts
CHANGED
|
@@ -12,5 +12,10 @@ export {
|
|
|
12
12
|
} from './action';
|
|
13
13
|
export { initScheduler } from './factory';
|
|
14
14
|
export { NoopSchedulerAdapter } from './noop';
|
|
15
|
-
export type {
|
|
15
|
+
export type {
|
|
16
|
+
ScheduledAction,
|
|
17
|
+
ScheduledActionOptions,
|
|
18
|
+
SchedulerAdapter,
|
|
19
|
+
SchedulerJobConfig,
|
|
20
|
+
} from './types';
|
|
16
21
|
export { wrapScheduledHandler } from './wrap-handler';
|
package/src/scheduler/node.ts
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* internal `scheduler/cron.ts` grammar shared with `application-node.ts`.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { type ExecutionDeadlineMs, resolveExecutionTimeoutMs, runWithExecutionDeadline } from '../execution-policy';
|
|
9
10
|
import { settleWithin } from '../internals/drain';
|
|
11
|
+
import { getLogger } from '../logger';
|
|
10
12
|
import {
|
|
11
13
|
getSchedulerJobDuration,
|
|
12
14
|
getSchedulerJobExecutedTotal,
|
|
@@ -60,6 +62,7 @@ type ScheduledEntry =
|
|
|
60
62
|
cron: string;
|
|
61
63
|
action: ScheduledAction;
|
|
62
64
|
intervalMs: number;
|
|
65
|
+
timeoutMs: ExecutionDeadlineMs;
|
|
63
66
|
timer?: ReturnType<typeof setInterval>;
|
|
64
67
|
}
|
|
65
68
|
| {
|
|
@@ -68,11 +71,18 @@ type ScheduledEntry =
|
|
|
68
71
|
action: ScheduledAction;
|
|
69
72
|
expr: CronExpression;
|
|
70
73
|
target: number;
|
|
74
|
+
timeoutMs: ExecutionDeadlineMs;
|
|
71
75
|
timer?: ReturnType<typeof setTimeout>;
|
|
72
76
|
};
|
|
73
77
|
|
|
74
78
|
/** Constructor options for {@link NodeSchedulerAdapter}. */
|
|
75
79
|
export interface NodeSchedulerAdapterConfig {
|
|
80
|
+
/**
|
|
81
|
+
* Default execution policy (A21) for entries registered without their own
|
|
82
|
+
* `timeoutMs` option: a positive integer ms deadline or explicit `null` =
|
|
83
|
+
* unlimited. Invalid values throw at construction.
|
|
84
|
+
*/
|
|
85
|
+
readonly timeoutMs?: number | null;
|
|
76
86
|
/**
|
|
77
87
|
* Upper bound (ms) on how long `stop()` waits for an in-flight tick to settle.
|
|
78
88
|
* A hung action is abandoned at this deadline so it cannot block shutdown.
|
|
@@ -98,6 +108,7 @@ export interface NodeSchedulerAdapterConfig {
|
|
|
98
108
|
export class NodeSchedulerAdapter implements SchedulerAdapter {
|
|
99
109
|
private readonly entries: ScheduledEntry[] = [];
|
|
100
110
|
private readonly drainTimeoutMs: number;
|
|
111
|
+
private readonly defaultTimeoutMs: ExecutionDeadlineMs;
|
|
101
112
|
private readonly now: () => number;
|
|
102
113
|
private running = false;
|
|
103
114
|
private readonly inflight = new Set<Promise<void>>();
|
|
@@ -110,30 +121,39 @@ export class NodeSchedulerAdapter implements SchedulerAdapter {
|
|
|
110
121
|
);
|
|
111
122
|
}
|
|
112
123
|
this.drainTimeoutMs = drainTimeoutMs ?? 30_000;
|
|
124
|
+
this.defaultTimeoutMs = resolveExecutionTimeoutMs('NodeSchedulerAdapter', config.timeoutMs ?? undefined);
|
|
113
125
|
this.now = now ?? (() => Date.now());
|
|
114
126
|
}
|
|
115
127
|
|
|
116
|
-
register(cron: string, action: ScheduledAction): void {
|
|
128
|
+
register(cron: string, action: ScheduledAction, options?: { timeoutMs?: number | null }): void {
|
|
117
129
|
// Fail at registration time: an unsupported expression must never reach
|
|
118
130
|
// start(), where it would otherwise create a silently-wrong interval
|
|
119
131
|
// (task 0060 F7) or a never-firing cron (task 0734 R1).
|
|
120
|
-
const entry = this.parseEntry(cron, action);
|
|
132
|
+
const entry = this.parseEntry(cron, action, options);
|
|
121
133
|
this.entries.push(entry);
|
|
122
134
|
if (this.running) {
|
|
123
135
|
this.startEntry(entry);
|
|
124
136
|
}
|
|
125
137
|
}
|
|
126
138
|
|
|
127
|
-
private parseEntry(cron: string, action: ScheduledAction): ScheduledEntry {
|
|
139
|
+
private parseEntry(cron: string, action: ScheduledAction, options?: { timeoutMs?: number | null }): ScheduledEntry {
|
|
140
|
+
// Resolve the entry's execution policy now: an explicit option (including
|
|
141
|
+
// explicit null) wins, absence inherits the adapter default. Invalid
|
|
142
|
+
// options throw here rather than leaving a tick to discover it mid-flight.
|
|
143
|
+
const timeoutMs = resolveExecutionTimeoutMs(
|
|
144
|
+
`scheduler entry "${cron}"`,
|
|
145
|
+
options?.timeoutMs,
|
|
146
|
+
this.defaultTimeoutMs,
|
|
147
|
+
);
|
|
128
148
|
const intervalMs = parseInterval(cron);
|
|
129
149
|
if (intervalMs !== undefined) {
|
|
130
|
-
return { kind: 'interval', cron, action, intervalMs };
|
|
150
|
+
return { kind: 'interval', cron, action, intervalMs, timeoutMs };
|
|
131
151
|
}
|
|
132
152
|
// Real five-field cron. Validate and verify a next occurrence exists at
|
|
133
153
|
// registration (bounded scan) so an unsatisfiable expression fails here.
|
|
134
154
|
const expr = parseCronExpression(cron);
|
|
135
155
|
const target = nextCronTime(expr, this.now()).getTime();
|
|
136
|
-
return { kind: 'cron', cron, action, expr, target };
|
|
156
|
+
return { kind: 'cron', cron, action, expr, target, timeoutMs };
|
|
137
157
|
}
|
|
138
158
|
|
|
139
159
|
async start(): Promise<void> {
|
|
@@ -247,7 +267,20 @@ export class NodeSchedulerAdapter implements SchedulerAdapter {
|
|
|
247
267
|
const startMs = performance.now();
|
|
248
268
|
getSchedulerJobExecutedTotal().add(1, { cron: entry.cron });
|
|
249
269
|
try {
|
|
250
|
-
|
|
270
|
+
// One shared deadline clock per tick (A21): expiry aborts the
|
|
271
|
+
// context and the action is always awaited to settlement. Timeout
|
|
272
|
+
// counts as a failure; the adapter keeps ticking.
|
|
273
|
+
const outcome = await runWithExecutionDeadline(entry.action, { timeoutMs: entry.timeoutMs });
|
|
274
|
+
if (outcome.timedOut) {
|
|
275
|
+
getSchedulerJobFailedTotal().add(1, { cron: entry.cron });
|
|
276
|
+
getLogger('scheduler').warn('scheduled tick exceeded its execution deadline', {
|
|
277
|
+
cron: entry.cron,
|
|
278
|
+
timeoutMs: entry.timeoutMs,
|
|
279
|
+
elapsedMs: Math.round(outcome.elapsedMs),
|
|
280
|
+
});
|
|
281
|
+
} else if (outcome.outcome === 'error') {
|
|
282
|
+
getSchedulerJobFailedTotal().add(1, { cron: entry.cron });
|
|
283
|
+
}
|
|
251
284
|
} catch {
|
|
252
285
|
// Swallow — scheduler errors should not crash the process
|
|
253
286
|
getSchedulerJobFailedTotal().add(1, { cron: entry.cron });
|
package/src/scheduler/types.ts
CHANGED
|
@@ -2,8 +2,25 @@
|
|
|
2
2
|
* Scheduler types and interface.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
import type { ExecutionContext } from '../execution-policy';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Signature for scheduled action handlers. The context carries the shared
|
|
9
|
+
* execution-deadline clock for this tick (A21); zero-argument handlers stay
|
|
10
|
+
* assignable — notably the Cloudflare adapter, which invokes actions with an
|
|
11
|
+
* unlimited execution context (no wall-clock deadline on that runtime).
|
|
12
|
+
*/
|
|
13
|
+
export type ScheduledAction = (context: ExecutionContext) => Promise<void>;
|
|
14
|
+
|
|
15
|
+
/** Per-entry execution policy options for {@link SchedulerAdapter.register}. */
|
|
16
|
+
export interface ScheduledActionOptions {
|
|
17
|
+
/**
|
|
18
|
+
* Execution policy for this entry's ticks: a positive integer ms deadline,
|
|
19
|
+
* explicit `null` = unlimited, omitted = inherit the adapter default.
|
|
20
|
+
* Invalid explicit values throw at registration time.
|
|
21
|
+
*/
|
|
22
|
+
timeoutMs?: number | null;
|
|
23
|
+
}
|
|
7
24
|
|
|
8
25
|
/**
|
|
9
26
|
* Abstract scheduler interface — implementations for Node and Cloudflare.
|
|
@@ -13,7 +30,7 @@ export type ScheduledAction = () => Promise<void>;
|
|
|
13
30
|
* action is abandoned at the deadline rather than blocking shutdown forever.
|
|
14
31
|
*/
|
|
15
32
|
export interface SchedulerAdapter {
|
|
16
|
-
register(cron: string, action: ScheduledAction): void;
|
|
33
|
+
register(cron: string, action: ScheduledAction, options?: ScheduledActionOptions): void;
|
|
17
34
|
start(): Promise<void>;
|
|
18
35
|
stop(): Promise<void>;
|
|
19
36
|
}
|
|
@@ -28,5 +45,17 @@ export interface SchedulerAdapter {
|
|
|
28
45
|
* command handler.
|
|
29
46
|
*/
|
|
30
47
|
export type SchedulerJobConfig =
|
|
31
|
-
| {
|
|
32
|
-
|
|
48
|
+
| {
|
|
49
|
+
readonly name: string;
|
|
50
|
+
readonly command: string;
|
|
51
|
+
readonly intervalMinutes: number;
|
|
52
|
+
readonly cron?: never;
|
|
53
|
+
readonly timeoutMs?: number | null;
|
|
54
|
+
}
|
|
55
|
+
| {
|
|
56
|
+
readonly name: string;
|
|
57
|
+
readonly command: string;
|
|
58
|
+
readonly cron: string;
|
|
59
|
+
readonly intervalMinutes?: never;
|
|
60
|
+
readonly timeoutMs?: number | null;
|
|
61
|
+
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { EventBus } from '../event-bus/event-bus';
|
|
2
2
|
import type { SchedulerEvents } from '../events';
|
|
3
|
+
import type { ExecutionContext } from '../execution-policy';
|
|
3
4
|
import { addSpanAttributes, addSpanEvent, traceAsync } from '../telemetry/tracing';
|
|
4
5
|
import type { ScheduledAction } from './types';
|
|
5
6
|
|
|
@@ -25,7 +26,7 @@ export function wrapScheduledHandler(
|
|
|
25
26
|
action: ScheduledAction,
|
|
26
27
|
systemBus?: EventBus<SchedulerEvents> | null,
|
|
27
28
|
): ScheduledAction {
|
|
28
|
-
return async () => {
|
|
29
|
+
return async (context: ExecutionContext) => {
|
|
29
30
|
const startTime = performance.now();
|
|
30
31
|
|
|
31
32
|
return traceAsync('scheduler.job', async (span) => {
|
|
@@ -33,7 +34,7 @@ export function wrapScheduledHandler(
|
|
|
33
34
|
|
|
34
35
|
let execError: string | undefined;
|
|
35
36
|
try {
|
|
36
|
-
await action();
|
|
37
|
+
await action(context);
|
|
37
38
|
} catch (error) {
|
|
38
39
|
execError = error instanceof Error ? error.message : String(error);
|
|
39
40
|
addSpanEvent('scheduler.job.error', { 'scheduler.error': execError });
|