@zudojs/queue 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +40 -8
- package/dist/deadLetter/deadLetter.core.js +1 -1
- package/dist/inMemoryQueue/inMemoryQueue.core.d.ts +95 -4
- package/dist/inMemoryQueue/inMemoryQueue.core.js +398 -73
- package/dist/inMemoryQueue/inMemoryQueue.processing.d.ts +53 -8
- package/dist/inMemoryQueue/inMemoryQueue.processing.js +83 -36
- package/dist/inMemoryQueue/inMemoryQueue.scheduling.d.ts +17 -1
- package/dist/inMemoryQueue/inMemoryQueue.scheduling.js +43 -11
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -2
- package/dist/job/job.core.js +20 -12
- package/dist/job/job.type.d.ts +3 -2
- package/dist/jobOptions/jobOptions.type.d.ts +12 -3
- package/dist/jobTypes/index.d.ts +1 -1
- package/dist/jobTypes/jobTypes.type.d.ts +9 -0
- package/dist/middleware/middleware.core.d.ts +10 -1
- package/dist/middleware/middleware.core.js +40 -17
- package/dist/processor/index.d.ts +1 -1
- package/dist/processor/index.js +1 -1
- package/dist/processor/processor.core.js +2 -0
- package/dist/processor/processor.type.d.ts +18 -1
- package/dist/processor/processor.type.js +15 -0
- package/dist/queue/index.d.ts +1 -1
- package/dist/queue/queue.core.js +2 -2
- package/dist/queue/queue.type.d.ts +108 -2
- package/dist/queueEmitter/queueEmitter.core.d.ts +22 -2
- package/dist/queueEmitter/queueEmitter.core.js +47 -7
- package/dist/queueManager/queueManager.core.js +24 -5
- package/dist/queueRegistry/queueRegistry.core.js +15 -0
- package/dist/queueRegistry/queueRegistry.type.d.ts +2 -0
- package/dist/retryPolicy/retryPolicy.core.d.ts +42 -3
- package/dist/retryPolicy/retryPolicy.core.js +82 -13
- package/dist/serializer/serializer.core.d.ts +2 -1
- package/dist/worker/worker.core.d.ts +4 -0
- package/dist/worker/worker.core.js +87 -23
- package/dist/worker/worker.type.d.ts +17 -3
- package/package.json +21 -14
- package/dist/.tsbuildinfo +0 -1
- package/dist/deadLetter/deadLetter.core.d.ts.map +0 -1
- package/dist/deadLetter/deadLetter.core.js.map +0 -1
- package/dist/deadLetter/deadLetter.type.d.ts.map +0 -1
- package/dist/deadLetter/deadLetter.type.js.map +0 -1
- package/dist/deadLetter/index.d.ts.map +0 -1
- package/dist/deadLetter/index.js.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.core.d.ts.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.core.js.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.factory.d.ts.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.factory.js.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.processing.d.ts.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.processing.js.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.scheduling.d.ts.map +0 -1
- package/dist/inMemoryQueue/inMemoryQueue.scheduling.js.map +0 -1
- package/dist/inMemoryQueue/index.d.ts.map +0 -1
- package/dist/inMemoryQueue/index.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/job/index.d.ts.map +0 -1
- package/dist/job/index.js.map +0 -1
- package/dist/job/job.core.d.ts.map +0 -1
- package/dist/job/job.core.js.map +0 -1
- package/dist/job/job.type.d.ts.map +0 -1
- package/dist/job/job.type.js.map +0 -1
- package/dist/jobContext/index.d.ts.map +0 -1
- package/dist/jobContext/index.js.map +0 -1
- package/dist/jobContext/jobContext.core.d.ts.map +0 -1
- package/dist/jobContext/jobContext.core.js.map +0 -1
- package/dist/jobContext/jobContext.type.d.ts.map +0 -1
- package/dist/jobContext/jobContext.type.js.map +0 -1
- package/dist/jobOptions/index.d.ts.map +0 -1
- package/dist/jobOptions/index.js.map +0 -1
- package/dist/jobOptions/jobOptions.core.d.ts.map +0 -1
- package/dist/jobOptions/jobOptions.core.js.map +0 -1
- package/dist/jobOptions/jobOptions.type.d.ts.map +0 -1
- package/dist/jobOptions/jobOptions.type.js.map +0 -1
- package/dist/jobResult/index.d.ts.map +0 -1
- package/dist/jobResult/index.js.map +0 -1
- package/dist/jobResult/jobResult.core.d.ts.map +0 -1
- package/dist/jobResult/jobResult.core.js.map +0 -1
- package/dist/jobResult/jobResult.type.d.ts.map +0 -1
- package/dist/jobResult/jobResult.type.js.map +0 -1
- package/dist/jobTypes/index.d.ts.map +0 -1
- package/dist/jobTypes/index.js.map +0 -1
- package/dist/jobTypes/jobTypes.type.d.ts.map +0 -1
- package/dist/jobTypes/jobTypes.type.js.map +0 -1
- package/dist/middleware/index.d.ts.map +0 -1
- package/dist/middleware/index.js.map +0 -1
- package/dist/middleware/middleware.core.d.ts.map +0 -1
- package/dist/middleware/middleware.core.js.map +0 -1
- package/dist/middleware/middleware.type.d.ts.map +0 -1
- package/dist/middleware/middleware.type.js.map +0 -1
- package/dist/processor/index.d.ts.map +0 -1
- package/dist/processor/index.js.map +0 -1
- package/dist/processor/processor.core.d.ts.map +0 -1
- package/dist/processor/processor.core.js.map +0 -1
- package/dist/processor/processor.type.d.ts.map +0 -1
- package/dist/processor/processor.type.js.map +0 -1
- package/dist/queue/index.d.ts.map +0 -1
- package/dist/queue/index.js.map +0 -1
- package/dist/queue/queue.core.d.ts.map +0 -1
- package/dist/queue/queue.core.js.map +0 -1
- package/dist/queue/queue.type.d.ts.map +0 -1
- package/dist/queue/queue.type.js.map +0 -1
- package/dist/queueEmitter/index.d.ts.map +0 -1
- package/dist/queueEmitter/index.js.map +0 -1
- package/dist/queueEmitter/queueEmitter.core.d.ts.map +0 -1
- package/dist/queueEmitter/queueEmitter.core.js.map +0 -1
- package/dist/queueEmitter/queueEmitter.type.d.ts.map +0 -1
- package/dist/queueEmitter/queueEmitter.type.js.map +0 -1
- package/dist/queueEvents/index.d.ts +0 -8
- package/dist/queueEvents/index.d.ts.map +0 -1
- package/dist/queueEvents/index.js +0 -7
- package/dist/queueEvents/index.js.map +0 -1
- package/dist/queueEvents/queueEvents.core.d.ts +0 -6
- package/dist/queueEvents/queueEvents.core.d.ts.map +0 -1
- package/dist/queueEvents/queueEvents.core.js +0 -12
- package/dist/queueEvents/queueEvents.core.js.map +0 -1
- package/dist/queueEvents/queueEvents.type.d.ts +0 -97
- package/dist/queueEvents/queueEvents.type.d.ts.map +0 -1
- package/dist/queueEvents/queueEvents.type.js +0 -2
- package/dist/queueEvents/queueEvents.type.js.map +0 -1
- package/dist/queueManager/index.d.ts.map +0 -1
- package/dist/queueManager/index.js.map +0 -1
- package/dist/queueManager/queueManager.core.d.ts.map +0 -1
- package/dist/queueManager/queueManager.core.js.map +0 -1
- package/dist/queueManager/queueManager.type.d.ts.map +0 -1
- package/dist/queueManager/queueManager.type.js.map +0 -1
- package/dist/queueRegistry/index.d.ts.map +0 -1
- package/dist/queueRegistry/index.js.map +0 -1
- package/dist/queueRegistry/queueRegistry.core.d.ts.map +0 -1
- package/dist/queueRegistry/queueRegistry.core.js.map +0 -1
- package/dist/queueRegistry/queueRegistry.type.d.ts.map +0 -1
- package/dist/queueRegistry/queueRegistry.type.js.map +0 -1
- package/dist/retryPolicy/index.d.ts.map +0 -1
- package/dist/retryPolicy/index.js.map +0 -1
- package/dist/retryPolicy/retryPolicy.core.d.ts.map +0 -1
- package/dist/retryPolicy/retryPolicy.core.js.map +0 -1
- package/dist/serializer/index.d.ts.map +0 -1
- package/dist/serializer/index.js.map +0 -1
- package/dist/serializer/serializer.core.d.ts.map +0 -1
- package/dist/serializer/serializer.core.js.map +0 -1
- package/dist/serializer/serializer.type.d.ts.map +0 -1
- package/dist/serializer/serializer.type.js.map +0 -1
- package/dist/worker/index.d.ts.map +0 -1
- package/dist/worker/index.js.map +0 -1
- package/dist/worker/worker.core.d.ts.map +0 -1
- package/dist/worker/worker.core.js.map +0 -1
- package/dist/worker/worker.type.d.ts.map +0 -1
- package/dist/worker/worker.type.js.map +0 -1
|
@@ -1,31 +1,61 @@
|
|
|
1
|
-
import { updateJobState } from "../job/job.core.js";
|
|
1
|
+
import { updateJobState, incrementJobAttempt } from "../job/job.core.js";
|
|
2
|
+
import { DEFAULT_JOB_OPTIONS } from "../jobOptions/jobOptions.core.js";
|
|
2
3
|
import { JobState as JobStateEnum } from "../jobTypes/jobTypes.type.js";
|
|
3
4
|
import { createJobContext } from "../jobContext/jobContext.core.js";
|
|
4
5
|
import { createMiddlewareChain, createTimeoutMiddleware, } from "../middleware/middleware.core.js";
|
|
5
|
-
import { calculateRetryDelay, shouldRetry, } from "../retryPolicy/retryPolicy.core.js";
|
|
6
|
+
import { calculateRetryDelay, resolveBackoff, shouldRetry, } from "../retryPolicy/retryPolicy.core.js";
|
|
6
7
|
import { moveToDeadLetter } from "../deadLetter/deadLetter.core.js";
|
|
7
8
|
import { JobMaxAttemptsError } from "@zudojs/errors";
|
|
9
|
+
/**
|
|
10
|
+
* Narrows a processor's return value to a `JobResult`.
|
|
11
|
+
*
|
|
12
|
+
* A processor may return a `JobResult`, a plain value, or nothing.
|
|
13
|
+
* Testing `"success" in result` without this guard throws a `TypeError`
|
|
14
|
+
* for a primitive return, which surfaced as the job failing and being
|
|
15
|
+
* dead-lettered.
|
|
16
|
+
*/
|
|
17
|
+
function isJobResult(value) {
|
|
18
|
+
return typeof value === "object" && value !== null && "success" in value;
|
|
19
|
+
}
|
|
8
20
|
/**
|
|
9
21
|
* Process a single job with middleware, retry, and failure handling.
|
|
22
|
+
*
|
|
23
|
+
* The job is expected to already be claimed (`active`); this records the
|
|
24
|
+
* start time, runs the middleware chain, and routes the outcome to
|
|
25
|
+
* completion, retry, or the dead letter store.
|
|
10
26
|
*/
|
|
11
|
-
export async function processJob(job, processor,
|
|
27
|
+
export async function processJob(job, processor, options, deps) {
|
|
28
|
+
const { jobs, emitter, counters } = deps;
|
|
12
29
|
const updatedJob = updateJobState(job, JobStateEnum.ACTIVE, {
|
|
13
30
|
startedAt: new Date().toISOString(),
|
|
14
31
|
});
|
|
15
32
|
jobs.set(updatedJob.id, updatedJob);
|
|
16
33
|
emitter.emit("job:started", { job: updatedJob });
|
|
17
|
-
const abortController = new AbortController();
|
|
34
|
+
const abortController = options.abortController ?? new AbortController();
|
|
18
35
|
const context = createJobContext(updatedJob, abortController.signal, {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
jobs.
|
|
36
|
+
...(deps.logger ? { logger: deps.logger } : {}),
|
|
37
|
+
onProgress: async (progress) => {
|
|
38
|
+
const current = jobs.get(updatedJob.id);
|
|
39
|
+
if (current === undefined) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
emitter.emit("job:progress", {
|
|
43
|
+
job: current,
|
|
44
|
+
progress: progress.percent,
|
|
45
|
+
});
|
|
22
46
|
},
|
|
23
47
|
});
|
|
24
|
-
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
25
|
-
const timeoutMiddleware = createTimeoutMiddleware(timeoutMs)
|
|
48
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_JOB_OPTIONS.timeout ?? 30_000;
|
|
49
|
+
const timeoutMiddleware = createTimeoutMiddleware(timeoutMs, () => {
|
|
50
|
+
// Let a cooperative processor observe the timeout and stop working
|
|
51
|
+
// instead of running on with its result discarded.
|
|
52
|
+
if (!abortController.signal.aborted) {
|
|
53
|
+
abortController.abort(new Error(`Job "${updatedJob.id}" timed out after ${timeoutMs}ms.`));
|
|
54
|
+
}
|
|
55
|
+
});
|
|
26
56
|
const middlewareChain = createMiddlewareChain([
|
|
27
57
|
timeoutMiddleware,
|
|
28
|
-
...middleware,
|
|
58
|
+
...deps.middleware,
|
|
29
59
|
]);
|
|
30
60
|
try {
|
|
31
61
|
const result = await middlewareChain({
|
|
@@ -35,16 +65,8 @@ export async function processJob(job, processor, middleware, jobs, options, emit
|
|
|
35
65
|
return processor(updatedJob, context);
|
|
36
66
|
},
|
|
37
67
|
});
|
|
38
|
-
if (result &&
|
|
39
|
-
|
|
40
|
-
completedAt: new Date().toISOString(),
|
|
41
|
-
});
|
|
42
|
-
jobs.set(updatedJob.id, completedJob);
|
|
43
|
-
counters.succeededCount++;
|
|
44
|
-
emitter.emit("job:completed", { job: completedJob, result: result.data });
|
|
45
|
-
}
|
|
46
|
-
else if (result && "success" in result && !result.success) {
|
|
47
|
-
await handleJobFailure(updatedJob, result.error ?? "Job failed", processor, abortController, jobs, emitter, deadLetterStore, counters);
|
|
68
|
+
if (isJobResult(result) && !result.success) {
|
|
69
|
+
await handleJobFailure(updatedJob, result.error ?? "Job failed", deps);
|
|
48
70
|
}
|
|
49
71
|
else {
|
|
50
72
|
const completedJob = updateJobState(updatedJob, JobStateEnum.COMPLETED, {
|
|
@@ -52,12 +74,18 @@ export async function processJob(job, processor, middleware, jobs, options, emit
|
|
|
52
74
|
});
|
|
53
75
|
jobs.set(updatedJob.id, completedJob);
|
|
54
76
|
counters.succeededCount++;
|
|
55
|
-
emitter.emit("job:completed", {
|
|
77
|
+
emitter.emit("job:completed", {
|
|
78
|
+
job: completedJob,
|
|
79
|
+
// A processor may return a `JobResult`, a plain value, or
|
|
80
|
+
// nothing. Only the first carries its payload under `data`.
|
|
81
|
+
result: isJobResult(result) ? result.data : result,
|
|
82
|
+
});
|
|
83
|
+
deps.onSettled?.(completedJob);
|
|
56
84
|
}
|
|
57
85
|
}
|
|
58
86
|
catch (error) {
|
|
59
87
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
60
|
-
await handleJobFailure(updatedJob, errorMessage,
|
|
88
|
+
await handleJobFailure(updatedJob, errorMessage, deps);
|
|
61
89
|
}
|
|
62
90
|
finally {
|
|
63
91
|
counters.processedCount++;
|
|
@@ -65,15 +93,18 @@ export async function processJob(job, processor, middleware, jobs, options, emit
|
|
|
65
93
|
}
|
|
66
94
|
/**
|
|
67
95
|
* Handle job failure with retry logic.
|
|
96
|
+
*
|
|
97
|
+
* A failure that will be retried is reported as `job:retrying` and counted
|
|
98
|
+
* separately; only a terminal failure increments `failedCount`, so the
|
|
99
|
+
* counters describe outcomes rather than attempts.
|
|
68
100
|
*/
|
|
69
|
-
export async function handleJobFailure(job, errorMessage,
|
|
70
|
-
const {
|
|
71
|
-
const failedJob =
|
|
101
|
+
export async function handleJobFailure(job, errorMessage, deps) {
|
|
102
|
+
const { jobs, emitter, counters, deadLetterStore } = deps;
|
|
103
|
+
const failedJob = updateJobState(job, JobStateEnum.FAILED, {
|
|
72
104
|
error: errorMessage,
|
|
73
105
|
failedAt: new Date().toISOString(),
|
|
74
106
|
});
|
|
75
107
|
jobs.set(job.id, failedJob);
|
|
76
|
-
counters.failedCount++;
|
|
77
108
|
emitter.emit("job:failed", {
|
|
78
109
|
job: failedJob,
|
|
79
110
|
error: new Error(errorMessage),
|
|
@@ -81,32 +112,48 @@ export async function handleJobFailure(job, errorMessage, processor, abortContro
|
|
|
81
112
|
const incrementedJob = incrementJobAttempt(failedJob);
|
|
82
113
|
jobs.set(job.id, incrementedJob);
|
|
83
114
|
if (shouldRetry(incrementedJob.attempt, incrementedJob.maxAttempts)) {
|
|
84
|
-
const retryingJob =
|
|
115
|
+
const retryingJob = updateJobState(incrementedJob, JobStateEnum.RETRYING);
|
|
85
116
|
jobs.set(job.id, retryingJob);
|
|
117
|
+
counters.retriedCount++;
|
|
86
118
|
emitter.emit("job:retrying", {
|
|
87
119
|
job: retryingJob,
|
|
88
120
|
attempt: incrementedJob.attempt,
|
|
89
121
|
});
|
|
90
|
-
const backoff = incrementedJob.backoff;
|
|
122
|
+
const backoff = resolveBackoff(incrementedJob.backoff);
|
|
91
123
|
const delay = calculateRetryDelay(incrementedJob.attempt, backoff);
|
|
92
|
-
setTimeout(() => {
|
|
124
|
+
const timer = setTimeout(() => {
|
|
125
|
+
if (deps.isDisposed()) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
93
128
|
const currentJob = jobs.get(job.id);
|
|
94
129
|
if (currentJob && currentJob.state === JobStateEnum.RETRYING) {
|
|
95
|
-
const waitingJob =
|
|
130
|
+
const waitingJob = updateJobState(currentJob, JobStateEnum.WAITING, {
|
|
96
131
|
error: undefined,
|
|
97
132
|
failedAt: undefined,
|
|
98
133
|
});
|
|
99
134
|
jobs.set(job.id, waitingJob);
|
|
100
135
|
}
|
|
101
136
|
}, delay);
|
|
137
|
+
// `unref` keeps a pending retry from holding the process open; the
|
|
138
|
+
// queue clears the timer explicitly on close.
|
|
139
|
+
timer.unref?.();
|
|
140
|
+
deps.registerRetryTimer(job.id, timer);
|
|
141
|
+
return;
|
|
102
142
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
143
|
+
counters.failedCount++;
|
|
144
|
+
counters.deadLetteredCount++;
|
|
145
|
+
const maxAttemptsError = new JobMaxAttemptsError(job.id, incrementedJob.attempt, incrementedJob.maxAttempts, { queueName: job.queueName });
|
|
146
|
+
try {
|
|
147
|
+
await moveToDeadLetter(deadLetterStore, incrementedJob, maxAttemptsError, {
|
|
148
|
+
reason: errorMessage,
|
|
108
149
|
});
|
|
109
|
-
jobs.set(job.id, deadLetterJob);
|
|
110
150
|
}
|
|
151
|
+
catch {
|
|
152
|
+
// A dead letter store that rejects must not mask the original
|
|
153
|
+
// failure or strand the job in `failed`.
|
|
154
|
+
}
|
|
155
|
+
const deadLetterJob = updateJobState(incrementedJob, JobStateEnum.DEAD_LETTER, { error: maxAttemptsError.message });
|
|
156
|
+
jobs.set(job.id, deadLetterJob);
|
|
157
|
+
deps.onSettled?.(deadLetterJob);
|
|
111
158
|
}
|
|
112
159
|
//# sourceMappingURL=inMemoryQueue.processing.js.map
|
|
@@ -1,10 +1,26 @@
|
|
|
1
1
|
import type { Job } from "../job/job.type.js";
|
|
2
|
+
import type { JobId } from "../jobTypes/jobTypes.type.js";
|
|
2
3
|
/**
|
|
3
4
|
* Schedule a job for future execution.
|
|
5
|
+
*
|
|
6
|
+
* The timer is registered so the queue can clear it on close, and
|
|
7
|
+
* unreferenced so a scheduled job never by itself keeps the process
|
|
8
|
+
* alive.
|
|
4
9
|
*/
|
|
5
|
-
export declare function scheduleJob<TData>(job: Job<TData>, scheduledTimers: Map<
|
|
10
|
+
export declare function scheduleJob<TData>(job: Job<TData>, scheduledTimers: Map<JobId, ReturnType<typeof setTimeout>>, jobs: Map<string, Job<TData>>): void;
|
|
11
|
+
/**
|
|
12
|
+
* Promotes scheduled jobs whose time has arrived.
|
|
13
|
+
*
|
|
14
|
+
* This backstops the per-job timers: a timer that was never armed, or
|
|
15
|
+
* that was delayed by a blocked event loop, still gets promoted on the
|
|
16
|
+
* next poll.
|
|
17
|
+
*/
|
|
18
|
+
export declare function promoteDueScheduledJobs<TData>(jobs: Map<string, Job<TData>>, scheduledTimers?: Map<JobId, ReturnType<typeof setTimeout>>): void;
|
|
6
19
|
/**
|
|
7
20
|
* Promote scheduled jobs that are ready to run.
|
|
21
|
+
*
|
|
22
|
+
* @deprecated Use {@link promoteDueScheduledJobs}, which also clears the
|
|
23
|
+
* timer belonging to a promoted job.
|
|
8
24
|
*/
|
|
9
25
|
export declare function scheduleDelayedJobs<TData>(jobs: Map<string, Job<TData>>): void;
|
|
10
26
|
//# sourceMappingURL=inMemoryQueue.scheduling.d.ts.map
|
|
@@ -1,38 +1,70 @@
|
|
|
1
1
|
import { updateJobState } from "../job/job.core.js";
|
|
2
2
|
import { JobState as JobStateEnum } from "../jobTypes/jobTypes.type.js";
|
|
3
|
+
import { MAX_TIMER_DELAY } from "../retryPolicy/retryPolicy.core.js";
|
|
3
4
|
/**
|
|
4
5
|
* Schedule a job for future execution.
|
|
6
|
+
*
|
|
7
|
+
* The timer is registered so the queue can clear it on close, and
|
|
8
|
+
* unreferenced so a scheduled job never by itself keeps the process
|
|
9
|
+
* alive.
|
|
5
10
|
*/
|
|
6
11
|
export function scheduleJob(job, scheduledTimers, jobs) {
|
|
7
12
|
if (!job.scheduledAt) {
|
|
8
13
|
return;
|
|
9
14
|
}
|
|
10
15
|
const scheduledTime = new Date(job.scheduledAt).getTime();
|
|
11
|
-
|
|
12
|
-
|
|
16
|
+
if (Number.isNaN(scheduledTime)) {
|
|
17
|
+
// An unparseable schedule would otherwise fire immediately via NaN
|
|
18
|
+
// coercion; promote the job instead of guessing at a delay.
|
|
19
|
+
jobs.set(job.id, updateJobState(job, JobStateEnum.WAITING));
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const delay = Math.min(Math.max(0, scheduledTime - Date.now()), MAX_TIMER_DELAY);
|
|
23
|
+
const existing = scheduledTimers.get(job.id);
|
|
24
|
+
if (existing) {
|
|
25
|
+
clearTimeout(existing);
|
|
26
|
+
}
|
|
13
27
|
const timer = setTimeout(() => {
|
|
14
28
|
scheduledTimers.delete(job.id);
|
|
15
29
|
const currentJob = jobs.get(job.id);
|
|
16
30
|
if (currentJob && currentJob.state === JobStateEnum.SCHEDULED) {
|
|
17
|
-
|
|
18
|
-
jobs.set(job.id, waitingJob);
|
|
31
|
+
jobs.set(job.id, updateJobState(currentJob, JobStateEnum.WAITING));
|
|
19
32
|
}
|
|
20
33
|
}, delay);
|
|
34
|
+
timer.unref?.();
|
|
21
35
|
scheduledTimers.set(job.id, timer);
|
|
22
36
|
}
|
|
23
37
|
/**
|
|
24
|
-
*
|
|
38
|
+
* Promotes scheduled jobs whose time has arrived.
|
|
39
|
+
*
|
|
40
|
+
* This backstops the per-job timers: a timer that was never armed, or
|
|
41
|
+
* that was delayed by a blocked event loop, still gets promoted on the
|
|
42
|
+
* next poll.
|
|
25
43
|
*/
|
|
26
|
-
export function
|
|
44
|
+
export function promoteDueScheduledJobs(jobs, scheduledTimers) {
|
|
27
45
|
const now = Date.now();
|
|
28
46
|
for (const job of jobs.values()) {
|
|
29
|
-
if (job.state
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
47
|
+
if (job.state !== JobStateEnum.SCHEDULED || !job.scheduledAt) {
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const scheduledTime = new Date(job.scheduledAt).getTime();
|
|
51
|
+
if (Number.isNaN(scheduledTime) || scheduledTime <= now) {
|
|
52
|
+
jobs.set(job.id, updateJobState(job, JobStateEnum.WAITING));
|
|
53
|
+
const timer = scheduledTimers?.get(job.id);
|
|
54
|
+
if (timer) {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
scheduledTimers?.delete(job.id);
|
|
34
57
|
}
|
|
35
58
|
}
|
|
36
59
|
}
|
|
37
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Promote scheduled jobs that are ready to run.
|
|
63
|
+
*
|
|
64
|
+
* @deprecated Use {@link promoteDueScheduledJobs}, which also clears the
|
|
65
|
+
* timer belonging to a promoted job.
|
|
66
|
+
*/
|
|
67
|
+
export function scheduleDelayedJobs(jobs) {
|
|
68
|
+
promoteDueScheduledJobs(jobs);
|
|
69
|
+
}
|
|
38
70
|
//# sourceMappingURL=inMemoryQueue.scheduling.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -16,7 +16,6 @@ export * from "./processor/index.js";
|
|
|
16
16
|
export * from "./queue/index.js";
|
|
17
17
|
export * from "./queueManager/index.js";
|
|
18
18
|
export * from "./queueRegistry/index.js";
|
|
19
|
-
export * from "./queueEvents/index.js";
|
|
20
19
|
export * from "./queueEmitter/index.js";
|
|
21
20
|
export * from "./retryPolicy/index.js";
|
|
22
21
|
export * from "./serializer/index.js";
|
package/dist/index.js
CHANGED
|
@@ -25,8 +25,6 @@ export * from "./queue/index.js";
|
|
|
25
25
|
export * from "./queueManager/index.js";
|
|
26
26
|
// Queue registry
|
|
27
27
|
export * from "./queueRegistry/index.js";
|
|
28
|
-
// Queue events
|
|
29
|
-
export * from "./queueEvents/index.js";
|
|
30
28
|
// Queue emitter
|
|
31
29
|
export * from "./queueEmitter/index.js";
|
|
32
30
|
// Retry policy
|
package/dist/job/job.core.js
CHANGED
|
@@ -1,39 +1,47 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { createJobId, JobState as JobStateEnum, } from "../jobTypes/jobTypes.type.js";
|
|
3
|
+
import { mergeJobOptions } from "../jobOptions/jobOptions.core.js";
|
|
3
4
|
/**
|
|
4
5
|
* Creates a new job instance.
|
|
5
6
|
*/
|
|
6
7
|
export function createJob(input, id) {
|
|
7
8
|
const now = new Date().toISOString();
|
|
8
|
-
|
|
9
|
-
|
|
9
|
+
// Defaults live in `mergeJobOptions`; duplicating them here is how
|
|
10
|
+
// `DEFAULT_JOB_OPTIONS.timeout` came to be exported, documented, and
|
|
11
|
+
// ignored by every job this factory produced.
|
|
12
|
+
const options = mergeJobOptions(input.options);
|
|
13
|
+
const jobId = id ?? createJobId(`job_${Date.now()}_${randomUUID().replace(/-/g, "")}`);
|
|
10
14
|
return {
|
|
11
15
|
id: jobId,
|
|
12
16
|
name: input.name,
|
|
13
17
|
queueName: input.queueName,
|
|
14
18
|
data: input.data,
|
|
15
|
-
state: options
|
|
19
|
+
state: options.delay || options.scheduledAt
|
|
16
20
|
? JobStateEnum.SCHEDULED
|
|
17
21
|
: JobStateEnum.WAITING,
|
|
18
22
|
attempt: 0,
|
|
19
|
-
maxAttempts: options
|
|
20
|
-
priority: options
|
|
23
|
+
maxAttempts: options.attempts,
|
|
24
|
+
priority: options.priority ?? 50,
|
|
21
25
|
createdAt: now,
|
|
22
26
|
updatedAt: now,
|
|
23
|
-
scheduledAt: options
|
|
27
|
+
scheduledAt: options.scheduledAt
|
|
24
28
|
? options.scheduledAt.toISOString()
|
|
25
|
-
: options
|
|
29
|
+
: options.delay
|
|
26
30
|
? new Date(Date.now() + options.delay).toISOString()
|
|
27
31
|
: undefined,
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
// Deliberately the caller's own value, not the merged default: a job
|
|
33
|
+
// that carries no timeout is what lets `WorkerOptions.timeoutMs` and the
|
|
34
|
+
// queue-wide fallback apply.
|
|
35
|
+
timeoutMs: input.options?.timeout,
|
|
36
|
+
deduplicationKey: options.deduplicationKey,
|
|
37
|
+
metadata: options.metadata,
|
|
38
|
+
backoff: options.backoff
|
|
32
39
|
? {
|
|
33
40
|
type: options.backoff.type,
|
|
34
41
|
delay: options.backoff.delay,
|
|
35
42
|
maxDelay: options.backoff.maxDelay,
|
|
36
43
|
multiplier: options.backoff.multiplier,
|
|
44
|
+
jitter: options.backoff.jitter,
|
|
37
45
|
}
|
|
38
46
|
: undefined,
|
|
39
47
|
};
|
package/dist/job/job.type.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Timestamp } from "@zudojs/constants";
|
|
2
|
-
import type { JobId, JobName, JobState, JobPriority,
|
|
2
|
+
import type { JobId, JobName, JobState, JobPriority, BackoffStrategy } from "../jobTypes/jobTypes.type.js";
|
|
3
3
|
import type { JobOptions } from "../jobOptions/jobOptions.type.js";
|
|
4
4
|
/**
|
|
5
5
|
* A unit of work in the queue system.
|
|
@@ -43,10 +43,11 @@ export interface Job<TData = unknown> {
|
|
|
43
43
|
readonly metadata?: Record<string, unknown>;
|
|
44
44
|
/** Backoff configuration. */
|
|
45
45
|
readonly backoff?: {
|
|
46
|
-
readonly type:
|
|
46
|
+
readonly type: BackoffStrategy;
|
|
47
47
|
readonly delay: number;
|
|
48
48
|
readonly maxDelay?: number;
|
|
49
49
|
readonly multiplier?: number;
|
|
50
|
+
readonly jitter?: "none" | "full" | "equal";
|
|
50
51
|
};
|
|
51
52
|
}
|
|
52
53
|
/**
|
|
@@ -1,16 +1,25 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { BackoffStrategy } from "../jobTypes/jobTypes.type.js";
|
|
2
2
|
/**
|
|
3
3
|
* Backoff configuration for retry attempts.
|
|
4
4
|
*/
|
|
5
5
|
export interface BackoffOptions {
|
|
6
|
-
/** Backoff strategy type. */
|
|
7
|
-
readonly type:
|
|
6
|
+
/** Backoff strategy type. Accepts `BackoffType` or its string value. */
|
|
7
|
+
readonly type: BackoffStrategy;
|
|
8
8
|
/** Initial delay in milliseconds. */
|
|
9
9
|
readonly delay: number;
|
|
10
10
|
/** Maximum delay in milliseconds (for exponential). */
|
|
11
11
|
readonly maxDelay?: number;
|
|
12
12
|
/** Multiplier for exponential backoff. */
|
|
13
13
|
readonly multiplier?: number;
|
|
14
|
+
/**
|
|
15
|
+
* Randomisation applied to the computed delay.
|
|
16
|
+
*
|
|
17
|
+
* `"full"` spreads retries uniformly over `[0, delay]`, `"equal"` keeps
|
|
18
|
+
* half the delay fixed, `"none"` (the default) leaves the delay
|
|
19
|
+
* deterministic. Jitter prevents jobs that failed together from
|
|
20
|
+
* retrying together.
|
|
21
|
+
*/
|
|
22
|
+
readonly jitter?: "none" | "full" | "equal";
|
|
14
23
|
}
|
|
15
24
|
/**
|
|
16
25
|
* Options for adding a job to the queue.
|
package/dist/jobTypes/index.d.ts
CHANGED
|
@@ -5,5 +5,5 @@
|
|
|
5
5
|
* as well as lifecycle state enums and priority constants.
|
|
6
6
|
*/
|
|
7
7
|
export { JobState, JobPriorityLevels, BackoffType, WorkerState, createJobId, createQueueName, createJobName, isJobId, isQueueName, isJobName, } from "./jobTypes.type.js";
|
|
8
|
-
export type { JobId, QueueName, JobName, JobPriority, } from "./jobTypes.type.js";
|
|
8
|
+
export type { JobId, QueueName, JobName, JobPriority, BackoffStrategy, } from "./jobTypes.type.js";
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -51,6 +51,15 @@ export declare enum BackoffType {
|
|
|
51
51
|
FIXED = "fixed",
|
|
52
52
|
EXPONENTIAL = "exponential"
|
|
53
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* A backoff strategy as either the enum member or its string value.
|
|
56
|
+
*
|
|
57
|
+
* Job options routinely arrive from JSON — a config file, a broker payload,
|
|
58
|
+
* an HTTP body — where `"fixed"` is the only spelling available. The runtime
|
|
59
|
+
* has always compared against the string values, but a bare `BackoffType`
|
|
60
|
+
* annotation rejected them, so correct calls failed to typecheck.
|
|
61
|
+
*/
|
|
62
|
+
export type BackoffStrategy = `${BackoffType}`;
|
|
54
63
|
/**
|
|
55
64
|
* Worker lifecycle states.
|
|
56
65
|
*/
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import type { QueueMiddleware } from "./middleware.type.js";
|
|
2
2
|
/**
|
|
3
3
|
* Creates a middleware chain from an array of middleware.
|
|
4
|
+
*
|
|
5
|
+
* Each middleware may call `next()` exactly once. A second call throws
|
|
6
|
+
* rather than silently re-entering the chain and running the processor
|
|
7
|
+
* twice.
|
|
4
8
|
*/
|
|
5
9
|
export declare function createMiddlewareChain(middleware: QueueMiddleware[]): QueueMiddleware;
|
|
6
10
|
/**
|
|
@@ -11,6 +15,11 @@ export declare function createLoggingMiddleware(logger?: {
|
|
|
11
15
|
}): QueueMiddleware;
|
|
12
16
|
/**
|
|
13
17
|
* Timeout middleware for queue processing.
|
|
18
|
+
*
|
|
19
|
+
* The timer is always cleared once the race settles, so a completed job
|
|
20
|
+
* never leaves an armed timer holding the event loop open. When the
|
|
21
|
+
* timeout wins, `onTimeout` is invoked so the caller can abort the job's
|
|
22
|
+
* `AbortSignal` and let a cooperative processor stop its own work.
|
|
14
23
|
*/
|
|
15
|
-
export declare function createTimeoutMiddleware(timeoutMs: number): QueueMiddleware;
|
|
24
|
+
export declare function createTimeoutMiddleware(timeoutMs: number, onTimeout?: () => void): QueueMiddleware;
|
|
16
25
|
//# sourceMappingURL=middleware.core.d.ts.map
|
|
@@ -1,22 +1,30 @@
|
|
|
1
1
|
import { JobTimeoutError } from "@zudojs/errors";
|
|
2
|
+
import { MAX_TIMER_DELAY } from "../retryPolicy/retryPolicy.core.js";
|
|
2
3
|
/**
|
|
3
4
|
* Creates a middleware chain from an array of middleware.
|
|
5
|
+
*
|
|
6
|
+
* Each middleware may call `next()` exactly once. A second call throws
|
|
7
|
+
* rather than silently re-entering the chain and running the processor
|
|
8
|
+
* twice.
|
|
4
9
|
*/
|
|
5
10
|
export function createMiddlewareChain(middleware) {
|
|
6
11
|
return async (ctx) => {
|
|
7
12
|
let index = -1;
|
|
8
|
-
const dispatch = async () => {
|
|
9
|
-
index
|
|
10
|
-
|
|
11
|
-
|
|
13
|
+
const dispatch = async (position) => {
|
|
14
|
+
if (position <= index) {
|
|
15
|
+
throw new Error(`Queue middleware at index ${position} called next() more than once.`);
|
|
16
|
+
}
|
|
17
|
+
index = position;
|
|
18
|
+
if (position < middleware.length) {
|
|
19
|
+
const current = middleware[position];
|
|
12
20
|
return current({
|
|
13
21
|
...ctx,
|
|
14
|
-
next: dispatch,
|
|
22
|
+
next: () => dispatch(position + 1),
|
|
15
23
|
});
|
|
16
24
|
}
|
|
17
25
|
return ctx.next();
|
|
18
26
|
};
|
|
19
|
-
return dispatch();
|
|
27
|
+
return dispatch(0);
|
|
20
28
|
};
|
|
21
29
|
}
|
|
22
30
|
/**
|
|
@@ -58,19 +66,34 @@ export function createLoggingMiddleware(logger) {
|
|
|
58
66
|
}
|
|
59
67
|
/**
|
|
60
68
|
* Timeout middleware for queue processing.
|
|
69
|
+
*
|
|
70
|
+
* The timer is always cleared once the race settles, so a completed job
|
|
71
|
+
* never leaves an armed timer holding the event loop open. When the
|
|
72
|
+
* timeout wins, `onTimeout` is invoked so the caller can abort the job's
|
|
73
|
+
* `AbortSignal` and let a cooperative processor stop its own work.
|
|
61
74
|
*/
|
|
62
|
-
export function createTimeoutMiddleware(timeoutMs) {
|
|
75
|
+
export function createTimeoutMiddleware(timeoutMs, onTimeout) {
|
|
76
|
+
const delay = Math.min(Math.max(0, timeoutMs), MAX_TIMER_DELAY);
|
|
63
77
|
return async (ctx) => {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
78
|
+
let timer;
|
|
79
|
+
try {
|
|
80
|
+
return await Promise.race([
|
|
81
|
+
ctx.next(),
|
|
82
|
+
new Promise((_, reject) => {
|
|
83
|
+
timer = setTimeout(() => {
|
|
84
|
+
onTimeout?.();
|
|
85
|
+
reject(new JobTimeoutError(ctx.job.id, timeoutMs, {
|
|
86
|
+
queueName: ctx.job.queueName,
|
|
87
|
+
}));
|
|
88
|
+
}, delay);
|
|
89
|
+
}),
|
|
90
|
+
]);
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
if (timer !== undefined) {
|
|
94
|
+
clearTimeout(timer);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
74
97
|
};
|
|
75
98
|
}
|
|
76
99
|
//# sourceMappingURL=middleware.core.js.map
|
|
@@ -5,6 +5,6 @@
|
|
|
5
5
|
* managing job processors.
|
|
6
6
|
*/
|
|
7
7
|
export { createProcessorRegistry } from "./processor.core.js";
|
|
8
|
-
export { isProcessor } from "./processor.type.js";
|
|
8
|
+
export { assertProcessor, isProcessor } from "./processor.type.js";
|
|
9
9
|
export type { Processor, ProcessorInfo, ProcessorRegistry, } from "./processor.type.js";
|
|
10
10
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/processor/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { JobError } from "@zudojs/errors";
|
|
2
|
+
import { assertProcessor } from "./processor.type.js";
|
|
2
3
|
/**
|
|
3
4
|
* Creates a new ProcessorRegistry.
|
|
4
5
|
*/
|
|
@@ -7,6 +8,7 @@ export function createProcessorRegistry() {
|
|
|
7
8
|
const infoMap = new Map();
|
|
8
9
|
return {
|
|
9
10
|
register(jobName, processor, options) {
|
|
11
|
+
assertProcessor(processor, jobName);
|
|
10
12
|
if (processors.has(jobName)) {
|
|
11
13
|
throw new JobError(`Processor already registered for job "${jobName}".`, { jobId: jobName });
|
|
12
14
|
}
|
|
@@ -3,8 +3,14 @@ import type { JobContext } from "../jobContext/jobContext.type.js";
|
|
|
3
3
|
import type { JobResult } from "../jobResult/jobResult.type.js";
|
|
4
4
|
/**
|
|
5
5
|
* A function that processes a job.
|
|
6
|
+
*
|
|
7
|
+
* A processor may return a {@link JobResult} to report success or failure
|
|
8
|
+
* explicitly, a plain value to be carried on `job:completed`, or nothing at
|
|
9
|
+
* all. The queue narrows the three cases at runtime; the previous signature
|
|
10
|
+
* admitted only the first two, so a processor returning its own value — the
|
|
11
|
+
* common case — failed to typecheck against a queue that handled it happily.
|
|
6
12
|
*/
|
|
7
|
-
export type Processor<TData = unknown, TResult = unknown> = (job: Job<TData>, context: JobContext<TData>) => Promise<JobResult<TResult
|
|
13
|
+
export type Processor<TData = unknown, TResult = unknown> = (job: Job<TData>, context: JobContext<TData>) => Promise<JobResult<TResult> | TResult | void>;
|
|
8
14
|
/**
|
|
9
15
|
* Metadata about a registered processor.
|
|
10
16
|
*/
|
|
@@ -39,4 +45,15 @@ export interface ProcessorRegistry {
|
|
|
39
45
|
* Checks if a value is a valid Processor.
|
|
40
46
|
*/
|
|
41
47
|
export declare function isProcessor(value: unknown): value is Processor;
|
|
48
|
+
/**
|
|
49
|
+
* Assert that a value can actually process a job.
|
|
50
|
+
*
|
|
51
|
+
* Registering a non-function succeeded silently and failed much later, deep
|
|
52
|
+
* inside job processing, as a job that "failed" and was dead-lettered.
|
|
53
|
+
*
|
|
54
|
+
* @param value - The candidate processor.
|
|
55
|
+
* @param jobName - The job name it is being registered for.
|
|
56
|
+
* @throws {TypeError} when the value is not callable.
|
|
57
|
+
*/
|
|
58
|
+
export declare function assertProcessor(value: unknown, jobName: string): asserts value is Processor;
|
|
42
59
|
//# sourceMappingURL=processor.type.d.ts.map
|
|
@@ -4,4 +4,19 @@
|
|
|
4
4
|
export function isProcessor(value) {
|
|
5
5
|
return typeof value === "function";
|
|
6
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* Assert that a value can actually process a job.
|
|
9
|
+
*
|
|
10
|
+
* Registering a non-function succeeded silently and failed much later, deep
|
|
11
|
+
* inside job processing, as a job that "failed" and was dead-lettered.
|
|
12
|
+
*
|
|
13
|
+
* @param value - The candidate processor.
|
|
14
|
+
* @param jobName - The job name it is being registered for.
|
|
15
|
+
* @throws {TypeError} when the value is not callable.
|
|
16
|
+
*/
|
|
17
|
+
export function assertProcessor(value, jobName) {
|
|
18
|
+
if (!isProcessor(value)) {
|
|
19
|
+
throw new TypeError(`Processor for job "${jobName}" must be a function; received ${value === null ? "null" : typeof value}.`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
7
22
|
//# sourceMappingURL=processor.type.js.map
|