@boringnode/queue 0.0.1-alpha.2 → 0.0.1-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +145 -34
- package/build/chunk-5PDDRF5O.js +26 -0
- package/build/chunk-5PDDRF5O.js.map +1 -0
- package/build/chunk-CD45GT6E.js +357 -0
- package/build/chunk-CD45GT6E.js.map +1 -0
- package/build/chunk-HMGNQSSG.js +103 -0
- package/build/chunk-HMGNQSSG.js.map +1 -0
- package/build/index-C3_tlebh.d.ts +776 -0
- package/build/index.d.ts +352 -4
- package/build/index.js +526 -137
- package/build/index.js.map +1 -1
- package/build/src/contracts/adapter.d.ts +1 -1
- package/build/src/drivers/knex_adapter.d.ts +2 -1
- package/build/src/drivers/knex_adapter.js +76 -17
- package/build/src/drivers/knex_adapter.js.map +1 -1
- package/build/src/drivers/redis_adapter.d.ts +2 -2
- package/build/src/drivers/redis_adapter.js +74 -27
- package/build/src/drivers/redis_adapter.js.map +1 -1
- package/build/src/drivers/sync_adapter.d.ts +7 -3
- package/build/src/drivers/sync_adapter.js +27 -7
- package/build/src/drivers/sync_adapter.js.map +1 -1
- package/build/src/types/index.d.ts +1 -0
- package/build/src/types/index.js +1 -0
- package/build/src/types/index.js.map +1 -0
- package/build/src/types/main.d.ts +1 -1
- package/package.json +20 -15
- package/build/job-Bd_c2lFK.d.ts +0 -149
package/build/index.js
CHANGED
|
@@ -1,14 +1,237 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parse
|
|
3
|
+
} from "./chunk-5PDDRF5O.js";
|
|
4
|
+
import {
|
|
5
|
+
Locator,
|
|
6
|
+
QueueManager,
|
|
7
|
+
debug_default
|
|
8
|
+
} from "./chunk-CD45GT6E.js";
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_ERROR_RETRY_DELAY,
|
|
11
|
+
DEFAULT_IDLE_DELAY,
|
|
12
|
+
DEFAULT_PRIORITY,
|
|
13
|
+
DEFAULT_STALLED_INTERVAL,
|
|
14
|
+
DEFAULT_STALLED_THRESHOLD,
|
|
15
|
+
E_INVALID_BASE_DELAY,
|
|
16
|
+
E_INVALID_MAX_DELAY,
|
|
17
|
+
E_INVALID_MULTIPLIER,
|
|
18
|
+
E_JOB_MAX_ATTEMPTS_REACHED,
|
|
19
|
+
E_JOB_TIMEOUT,
|
|
20
|
+
exceptions_exports
|
|
21
|
+
} from "./chunk-HMGNQSSG.js";
|
|
22
|
+
|
|
23
|
+
// src/job_dispatcher.ts
|
|
24
|
+
import { randomUUID } from "crypto";
|
|
25
|
+
var JobDispatcher = class {
|
|
26
|
+
#name;
|
|
27
|
+
#payload;
|
|
28
|
+
#queue = "default";
|
|
29
|
+
#adapter;
|
|
30
|
+
#delay;
|
|
31
|
+
#priority;
|
|
32
|
+
/**
|
|
33
|
+
* Create a new job dispatcher.
|
|
34
|
+
*
|
|
35
|
+
* @param name - The job class name (used to locate the class at runtime)
|
|
36
|
+
* @param payload - The data to pass to the job
|
|
37
|
+
*/
|
|
38
|
+
constructor(name, payload) {
|
|
39
|
+
this.#name = name;
|
|
40
|
+
this.#payload = payload;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Set the target queue for this job.
|
|
44
|
+
*
|
|
45
|
+
* @param queue - Queue name (default: 'default')
|
|
46
|
+
* @returns This dispatcher for chaining
|
|
47
|
+
*
|
|
48
|
+
* @example
|
|
49
|
+
* ```typescript
|
|
50
|
+
* await SendEmailJob.dispatch(payload).toQueue('emails')
|
|
51
|
+
* ```
|
|
52
|
+
*/
|
|
53
|
+
toQueue(queue) {
|
|
54
|
+
this.#queue = queue;
|
|
55
|
+
return this;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Delay the job execution.
|
|
59
|
+
*
|
|
60
|
+
* The job will be stored in a delayed state and moved to pending
|
|
61
|
+
* after the delay expires.
|
|
62
|
+
*
|
|
63
|
+
* @param delay - Delay as milliseconds or duration string ('5s', '1h', '7d')
|
|
64
|
+
* @returns This dispatcher for chaining
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```typescript
|
|
68
|
+
* // Send reminder in 24 hours
|
|
69
|
+
* await ReminderJob.dispatch(payload).in('24h')
|
|
70
|
+
*
|
|
71
|
+
* // Process in 5 minutes
|
|
72
|
+
* await CleanupJob.dispatch(payload).in('5m')
|
|
73
|
+
* ```
|
|
74
|
+
*/
|
|
75
|
+
in(delay) {
|
|
76
|
+
this.#delay = delay;
|
|
77
|
+
return this;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Set the job priority.
|
|
81
|
+
*
|
|
82
|
+
* Lower numbers = higher priority. Jobs with lower priority values
|
|
83
|
+
* are processed before jobs with higher values.
|
|
84
|
+
*
|
|
85
|
+
* @param priority - Priority level (1-10, default: 5)
|
|
86
|
+
* @returns This dispatcher for chaining
|
|
87
|
+
*
|
|
88
|
+
* @example
|
|
89
|
+
* ```typescript
|
|
90
|
+
* // High priority job
|
|
91
|
+
* await UrgentJob.dispatch(payload).priority(1)
|
|
92
|
+
*
|
|
93
|
+
* // Low priority job
|
|
94
|
+
* await BackgroundJob.dispatch(payload).priority(10)
|
|
95
|
+
* ```
|
|
96
|
+
*/
|
|
97
|
+
priority(priority) {
|
|
98
|
+
this.#priority = priority;
|
|
99
|
+
return this;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Use a specific adapter for this job.
|
|
103
|
+
*
|
|
104
|
+
* @param adapter - Adapter name or factory function
|
|
105
|
+
* @returns This dispatcher for chaining
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```typescript
|
|
109
|
+
* // Use named adapter
|
|
110
|
+
* await Job.dispatch(payload).with('redis')
|
|
111
|
+
*
|
|
112
|
+
* // Use custom adapter instance
|
|
113
|
+
* await Job.dispatch(payload).with(() => new CustomAdapter())
|
|
114
|
+
* ```
|
|
115
|
+
*/
|
|
116
|
+
with(adapter) {
|
|
117
|
+
this.#adapter = adapter;
|
|
118
|
+
return this;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Dispatch the job to the queue.
|
|
122
|
+
*
|
|
123
|
+
* @returns The unique job ID
|
|
124
|
+
*
|
|
125
|
+
* @example
|
|
126
|
+
* ```typescript
|
|
127
|
+
* const jobId = await SendEmailJob.dispatch(payload).run()
|
|
128
|
+
* console.log(`Dispatched job: ${jobId}`)
|
|
129
|
+
* ```
|
|
130
|
+
*/
|
|
131
|
+
async run() {
|
|
132
|
+
const id = randomUUID();
|
|
133
|
+
debug_default("dispatching job %s with id %s using payload %s", this.#name, id, this.#payload);
|
|
134
|
+
const adapter = this.#getAdapterInstance();
|
|
135
|
+
const payload = {
|
|
136
|
+
id,
|
|
137
|
+
name: this.#name,
|
|
138
|
+
payload: this.#payload,
|
|
139
|
+
attempts: 0,
|
|
140
|
+
priority: this.#priority
|
|
141
|
+
};
|
|
142
|
+
if (this.#delay) {
|
|
143
|
+
const parsedDelay = parse(this.#delay);
|
|
144
|
+
await adapter.pushLaterOn(this.#queue, payload, parsedDelay);
|
|
145
|
+
} else {
|
|
146
|
+
await adapter.pushOn(this.#queue, payload);
|
|
147
|
+
}
|
|
148
|
+
return id;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Thenable implementation for auto-dispatch when awaited.
|
|
152
|
+
*
|
|
153
|
+
* Allows `await Job.dispatch(payload)` without explicit `.run()`.
|
|
154
|
+
*
|
|
155
|
+
* @param onFulfilled - Success callback
|
|
156
|
+
* @param onRejected - Error callback
|
|
157
|
+
* @returns Promise resolving to the job ID
|
|
158
|
+
*/
|
|
159
|
+
then(onFulfilled, onRejected) {
|
|
160
|
+
return this.run().then(onFulfilled, onRejected);
|
|
161
|
+
}
|
|
162
|
+
#getAdapterInstance() {
|
|
163
|
+
if (!this.#adapter) {
|
|
164
|
+
return QueueManager.use();
|
|
165
|
+
}
|
|
166
|
+
if (typeof this.#adapter === "string") {
|
|
167
|
+
return QueueManager.use(this.#adapter);
|
|
168
|
+
}
|
|
169
|
+
return this.#adapter();
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
|
|
1
173
|
// src/job.ts
|
|
2
|
-
import { JobDispatcher } from "#src/job_dispatcher";
|
|
3
174
|
var Job = class {
|
|
4
175
|
#payload;
|
|
176
|
+
#context;
|
|
177
|
+
/** Static options for this job class (queue, retries, timeout, etc.) */
|
|
5
178
|
static options = {};
|
|
179
|
+
/** The payload data passed to this job instance */
|
|
6
180
|
get payload() {
|
|
7
181
|
return this.#payload;
|
|
8
182
|
}
|
|
9
|
-
|
|
183
|
+
/**
|
|
184
|
+
* Context information for the current job execution.
|
|
185
|
+
*
|
|
186
|
+
* Provides metadata such as job ID, current attempt number,
|
|
187
|
+
* queue name, priority, and timing information.
|
|
188
|
+
*
|
|
189
|
+
* @example
|
|
190
|
+
* ```typescript
|
|
191
|
+
* async execute() {
|
|
192
|
+
* if (this.context.attempt > 1) {
|
|
193
|
+
* console.log(`Retry attempt ${this.context.attempt}`)
|
|
194
|
+
* }
|
|
195
|
+
* console.log(`Processing job ${this.context.jobId} on queue ${this.context.queue}`)
|
|
196
|
+
* }
|
|
197
|
+
* ```
|
|
198
|
+
*/
|
|
199
|
+
get context() {
|
|
200
|
+
return this.#context;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Create a new job instance.
|
|
204
|
+
*
|
|
205
|
+
* @param payload - The data to be processed by this job
|
|
206
|
+
* @param context - The job execution context (provided by the worker)
|
|
207
|
+
*/
|
|
208
|
+
constructor(payload, context) {
|
|
10
209
|
this.#payload = payload;
|
|
210
|
+
this.#context = Object.freeze(context);
|
|
11
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* Dispatch this job to the queue.
|
|
214
|
+
*
|
|
215
|
+
* Returns a JobDispatcher for fluent configuration before dispatching.
|
|
216
|
+
* The job is not actually dispatched until `.run()` is called or the
|
|
217
|
+
* dispatcher is awaited.
|
|
218
|
+
*
|
|
219
|
+
* @param payload - The data to pass to the job
|
|
220
|
+
* @returns A JobDispatcher for fluent configuration
|
|
221
|
+
*
|
|
222
|
+
* @example
|
|
223
|
+
* ```typescript
|
|
224
|
+
* // Simple dispatch
|
|
225
|
+
* await SendEmailJob.dispatch({ to: 'user@example.com', subject: 'Hello' })
|
|
226
|
+
*
|
|
227
|
+
* // With options
|
|
228
|
+
* await SendEmailJob.dispatch({ to: 'user@example.com' })
|
|
229
|
+
* .toQueue('high-priority')
|
|
230
|
+
* .priority(1)
|
|
231
|
+
* .in('5m')
|
|
232
|
+
* .run()
|
|
233
|
+
* ```
|
|
234
|
+
*/
|
|
12
235
|
static dispatch(payload) {
|
|
13
236
|
const dispatcher = new JobDispatcher(
|
|
14
237
|
this.jobName,
|
|
@@ -28,50 +251,164 @@ var Job = class {
|
|
|
28
251
|
};
|
|
29
252
|
|
|
30
253
|
// src/worker.ts
|
|
31
|
-
import { randomUUID } from "crypto";
|
|
254
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
32
255
|
import { setTimeout } from "timers/promises";
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
256
|
+
|
|
257
|
+
// src/job_pool.ts
|
|
258
|
+
var JobPool = class {
|
|
259
|
+
#activeJobs = /* @__PURE__ */ new Map();
|
|
260
|
+
/** Number of currently running jobs */
|
|
261
|
+
get size() {
|
|
262
|
+
return this.#activeJobs.size;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Check if the pool has no running jobs.
|
|
266
|
+
*
|
|
267
|
+
* @returns True if no jobs are running
|
|
268
|
+
*/
|
|
269
|
+
isEmpty() {
|
|
270
|
+
return this.#activeJobs.size === 0;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Check if the pool can accept more jobs.
|
|
274
|
+
*
|
|
275
|
+
* @param concurrency - Maximum number of concurrent jobs
|
|
276
|
+
* @returns True if there's room for more jobs
|
|
277
|
+
*/
|
|
278
|
+
hasCapacity(concurrency) {
|
|
279
|
+
return this.#activeJobs.size < concurrency;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Add a job to the pool.
|
|
283
|
+
*
|
|
284
|
+
* @param job - The acquired job data
|
|
285
|
+
* @param queue - The queue the job came from
|
|
286
|
+
* @param promise - Promise that resolves when the job completes
|
|
287
|
+
*/
|
|
288
|
+
add(job, queue, promise) {
|
|
289
|
+
this.#activeJobs.set(job.id, { promise, job, queue });
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Wait for the next job to complete and return it.
|
|
293
|
+
*
|
|
294
|
+
* Uses `Promise.race()` internally, so the fastest job wins.
|
|
295
|
+
* The completed job is removed from the pool.
|
|
296
|
+
*
|
|
297
|
+
* @returns The first job to complete (success or failure)
|
|
298
|
+
*/
|
|
299
|
+
async waitForNextCompletion() {
|
|
300
|
+
const completedJobId = await Promise.race(
|
|
301
|
+
[...this.#activeJobs.entries()].map(async ([id, { promise }]) => {
|
|
302
|
+
try {
|
|
303
|
+
await promise;
|
|
304
|
+
} catch {
|
|
305
|
+
}
|
|
306
|
+
return id;
|
|
307
|
+
})
|
|
308
|
+
);
|
|
309
|
+
const completed = this.#activeJobs.get(completedJobId);
|
|
310
|
+
this.#activeJobs.delete(completedJobId);
|
|
311
|
+
return completed;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Wait for all running jobs to complete.
|
|
315
|
+
*
|
|
316
|
+
* Used during graceful shutdown to ensure no jobs are abandoned.
|
|
317
|
+
* Clears the pool after all jobs finish.
|
|
318
|
+
*/
|
|
319
|
+
async drain() {
|
|
320
|
+
const promises = [...this.#activeJobs.values()].map(async ({ promise }) => {
|
|
321
|
+
try {
|
|
322
|
+
await promise;
|
|
323
|
+
} catch {
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
await Promise.all(promises);
|
|
327
|
+
this.#activeJobs.clear();
|
|
328
|
+
}
|
|
329
|
+
};
|
|
330
|
+
|
|
331
|
+
// src/worker.ts
|
|
39
332
|
var Worker = class {
|
|
40
333
|
#id;
|
|
41
334
|
#config;
|
|
335
|
+
#idleDelay;
|
|
336
|
+
#stalledInterval;
|
|
337
|
+
#stalledThreshold;
|
|
338
|
+
#maxStalledCount;
|
|
339
|
+
#concurrency;
|
|
340
|
+
#gracefulShutdown;
|
|
341
|
+
#onShutdownSignal;
|
|
42
342
|
#adapter;
|
|
43
343
|
#running = false;
|
|
44
344
|
#initialized = false;
|
|
45
345
|
#generator;
|
|
46
346
|
#pool;
|
|
347
|
+
#lastStalledCheck = 0;
|
|
348
|
+
#shutdownHandler;
|
|
349
|
+
/** Unique identifier for this worker instance */
|
|
47
350
|
get id() {
|
|
48
351
|
return this.#id;
|
|
49
352
|
}
|
|
353
|
+
/**
|
|
354
|
+
* Create a new worker instance.
|
|
355
|
+
*
|
|
356
|
+
* @param config - Queue configuration including adapter and worker settings
|
|
357
|
+
*/
|
|
50
358
|
constructor(config) {
|
|
51
359
|
this.#config = config;
|
|
52
|
-
this.#id =
|
|
53
|
-
|
|
360
|
+
this.#id = randomUUID2();
|
|
361
|
+
this.#idleDelay = parse(config.worker?.idleDelay ?? DEFAULT_IDLE_DELAY);
|
|
362
|
+
this.#stalledInterval = parse(config.worker?.stalledInterval ?? DEFAULT_STALLED_INTERVAL);
|
|
363
|
+
this.#stalledThreshold = parse(config.worker?.stalledThreshold ?? DEFAULT_STALLED_THRESHOLD);
|
|
364
|
+
this.#maxStalledCount = config.worker?.maxStalledCount ?? 1;
|
|
365
|
+
this.#concurrency = config.worker?.concurrency ?? 1;
|
|
366
|
+
this.#gracefulShutdown = config.worker?.gracefulShutdown ?? true;
|
|
367
|
+
this.#onShutdownSignal = config.worker?.onShutdownSignal;
|
|
368
|
+
debug_default("created worker with id %s and config %O", this.#id, config);
|
|
54
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* Initialize the worker (called automatically by `start()`).
|
|
372
|
+
*
|
|
373
|
+
* Sets up the QueueManager and adapter connection.
|
|
374
|
+
*/
|
|
55
375
|
async init() {
|
|
56
376
|
if (this.#initialized) {
|
|
57
377
|
return;
|
|
58
378
|
}
|
|
59
|
-
|
|
379
|
+
debug_default("initializing worker %s", this.#id);
|
|
60
380
|
await QueueManager.init(this.#config);
|
|
61
381
|
this.#adapter = QueueManager.use();
|
|
62
382
|
this.#adapter.setWorkerId(this.#id);
|
|
63
383
|
this.#initialized = true;
|
|
64
|
-
|
|
384
|
+
debug_default("worker %s initialized", this.#id);
|
|
65
385
|
}
|
|
386
|
+
/**
|
|
387
|
+
* Start processing jobs from the specified queues.
|
|
388
|
+
*
|
|
389
|
+
* This method blocks until the worker is stopped (via `stop()` or signal).
|
|
390
|
+
* Jobs are processed concurrently up to the configured concurrency limit.
|
|
391
|
+
*
|
|
392
|
+
* @param queues - Queue names to process (default: ['default'])
|
|
393
|
+
*
|
|
394
|
+
* @example
|
|
395
|
+
* ```typescript
|
|
396
|
+
* // Process single queue
|
|
397
|
+
* await worker.start()
|
|
398
|
+
*
|
|
399
|
+
* // Process multiple queues (priority order)
|
|
400
|
+
* await worker.start(['high-priority', 'default', 'low-priority'])
|
|
401
|
+
* ```
|
|
402
|
+
*/
|
|
66
403
|
async start(queues = ["default"]) {
|
|
67
404
|
await this.init();
|
|
68
405
|
if (this.#running) {
|
|
69
|
-
|
|
406
|
+
debug_default("worker %s is already running", this.#id);
|
|
70
407
|
return;
|
|
71
408
|
}
|
|
72
409
|
this.#running = true;
|
|
73
|
-
|
|
74
|
-
|
|
410
|
+
debug_default("starting worker %s on queues: %O", this.#id, queues);
|
|
411
|
+
this.#setupGracefulShutdown();
|
|
75
412
|
for await (const cycle of this.process(queues)) {
|
|
76
413
|
if (["started", "completed"].includes(cycle.type)) {
|
|
77
414
|
continue;
|
|
@@ -79,25 +416,53 @@ var Worker = class {
|
|
|
79
416
|
if (["idle", "error"].includes(cycle.type)) {
|
|
80
417
|
const delay = parse(cycle.suggestedDelay);
|
|
81
418
|
if (cycle.type === "error") {
|
|
82
|
-
|
|
419
|
+
debug_default("worker %s encountered an error: %O", this.#id, cycle.error);
|
|
83
420
|
} else {
|
|
84
|
-
|
|
421
|
+
debug_default("worker %s is idle, waiting for %dms", this.#id, delay);
|
|
85
422
|
}
|
|
86
423
|
await setTimeout(delay);
|
|
87
424
|
}
|
|
88
425
|
}
|
|
89
426
|
}
|
|
427
|
+
/**
|
|
428
|
+
* Stop the worker gracefully.
|
|
429
|
+
*
|
|
430
|
+
* Waits for all running jobs to complete before shutting down.
|
|
431
|
+
* Called automatically on SIGINT/SIGTERM if gracefulShutdown is enabled.
|
|
432
|
+
*/
|
|
90
433
|
async stop() {
|
|
91
|
-
|
|
434
|
+
debug_default("stopping worker %s", this.#id);
|
|
92
435
|
this.#running = false;
|
|
93
436
|
if (this.#pool) {
|
|
94
|
-
|
|
437
|
+
debug_default("worker %s: waiting for %d running jobs to complete", this.#id, this.#pool.size);
|
|
95
438
|
await this.#pool.drain();
|
|
96
439
|
}
|
|
97
440
|
if (this.#adapter) {
|
|
98
441
|
await this.#adapter.destroy();
|
|
99
442
|
}
|
|
443
|
+
this.#removeShutdownHandlers();
|
|
100
444
|
}
|
|
445
|
+
/**
|
|
446
|
+
* Process a single cycle and return the result.
|
|
447
|
+
*
|
|
448
|
+
* Useful for testing or when you need fine-grained control.
|
|
449
|
+
* Each cycle may start new jobs, complete a job, or return idle.
|
|
450
|
+
*
|
|
451
|
+
* @param queues - Queue names to process
|
|
452
|
+
* @returns The cycle result, or null if the worker was stopped
|
|
453
|
+
*
|
|
454
|
+
* @example
|
|
455
|
+
* ```typescript
|
|
456
|
+
* const worker = new Worker(config)
|
|
457
|
+
*
|
|
458
|
+
* // Process cycles manually
|
|
459
|
+
* let cycle = await worker.processCycle(['default'])
|
|
460
|
+
* while (cycle) {
|
|
461
|
+
* console.log('Cycle:', cycle.type)
|
|
462
|
+
* cycle = await worker.processCycle(['default'])
|
|
463
|
+
* }
|
|
464
|
+
* ```
|
|
465
|
+
*/
|
|
101
466
|
async processCycle(queues) {
|
|
102
467
|
await this.init();
|
|
103
468
|
this.#running = true;
|
|
@@ -111,26 +476,58 @@ var Worker = class {
|
|
|
111
476
|
}
|
|
112
477
|
return result.value;
|
|
113
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* Generator that yields worker cycle events.
|
|
481
|
+
*
|
|
482
|
+
* Low-level API for processing jobs. Yields events for:
|
|
483
|
+
* - `started`: A new job began execution
|
|
484
|
+
* - `completed`: A job finished (success or failure)
|
|
485
|
+
* - `idle`: No jobs available, suggest waiting
|
|
486
|
+
* - `error`: An error occurred during processing
|
|
487
|
+
*
|
|
488
|
+
* @param queues - Queue names to process
|
|
489
|
+
* @yields WorkerCycle events
|
|
490
|
+
*
|
|
491
|
+
* @example
|
|
492
|
+
* ```typescript
|
|
493
|
+
* for await (const cycle of worker.process(['default'])) {
|
|
494
|
+
* switch (cycle.type) {
|
|
495
|
+
* case 'started':
|
|
496
|
+
* console.log(`Started job ${cycle.job.id}`)
|
|
497
|
+
* break
|
|
498
|
+
* case 'completed':
|
|
499
|
+
* console.log(`Completed job ${cycle.job.id}`)
|
|
500
|
+
* break
|
|
501
|
+
* case 'idle':
|
|
502
|
+
* await sleep(cycle.suggestedDelay)
|
|
503
|
+
* break
|
|
504
|
+
* }
|
|
505
|
+
* }
|
|
506
|
+
* ```
|
|
507
|
+
*/
|
|
114
508
|
async *process(queues) {
|
|
115
|
-
const pollingInterval = parse(this.#config.worker?.pollingInterval || "2s");
|
|
116
509
|
this.#pool = new JobPool();
|
|
117
510
|
while (this.#running) {
|
|
118
511
|
try {
|
|
512
|
+
await this.#checkStalledJobs(queues);
|
|
119
513
|
yield* this.#fillPool(queues);
|
|
120
514
|
if (this.#pool.isEmpty()) {
|
|
121
|
-
yield { type: "idle", suggestedDelay:
|
|
515
|
+
yield { type: "idle", suggestedDelay: this.#idleDelay };
|
|
122
516
|
continue;
|
|
123
517
|
}
|
|
124
518
|
const completed = await this.#pool.waitForNextCompletion();
|
|
125
519
|
yield { type: "completed", queue: completed.queue, job: completed.job };
|
|
126
520
|
} catch (error) {
|
|
127
|
-
yield {
|
|
521
|
+
yield {
|
|
522
|
+
type: "error",
|
|
523
|
+
error,
|
|
524
|
+
suggestedDelay: parse(DEFAULT_ERROR_RETRY_DELAY)
|
|
525
|
+
};
|
|
128
526
|
}
|
|
129
527
|
}
|
|
130
528
|
}
|
|
131
529
|
async *#fillPool(queues) {
|
|
132
|
-
const
|
|
133
|
-
const slotsAvailable = concurrency - this.#pool.size;
|
|
530
|
+
const slotsAvailable = this.#concurrency - this.#pool.size;
|
|
134
531
|
if (slotsAvailable <= 0) return;
|
|
135
532
|
const popPromises = Array.from({ length: slotsAvailable }, () => this.#acquireNextJob(queues));
|
|
136
533
|
const results = await Promise.all(popPromises);
|
|
@@ -144,44 +541,44 @@ var Worker = class {
|
|
|
144
541
|
}
|
|
145
542
|
async #execute(job, queue) {
|
|
146
543
|
const startTime = performance.now();
|
|
147
|
-
|
|
544
|
+
debug_default("worker %s: executing job %s (%s)", this.#id, job.id, job.name);
|
|
148
545
|
const { instance, options, timeout } = await this.#initJob(job, queue);
|
|
149
546
|
try {
|
|
150
547
|
await this.#executeWithTimeout(instance, timeout);
|
|
151
548
|
await this.#adapter.completeJob(job.id, queue);
|
|
152
549
|
const duration = (performance.now() - startTime).toFixed(2);
|
|
153
|
-
|
|
550
|
+
debug_default("worker %s: successfully executed job %s in %dms", this.#id, job.id, duration);
|
|
154
551
|
} catch (e) {
|
|
155
|
-
const isTimeout = e instanceof
|
|
552
|
+
const isTimeout = e instanceof E_JOB_TIMEOUT;
|
|
156
553
|
if (isTimeout && options.failOnTimeout) {
|
|
157
|
-
|
|
554
|
+
debug_default("worker %s: job %s timed out and failOnTimeout is set", this.#id, job.id);
|
|
158
555
|
await this.#adapter.failJob(job.id, queue, e);
|
|
159
556
|
await instance.failed?.(e);
|
|
160
557
|
return;
|
|
161
558
|
}
|
|
162
559
|
const mergedConfig = QueueManager.getMergedRetryConfig(queue, options.retry);
|
|
163
560
|
if (typeof mergedConfig.maxRetries === "undefined" || mergedConfig.maxRetries <= 0) {
|
|
164
|
-
|
|
561
|
+
debug_default("worker %s: job %s has no retries configured, marking as failed", this.#id, job.id);
|
|
165
562
|
await this.#adapter.failJob(job.id, queue, e);
|
|
166
563
|
await instance.failed?.(e);
|
|
167
564
|
return;
|
|
168
565
|
}
|
|
169
566
|
if (job.attempts >= mergedConfig.maxRetries) {
|
|
170
|
-
|
|
567
|
+
debug_default(
|
|
171
568
|
"worker %s: job %s has exceeded max retries (%d), marking as failed",
|
|
172
569
|
this.#id,
|
|
173
570
|
job.id,
|
|
174
571
|
mergedConfig.maxRetries
|
|
175
572
|
);
|
|
176
573
|
await this.#adapter.failJob(job.id, queue, e);
|
|
177
|
-
const exception = new
|
|
574
|
+
const exception = new E_JOB_MAX_ATTEMPTS_REACHED([job.name]);
|
|
178
575
|
await instance.failed?.(exception);
|
|
179
576
|
return;
|
|
180
577
|
}
|
|
181
578
|
if (mergedConfig.backoff) {
|
|
182
579
|
const strategy = mergedConfig.backoff();
|
|
183
580
|
const nextRetryAt = strategy.getNextRetryAt(job.attempts + 1);
|
|
184
|
-
|
|
581
|
+
debug_default("worker %s: job %s will retry at %s", this.#id, job.id, nextRetryAt.toISOString());
|
|
185
582
|
await this.#adapter.retryJob(job.id, queue, nextRetryAt);
|
|
186
583
|
return;
|
|
187
584
|
}
|
|
@@ -191,12 +588,22 @@ var Worker = class {
|
|
|
191
588
|
async #initJob(job, queue) {
|
|
192
589
|
try {
|
|
193
590
|
const JobClass = Locator.getOrThrow(job.name);
|
|
194
|
-
const
|
|
591
|
+
const context = Object.freeze({
|
|
592
|
+
jobId: job.id,
|
|
593
|
+
name: job.name,
|
|
594
|
+
attempt: job.attempts + 1,
|
|
595
|
+
queue,
|
|
596
|
+
priority: job.priority ?? DEFAULT_PRIORITY,
|
|
597
|
+
acquiredAt: new Date(job.acquiredAt),
|
|
598
|
+
stalledCount: job.stalledCount ?? 0
|
|
599
|
+
});
|
|
600
|
+
const jobFactory = QueueManager.getJobFactory();
|
|
601
|
+
const instance = jobFactory ? await jobFactory(JobClass, job.payload, context) : new JobClass(job.payload, context);
|
|
195
602
|
const options = JobClass.options || {};
|
|
196
603
|
const timeout = this.#getJobTimeout(options);
|
|
197
604
|
return { instance, options, timeout };
|
|
198
605
|
} catch (error) {
|
|
199
|
-
|
|
606
|
+
debug_default("worker %s: failed to initialize job %s (%s)", this.#id, job.id, job.name);
|
|
200
607
|
await this.#adapter.failJob(job.id, queue, error);
|
|
201
608
|
throw error;
|
|
202
609
|
}
|
|
@@ -217,7 +624,7 @@ var Worker = class {
|
|
|
217
624
|
const signal = AbortSignal.timeout(timeout);
|
|
218
625
|
const abortPromise = new Promise((_, reject) => {
|
|
219
626
|
signal.addEventListener("abort", () => {
|
|
220
|
-
reject(new
|
|
627
|
+
reject(new E_JOB_TIMEOUT([instance.constructor.name, timeout]));
|
|
221
628
|
});
|
|
222
629
|
});
|
|
223
630
|
await Promise.race([instance.execute(signal), abortPromise]);
|
|
@@ -228,126 +635,89 @@ var Worker = class {
|
|
|
228
635
|
if (!job) {
|
|
229
636
|
continue;
|
|
230
637
|
}
|
|
231
|
-
|
|
638
|
+
debug_default("worker %s: acquired job %s", this.#id, job.id);
|
|
232
639
|
return { job, queue };
|
|
233
640
|
}
|
|
234
641
|
return null;
|
|
235
642
|
}
|
|
236
|
-
async #
|
|
237
|
-
const
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
process.exit(0);
|
|
241
|
-
};
|
|
242
|
-
process.on("SIGINT", shutdown);
|
|
243
|
-
process.on("SIGTERM", shutdown);
|
|
244
|
-
}
|
|
245
|
-
};
|
|
246
|
-
|
|
247
|
-
// src/queue_manager.ts
|
|
248
|
-
import * as errors2 from "#src/exceptions";
|
|
249
|
-
import debug2 from "#src/debug";
|
|
250
|
-
import { Locator as Locator2 } from "#src/locator";
|
|
251
|
-
var QueueManagerSingleton = class {
|
|
252
|
-
#defaultAdapter;
|
|
253
|
-
#adapters = {};
|
|
254
|
-
#adapterInstances = /* @__PURE__ */ new Map();
|
|
255
|
-
#globalRetryConfig;
|
|
256
|
-
#queueConfigs = /* @__PURE__ */ new Map();
|
|
257
|
-
async init(config) {
|
|
258
|
-
debug2("initializing queue manager with config: %O", config);
|
|
259
|
-
this.#validateConfig(config);
|
|
260
|
-
this.#adapterInstances.clear();
|
|
261
|
-
this.#defaultAdapter = config.default;
|
|
262
|
-
this.#adapters = config.adapters;
|
|
263
|
-
this.#globalRetryConfig = config.retry;
|
|
264
|
-
if (config.queues) {
|
|
265
|
-
for (const [queue, queueConfig] of Object.entries(config.queues)) {
|
|
266
|
-
this.#queueConfigs.set(queue, queueConfig);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
await Locator2.registerFromGlob(config.locations);
|
|
270
|
-
return this;
|
|
271
|
-
}
|
|
272
|
-
use(adapter) {
|
|
273
|
-
if (!adapter) {
|
|
274
|
-
adapter = this.#defaultAdapter;
|
|
275
|
-
}
|
|
276
|
-
const cached = this.#adapterInstances.get(adapter);
|
|
277
|
-
if (cached) {
|
|
278
|
-
return cached;
|
|
279
|
-
}
|
|
280
|
-
const adapterFactory = this.#adapters[adapter];
|
|
281
|
-
if (!adapterFactory) {
|
|
282
|
-
throw new errors2.E_CONFIGURATION_ERROR([`Adapter "${adapter}" is not registered`]);
|
|
643
|
+
async #checkStalledJobs(queues) {
|
|
644
|
+
const now = Date.now();
|
|
645
|
+
if (now - this.#lastStalledCheck < this.#stalledInterval) {
|
|
646
|
+
return;
|
|
283
647
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
648
|
+
this.#lastStalledCheck = now;
|
|
649
|
+
for (const queue of queues) {
|
|
650
|
+
const recovered = await this.#adapter.recoverStalledJobs(
|
|
651
|
+
queue,
|
|
652
|
+
this.#stalledThreshold,
|
|
653
|
+
this.#maxStalledCount
|
|
654
|
+
);
|
|
655
|
+
if (recovered > 0) {
|
|
656
|
+
debug_default("worker %s: recovered %d stalled jobs from queue %s", this.#id, recovered, queue);
|
|
657
|
+
}
|
|
291
658
|
}
|
|
292
659
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
getMergedRetryConfig(queue, jobRetryConfig) {
|
|
297
|
-
const queueConfig = this.#queueConfigs.get(queue);
|
|
298
|
-
const queueRetryConfig = queueConfig?.retry || {};
|
|
299
|
-
let maxRetries = jobRetryConfig?.maxRetries || queueRetryConfig.maxRetries || this.#globalRetryConfig?.maxRetries || 0;
|
|
300
|
-
let backoff = jobRetryConfig?.backoff || queueRetryConfig.backoff || this.#globalRetryConfig?.backoff;
|
|
301
|
-
return { maxRetries, backoff };
|
|
302
|
-
}
|
|
303
|
-
#validateConfig(config) {
|
|
304
|
-
if (!config.adapters || Object.keys(config.adapters).length === 0) {
|
|
305
|
-
throw new errors2.E_CONFIGURATION_ERROR(["At least one adapter must be configured"]);
|
|
306
|
-
}
|
|
307
|
-
if (!config.default) {
|
|
308
|
-
throw new errors2.E_CONFIGURATION_ERROR(["Default adapter must be specified"]);
|
|
309
|
-
}
|
|
310
|
-
if (!config.locations || config.locations.length === 0) {
|
|
311
|
-
throw new errors2.E_CONFIGURATION_ERROR(["Job locations must be specified"]);
|
|
312
|
-
}
|
|
313
|
-
if (!config.adapters[config.default]) {
|
|
314
|
-
throw new errors2.E_CONFIGURATION_ERROR([
|
|
315
|
-
`Default adapter "${config.default}" not found in adapters configuration`
|
|
316
|
-
]);
|
|
660
|
+
#setupGracefulShutdown() {
|
|
661
|
+
if (!this.#gracefulShutdown) {
|
|
662
|
+
return;
|
|
317
663
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
664
|
+
this.#shutdownHandler = async () => {
|
|
665
|
+
debug_default("received shutdown signal, stopping worker...");
|
|
666
|
+
if (this.#onShutdownSignal) {
|
|
667
|
+
await this.#onShutdownSignal();
|
|
321
668
|
}
|
|
322
|
-
|
|
669
|
+
await this.stop();
|
|
670
|
+
};
|
|
671
|
+
process.on("SIGINT", this.#shutdownHandler);
|
|
672
|
+
process.on("SIGTERM", this.#shutdownHandler);
|
|
323
673
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
674
|
+
#removeShutdownHandlers() {
|
|
675
|
+
if (this.#shutdownHandler) {
|
|
676
|
+
process.off("SIGINT", this.#shutdownHandler);
|
|
677
|
+
process.off("SIGTERM", this.#shutdownHandler);
|
|
678
|
+
this.#shutdownHandler = void 0;
|
|
328
679
|
}
|
|
329
|
-
this.#adapterInstances.clear();
|
|
330
680
|
}
|
|
331
681
|
};
|
|
332
|
-
var QueueManager2 = new QueueManagerSingleton();
|
|
333
682
|
|
|
334
683
|
// src/strategies/backoff_strategy.ts
|
|
335
|
-
import * as errors3 from "#src/exceptions";
|
|
336
|
-
import { parse as parse2 } from "#src/utils";
|
|
337
684
|
import { RuntimeException } from "@poppinss/utils";
|
|
338
685
|
import { assertUnreachable } from "@poppinss/utils/assert";
|
|
339
686
|
var BackoffStrategy = class {
|
|
340
687
|
#config;
|
|
688
|
+
/**
|
|
689
|
+
* Create a new backoff strategy.
|
|
690
|
+
*
|
|
691
|
+
* @param config - Backoff configuration
|
|
692
|
+
* @throws {E_INVALID_BASE_DELAY} If baseDelay is not positive
|
|
693
|
+
* @throws {E_INVALID_MAX_DELAY} If maxDelay is invalid
|
|
694
|
+
* @throws {E_INVALID_MULTIPLIER} If multiplier is invalid
|
|
695
|
+
*/
|
|
341
696
|
constructor(config) {
|
|
342
697
|
this.#config = config;
|
|
343
698
|
this.#validateConfig();
|
|
344
699
|
}
|
|
700
|
+
/**
|
|
701
|
+
* Calculate the delay for a given attempt number.
|
|
702
|
+
*
|
|
703
|
+
* @param attempt - The attempt number (1-based)
|
|
704
|
+
* @returns Delay in milliseconds
|
|
705
|
+
* @throws {RuntimeException} If attempt is less than 1
|
|
706
|
+
*
|
|
707
|
+
* @example
|
|
708
|
+
* ```typescript
|
|
709
|
+
* // Exponential: 1s, 2s, 4s, 8s, 16s, ...
|
|
710
|
+
* strategy.calculateDelay(1) // 1000
|
|
711
|
+
* strategy.calculateDelay(2) // 2000
|
|
712
|
+
* strategy.calculateDelay(3) // 4000
|
|
713
|
+
* ```
|
|
714
|
+
*/
|
|
345
715
|
calculateDelay(attempt) {
|
|
346
716
|
if (attempt < 1) {
|
|
347
717
|
throw new RuntimeException("Attempt number must be >= 1");
|
|
348
718
|
}
|
|
349
|
-
const baseDelayMs =
|
|
350
|
-
const maxDelayMs = this.#config.maxDelay ?
|
|
719
|
+
const baseDelayMs = parse(this.#config.baseDelay);
|
|
720
|
+
const maxDelayMs = this.#config.maxDelay ? parse(this.#config.maxDelay) : Infinity;
|
|
351
721
|
const multiplier = this.#config.multiplier ?? 2;
|
|
352
722
|
let delay;
|
|
353
723
|
switch (this.#config.strategy) {
|
|
@@ -369,39 +739,56 @@ var BackoffStrategy = class {
|
|
|
369
739
|
}
|
|
370
740
|
return Math.floor(delay);
|
|
371
741
|
}
|
|
742
|
+
/**
|
|
743
|
+
* Get the Date when the next retry should occur.
|
|
744
|
+
*
|
|
745
|
+
* @param attempt - The attempt number (1-based)
|
|
746
|
+
* @returns Date for the next retry
|
|
747
|
+
*
|
|
748
|
+
* @example
|
|
749
|
+
* ```typescript
|
|
750
|
+
* const nextRetry = strategy.getNextRetryAt(3)
|
|
751
|
+
* console.log(`Retry at: ${nextRetry.toISOString()}`)
|
|
752
|
+
* ```
|
|
753
|
+
*/
|
|
372
754
|
getNextRetryAt(attempt) {
|
|
373
755
|
const delay = this.calculateDelay(attempt);
|
|
374
756
|
return new Date(Date.now() + delay);
|
|
375
757
|
}
|
|
758
|
+
/**
|
|
759
|
+
* Get a frozen copy of the configuration.
|
|
760
|
+
*
|
|
761
|
+
* @returns Readonly configuration object
|
|
762
|
+
*/
|
|
376
763
|
getConfig() {
|
|
377
764
|
return Object.freeze({ ...this.#config });
|
|
378
765
|
}
|
|
379
766
|
#validateConfig() {
|
|
380
|
-
const baseDelayMs =
|
|
767
|
+
const baseDelayMs = parse(this.#config.baseDelay);
|
|
381
768
|
if (baseDelayMs <= 0) {
|
|
382
|
-
throw new
|
|
769
|
+
throw new E_INVALID_BASE_DELAY([
|
|
383
770
|
"Base delay must be a positive integer greater than zero"
|
|
384
771
|
]);
|
|
385
772
|
}
|
|
386
773
|
if (this.#config.maxDelay) {
|
|
387
|
-
const maxDelayMs =
|
|
774
|
+
const maxDelayMs = parse(this.#config.maxDelay);
|
|
388
775
|
if (maxDelayMs <= 0) {
|
|
389
|
-
throw new
|
|
776
|
+
throw new E_INVALID_MAX_DELAY([
|
|
390
777
|
"Max delay must be a positive integer greater than zero"
|
|
391
778
|
]);
|
|
392
779
|
}
|
|
393
780
|
if (maxDelayMs <= baseDelayMs) {
|
|
394
|
-
throw new
|
|
781
|
+
throw new E_INVALID_MAX_DELAY(["Max delay should be greater than base delay"]);
|
|
395
782
|
}
|
|
396
783
|
}
|
|
397
784
|
if (this.#config.multiplier !== void 0) {
|
|
398
785
|
if (this.#config.multiplier <= 0) {
|
|
399
|
-
throw new
|
|
786
|
+
throw new E_INVALID_MULTIPLIER([
|
|
400
787
|
"Multiplier must be a positive number greater than zero"
|
|
401
788
|
]);
|
|
402
789
|
}
|
|
403
790
|
if (this.#config.strategy === "exponential" && this.#config.multiplier < 1) {
|
|
404
|
-
throw new
|
|
791
|
+
throw new E_INVALID_MULTIPLIER(["Exponential strategy multiplier should be >= 1"]);
|
|
405
792
|
}
|
|
406
793
|
}
|
|
407
794
|
}
|
|
@@ -440,9 +827,11 @@ function customBackoff(config) {
|
|
|
440
827
|
}
|
|
441
828
|
export {
|
|
442
829
|
Job,
|
|
443
|
-
|
|
830
|
+
Locator,
|
|
831
|
+
QueueManager,
|
|
444
832
|
Worker,
|
|
445
833
|
customBackoff,
|
|
834
|
+
exceptions_exports as errors,
|
|
446
835
|
exponentialBackoff,
|
|
447
836
|
fixedBackoff,
|
|
448
837
|
linearBackoff
|