@bymax-one/nest-queue 1.2.14 → 1.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -1
- package/README.md +77 -68
- package/dist/server/index.cjs +219 -107
- package/dist/server/index.d.cts +57 -23
- package/dist/server/index.d.ts +57 -23
- package/dist/server/index.mjs +219 -107
- package/package.json +1 -1
package/dist/server/index.cjs
CHANGED
|
@@ -90,6 +90,11 @@ function invalid(reason) {
|
|
|
90
90
|
reason
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
|
+
function checkTimerDelay(name, value) {
|
|
94
|
+
if (value !== void 0 && !isTimerDelayMs(value)) {
|
|
95
|
+
invalid(`${name} must be a whole number from 1 to ${String(MAX_TIMER_DELAY_MS)}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
93
98
|
function isParseableUrl(url) {
|
|
94
99
|
const candidate = /^rediss?:\/\//i.test(url) ? url : url.startsWith("//") ? `redis:${url}` : `redis://${url}`;
|
|
95
100
|
if (!URL.canParse(candidate)) return false;
|
|
@@ -130,15 +135,13 @@ function validateOptions(opts) {
|
|
|
130
135
|
if (drainTimeoutMs !== void 0 && drainTimeoutMs <= 0) {
|
|
131
136
|
invalid("shutdown.drainTimeoutMs must be > 0");
|
|
132
137
|
}
|
|
138
|
+
checkTimerDelay("shutdown.drainTimeoutMs", drainTimeoutMs);
|
|
133
139
|
const cacheTtlMs = opts.metrics?.cacheTtlMs;
|
|
134
140
|
if (cacheTtlMs !== void 0 && cacheTtlMs < 0) {
|
|
135
141
|
invalid("metrics.cacheTtlMs must be >= 0");
|
|
136
142
|
}
|
|
137
|
-
for (const key of ["commandTimeoutMs", "enqueueTimeoutMs"]) {
|
|
138
|
-
|
|
139
|
-
if (value !== void 0 && !isTimerDelayMs(value)) {
|
|
140
|
-
invalid(`${key} must be a whole number from 1 to ${String(MAX_TIMER_DELAY_MS)}`);
|
|
141
|
-
}
|
|
143
|
+
for (const key of ["commandTimeoutMs", "enqueueTimeoutMs", "connectionReadyTimeoutMs"]) {
|
|
144
|
+
checkTimerDelay(key, opts[key]);
|
|
142
145
|
}
|
|
143
146
|
}
|
|
144
147
|
|
|
@@ -157,6 +160,7 @@ var DEFAULT_ENQUEUE_TIMEOUT_MS = 1e4;
|
|
|
157
160
|
var MAX_JOBS_PAGE_SIZE = 1e3;
|
|
158
161
|
var DEFAULT_METRICS_CACHE_TTL_MS = 5e3;
|
|
159
162
|
var DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS = 3e4;
|
|
163
|
+
var MAX_BULK_SIZE = 1e3;
|
|
160
164
|
|
|
161
165
|
// src/server/config/resolved-options.ts
|
|
162
166
|
function applyDefaults(opts) {
|
|
@@ -250,13 +254,14 @@ function duplicateConnection(client) {
|
|
|
250
254
|
}
|
|
251
255
|
|
|
252
256
|
// src/server/services/connection-resolver.service.ts
|
|
253
|
-
var _logger, _client, _options;
|
|
257
|
+
var _logger, _client, _options, _pendingReady;
|
|
254
258
|
exports.ConnectionResolver = class ConnectionResolver {
|
|
255
259
|
constructor(options) {
|
|
256
260
|
__privateAdd(this, _logger, new common.Logger(exports.ConnectionResolver.name));
|
|
257
261
|
__privateAdd(this, _client);
|
|
258
262
|
__privateAdd(this, _options);
|
|
259
263
|
__publicField(this, "mode");
|
|
264
|
+
__privateAdd(this, _pendingReady);
|
|
260
265
|
__privateSet(this, _options, options);
|
|
261
266
|
}
|
|
262
267
|
async init() {
|
|
@@ -312,6 +317,21 @@ exports.ConnectionResolver = class ConnectionResolver {
|
|
|
312
317
|
isOwned() {
|
|
313
318
|
return this.mode === "mode-b-owned";
|
|
314
319
|
}
|
|
320
|
+
awaitReady() {
|
|
321
|
+
const client = this.getClient();
|
|
322
|
+
if (!WAITING_STATUSES.has(client.status)) return Promise.resolve();
|
|
323
|
+
__privateGet(this, _pendingReady) ?? __privateSet(this, _pendingReady, new Promise((resolve) => {
|
|
324
|
+
const release = () => {
|
|
325
|
+
client.off("ready", release);
|
|
326
|
+
client.off("end", release);
|
|
327
|
+
__privateSet(this, _pendingReady, void 0);
|
|
328
|
+
resolve();
|
|
329
|
+
};
|
|
330
|
+
client.on("ready", release);
|
|
331
|
+
client.on("end", release);
|
|
332
|
+
}));
|
|
333
|
+
return __privateGet(this, _pendingReady);
|
|
334
|
+
}
|
|
315
335
|
async teardown() {
|
|
316
336
|
if (this.isOwned() && __privateGet(this, _client)) {
|
|
317
337
|
const client = __privateGet(this, _client);
|
|
@@ -332,7 +352,7 @@ exports.ConnectionResolver = class ConnectionResolver {
|
|
|
332
352
|
}
|
|
333
353
|
if (client.options.maxRetriesPerRequest === null) {
|
|
334
354
|
__privateGet(this, _logger).warn(
|
|
335
|
-
"The shared client has maxRetriesPerRequest: null, so a
|
|
355
|
+
"The shared client has maxRetriesPerRequest: null, so a command issued while Redis is unreachable stays queued in the client for as long as the outage lasts. QueueService calls are bounded by enqueueTimeoutMs and the caller is released, but the command is not cancelled. Give the client a finite limit to fail fast."
|
|
336
356
|
);
|
|
337
357
|
}
|
|
338
358
|
const probe = duplicateConnection(client);
|
|
@@ -382,10 +402,17 @@ exports.ConnectionResolver = class ConnectionResolver {
|
|
|
382
402
|
_logger = new WeakMap();
|
|
383
403
|
_client = new WeakMap();
|
|
384
404
|
_options = new WeakMap();
|
|
405
|
+
_pendingReady = new WeakMap();
|
|
385
406
|
exports.ConnectionResolver = __decorateClass([
|
|
386
407
|
common.Injectable(),
|
|
387
408
|
__decorateParam(0, common.Inject(BYMAX_QUEUE_OPTIONS))
|
|
388
409
|
], exports.ConnectionResolver);
|
|
410
|
+
var WAITING_STATUSES = /* @__PURE__ */ new Set([
|
|
411
|
+
"connecting",
|
|
412
|
+
"connect",
|
|
413
|
+
"reconnecting",
|
|
414
|
+
"close"
|
|
415
|
+
]);
|
|
389
416
|
function isReplyError(err) {
|
|
390
417
|
return typeof err === "object" && err !== null && err.name === "ReplyError";
|
|
391
418
|
}
|
|
@@ -434,6 +461,13 @@ function attachDefaultErrorListener(emitter, logger, kind, queueName) {
|
|
|
434
461
|
logger.error(`${kind} for queue "${queueName}" emitted an error: ${describeError(err)}`);
|
|
435
462
|
});
|
|
436
463
|
}
|
|
464
|
+
function assertPageWindow(operation, start, end) {
|
|
465
|
+
if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start || end - start >= MAX_JOBS_PAGE_SIZE) {
|
|
466
|
+
throw new QueueException(QUEUE_ERROR_CODES.INVALID_OPTIONS, common.HttpStatus.BAD_REQUEST, {
|
|
467
|
+
reason: `${operation} needs integers with 0 <= start <= end and at most ${String(MAX_JOBS_PAGE_SIZE)} jobs per call`
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
}
|
|
437
471
|
|
|
438
472
|
// src/server/utils/with-deadline.ts
|
|
439
473
|
function withDeadline(work, timeoutMs, onTimeout) {
|
|
@@ -456,6 +490,18 @@ function timeoutException(operation, timeoutMs, cause) {
|
|
|
456
490
|
if (cause !== void 0) exception.cause = cause;
|
|
457
491
|
return exception;
|
|
458
492
|
}
|
|
493
|
+
function unavailableException(operation, cause) {
|
|
494
|
+
const exception = new QueueException(
|
|
495
|
+
QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
|
|
496
|
+
common.HttpStatus.INTERNAL_SERVER_ERROR,
|
|
497
|
+
{ operation, reason: "offline" }
|
|
498
|
+
);
|
|
499
|
+
exception.cause = cause;
|
|
500
|
+
return exception;
|
|
501
|
+
}
|
|
502
|
+
function isOfflineRejection(err) {
|
|
503
|
+
return err instanceof Error && err.message === "Stream isn't writeable and enableOfflineQueue options is false";
|
|
504
|
+
}
|
|
459
505
|
function isCommandTimeout(err) {
|
|
460
506
|
return err instanceof Error && err.message === "Command timed out";
|
|
461
507
|
}
|
|
@@ -463,12 +509,12 @@ async function withCommandTimeoutCode(operation, commandTimeoutMs, work) {
|
|
|
463
509
|
try {
|
|
464
510
|
return await work;
|
|
465
511
|
} catch (err) {
|
|
466
|
-
|
|
512
|
+
if (isCommandTimeout(err)) throw timeoutException(operation, commandTimeoutMs(), err);
|
|
513
|
+
throw isOfflineRejection(err) ? unavailableException(operation, err) : err;
|
|
467
514
|
}
|
|
468
515
|
}
|
|
469
516
|
|
|
470
517
|
// src/server/services/queue.service.ts
|
|
471
|
-
var MAX_BULK_SIZE = 1e3;
|
|
472
518
|
function assertValidCron(repeat, queueOptions) {
|
|
473
519
|
if (queueOptions.settings?.repeatStrategy !== void 0) return;
|
|
474
520
|
try {
|
|
@@ -479,13 +525,14 @@ function assertValidCron(repeat, queueOptions) {
|
|
|
479
525
|
});
|
|
480
526
|
}
|
|
481
527
|
}
|
|
482
|
-
var _queues;
|
|
528
|
+
var _queues, _warm;
|
|
483
529
|
exports.QueueService = class QueueService {
|
|
484
530
|
constructor(connection, options) {
|
|
485
531
|
this.connection = connection;
|
|
486
532
|
this.options = options;
|
|
487
533
|
__publicField(this, "logger", new common.Logger(exports.QueueService.name));
|
|
488
534
|
__privateAdd(this, _queues, /* @__PURE__ */ new Map());
|
|
535
|
+
__privateAdd(this, _warm, /* @__PURE__ */ new Set());
|
|
489
536
|
}
|
|
490
537
|
connection;
|
|
491
538
|
options;
|
|
@@ -504,27 +551,38 @@ exports.QueueService = class QueueService {
|
|
|
504
551
|
__privateGet(this, _queues).set(queueName, queue);
|
|
505
552
|
return queue;
|
|
506
553
|
}
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
554
|
+
call(operation, queueName, run) {
|
|
555
|
+
let expired = false;
|
|
556
|
+
const guarded = withDeadline(
|
|
557
|
+
this.afterReady(queueName).then(() => {
|
|
558
|
+
if (expired) throw new Error("the call had already timed out");
|
|
559
|
+
return run();
|
|
560
|
+
}),
|
|
561
|
+
this.options.enqueueTimeoutMs,
|
|
562
|
+
() => {
|
|
563
|
+
expired = true;
|
|
564
|
+
return timeoutException(operation, this.options.enqueueTimeoutMs);
|
|
565
|
+
}
|
|
566
|
+
);
|
|
511
567
|
return withCommandTimeoutCode(
|
|
512
568
|
operation,
|
|
513
569
|
() => this.connection.getClient().options.commandTimeout ?? this.options.commandTimeoutMs,
|
|
514
|
-
|
|
570
|
+
guarded
|
|
515
571
|
);
|
|
516
572
|
}
|
|
573
|
+
async afterReady(queueName) {
|
|
574
|
+
if (__privateGet(this, _warm).has(queueName)) return;
|
|
575
|
+
await this.connection.awaitReady();
|
|
576
|
+
__privateGet(this, _warm).add(queueName);
|
|
577
|
+
}
|
|
517
578
|
queueFor(queueName) {
|
|
518
579
|
return this.getOrCreateQueue(queueName);
|
|
519
580
|
}
|
|
520
581
|
async enqueue(queueName, jobName, data, options) {
|
|
521
582
|
const job = await this.call(
|
|
522
583
|
"enqueue",
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
this.options.enqueueTimeoutMs,
|
|
526
|
-
() => this.enqueueTimeout("enqueue")
|
|
527
|
-
)
|
|
584
|
+
queueName,
|
|
585
|
+
() => this.queueFor(queueName).add(jobName, data, options)
|
|
528
586
|
);
|
|
529
587
|
return job;
|
|
530
588
|
}
|
|
@@ -543,16 +601,13 @@ exports.QueueService = class QueueService {
|
|
|
543
601
|
try {
|
|
544
602
|
const created = await this.call(
|
|
545
603
|
"enqueueBulk",
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
),
|
|
554
|
-
this.options.enqueueTimeoutMs,
|
|
555
|
-
() => this.enqueueTimeout("enqueueBulk")
|
|
604
|
+
queueName,
|
|
605
|
+
() => this.queueFor(queueName).addBulk(
|
|
606
|
+
jobs.map((job) => ({
|
|
607
|
+
name: job.name,
|
|
608
|
+
data: job.data,
|
|
609
|
+
...job.options === void 0 ? {} : { opts: job.options }
|
|
610
|
+
}))
|
|
556
611
|
)
|
|
557
612
|
);
|
|
558
613
|
return created;
|
|
@@ -570,22 +625,23 @@ exports.QueueService = class QueueService {
|
|
|
570
625
|
}
|
|
571
626
|
}
|
|
572
627
|
async getJob(queueName, jobId) {
|
|
573
|
-
const job = await this.call("getJob", this.queueFor(queueName).getJob(jobId));
|
|
628
|
+
const job = await this.call("getJob", queueName, () => this.queueFor(queueName).getJob(jobId));
|
|
574
629
|
return job ?? null;
|
|
575
630
|
}
|
|
576
631
|
async getJobs(queueName, status, start = 0, end = 50) {
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
632
|
+
assertPageWindow("getJobs", start, end);
|
|
633
|
+
const jobs = await this.call(
|
|
634
|
+
"getJobs",
|
|
635
|
+
queueName,
|
|
636
|
+
() => this.queueFor(queueName).getJobs([status], start, end)
|
|
637
|
+
);
|
|
583
638
|
return jobs;
|
|
584
639
|
}
|
|
585
640
|
async getMetrics(queueName) {
|
|
586
641
|
const counts = await this.call(
|
|
587
642
|
"getMetrics",
|
|
588
|
-
|
|
643
|
+
queueName,
|
|
644
|
+
() => this.queueFor(queueName).getJobCounts("waiting", "active", "completed", "failed", "delayed")
|
|
589
645
|
);
|
|
590
646
|
return {
|
|
591
647
|
queue: queueName,
|
|
@@ -604,24 +660,38 @@ exports.QueueService = class QueueService {
|
|
|
604
660
|
if ("pattern" in repeat) assertValidCron(repeat, this.options.queueOptions);
|
|
605
661
|
const job = await this.call(
|
|
606
662
|
"upsertJobScheduler",
|
|
607
|
-
|
|
663
|
+
queueName,
|
|
664
|
+
() => queue.upsertJobScheduler(schedulerId, repeat, jobTemplate)
|
|
608
665
|
);
|
|
609
666
|
return job;
|
|
610
667
|
}
|
|
611
668
|
async removeJobScheduler(queueName, schedulerId) {
|
|
612
|
-
return this.call(
|
|
669
|
+
return this.call(
|
|
670
|
+
"removeJobScheduler",
|
|
671
|
+
queueName,
|
|
672
|
+
() => this.queueFor(queueName).removeJobScheduler(schedulerId)
|
|
673
|
+
);
|
|
613
674
|
}
|
|
614
675
|
async getJobSchedulers(queueName, start = 0, end = 50, asc = true) {
|
|
615
|
-
|
|
676
|
+
assertPageWindow("getJobSchedulers", start, end);
|
|
677
|
+
return this.call(
|
|
678
|
+
"getJobSchedulers",
|
|
679
|
+
queueName,
|
|
680
|
+
() => this.queueFor(queueName).getJobSchedulers(start, end, asc)
|
|
681
|
+
);
|
|
616
682
|
}
|
|
617
683
|
async pauseQueue(queueName) {
|
|
618
|
-
await this.call("pauseQueue", this.queueFor(queueName).pause());
|
|
684
|
+
await this.call("pauseQueue", queueName, () => this.queueFor(queueName).pause());
|
|
619
685
|
}
|
|
620
686
|
async resumeQueue(queueName) {
|
|
621
|
-
await this.call("resumeQueue", this.queueFor(queueName).resume());
|
|
687
|
+
await this.call("resumeQueue", queueName, () => this.queueFor(queueName).resume());
|
|
622
688
|
}
|
|
623
689
|
async cleanQueue(queueName, gracePeriodMs, limit, status) {
|
|
624
|
-
return this.call(
|
|
690
|
+
return this.call(
|
|
691
|
+
"cleanQueue",
|
|
692
|
+
queueName,
|
|
693
|
+
() => this.queueFor(queueName).clean(gracePeriodMs, limit, status)
|
|
694
|
+
);
|
|
625
695
|
}
|
|
626
696
|
getCachedQueues() {
|
|
627
697
|
return __privateGet(this, _queues);
|
|
@@ -631,9 +701,11 @@ exports.QueueService = class QueueService {
|
|
|
631
701
|
await queue.close().catch(() => void 0);
|
|
632
702
|
}
|
|
633
703
|
__privateGet(this, _queues).clear();
|
|
704
|
+
__privateGet(this, _warm).clear();
|
|
634
705
|
}
|
|
635
706
|
};
|
|
636
707
|
_queues = new WeakMap();
|
|
708
|
+
_warm = new WeakMap();
|
|
637
709
|
exports.QueueService = __decorateClass([
|
|
638
710
|
common.Injectable(),
|
|
639
711
|
__decorateParam(0, common.Inject(exports.ConnectionResolver)),
|
|
@@ -642,13 +714,15 @@ exports.QueueService = __decorateClass([
|
|
|
642
714
|
function errorCodeOf(err) {
|
|
643
715
|
return err.getResponse().error.code;
|
|
644
716
|
}
|
|
645
|
-
var _connection;
|
|
717
|
+
var _connection, _deadlineMs;
|
|
646
718
|
exports.FlowService = class FlowService {
|
|
647
|
-
constructor(connection, enabled, prefix, telemetry) {
|
|
719
|
+
constructor(connection, enabled, prefix, telemetry, enqueueTimeoutMs = DEFAULT_ENQUEUE_TIMEOUT_MS) {
|
|
648
720
|
__publicField(this, "logger", new common.Logger(exports.FlowService.name));
|
|
649
721
|
__publicField(this, "producer");
|
|
650
722
|
__privateAdd(this, _connection);
|
|
723
|
+
__privateAdd(this, _deadlineMs);
|
|
651
724
|
__privateSet(this, _connection, connection);
|
|
725
|
+
__privateSet(this, _deadlineMs, enqueueTimeoutMs);
|
|
652
726
|
if (enabled) {
|
|
653
727
|
this.producer = new bullmq.FlowProducer({
|
|
654
728
|
connection: connection.getClient(),
|
|
@@ -659,17 +733,24 @@ exports.FlowService = class FlowService {
|
|
|
659
733
|
}
|
|
660
734
|
}
|
|
661
735
|
async add(flow) {
|
|
662
|
-
return
|
|
663
|
-
"addFlow",
|
|
664
|
-
() => this.commandTimeout(),
|
|
665
|
-
this.requireProducer().add(flow)
|
|
666
|
-
);
|
|
736
|
+
return this.bounded("addFlow", this.requireProducer().add(flow));
|
|
667
737
|
}
|
|
668
738
|
async addBulk(flows) {
|
|
739
|
+
const producer = this.requireProducer();
|
|
740
|
+
if (flows.length > MAX_BULK_SIZE) {
|
|
741
|
+
throw new QueueException(
|
|
742
|
+
QUEUE_ERROR_CODES.BULK_ENQUEUE_FAILED,
|
|
743
|
+
common.HttpStatus.INTERNAL_SERVER_ERROR,
|
|
744
|
+
{ reason: "batch size exceeds limit", limit: MAX_BULK_SIZE, received: flows.length }
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
return this.bounded("addFlowBulk", producer.addBulk([...flows]));
|
|
748
|
+
}
|
|
749
|
+
bounded(operation, work) {
|
|
669
750
|
return withCommandTimeoutCode(
|
|
670
|
-
|
|
751
|
+
operation,
|
|
671
752
|
() => this.commandTimeout(),
|
|
672
|
-
this
|
|
753
|
+
withDeadline(work, __privateGet(this, _deadlineMs), () => timeoutException(operation, __privateGet(this, _deadlineMs)))
|
|
673
754
|
);
|
|
674
755
|
}
|
|
675
756
|
commandTimeout() {
|
|
@@ -691,6 +772,7 @@ exports.FlowService = class FlowService {
|
|
|
691
772
|
}
|
|
692
773
|
};
|
|
693
774
|
_connection = new WeakMap();
|
|
775
|
+
_deadlineMs = new WeakMap();
|
|
694
776
|
exports.FlowService = __decorateClass([
|
|
695
777
|
common.Injectable()
|
|
696
778
|
], exports.FlowService);
|
|
@@ -736,6 +818,41 @@ exports.MetricsService = class MetricsService {
|
|
|
736
818
|
exports.MetricsService = __decorateClass([
|
|
737
819
|
common.Injectable()
|
|
738
820
|
], exports.MetricsService);
|
|
821
|
+
|
|
822
|
+
// src/server/utils/worker-option-problem.ts
|
|
823
|
+
var WORKER_TIMER_MIN_MS = 1e3;
|
|
824
|
+
function countProblem(name, value, min, max) {
|
|
825
|
+
if (!Number.isInteger(value)) return { reason: `${name} must be a whole number`, actual: value };
|
|
826
|
+
if (value < min) return { reason: `${name} must be >= ${String(min)}`, actual: value };
|
|
827
|
+
if (value > max) return { reason: `${name} must be <= ${String(max)}`, actual: value };
|
|
828
|
+
return void 0;
|
|
829
|
+
}
|
|
830
|
+
function timerProblem(name, value) {
|
|
831
|
+
if (value === void 0) return void 0;
|
|
832
|
+
if (Number.isInteger(value) && value >= WORKER_TIMER_MIN_MS && value <= MAX_TIMER_DELAY_MS) {
|
|
833
|
+
return void 0;
|
|
834
|
+
}
|
|
835
|
+
return {
|
|
836
|
+
reason: `${name} must be a whole number from ${String(WORKER_TIMER_MIN_MS)} to ${String(MAX_TIMER_DELAY_MS)}`,
|
|
837
|
+
actual: value
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
function workerOptionProblem(opts) {
|
|
841
|
+
if (opts === void 0) return void 0;
|
|
842
|
+
const { concurrency, limiter, lockDuration, stalledInterval } = opts;
|
|
843
|
+
return (concurrency === void 0 ? void 0 : countProblem("concurrency", concurrency, 1, MAX_WORKER_CONCURRENCY)) ?? (limiter === void 0 ? void 0 : countProblem("limiter.max", limiter.max, 1, Number.POSITIVE_INFINITY) ?? countProblem("limiter.duration", limiter.duration, 1, Number.POSITIVE_INFINITY)) ?? timerProblem("lockDuration", lockDuration) ?? timerProblem("stalledInterval", stalledInterval);
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
// src/server/utils/abandoned-workers.ts
|
|
847
|
+
var abandoned = /* @__PURE__ */ new WeakSet();
|
|
848
|
+
function abandonWorker(worker) {
|
|
849
|
+
abandoned.add(worker);
|
|
850
|
+
}
|
|
851
|
+
function isAbandoned(worker) {
|
|
852
|
+
return abandoned.has(worker);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// src/server/services/worker-registry.service.ts
|
|
739
856
|
var _entries;
|
|
740
857
|
exports.WorkerRegistry = class WorkerRegistry {
|
|
741
858
|
constructor(connection, options) {
|
|
@@ -756,7 +873,7 @@ exports.WorkerRegistry = class WorkerRegistry {
|
|
|
756
873
|
try {
|
|
757
874
|
worker = new bullmq.Worker(
|
|
758
875
|
queueName,
|
|
759
|
-
toProcessor(handler, options?.jobTimeoutMs),
|
|
876
|
+
toProcessor(handler, options?.jobTimeoutMs, () => isAbandoned(worker)),
|
|
760
877
|
bullOpts
|
|
761
878
|
);
|
|
762
879
|
} catch (cause) {
|
|
@@ -849,51 +966,15 @@ exports.WorkerRegistry = class WorkerRegistry {
|
|
|
849
966
|
}
|
|
850
967
|
}
|
|
851
968
|
validateOptions(opts) {
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
actual: opts.concurrency
|
|
860
|
-
}
|
|
861
|
-
);
|
|
862
|
-
}
|
|
863
|
-
if (opts.concurrency > MAX_WORKER_CONCURRENCY) {
|
|
864
|
-
throw new QueueException(
|
|
865
|
-
QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
|
|
866
|
-
common.HttpStatus.INTERNAL_SERVER_ERROR,
|
|
867
|
-
{
|
|
868
|
-
reason: `concurrency must be <= ${MAX_WORKER_CONCURRENCY.toString()}`,
|
|
869
|
-
actual: opts.concurrency
|
|
870
|
-
}
|
|
871
|
-
);
|
|
872
|
-
}
|
|
969
|
+
const problem = workerOptionProblem(opts);
|
|
970
|
+
if (problem !== void 0) {
|
|
971
|
+
throw new QueueException(
|
|
972
|
+
QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
|
|
973
|
+
common.HttpStatus.INTERNAL_SERVER_ERROR,
|
|
974
|
+
problem
|
|
975
|
+
);
|
|
873
976
|
}
|
|
874
977
|
this.validateJobTimeout(opts);
|
|
875
|
-
if (opts?.limiter !== void 0) {
|
|
876
|
-
if (opts.limiter.max < 1) {
|
|
877
|
-
throw new QueueException(
|
|
878
|
-
QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
|
|
879
|
-
common.HttpStatus.INTERNAL_SERVER_ERROR,
|
|
880
|
-
{
|
|
881
|
-
reason: "limiter.max must be >= 1",
|
|
882
|
-
actual: opts.limiter.max
|
|
883
|
-
}
|
|
884
|
-
);
|
|
885
|
-
}
|
|
886
|
-
if (opts.limiter.duration < 1) {
|
|
887
|
-
throw new QueueException(
|
|
888
|
-
QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
|
|
889
|
-
common.HttpStatus.INTERNAL_SERVER_ERROR,
|
|
890
|
-
{
|
|
891
|
-
reason: "limiter.duration must be >= 1",
|
|
892
|
-
actual: opts.limiter.duration
|
|
893
|
-
}
|
|
894
|
-
);
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
978
|
}
|
|
898
979
|
validateJobTimeout(opts) {
|
|
899
980
|
if (opts?.jobTimeoutMs !== void 0 && !isTimerDelayMs(opts.jobTimeoutMs)) {
|
|
@@ -970,7 +1051,7 @@ var JobTimeoutError = class extends Error {
|
|
|
970
1051
|
this.name = "JobTimeoutError";
|
|
971
1052
|
}
|
|
972
1053
|
};
|
|
973
|
-
function
|
|
1054
|
+
function adaptHandler(handler, timeoutMs) {
|
|
974
1055
|
if (timeoutMs === void 0) return (job, _token, signal) => handler(job, signal);
|
|
975
1056
|
return (job, _token, signal) => {
|
|
976
1057
|
const own = new AbortController();
|
|
@@ -992,6 +1073,16 @@ function toProcessor(handler, timeoutMs) {
|
|
|
992
1073
|
});
|
|
993
1074
|
};
|
|
994
1075
|
}
|
|
1076
|
+
function toProcessor(handler, timeoutMs, abandoned2) {
|
|
1077
|
+
const run = adaptHandler(handler, timeoutMs);
|
|
1078
|
+
return (job, token, signal) => run(job, token, signal).then(
|
|
1079
|
+
(value) => abandoned2() ? new Promise(() => void 0) : value,
|
|
1080
|
+
(error) => {
|
|
1081
|
+
if (abandoned2()) return new Promise(() => void 0);
|
|
1082
|
+
throw error;
|
|
1083
|
+
}
|
|
1084
|
+
);
|
|
1085
|
+
}
|
|
995
1086
|
var _events, _connections;
|
|
996
1087
|
exports.QueueEventsRegistry = class QueueEventsRegistry {
|
|
997
1088
|
constructor(connection, options) {
|
|
@@ -1170,6 +1261,7 @@ ProcessorDiscoveryService = __decorateClass([
|
|
|
1170
1261
|
__decorateParam(1, common.Inject(exports.WorkerRegistry)),
|
|
1171
1262
|
__decorateParam(2, common.Inject(exports.QueueEventsRegistry))
|
|
1172
1263
|
], ProcessorDiscoveryService);
|
|
1264
|
+
var FORCE_CLOSE_GRACE_MS = 1e3;
|
|
1173
1265
|
var DrainTimeoutError = class extends Error {
|
|
1174
1266
|
};
|
|
1175
1267
|
var QueueLifecycle = class {
|
|
@@ -1203,25 +1295,33 @@ var QueueLifecycle = class {
|
|
|
1203
1295
|
await this.closeQueues();
|
|
1204
1296
|
await this.teardownConnection();
|
|
1205
1297
|
this.logger.log(
|
|
1206
|
-
`Queue shutdown complete in ${String(Date.now() - (this.startedAt ?? Date.now()))}ms (
|
|
1298
|
+
`Queue shutdown complete in ${String(Date.now() - (this.startedAt ?? Date.now()))}ms (${String(this.forcedWorkers)} worker(s) stopped without draining)`
|
|
1207
1299
|
);
|
|
1208
1300
|
}
|
|
1209
1301
|
async closeWorkers(deadline) {
|
|
1210
|
-
let
|
|
1302
|
+
let undrained = 0;
|
|
1211
1303
|
await Promise.all(
|
|
1212
1304
|
Array.from(this.workers.getAll(), async ([name, worker]) => {
|
|
1213
1305
|
try {
|
|
1214
|
-
await bounded(
|
|
1306
|
+
await bounded(
|
|
1307
|
+
worker.pause().then(() => worker.close()),
|
|
1308
|
+
deadline
|
|
1309
|
+
);
|
|
1215
1310
|
} catch (cause) {
|
|
1216
|
-
|
|
1311
|
+
undrained++;
|
|
1312
|
+
abandonWorker(worker);
|
|
1313
|
+
const stopped = "it is stopped without waiting for its in-flight job, which becomes stalled and is retried";
|
|
1217
1314
|
this.logger.warn(
|
|
1218
|
-
cause instanceof DrainTimeoutError ? `Worker "${name}" exceeded the ${String(this.resolved.shutdown.drainTimeoutMs)}ms drain budget
|
|
1315
|
+
cause instanceof DrainTimeoutError ? `Worker "${name}" exceeded the ${String(this.resolved.shutdown.drainTimeoutMs)}ms drain budget (${QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED}); ${stopped}` : `Worker "${name}" failed to close gracefully (${describeError(cause)}); ${stopped}`
|
|
1316
|
+
);
|
|
1317
|
+
const deadlineError = () => new DrainTimeoutError();
|
|
1318
|
+
await withDeadline(worker.close(true), FORCE_CLOSE_GRACE_MS, deadlineError).catch(
|
|
1319
|
+
() => void 0
|
|
1219
1320
|
);
|
|
1220
|
-
void worker.close(true).catch(() => void 0);
|
|
1221
1321
|
}
|
|
1222
1322
|
})
|
|
1223
1323
|
);
|
|
1224
|
-
return
|
|
1324
|
+
return undrained;
|
|
1225
1325
|
}
|
|
1226
1326
|
async closeQueueEvents(deadline) {
|
|
1227
1327
|
await Promise.all(
|
|
@@ -1296,7 +1396,13 @@ exports.BymaxQueueModule = class BymaxQueueModule extends ConfigurableModuleClas
|
|
|
1296
1396
|
{ provide: BYMAX_QUEUE_RESOLVED_OPTIONS, useValue: resolved },
|
|
1297
1397
|
{
|
|
1298
1398
|
provide: exports.FlowService,
|
|
1299
|
-
useFactory: (conn) => new exports.FlowService(
|
|
1399
|
+
useFactory: (conn) => new exports.FlowService(
|
|
1400
|
+
conn,
|
|
1401
|
+
resolved.flows.enabled,
|
|
1402
|
+
resolved.prefix,
|
|
1403
|
+
resolved.telemetry,
|
|
1404
|
+
resolved.enqueueTimeoutMs
|
|
1405
|
+
),
|
|
1300
1406
|
inject: [exports.ConnectionResolver]
|
|
1301
1407
|
},
|
|
1302
1408
|
{
|
|
@@ -1326,7 +1432,13 @@ exports.BymaxQueueModule = class BymaxQueueModule extends ConfigurableModuleClas
|
|
|
1326
1432
|
},
|
|
1327
1433
|
{
|
|
1328
1434
|
provide: exports.FlowService,
|
|
1329
|
-
useFactory: (conn, resolved) => new exports.FlowService(
|
|
1435
|
+
useFactory: (conn, resolved) => new exports.FlowService(
|
|
1436
|
+
conn,
|
|
1437
|
+
resolved.flows.enabled,
|
|
1438
|
+
resolved.prefix,
|
|
1439
|
+
resolved.telemetry,
|
|
1440
|
+
resolved.enqueueTimeoutMs
|
|
1441
|
+
),
|
|
1330
1442
|
inject: [exports.ConnectionResolver, BYMAX_QUEUE_RESOLVED_OPTIONS]
|
|
1331
1443
|
},
|
|
1332
1444
|
{
|