@bymax-one/nest-queue 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +123 -0
- package/LICENSE +21 -0
- package/README.md +858 -0
- package/dist/server/index.cjs +1518 -0
- package/dist/server/index.d.cts +1386 -0
- package/dist/server/index.d.ts +1386 -0
- package/dist/server/index.mjs +1500 -0
- package/dist/shared/index.cjs +32 -0
- package/dist/shared/index.d.cts +132 -0
- package/dist/shared/index.d.ts +132 -0
- package/dist/shared/index.mjs +29 -0
- package/package.json +153 -0
|
@@ -0,0 +1,1500 @@
|
|
|
1
|
+
import { Injectable, Inject, ConfigurableModuleBuilder, Module, Logger, HttpException, HttpStatus } from '@nestjs/common';
|
|
2
|
+
import { DiscoveryService, DiscoveryModule } from '@nestjs/core';
|
|
3
|
+
import { Redis } from 'ioredis';
|
|
4
|
+
import { Worker, QueueEvents, Queue, FlowProducer } from 'bullmq';
|
|
5
|
+
import { isAbsolute, extname } from 'path';
|
|
6
|
+
import 'reflect-metadata';
|
|
7
|
+
|
|
8
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
9
|
+
var __decorateClass = (decorators, target, key, kind) => {
|
|
10
|
+
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
11
|
+
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
12
|
+
if (decorator = decorators[i])
|
|
13
|
+
result = (decorator(result)) || result;
|
|
14
|
+
return result;
|
|
15
|
+
};
|
|
16
|
+
var __decorateParam = (index, decorator) => (target, key) => decorator(target, key, index);
|
|
17
|
+
|
|
18
|
+
// src/shared/constants/error-codes.ts
|
|
19
|
+
var QUEUE_ERROR_CODES = {
|
|
20
|
+
CONNECTION_INVALID: "queue.connection_invalid",
|
|
21
|
+
CONNECTION_REQUIRES_NULL_RETRIES: "queue.connection_requires_null_retries",
|
|
22
|
+
CONNECTION_TIMEOUT: "queue.connection_timeout",
|
|
23
|
+
QUEUE_NOT_FOUND: "queue.queue_not_found",
|
|
24
|
+
JOB_NOT_FOUND: "queue.job_not_found",
|
|
25
|
+
INVALID_JOB_DATA: "queue.invalid_job_data",
|
|
26
|
+
INVALID_REPEAT_OPTIONS: "queue.invalid_repeat_options",
|
|
27
|
+
DUPLICATE_PROCESSOR: "queue.duplicate_processor",
|
|
28
|
+
FLOW_DISABLED: "queue.flow_disabled",
|
|
29
|
+
METRICS_DISABLED: "queue.metrics_disabled",
|
|
30
|
+
SHUTDOWN_TIMEOUT_EXCEEDED: "queue.shutdown_timeout_exceeded",
|
|
31
|
+
BULK_ENQUEUE_FAILED: "queue.bulk_enqueue_failed",
|
|
32
|
+
WORKER_REGISTRATION_FAILED: "queue.worker_registration_failed",
|
|
33
|
+
INVALID_OPTIONS: "queue.invalid_options"
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
// src/server/constants/error-codes.ts
|
|
37
|
+
var QUEUE_ERROR_MESSAGES = {
|
|
38
|
+
[QUEUE_ERROR_CODES.CONNECTION_INVALID]: "Invalid Redis connection configuration",
|
|
39
|
+
[QUEUE_ERROR_CODES.CONNECTION_REQUIRES_NULL_RETRIES]: "Worker connection must have maxRetriesPerRequest=null",
|
|
40
|
+
[QUEUE_ERROR_CODES.CONNECTION_TIMEOUT]: "Redis connection timeout",
|
|
41
|
+
[QUEUE_ERROR_CODES.QUEUE_NOT_FOUND]: "Queue not found",
|
|
42
|
+
[QUEUE_ERROR_CODES.JOB_NOT_FOUND]: "Job not found",
|
|
43
|
+
[QUEUE_ERROR_CODES.INVALID_JOB_DATA]: "Invalid job data",
|
|
44
|
+
[QUEUE_ERROR_CODES.INVALID_REPEAT_OPTIONS]: "Invalid repeat options",
|
|
45
|
+
[QUEUE_ERROR_CODES.DUPLICATE_PROCESSOR]: "Multiple processor decorators target the same queue",
|
|
46
|
+
[QUEUE_ERROR_CODES.FLOW_DISABLED]: "Flow support is disabled",
|
|
47
|
+
[QUEUE_ERROR_CODES.METRICS_DISABLED]: "Metrics support is disabled",
|
|
48
|
+
[QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED]: "Shutdown timeout exceeded",
|
|
49
|
+
[QUEUE_ERROR_CODES.BULK_ENQUEUE_FAILED]: "Bulk enqueue failed",
|
|
50
|
+
[QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED]: "Failed to register worker",
|
|
51
|
+
[QUEUE_ERROR_CODES.INVALID_OPTIONS]: "Invalid module options"
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// src/server/errors/queue-exception.ts
|
|
55
|
+
var QueueException = class extends HttpException {
|
|
56
|
+
constructor(code, statusCode = HttpStatus.INTERNAL_SERVER_ERROR, details) {
|
|
57
|
+
super(
|
|
58
|
+
{
|
|
59
|
+
error: {
|
|
60
|
+
code,
|
|
61
|
+
message: QUEUE_ERROR_MESSAGES[code] ?? "Queue error",
|
|
62
|
+
details: details ?? null
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
statusCode
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
// src/server/config/validate-options.ts
|
|
71
|
+
function invalid(reason) {
|
|
72
|
+
throw new QueueException(QUEUE_ERROR_CODES.INVALID_OPTIONS, 500, { reason });
|
|
73
|
+
}
|
|
74
|
+
function validateOptions(opts) {
|
|
75
|
+
const cfg = opts.connection;
|
|
76
|
+
if (cfg === null || typeof cfg !== "object") {
|
|
77
|
+
invalid("connection is required");
|
|
78
|
+
}
|
|
79
|
+
const hasClient = "client" in cfg;
|
|
80
|
+
const hasUrl = "url" in cfg;
|
|
81
|
+
const hasOptions = "options" in cfg;
|
|
82
|
+
if (!hasClient && !hasUrl && !hasOptions) {
|
|
83
|
+
invalid("connection must specify client | url | options");
|
|
84
|
+
}
|
|
85
|
+
if (hasClient && (hasUrl || hasOptions)) {
|
|
86
|
+
invalid("connection.client is mutually exclusive with url/options");
|
|
87
|
+
}
|
|
88
|
+
const drainTimeoutMs = opts.shutdown?.drainTimeoutMs;
|
|
89
|
+
if (drainTimeoutMs !== void 0 && drainTimeoutMs <= 0) {
|
|
90
|
+
invalid("shutdown.drainTimeoutMs must be > 0");
|
|
91
|
+
}
|
|
92
|
+
const cacheTtlMs = opts.metrics?.cacheTtlMs;
|
|
93
|
+
if (cacheTtlMs !== void 0 && cacheTtlMs < 0) {
|
|
94
|
+
invalid("metrics.cacheTtlMs must be >= 0");
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// src/server/constants/default-options.ts
|
|
99
|
+
var DEFAULT_WORKER_CONCURRENCY = 2;
|
|
100
|
+
var MAX_WORKER_CONCURRENCY = 1e3;
|
|
101
|
+
var DEFAULT_JOB_OPTIONS = {
|
|
102
|
+
attempts: 3,
|
|
103
|
+
backoff: { type: "exponential", delay: 2e3 },
|
|
104
|
+
removeOnComplete: { age: 24 * 3600, count: 1e3 },
|
|
105
|
+
removeOnFail: { age: 7 * 24 * 3600, count: 5e3 }
|
|
106
|
+
};
|
|
107
|
+
var DEFAULT_CONNECTION_READY_TIMEOUT_MS = 1e4;
|
|
108
|
+
var DEFAULT_METRICS_CACHE_TTL_MS = 5e3;
|
|
109
|
+
var DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS = 3e4;
|
|
110
|
+
|
|
111
|
+
// src/server/config/resolved-options.ts
|
|
112
|
+
function applyDefaults(opts) {
|
|
113
|
+
const base = {
|
|
114
|
+
connection: opts.connection,
|
|
115
|
+
defaultJobOptions: { ...DEFAULT_JOB_OPTIONS, ...opts.defaultJobOptions ?? {} },
|
|
116
|
+
prefix: opts.prefix ?? "bull",
|
|
117
|
+
queueOptions: opts.queueOptions ?? {},
|
|
118
|
+
flows: { enabled: opts.flows?.enabled ?? false },
|
|
119
|
+
metrics: {
|
|
120
|
+
enabled: opts.metrics?.enabled ?? false,
|
|
121
|
+
cacheTtlMs: opts.metrics?.cacheTtlMs ?? DEFAULT_METRICS_CACHE_TTL_MS
|
|
122
|
+
},
|
|
123
|
+
shutdown: {
|
|
124
|
+
drainTimeoutMs: opts.shutdown?.drainTimeoutMs ?? DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS,
|
|
125
|
+
drainOnShutdown: opts.shutdown?.drainOnShutdown ?? false
|
|
126
|
+
},
|
|
127
|
+
connectionReadyTimeoutMs: opts.connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
|
|
128
|
+
};
|
|
129
|
+
if (opts.telemetry !== void 0) base.telemetry = opts.telemetry;
|
|
130
|
+
return Object.freeze(base);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// src/server/bymax-queue.constants.ts
|
|
134
|
+
var BYMAX_QUEUE_OPTIONS = /* @__PURE__ */ Symbol("BYMAX_QUEUE_OPTIONS");
|
|
135
|
+
var BYMAX_QUEUE_REDIS_CLIENT = /* @__PURE__ */ Symbol("BYMAX_QUEUE_REDIS_CLIENT");
|
|
136
|
+
var BYMAX_QUEUE_CONNECTION_MODE = /* @__PURE__ */ Symbol("BYMAX_QUEUE_CONNECTION_MODE");
|
|
137
|
+
var BYMAX_QUEUE_RESOLVED_OPTIONS = /* @__PURE__ */ Symbol("BYMAX_QUEUE_RESOLVED_OPTIONS");
|
|
138
|
+
|
|
139
|
+
// src/server/utils/validate-connection.ts
|
|
140
|
+
function assertBlockingConnection(client) {
|
|
141
|
+
const actualValue = client.options?.maxRetriesPerRequest;
|
|
142
|
+
if (actualValue !== null) {
|
|
143
|
+
throw new QueueException(QUEUE_ERROR_CODES.CONNECTION_REQUIRES_NULL_RETRIES, 500, {
|
|
144
|
+
actualValue: actualValue ?? null,
|
|
145
|
+
expectedValue: null
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
function isClientUsable(client) {
|
|
150
|
+
return client.status === "ready" || client.status === "connecting";
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// src/server/utils/duplicate-connection.ts
|
|
154
|
+
function duplicateConnection(client) {
|
|
155
|
+
return client.duplicate({ maxRetriesPerRequest: null });
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// src/server/services/connection-resolver.service.ts
|
|
159
|
+
var ConnectionResolver = class {
|
|
160
|
+
constructor(options) {
|
|
161
|
+
this.options = options;
|
|
162
|
+
}
|
|
163
|
+
options;
|
|
164
|
+
client;
|
|
165
|
+
mode;
|
|
166
|
+
/** Resolve and validate the connection. Call once during module bootstrap. */
|
|
167
|
+
async init() {
|
|
168
|
+
const cfg = this.options.connection;
|
|
169
|
+
if ("client" in cfg) {
|
|
170
|
+
this.initModeA(cfg.client);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
this.mode = "mode-b-owned";
|
|
174
|
+
this.client = "url" in cfg ? new Redis(cfg.url, { ...cfg.options ?? {}, lazyConnect: false }) : new Redis({ ...cfg.options, lazyConnect: false });
|
|
175
|
+
await this.waitReady(
|
|
176
|
+
this.options.connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
/** The resolved Queue-role client. */
|
|
180
|
+
getClient() {
|
|
181
|
+
if (!this.client) {
|
|
182
|
+
throw new QueueException(QUEUE_ERROR_CODES.CONNECTION_INVALID, 500, {
|
|
183
|
+
reason: "not initialized"
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
return this.client;
|
|
187
|
+
}
|
|
188
|
+
/** The resolved connection mode. */
|
|
189
|
+
getMode() {
|
|
190
|
+
if (!this.mode) {
|
|
191
|
+
throw new QueueException(QUEUE_ERROR_CODES.CONNECTION_INVALID, 500, {
|
|
192
|
+
reason: "not initialized"
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
return this.mode;
|
|
196
|
+
}
|
|
197
|
+
/** Whether the library owns (and must close) the connection. */
|
|
198
|
+
isOwned() {
|
|
199
|
+
return this.mode === "mode-b-owned";
|
|
200
|
+
}
|
|
201
|
+
/** Close the library-owned connection on shutdown; never touch a BYO client. */
|
|
202
|
+
async teardown() {
|
|
203
|
+
if (this.isOwned() && this.client) {
|
|
204
|
+
const client = this.client;
|
|
205
|
+
await client.quit().catch(() => {
|
|
206
|
+
client.disconnect();
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/** Validate and adopt a bring-your-own client for the Queue role. */
|
|
211
|
+
initModeA(client) {
|
|
212
|
+
this.mode = "mode-a-byo";
|
|
213
|
+
this.client = client;
|
|
214
|
+
if (!isClientUsable(client)) {
|
|
215
|
+
throw new QueueException(QUEUE_ERROR_CODES.CONNECTION_INVALID, 500, { status: client.status });
|
|
216
|
+
}
|
|
217
|
+
const probe = duplicateConnection(client);
|
|
218
|
+
try {
|
|
219
|
+
assertBlockingConnection(probe);
|
|
220
|
+
} finally {
|
|
221
|
+
probe.disconnect();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/** Resolve when the owned client reaches `ready`, or reject on timeout/error. */
|
|
225
|
+
async waitReady(timeoutMs) {
|
|
226
|
+
const client = this.getClient();
|
|
227
|
+
if (client.status === "ready") return;
|
|
228
|
+
await new Promise((resolve, reject) => {
|
|
229
|
+
const cleanup = () => {
|
|
230
|
+
clearTimeout(timer);
|
|
231
|
+
client.off("ready", onReady);
|
|
232
|
+
client.off("error", onError);
|
|
233
|
+
};
|
|
234
|
+
const onReady = () => {
|
|
235
|
+
cleanup();
|
|
236
|
+
resolve();
|
|
237
|
+
};
|
|
238
|
+
const onError = (err) => {
|
|
239
|
+
cleanup();
|
|
240
|
+
reject(err);
|
|
241
|
+
};
|
|
242
|
+
const timer = setTimeout(() => {
|
|
243
|
+
cleanup();
|
|
244
|
+
reject(new QueueException(QUEUE_ERROR_CODES.CONNECTION_TIMEOUT, 500, { timeoutMs }));
|
|
245
|
+
}, timeoutMs);
|
|
246
|
+
client.once("ready", onReady);
|
|
247
|
+
client.once("error", onError);
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
ConnectionResolver = __decorateClass([
|
|
252
|
+
Injectable(),
|
|
253
|
+
__decorateParam(0, Inject(BYMAX_QUEUE_OPTIONS))
|
|
254
|
+
], ConnectionResolver);
|
|
255
|
+
|
|
256
|
+
// src/server/utils/validate-job-scheduler-options.ts
|
|
257
|
+
function invalid2(reason) {
|
|
258
|
+
throw new QueueException(QUEUE_ERROR_CODES.INVALID_REPEAT_OPTIONS, 400, { reason });
|
|
259
|
+
}
|
|
260
|
+
function toEpochMs(date) {
|
|
261
|
+
return typeof date === "number" ? date : Date.parse(date);
|
|
262
|
+
}
|
|
263
|
+
function validateJobSchedulerOptions(repeat) {
|
|
264
|
+
const shape = repeat;
|
|
265
|
+
if (shape === null || typeof shape !== "object" || Array.isArray(shape)) {
|
|
266
|
+
invalid2("repeat must be an object declaring either pattern or every");
|
|
267
|
+
}
|
|
268
|
+
const hasPattern = "pattern" in repeat;
|
|
269
|
+
const hasEvery = "every" in repeat;
|
|
270
|
+
if (hasPattern === hasEvery) {
|
|
271
|
+
invalid2("exactly one of pattern | every is required");
|
|
272
|
+
}
|
|
273
|
+
if ("pattern" in repeat && (typeof repeat.pattern !== "string" || repeat.pattern.trim() === "")) {
|
|
274
|
+
invalid2("pattern must be a non-empty string");
|
|
275
|
+
}
|
|
276
|
+
if ("every" in repeat && (!Number.isInteger(repeat.every) || repeat.every <= 0)) {
|
|
277
|
+
invalid2("every must be a positive integer");
|
|
278
|
+
}
|
|
279
|
+
if (repeat.endDate !== void 0) {
|
|
280
|
+
const endMs = toEpochMs(repeat.endDate);
|
|
281
|
+
if (Number.isNaN(endMs) || endMs <= Date.now()) {
|
|
282
|
+
invalid2("endDate must be in the future");
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// src/server/utils/attach-error-listener.ts
|
|
288
|
+
function attachDefaultErrorListener(emitter, logger, kind, queueName) {
|
|
289
|
+
emitter.on("error", (err) => {
|
|
290
|
+
logger.error(`${kind} for queue "${queueName}" emitted an error: ${err.message}`);
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// src/server/services/queue.service.ts
|
|
295
|
+
var MAX_BULK_SIZE = 1e3;
|
|
296
|
+
function isOperationalError(err) {
|
|
297
|
+
return typeof err?.code === "string";
|
|
298
|
+
}
|
|
299
|
+
var QueueService = class {
|
|
300
|
+
constructor(connection, options) {
|
|
301
|
+
this.connection = connection;
|
|
302
|
+
this.options = options;
|
|
303
|
+
}
|
|
304
|
+
connection;
|
|
305
|
+
options;
|
|
306
|
+
logger = new Logger(QueueService.name);
|
|
307
|
+
queues = /* @__PURE__ */ new Map();
|
|
308
|
+
/**
|
|
309
|
+
* Return the cached `Queue` for `queueName`, or create one with the module
|
|
310
|
+
* defaults applied. Subsequent calls with the same name return the same
|
|
311
|
+
* instance.
|
|
312
|
+
*
|
|
313
|
+
* @param queueName - Unique queue name (used as the Redis key prefix).
|
|
314
|
+
* @param overrides - Optional per-queue option overrides.
|
|
315
|
+
*/
|
|
316
|
+
getOrCreateQueue(queueName, overrides) {
|
|
317
|
+
const existing = this.queues.get(queueName);
|
|
318
|
+
if (existing) return existing;
|
|
319
|
+
const queue = new Queue(queueName, {
|
|
320
|
+
connection: this.connection.getClient(),
|
|
321
|
+
prefix: this.options.prefix,
|
|
322
|
+
defaultJobOptions: this.options.defaultJobOptions,
|
|
323
|
+
...this.options.queueOptions,
|
|
324
|
+
...this.options.telemetry ? { telemetry: this.options.telemetry } : {},
|
|
325
|
+
...overrides
|
|
326
|
+
});
|
|
327
|
+
attachDefaultErrorListener(queue, this.logger, "Queue", queueName);
|
|
328
|
+
this.queues.set(queueName, queue);
|
|
329
|
+
return queue;
|
|
330
|
+
}
|
|
331
|
+
/** Resolve the raw cached queue (default generics) for internal delegation. */
|
|
332
|
+
queueFor(queueName) {
|
|
333
|
+
return this.getOrCreateQueue(queueName);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Add a single job to a queue. The queue is created lazily if needed.
|
|
337
|
+
*
|
|
338
|
+
* Delivery is **at-least-once** — a handler may run more than once for the same
|
|
339
|
+
* job (worker crash, lock expiry, or a shutdown force-close). Make handlers
|
|
340
|
+
* idempotent: use an idempotency key on writes, prefer upserts over inserts, or
|
|
341
|
+
* keep an "already-processed" marker keyed by `job.id`.
|
|
342
|
+
*
|
|
343
|
+
* `options` surfaces BullMQ natives directly — including `jobId` (idempotent
|
|
344
|
+
* insert: a second add with the same id is a no-op while the first exists) and
|
|
345
|
+
* `deduplication` (windowed deduplication). The library writes no deduplication
|
|
346
|
+
* code; BullMQ owns the behavior. The deduplication key is independent of
|
|
347
|
+
* `jobId`, so both can be set together or apart. Both collapse duplicate
|
|
348
|
+
* **producers** and do not change the at-least-once guarantee on the
|
|
349
|
+
* **consumer** side.
|
|
350
|
+
*
|
|
351
|
+
* Deduplication modes (`deduplication: { id, ttl?, extend?, replace?, keepLastIfActive? }`):
|
|
352
|
+
* - Simple `{ id }` — collapse duplicates until the in-flight job completes or fails.
|
|
353
|
+
* - Throttle `{ id, ttl }` — ignore duplicates for `ttl` milliseconds.
|
|
354
|
+
* - Debounce `{ id, ttl, extend: true, replace: true }` — keep only the latest
|
|
355
|
+
* data; each duplicate resets the TTL.
|
|
356
|
+
* - keep-last-if-active `{ id, keepLastIfActive: true }` — while a job is active,
|
|
357
|
+
* store the latest data and run a single follow-up when it finishes.
|
|
358
|
+
*
|
|
359
|
+
* @param queueName - Target queue (created lazily if absent).
|
|
360
|
+
* @param jobName - Job identifier used by `@Process` to dispatch.
|
|
361
|
+
* @param data - Typed job payload.
|
|
362
|
+
* @param options - Per-job BullMQ options (priority, delay, `jobId`, `deduplication`, ...).
|
|
363
|
+
* @returns The created job.
|
|
364
|
+
* @example Throttle: at most one reindex per term every 5 seconds.
|
|
365
|
+
* await queueService.enqueue('search', 'reindex', { term }, {
|
|
366
|
+
* deduplication: { id: `reindex:${term}`, ttl: 5_000 },
|
|
367
|
+
* })
|
|
368
|
+
*/
|
|
369
|
+
async enqueue(queueName, jobName, data, options) {
|
|
370
|
+
const job = await this.queueFor(queueName).add(jobName, data, options);
|
|
371
|
+
return job;
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Enqueue multiple jobs in a single Redis roundtrip. The batch is bounded by
|
|
375
|
+
* {@link MAX_BULK_SIZE} to guard against a self-inflicted memory spike; any
|
|
376
|
+
* failure is wrapped in a `QueueException`. Delivery is at-least-once, so every
|
|
377
|
+
* job's handler must be idempotent (see {@link QueueService.enqueue}).
|
|
378
|
+
*
|
|
379
|
+
* @param queueName - Target queue.
|
|
380
|
+
* @param jobs - Job descriptors (length must not exceed the bulk size cap).
|
|
381
|
+
*/
|
|
382
|
+
async enqueueBulk(queueName, jobs) {
|
|
383
|
+
if (jobs.length > MAX_BULK_SIZE) {
|
|
384
|
+
throw new QueueException(QUEUE_ERROR_CODES.BULK_ENQUEUE_FAILED, 500, {
|
|
385
|
+
reason: "batch size exceeds limit",
|
|
386
|
+
limit: MAX_BULK_SIZE,
|
|
387
|
+
received: jobs.length
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
try {
|
|
391
|
+
const created = await this.queueFor(queueName).addBulk(
|
|
392
|
+
jobs.map((job) => ({
|
|
393
|
+
name: job.name,
|
|
394
|
+
data: job.data,
|
|
395
|
+
...job.options === void 0 ? {} : { opts: job.options }
|
|
396
|
+
}))
|
|
397
|
+
);
|
|
398
|
+
return created;
|
|
399
|
+
} catch (err) {
|
|
400
|
+
throw new QueueException(QUEUE_ERROR_CODES.BULK_ENQUEUE_FAILED, 500, {
|
|
401
|
+
cause: err.message
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Fetch a job by id. Returns `null` when the job does not exist.
|
|
407
|
+
*
|
|
408
|
+
* @param queueName - Target queue.
|
|
409
|
+
* @param jobId - The job identifier.
|
|
410
|
+
*/
|
|
411
|
+
async getJob(queueName, jobId) {
|
|
412
|
+
const job = await this.queueFor(queueName).getJob(jobId);
|
|
413
|
+
return job ?? null;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Fetch jobs in a given status with pagination.
|
|
417
|
+
*
|
|
418
|
+
* @param queueName - Target queue.
|
|
419
|
+
* @param status - Status filter.
|
|
420
|
+
* @param start - Starting index (inclusive). Default: 0.
|
|
421
|
+
* @param end - Ending index (inclusive). Default: 50.
|
|
422
|
+
*/
|
|
423
|
+
async getJobs(queueName, status, start = 0, end = 50) {
|
|
424
|
+
const jobs = await this.queueFor(queueName).getJobs([status], start, end);
|
|
425
|
+
return jobs;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Return an uncached snapshot of job counts grouped by status.
|
|
429
|
+
*
|
|
430
|
+
* @param queueName - Target queue.
|
|
431
|
+
*/
|
|
432
|
+
async getMetrics(queueName) {
|
|
433
|
+
const counts = await this.queueFor(queueName).getJobCounts(
|
|
434
|
+
"waiting",
|
|
435
|
+
"active",
|
|
436
|
+
"completed",
|
|
437
|
+
"failed",
|
|
438
|
+
"delayed",
|
|
439
|
+
"paused"
|
|
440
|
+
);
|
|
441
|
+
return {
|
|
442
|
+
queue: queueName,
|
|
443
|
+
counts,
|
|
444
|
+
collectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Create or update a Job Scheduler — the current BullMQ recurring-jobs API.
|
|
449
|
+
* Idempotent by `schedulerId`: BullMQ performs an atomic `override: true`
|
|
450
|
+
* upsert, so re-registering on every boot updates the schedule in place and
|
|
451
|
+
* never duplicates it. This supersedes the repeatable-jobs surface removed in
|
|
452
|
+
* BullMQ v6.
|
|
453
|
+
*
|
|
454
|
+
* The schedule is structurally validated first; the cron string itself is
|
|
455
|
+
* parsed by BullMQ at registration time, and a parse failure is rethrown as
|
|
456
|
+
* `INVALID_REPEAT_OPTIONS` (400).
|
|
457
|
+
*
|
|
458
|
+
* @param queueName - Target queue.
|
|
459
|
+
* @param schedulerId - Stable, unique scheduler identifier (the upsert key).
|
|
460
|
+
* @param repeat - Cron `pattern` OR `every` (ms), plus optional tz/limit/start/end.
|
|
461
|
+
* @param template - Optional job template; `name` defaults to `schedulerId`, `data` to `{}`.
|
|
462
|
+
* @returns The first scheduled (delayed) job, or `undefined`.
|
|
463
|
+
* @throws {QueueException} `INVALID_REPEAT_OPTIONS` (400) for an invalid schedule.
|
|
464
|
+
*/
|
|
465
|
+
async upsertJobScheduler(queueName, schedulerId, repeat, template) {
|
|
466
|
+
validateJobSchedulerOptions(repeat);
|
|
467
|
+
const queue = this.queueFor(queueName);
|
|
468
|
+
const jobTemplate = {
|
|
469
|
+
name: template?.name ?? schedulerId,
|
|
470
|
+
data: template?.data ?? {},
|
|
471
|
+
...template?.opts === void 0 ? {} : { opts: template.opts }
|
|
472
|
+
};
|
|
473
|
+
if ("pattern" in repeat) {
|
|
474
|
+
try {
|
|
475
|
+
const job2 = await queue.upsertJobScheduler(schedulerId, repeat, jobTemplate);
|
|
476
|
+
return job2;
|
|
477
|
+
} catch (err) {
|
|
478
|
+
if (isOperationalError(err)) throw err;
|
|
479
|
+
throw new QueueException(QUEUE_ERROR_CODES.INVALID_REPEAT_OPTIONS, 400, {
|
|
480
|
+
reason: "pattern must be a valid cron expression (5- or 6-field)"
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
const job = await queue.upsertJobScheduler(schedulerId, repeat, jobTemplate);
|
|
485
|
+
return job;
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Remove a Job Scheduler by id.
|
|
489
|
+
*
|
|
490
|
+
* @param queueName - Target queue.
|
|
491
|
+
* @param schedulerId - The scheduler identifier to remove.
|
|
492
|
+
* @returns `true` when a scheduler was removed, `false` otherwise.
|
|
493
|
+
*/
|
|
494
|
+
async removeJobScheduler(queueName, schedulerId) {
|
|
495
|
+
return this.queueFor(queueName).removeJobScheduler(schedulerId);
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* List the Job Schedulers registered on a queue, for inspection or health checks.
|
|
499
|
+
*
|
|
500
|
+
* @param queueName - Target queue.
|
|
501
|
+
* @param start - Page start index (inclusive). Default: 0.
|
|
502
|
+
* @param end - Page end index (inclusive). Default: 50.
|
|
503
|
+
* @param asc - Sort ascending by next run time. Default: true.
|
|
504
|
+
* @returns The registered schedulers in the requested page.
|
|
505
|
+
*/
|
|
506
|
+
async getJobSchedulers(queueName, start = 0, end = 50, asc = true) {
|
|
507
|
+
return this.queueFor(queueName).getJobSchedulers(start, end, asc);
|
|
508
|
+
}
|
|
509
|
+
/** Pause a queue, halting the processing of new jobs. */
|
|
510
|
+
async pauseQueue(queueName) {
|
|
511
|
+
await this.queueFor(queueName).pause();
|
|
512
|
+
}
|
|
513
|
+
/** Resume a previously paused queue. */
|
|
514
|
+
async resumeQueue(queueName) {
|
|
515
|
+
await this.queueFor(queueName).resume();
|
|
516
|
+
}
|
|
517
|
+
/**
|
|
518
|
+
* Remove up to `limit` jobs older than `gracePeriodMs` in a given status. The
|
|
519
|
+
* argument order mirrors BullMQ's `Queue.clean(grace, limit, type?)` exactly;
|
|
520
|
+
* `limit` is required and `0` means no limit. Returns the removed job ids.
|
|
521
|
+
*
|
|
522
|
+
* @param queueName - Target queue.
|
|
523
|
+
* @param gracePeriodMs - Keep jobs younger than this many milliseconds.
|
|
524
|
+
* @param limit - Maximum number of jobs to remove (`0` = no limit).
|
|
525
|
+
* @param status - Status of jobs to clean.
|
|
526
|
+
*/
|
|
527
|
+
async cleanQueue(queueName, gracePeriodMs, limit, status) {
|
|
528
|
+
return this.queueFor(queueName).clean(gracePeriodMs, limit, status);
|
|
529
|
+
}
|
|
530
|
+
/** Return a read-only view of the cached queues. */
|
|
531
|
+
getCachedQueues() {
|
|
532
|
+
return this.queues;
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Close every cached queue (swallowing per-queue errors) and clear the cache.
|
|
536
|
+
*
|
|
537
|
+
* Deliberately NOT named `onModuleDestroy`: NestJS invokes lifecycle hooks by
|
|
538
|
+
* method name, so a hook here would run on Nest's own schedule, concurrently
|
|
539
|
+
* with — and possibly before — the bounded drain in `QueueLifecycle`. Closing a
|
|
540
|
+
* queue while its worker is still draining is exactly what the ordered sequence
|
|
541
|
+
* exists to prevent. `QueueLifecycle` is the only caller.
|
|
542
|
+
*/
|
|
543
|
+
async closeAll() {
|
|
544
|
+
for (const queue of this.queues.values()) {
|
|
545
|
+
await queue.close().catch(() => void 0);
|
|
546
|
+
}
|
|
547
|
+
this.queues.clear();
|
|
548
|
+
}
|
|
549
|
+
};
|
|
550
|
+
QueueService = __decorateClass([
|
|
551
|
+
Injectable(),
|
|
552
|
+
__decorateParam(0, Inject(ConnectionResolver)),
|
|
553
|
+
__decorateParam(1, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
|
|
554
|
+
], QueueService);
|
|
555
|
+
var FlowService = class {
|
|
556
|
+
logger = new Logger(FlowService.name);
|
|
557
|
+
producer;
|
|
558
|
+
constructor(connection, enabled, prefix, telemetry) {
|
|
559
|
+
if (enabled) {
|
|
560
|
+
this.producer = new FlowProducer({
|
|
561
|
+
connection: connection.getClient(),
|
|
562
|
+
// Match the producer Queue's key prefix so flow jobs land in the same
|
|
563
|
+
// keyspace the workers poll; a non-default prefix breaks flows otherwise.
|
|
564
|
+
prefix,
|
|
565
|
+
...telemetry ? { telemetry } : {}
|
|
566
|
+
});
|
|
567
|
+
attachDefaultErrorListener(this.producer, this.logger, "FlowProducer", "*");
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Add a flow tree (a parent with its descendant children). The root job
|
|
572
|
+
* becomes processable only after every descendant has completed.
|
|
573
|
+
*
|
|
574
|
+
* By BullMQ default a failed (retry-exhausted) child does NOT fail its parent:
|
|
575
|
+
* the parent stays in the `waiting-children` state until the child eventually
|
|
576
|
+
* completes. To make a child's failure propagate upward and fail its parent,
|
|
577
|
+
* set `failParentOnFailure: true` on that child (it cascades when ancestors
|
|
578
|
+
* also set it). To let a parent proceed despite a failed child, set
|
|
579
|
+
* `ignoreDependencyOnFailure: true` on that child. Bound each child's
|
|
580
|
+
* `attempts` so an unrecoverable child cannot leave the parent waiting forever.
|
|
581
|
+
*
|
|
582
|
+
* @param flow - Flow tree definition.
|
|
583
|
+
* @returns The root `JobNode` containing the created job and its children.
|
|
584
|
+
* @throws {QueueException} `FLOW_DISABLED` (503) when flows are not enabled.
|
|
585
|
+
*/
|
|
586
|
+
async add(flow) {
|
|
587
|
+
return this.requireProducer().add(flow);
|
|
588
|
+
}
|
|
589
|
+
/**
|
|
590
|
+
* Add multiple flow trees in a single Redis roundtrip.
|
|
591
|
+
*
|
|
592
|
+
* @param flows - Flow tree definitions.
|
|
593
|
+
* @returns The root `JobNode` for each flow, in input order.
|
|
594
|
+
* @throws {QueueException} `FLOW_DISABLED` (503) when flows are not enabled.
|
|
595
|
+
*/
|
|
596
|
+
async addBulk(flows) {
|
|
597
|
+
return this.requireProducer().addBulk([...flows]);
|
|
598
|
+
}
|
|
599
|
+
/**
|
|
600
|
+
* Escape hatch returning the underlying `FlowProducer` for advanced use cases.
|
|
601
|
+
* Prefer {@link FlowService.add} / {@link FlowService.addBulk} when possible.
|
|
602
|
+
*
|
|
603
|
+
* @returns The underlying BullMQ `FlowProducer`.
|
|
604
|
+
* @throws {QueueException} `FLOW_DISABLED` (503) when flows are not enabled.
|
|
605
|
+
*/
|
|
606
|
+
getProducer() {
|
|
607
|
+
return this.requireProducer();
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Close the producer when active; a no-op when inactive. A failed `close()` is
|
|
611
|
+
* swallowed so it never aborts the shutdown sequence.
|
|
612
|
+
*
|
|
613
|
+
* Deliberately NOT named `onModuleDestroy`: NestJS invokes lifecycle hooks by
|
|
614
|
+
* method name, so a hook here would run in addition to the call the ordered
|
|
615
|
+
* sequence already makes, closing the producer twice. `QueueLifecycle` is the
|
|
616
|
+
* only caller.
|
|
617
|
+
*/
|
|
618
|
+
async close() {
|
|
619
|
+
if (this.producer) {
|
|
620
|
+
await this.producer.close().catch(() => void 0);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
/**
|
|
624
|
+
* Return the active producer or throw when flows are disabled. The producer is
|
|
625
|
+
* present if and only if flows were enabled at construction time.
|
|
626
|
+
*
|
|
627
|
+
* @throws {QueueException} `FLOW_DISABLED` (503) when no producer exists.
|
|
628
|
+
*/
|
|
629
|
+
requireProducer() {
|
|
630
|
+
if (!this.producer) {
|
|
631
|
+
throw new QueueException(QUEUE_ERROR_CODES.FLOW_DISABLED, 503);
|
|
632
|
+
}
|
|
633
|
+
return this.producer;
|
|
634
|
+
}
|
|
635
|
+
};
|
|
636
|
+
FlowService = __decorateClass([
|
|
637
|
+
Injectable()
|
|
638
|
+
], FlowService);
|
|
639
|
+
var MetricsService = class {
|
|
640
|
+
constructor(queueService, enabled, ttlMs) {
|
|
641
|
+
this.queueService = queueService;
|
|
642
|
+
this.enabled = enabled;
|
|
643
|
+
this.ttlMs = ttlMs;
|
|
644
|
+
}
|
|
645
|
+
queueService;
|
|
646
|
+
enabled;
|
|
647
|
+
ttlMs;
|
|
648
|
+
cache = /* @__PURE__ */ new Map();
|
|
649
|
+
/**
|
|
650
|
+
* Return cached metrics for `queueName`, or fetch fresh (and cache them) on a
|
|
651
|
+
* miss or after expiry.
|
|
652
|
+
*
|
|
653
|
+
* @param queueName - Target queue.
|
|
654
|
+
* @returns The current metrics snapshot.
|
|
655
|
+
* @throws {QueueException} `METRICS_DISABLED` (503) when metrics are disabled.
|
|
656
|
+
*/
|
|
657
|
+
async get(queueName) {
|
|
658
|
+
this.ensureEnabled();
|
|
659
|
+
const entry = this.cache.get(queueName);
|
|
660
|
+
if (entry && entry.expiresAt > Date.now()) {
|
|
661
|
+
return entry.metrics;
|
|
662
|
+
}
|
|
663
|
+
const metrics = await this.queueService.getMetrics(queueName);
|
|
664
|
+
this.cache.set(queueName, { metrics, expiresAt: Date.now() + this.ttlMs });
|
|
665
|
+
return metrics;
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Return a metrics snapshot for every queue currently cached in `QueueService`.
|
|
669
|
+
*
|
|
670
|
+
* @returns Metrics for all known queues.
|
|
671
|
+
* @throws {QueueException} `METRICS_DISABLED` (503) when metrics are disabled.
|
|
672
|
+
*/
|
|
673
|
+
async getAll() {
|
|
674
|
+
this.ensureEnabled();
|
|
675
|
+
const names = Array.from(this.queueService.getCachedQueues().keys());
|
|
676
|
+
return Promise.all(names.map((name) => this.get(name)));
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Invalidate the cache for `queueName`, or the whole cache when omitted.
|
|
680
|
+
*
|
|
681
|
+
* @param queueName - The queue entry to drop; omit to clear everything.
|
|
682
|
+
* @throws {QueueException} `METRICS_DISABLED` (503) when metrics are disabled.
|
|
683
|
+
*/
|
|
684
|
+
invalidate(queueName) {
|
|
685
|
+
this.ensureEnabled();
|
|
686
|
+
if (queueName === void 0) {
|
|
687
|
+
this.cache.clear();
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
this.cache.delete(queueName);
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Guard every operation behind the opt-in flag.
|
|
694
|
+
*
|
|
695
|
+
* @throws {QueueException} `METRICS_DISABLED` (503) when metrics are disabled.
|
|
696
|
+
*/
|
|
697
|
+
ensureEnabled() {
|
|
698
|
+
if (!this.enabled) {
|
|
699
|
+
throw new QueueException(QUEUE_ERROR_CODES.METRICS_DISABLED, 503);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
};
|
|
703
|
+
MetricsService = __decorateClass([
|
|
704
|
+
Injectable()
|
|
705
|
+
], MetricsService);
|
|
706
|
+
var WorkerRegistry = class {
|
|
707
|
+
constructor(connection, options) {
|
|
708
|
+
this.connection = connection;
|
|
709
|
+
this.options = options;
|
|
710
|
+
}
|
|
711
|
+
connection;
|
|
712
|
+
options;
|
|
713
|
+
logger = new Logger(WorkerRegistry.name);
|
|
714
|
+
entries = /* @__PURE__ */ new Map();
|
|
715
|
+
/**
|
|
716
|
+
* Register an in-process worker with a handler function. The BullMQ `Worker`
|
|
717
|
+
* constructor is synchronous — no Redis handshake is performed at registration
|
|
718
|
+
* time. The duplicated connection is disconnected immediately if construction
|
|
719
|
+
* fails, preventing a Redis connection leak.
|
|
720
|
+
*
|
|
721
|
+
* @param config - Queue name, handler, and optional worker options.
|
|
722
|
+
* @returns The created BullMQ `Worker` instance.
|
|
723
|
+
* @throws {QueueException} `DUPLICATE_PROCESSOR` when a worker for
|
|
724
|
+
* `queueName` already exists.
|
|
725
|
+
* @throws {QueueException} `WORKER_REGISTRATION_FAILED` when options are
|
|
726
|
+
* invalid or the BullMQ `Worker` constructor throws.
|
|
727
|
+
*/
|
|
728
|
+
register(config) {
|
|
729
|
+
const { queueName, handler, options } = config;
|
|
730
|
+
this.guardDuplicate(queueName);
|
|
731
|
+
this.validateOptions(options);
|
|
732
|
+
const conn = duplicateConnection(this.connection.getClient());
|
|
733
|
+
const bullOpts = this.buildBullOptions(options, conn);
|
|
734
|
+
let worker;
|
|
735
|
+
try {
|
|
736
|
+
worker = new Worker(queueName, handler, bullOpts);
|
|
737
|
+
} catch (cause) {
|
|
738
|
+
conn.disconnect();
|
|
739
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
740
|
+
queueName,
|
|
741
|
+
cause: cause instanceof Error ? cause.message : String(cause)
|
|
742
|
+
});
|
|
743
|
+
}
|
|
744
|
+
attachDefaultErrorListener(worker, this.logger, "Worker", queueName);
|
|
745
|
+
this.entries.set(queueName, { worker, connection: conn });
|
|
746
|
+
return worker;
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Register a sandboxed (file-based, out-of-process) worker. The processor
|
|
750
|
+
* file must be an absolute path or a `file:` URL pointing to a compiled
|
|
751
|
+
* `.js`/`.cjs`/`.mjs` artifact. NestJS DI is **not** available inside the
|
|
752
|
+
* processor. The BullMQ `Worker` constructor is synchronous.
|
|
753
|
+
*
|
|
754
|
+
* @param config - Queue name, processor file path/URL, and optional options.
|
|
755
|
+
* @returns The created BullMQ `Worker` instance.
|
|
756
|
+
* @throws {QueueException} `WORKER_REGISTRATION_FAILED` when `processorFile`
|
|
757
|
+
* is a relative path, has a disallowed extension, or uses a non-`file:` URL protocol.
|
|
758
|
+
* @throws {QueueException} `DUPLICATE_PROCESSOR` when a worker for
|
|
759
|
+
* `queueName` already exists.
|
|
760
|
+
* @throws {QueueException} `WORKER_REGISTRATION_FAILED` when options are
|
|
761
|
+
* invalid or the BullMQ `Worker` constructor throws.
|
|
762
|
+
*/
|
|
763
|
+
registerSandboxed(config) {
|
|
764
|
+
const { queueName, processorFile, options } = config;
|
|
765
|
+
this.validateProcessorFile(processorFile);
|
|
766
|
+
this.guardDuplicate(queueName);
|
|
767
|
+
this.validateOptions(options);
|
|
768
|
+
const conn = duplicateConnection(this.connection.getClient());
|
|
769
|
+
const bullOpts = this.buildBullOptions(options, conn);
|
|
770
|
+
if (options?.useWorkerThreads !== void 0) {
|
|
771
|
+
bullOpts.useWorkerThreads = options.useWorkerThreads;
|
|
772
|
+
}
|
|
773
|
+
let worker;
|
|
774
|
+
try {
|
|
775
|
+
worker = new Worker(queueName, processorFile, bullOpts);
|
|
776
|
+
} catch (cause) {
|
|
777
|
+
conn.disconnect();
|
|
778
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
779
|
+
queueName,
|
|
780
|
+
cause: cause instanceof Error ? cause.message : String(cause)
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
attachDefaultErrorListener(worker, this.logger, "Worker", queueName);
|
|
784
|
+
this.entries.set(queueName, { worker, connection: conn });
|
|
785
|
+
return worker;
|
|
786
|
+
}
|
|
787
|
+
/**
|
|
788
|
+
* Stop and remove the worker for the given queue, releasing the duplicated
|
|
789
|
+
* connection the library opened for it. If no worker exists for `queueName`
|
|
790
|
+
* this is a no-op.
|
|
791
|
+
*
|
|
792
|
+
* @param queueName - The queue whose worker should be removed.
|
|
793
|
+
*/
|
|
794
|
+
async unregister(queueName) {
|
|
795
|
+
const entry = this.entries.get(queueName);
|
|
796
|
+
if (!entry) return;
|
|
797
|
+
await entry.worker.close();
|
|
798
|
+
await this.closeConnection(entry.connection);
|
|
799
|
+
this.entries.delete(queueName);
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Returns the queue names of all currently registered workers.
|
|
803
|
+
*
|
|
804
|
+
* @returns An immutable array of queue names.
|
|
805
|
+
*/
|
|
806
|
+
list() {
|
|
807
|
+
return Array.from(this.entries.keys());
|
|
808
|
+
}
|
|
809
|
+
/**
|
|
810
|
+
* Returns a read-only view of the registered workers keyed by queue name.
|
|
811
|
+
* Intended for the shutdown orchestrator only.
|
|
812
|
+
*
|
|
813
|
+
* @returns A read-only map of queue name to worker.
|
|
814
|
+
*/
|
|
815
|
+
getAll() {
|
|
816
|
+
return new Map(
|
|
817
|
+
Array.from(this.entries, ([name, entry]) => [name, entry.worker])
|
|
818
|
+
);
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* Returns a read-only view of the duplicated connections the library opened
|
|
822
|
+
* for its workers, keyed by queue name. Intended for the shutdown orchestrator
|
|
823
|
+
* so it can close exactly the connections the library created (Mode A never
|
|
824
|
+
* exposes the consumer's shared client here).
|
|
825
|
+
*
|
|
826
|
+
* @returns A read-only map of queue name to duplicated connection.
|
|
827
|
+
*/
|
|
828
|
+
getConnections() {
|
|
829
|
+
return new Map(
|
|
830
|
+
Array.from(this.entries, ([name, entry]) => [name, entry.connection])
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Gracefully quit a duplicated connection, falling back to a hard disconnect
|
|
835
|
+
* if the quit handshake fails so a socket is never leaked.
|
|
836
|
+
*
|
|
837
|
+
* @param connection - The duplicated connection to release.
|
|
838
|
+
*/
|
|
839
|
+
async closeConnection(connection) {
|
|
840
|
+
try {
|
|
841
|
+
await connection.quit();
|
|
842
|
+
} catch {
|
|
843
|
+
connection.disconnect();
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
/**
|
|
847
|
+
* Guard against registering two workers for the same queue.
|
|
848
|
+
*
|
|
849
|
+
* @param queueName - Queue name to check.
|
|
850
|
+
* @throws {QueueException} `DUPLICATE_PROCESSOR` if the queue is already registered.
|
|
851
|
+
*/
|
|
852
|
+
guardDuplicate(queueName) {
|
|
853
|
+
if (this.entries.has(queueName)) {
|
|
854
|
+
throw new QueueException(QUEUE_ERROR_CODES.DUPLICATE_PROCESSOR, 500, { queueName });
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Validate tunable options before constructing a Worker.
|
|
859
|
+
* Enforces `concurrency` in `[1, MAX_WORKER_CONCURRENCY]` and
|
|
860
|
+
* `limiter.max`/`limiter.duration` `>= 1`.
|
|
861
|
+
*
|
|
862
|
+
* @param opts - The caller-supplied options (may be undefined).
|
|
863
|
+
* @throws {QueueException} `WORKER_REGISTRATION_FAILED` when values are out of range.
|
|
864
|
+
*/
|
|
865
|
+
validateOptions(opts) {
|
|
866
|
+
if (opts?.concurrency !== void 0) {
|
|
867
|
+
if (opts.concurrency < 1) {
|
|
868
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
869
|
+
reason: "concurrency must be >= 1",
|
|
870
|
+
actual: opts.concurrency
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
if (opts.concurrency > MAX_WORKER_CONCURRENCY) {
|
|
874
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
875
|
+
reason: `concurrency must be <= ${MAX_WORKER_CONCURRENCY.toString()}`,
|
|
876
|
+
actual: opts.concurrency
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
if (opts?.limiter !== void 0) {
|
|
881
|
+
if (opts.limiter.max < 1) {
|
|
882
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
883
|
+
reason: "limiter.max must be >= 1",
|
|
884
|
+
actual: opts.limiter.max
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
if (opts.limiter.duration < 1) {
|
|
888
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
889
|
+
reason: "limiter.duration must be >= 1",
|
|
890
|
+
actual: opts.limiter.duration
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
/**
|
|
896
|
+
* Validate that `processorFile` is a safe, reachable artifact path. Accepts
|
|
897
|
+
* absolute paths with `.js`/`.cjs`/`.mjs` extensions and `file:` URLs. Rejects
|
|
898
|
+
* relative paths, non-JS extensions, and non-`file:` URL protocols to prevent
|
|
899
|
+
* path-traversal and arbitrary-file-execution attacks.
|
|
900
|
+
*
|
|
901
|
+
* @param processorFile - The path or URL supplied by the caller.
|
|
902
|
+
* @throws {QueueException} `WORKER_REGISTRATION_FAILED` when validation fails.
|
|
903
|
+
*/
|
|
904
|
+
validateProcessorFile(processorFile) {
|
|
905
|
+
if (processorFile instanceof URL) {
|
|
906
|
+
if (processorFile.protocol !== "file:") {
|
|
907
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
908
|
+
reason: "processorFile URL must use the file: protocol"
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
this.assertProcessorExtension(processorFile.pathname);
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
if (!isAbsolute(processorFile)) {
|
|
915
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
916
|
+
reason: "processorFile must be an absolute path"
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
this.assertProcessorExtension(processorFile);
|
|
920
|
+
}
|
|
921
|
+
/**
|
|
922
|
+
* Reject a processor path whose extension is not an executable JS module,
|
|
923
|
+
* the last line of defense against loading an arbitrary file as a worker.
|
|
924
|
+
*
|
|
925
|
+
* @param filePath - The filesystem path (or file: URL pathname) to check.
|
|
926
|
+
* @throws {QueueException} `WORKER_REGISTRATION_FAILED` for a disallowed extension.
|
|
927
|
+
*/
|
|
928
|
+
assertProcessorExtension(filePath) {
|
|
929
|
+
const allowedExtensions = /* @__PURE__ */ new Set([".js", ".cjs", ".mjs"]);
|
|
930
|
+
if (!allowedExtensions.has(extname(filePath))) {
|
|
931
|
+
throw new QueueException(QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED, 500, {
|
|
932
|
+
reason: "processorFile must have a .js, .cjs, or .mjs extension"
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* Build a BullMQ `WorkerOptions` object from the library's `WorkerOptions`
|
|
938
|
+
* shape. Only forwards optional fields that are explicitly set to preserve
|
|
939
|
+
* `exactOptionalPropertyTypes` correctness. Accepts the pre-created duplicated
|
|
940
|
+
* connection so callers can close it on construction failure.
|
|
941
|
+
*
|
|
942
|
+
* @param opts - Library-level worker options.
|
|
943
|
+
* @param conn - Pre-created duplicated Redis connection.
|
|
944
|
+
* @returns A BullMQ-compatible options object.
|
|
945
|
+
*/
|
|
946
|
+
buildBullOptions(opts, conn) {
|
|
947
|
+
const result = {
|
|
948
|
+
connection: conn,
|
|
949
|
+
concurrency: opts?.concurrency ?? DEFAULT_WORKER_CONCURRENCY,
|
|
950
|
+
autorun: opts?.autorun ?? true,
|
|
951
|
+
// Match the producer Queue's key prefix so the worker polls the same
|
|
952
|
+
// keyspace; without this a non-default prefix leaves jobs unconsumed.
|
|
953
|
+
prefix: this.options.prefix
|
|
954
|
+
};
|
|
955
|
+
if (opts?.limiter !== void 0) result.limiter = opts.limiter;
|
|
956
|
+
if (opts?.lockDuration !== void 0) result.lockDuration = opts.lockDuration;
|
|
957
|
+
if (opts?.stalledInterval !== void 0) result.stalledInterval = opts.stalledInterval;
|
|
958
|
+
if (this.options.telemetry !== void 0) result.telemetry = this.options.telemetry;
|
|
959
|
+
return result;
|
|
960
|
+
}
|
|
961
|
+
};
|
|
962
|
+
WorkerRegistry = __decorateClass([
|
|
963
|
+
Injectable(),
|
|
964
|
+
__decorateParam(0, Inject(ConnectionResolver)),
|
|
965
|
+
__decorateParam(1, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
|
|
966
|
+
], WorkerRegistry);
|
|
967
|
+
var QueueEventsRegistry = class {
|
|
968
|
+
constructor(connection, options) {
|
|
969
|
+
this.connection = connection;
|
|
970
|
+
this.options = options;
|
|
971
|
+
}
|
|
972
|
+
connection;
|
|
973
|
+
options;
|
|
974
|
+
logger = new Logger(QueueEventsRegistry.name);
|
|
975
|
+
events = /* @__PURE__ */ new Map();
|
|
976
|
+
connections = /* @__PURE__ */ new Map();
|
|
977
|
+
/**
|
|
978
|
+
* Returns the cached `QueueEvents` for `queueName`, creating one on first
|
|
979
|
+
* call. Subsequent calls for the same name return the same instance without
|
|
980
|
+
* opening a second Redis connection.
|
|
981
|
+
*
|
|
982
|
+
* @param queueName - The BullMQ queue to observe.
|
|
983
|
+
* @returns The `QueueEvents` instance for the queue.
|
|
984
|
+
*/
|
|
985
|
+
getOrCreate(queueName) {
|
|
986
|
+
const existing = this.events.get(queueName);
|
|
987
|
+
if (existing) return existing;
|
|
988
|
+
const conn = duplicateConnection(this.connection.getClient());
|
|
989
|
+
let qe;
|
|
990
|
+
try {
|
|
991
|
+
qe = new QueueEvents(queueName, { connection: conn, prefix: this.options.prefix });
|
|
992
|
+
} catch (err) {
|
|
993
|
+
conn.disconnect();
|
|
994
|
+
throw err;
|
|
995
|
+
}
|
|
996
|
+
attachDefaultErrorListener(qe, this.logger, "QueueEvents", queueName);
|
|
997
|
+
this.events.set(queueName, qe);
|
|
998
|
+
this.connections.set(queueName, conn);
|
|
999
|
+
return qe;
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* Returns the queue names for which a `QueueEvents` connection is open.
|
|
1003
|
+
*
|
|
1004
|
+
* @returns An immutable array of queue names.
|
|
1005
|
+
*/
|
|
1006
|
+
list() {
|
|
1007
|
+
return Array.from(this.events.keys());
|
|
1008
|
+
}
|
|
1009
|
+
/**
|
|
1010
|
+
* Returns the live `QueueEvents` map. Intended for the shutdown orchestrator.
|
|
1011
|
+
*
|
|
1012
|
+
* @returns A read-only view of the internal map.
|
|
1013
|
+
*/
|
|
1014
|
+
getAll() {
|
|
1015
|
+
return this.events;
|
|
1016
|
+
}
|
|
1017
|
+
/**
|
|
1018
|
+
* Returns a read-only view of the duplicated connections the library opened
|
|
1019
|
+
* for its `QueueEvents`, keyed by queue name. Intended for the shutdown
|
|
1020
|
+
* orchestrator so it can close exactly the connections the library created
|
|
1021
|
+
* (Mode A never exposes the consumer's shared client here).
|
|
1022
|
+
*
|
|
1023
|
+
* Returns a copy. `ReadonlyMap` is a compile-time claim only — handing back the
|
|
1024
|
+
* live registry lets any consumer cast it away and delete a connection the
|
|
1025
|
+
* shutdown sequence is about to close. The allocation costs nothing on a path
|
|
1026
|
+
* that runs once per shutdown.
|
|
1027
|
+
*
|
|
1028
|
+
* @returns A read-only snapshot of queue name to duplicated connection.
|
|
1029
|
+
*/
|
|
1030
|
+
getConnections() {
|
|
1031
|
+
return new Map(this.connections);
|
|
1032
|
+
}
|
|
1033
|
+
};
|
|
1034
|
+
QueueEventsRegistry = __decorateClass([
|
|
1035
|
+
Injectable(),
|
|
1036
|
+
__decorateParam(0, Inject(ConnectionResolver)),
|
|
1037
|
+
__decorateParam(1, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
|
|
1038
|
+
], QueueEventsRegistry);
|
|
1039
|
+
|
|
1040
|
+
// src/server/decorators/metadata-keys.constants.ts
|
|
1041
|
+
var PROCESSOR_METADATA_KEY = /* @__PURE__ */ Symbol("bymax_queue:processor");
|
|
1042
|
+
var PROCESS_HANDLERS_METADATA_KEY = /* @__PURE__ */ Symbol("bymax_queue:process_handlers");
|
|
1043
|
+
var WORKER_EVENT_LISTENERS_METADATA_KEY = /* @__PURE__ */ Symbol("bymax_queue:worker_event_listeners");
|
|
1044
|
+
var QUEUE_EVENT_LISTENERS_METADATA_KEY = /* @__PURE__ */ Symbol("bymax_queue:queue_event_listeners");
|
|
1045
|
+
|
|
1046
|
+
// src/server/services/processor-discovery.service.ts
|
|
1047
|
+
var ProcessorDiscoveryService = class {
|
|
1048
|
+
constructor(discovery, workers, events) {
|
|
1049
|
+
this.discovery = discovery;
|
|
1050
|
+
this.workers = workers;
|
|
1051
|
+
this.events = events;
|
|
1052
|
+
}
|
|
1053
|
+
discovery;
|
|
1054
|
+
workers;
|
|
1055
|
+
events;
|
|
1056
|
+
logger = new Logger(ProcessorDiscoveryService.name);
|
|
1057
|
+
registeredQueues = /* @__PURE__ */ new Set();
|
|
1058
|
+
/**
|
|
1059
|
+
* Entry point called by NestJS after the module is fully initialized.
|
|
1060
|
+
* Discovers all `@Processor`-annotated providers and wires them.
|
|
1061
|
+
*/
|
|
1062
|
+
onModuleInit() {
|
|
1063
|
+
const providers = this.discovery.getProviders();
|
|
1064
|
+
for (const wrapper of providers) {
|
|
1065
|
+
const instance = wrapper.instance;
|
|
1066
|
+
if (!instance || typeof instance !== "object") continue;
|
|
1067
|
+
const ctor = instance.constructor;
|
|
1068
|
+
const processorMeta = Reflect.getOwnMetadata(PROCESSOR_METADATA_KEY, ctor);
|
|
1069
|
+
if (!processorMeta) continue;
|
|
1070
|
+
this.wireProcessor(instance, processorMeta);
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
/**
|
|
1074
|
+
* Build a dispatcher function that routes `job.name` to the correct handler.
|
|
1075
|
+
* Named handlers take priority; a catch-all (`@Process()`) handles everything
|
|
1076
|
+
* else. Throws if no matching handler is found at job-processing time.
|
|
1077
|
+
*
|
|
1078
|
+
* @param instance - The provider instance whose methods serve as handlers.
|
|
1079
|
+
* @param handlers - Accumulated `@Process` metadata entries for the class.
|
|
1080
|
+
* @returns A dispatcher `(job: Job) => Promise<unknown>`.
|
|
1081
|
+
*/
|
|
1082
|
+
buildDispatcher(instance, handlers) {
|
|
1083
|
+
const namedHandlers = /* @__PURE__ */ new Map();
|
|
1084
|
+
let catchAll;
|
|
1085
|
+
for (const { jobName, methodKey } of handlers) {
|
|
1086
|
+
const method = instance[methodKey];
|
|
1087
|
+
if (typeof method !== "function") continue;
|
|
1088
|
+
const fn = method.bind(instance);
|
|
1089
|
+
if (jobName !== void 0) {
|
|
1090
|
+
namedHandlers.set(jobName, fn);
|
|
1091
|
+
} else {
|
|
1092
|
+
catchAll = fn;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
return (job) => {
|
|
1096
|
+
const specific = namedHandlers.get(job.name);
|
|
1097
|
+
if (specific) return specific(job);
|
|
1098
|
+
if (catchAll) return catchAll(job);
|
|
1099
|
+
return Promise.reject(
|
|
1100
|
+
new Error(
|
|
1101
|
+
`No handler registered for job name "${job.name}" on queue \u2014 add a matching @Process("name") or a catch-all @Process() method.`
|
|
1102
|
+
)
|
|
1103
|
+
);
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
/**
|
|
1107
|
+
* Wire a single discovered processor class: validate uniqueness, register a
|
|
1108
|
+
* Worker, bind worker-local and queue-global event listeners.
|
|
1109
|
+
*
|
|
1110
|
+
* @param instance - The resolved provider instance.
|
|
1111
|
+
* @param processorMeta - Metadata written by `@Processor`.
|
|
1112
|
+
*/
|
|
1113
|
+
wireProcessor(instance, processorMeta) {
|
|
1114
|
+
const { queueName, workerOptions, _warnedNoConcurrency } = processorMeta;
|
|
1115
|
+
const ctor = instance.constructor;
|
|
1116
|
+
if (this.registeredQueues.has(queueName)) {
|
|
1117
|
+
throw new QueueException(QUEUE_ERROR_CODES.DUPLICATE_PROCESSOR, 500, { queueName });
|
|
1118
|
+
}
|
|
1119
|
+
this.registeredQueues.add(queueName);
|
|
1120
|
+
if (_warnedNoConcurrency) {
|
|
1121
|
+
this.logger.warn(
|
|
1122
|
+
`Queue "${queueName}": concurrency was not specified \u2014 defaulting to concurrency=${String(DEFAULT_WORKER_CONCURRENCY)}. Set an explicit concurrency for production workloads.`
|
|
1123
|
+
);
|
|
1124
|
+
}
|
|
1125
|
+
const handlers = Reflect.getOwnMetadata(PROCESS_HANDLERS_METADATA_KEY, ctor) ?? [];
|
|
1126
|
+
const dispatcher = this.buildDispatcher(instance, handlers);
|
|
1127
|
+
const worker = this.workers.register({
|
|
1128
|
+
queueName,
|
|
1129
|
+
handler: dispatcher,
|
|
1130
|
+
options: workerOptions
|
|
1131
|
+
});
|
|
1132
|
+
const workerListeners = Reflect.getOwnMetadata(WORKER_EVENT_LISTENERS_METADATA_KEY, ctor) ?? [];
|
|
1133
|
+
const workerEmitter = worker;
|
|
1134
|
+
for (const { eventName, methodKey } of workerListeners) {
|
|
1135
|
+
const method = instance[methodKey];
|
|
1136
|
+
if (typeof method === "function") {
|
|
1137
|
+
workerEmitter.on(eventName, method.bind(instance));
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
const queueListeners = Reflect.getOwnMetadata(QUEUE_EVENT_LISTENERS_METADATA_KEY, ctor) ?? [];
|
|
1141
|
+
if (queueListeners.length > 0) {
|
|
1142
|
+
const qe = this.events.getOrCreate(queueName);
|
|
1143
|
+
const qeEmitter = qe;
|
|
1144
|
+
for (const { eventName, methodKey } of queueListeners) {
|
|
1145
|
+
const method = instance[methodKey];
|
|
1146
|
+
if (typeof method === "function") {
|
|
1147
|
+
qeEmitter.on(eventName, method.bind(instance));
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
};
|
|
1153
|
+
ProcessorDiscoveryService = __decorateClass([
|
|
1154
|
+
Injectable(),
|
|
1155
|
+
__decorateParam(0, Inject(DiscoveryService)),
|
|
1156
|
+
__decorateParam(1, Inject(WorkerRegistry)),
|
|
1157
|
+
__decorateParam(2, Inject(QueueEventsRegistry))
|
|
1158
|
+
], ProcessorDiscoveryService);
|
|
1159
|
+
var QueueLifecycle = class {
|
|
1160
|
+
constructor(workers, events, queues, flow, connection, resolved) {
|
|
1161
|
+
this.workers = workers;
|
|
1162
|
+
this.events = events;
|
|
1163
|
+
this.queues = queues;
|
|
1164
|
+
this.flow = flow;
|
|
1165
|
+
this.connection = connection;
|
|
1166
|
+
this.resolved = resolved;
|
|
1167
|
+
}
|
|
1168
|
+
workers;
|
|
1169
|
+
events;
|
|
1170
|
+
queues;
|
|
1171
|
+
flow;
|
|
1172
|
+
connection;
|
|
1173
|
+
resolved;
|
|
1174
|
+
logger = new Logger(QueueLifecycle.name);
|
|
1175
|
+
/**
|
|
1176
|
+
* Run the ordered shutdown sequence. Called by NestJS when the application
|
|
1177
|
+
* context closes (e.g. on `SIGTERM` with shutdown hooks enabled).
|
|
1178
|
+
*/
|
|
1179
|
+
async onModuleDestroy() {
|
|
1180
|
+
const start = Date.now();
|
|
1181
|
+
const forced = await this.closeWorkers();
|
|
1182
|
+
await this.closeQueueEvents();
|
|
1183
|
+
await this.closeDuplicatedConnections();
|
|
1184
|
+
await this.drainQueuesIfRequested();
|
|
1185
|
+
await this.closeFlowProducer();
|
|
1186
|
+
await this.closeQueues();
|
|
1187
|
+
await this.teardownConnection();
|
|
1188
|
+
this.logger.log(
|
|
1189
|
+
`Queue shutdown complete in ${String(Date.now() - start)}ms (forced ${String(forced)} worker(s))`
|
|
1190
|
+
);
|
|
1191
|
+
}
|
|
1192
|
+
/**
|
|
1193
|
+
* Close every worker with a bounded drain, force-closing any that exceed the
|
|
1194
|
+
* budget. Returns the number of workers that had to be force-closed.
|
|
1195
|
+
*
|
|
1196
|
+
* @returns The count of force-closed workers.
|
|
1197
|
+
*/
|
|
1198
|
+
async closeWorkers() {
|
|
1199
|
+
let forced = 0;
|
|
1200
|
+
const drainTimeoutMs = this.resolved.shutdown.drainTimeoutMs;
|
|
1201
|
+
for (const [name, worker] of this.workers.getAll()) {
|
|
1202
|
+
try {
|
|
1203
|
+
await this.closeWorkerWithTimeout(worker, drainTimeoutMs);
|
|
1204
|
+
} catch {
|
|
1205
|
+
forced++;
|
|
1206
|
+
this.logger.warn(
|
|
1207
|
+
`Worker "${name}" exceeded the ${String(drainTimeoutMs)}ms drain budget \u2014 forcing close (${QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED}); the in-flight job becomes stalled and is retried`
|
|
1208
|
+
);
|
|
1209
|
+
void worker.close(true).catch(() => void 0);
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
return forced;
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* Race a graceful `worker.close()` against a timer. `worker.close()` already
|
|
1216
|
+
* drains active jobs and takes no timeout argument, so the timer is the only
|
|
1217
|
+
* way to bound the wait. Rejects with a timeout error when the budget elapses.
|
|
1218
|
+
*
|
|
1219
|
+
* @param worker - The worker to close.
|
|
1220
|
+
* @param timeoutMs - The drain budget in milliseconds.
|
|
1221
|
+
*/
|
|
1222
|
+
async closeWorkerWithTimeout(worker, timeoutMs) {
|
|
1223
|
+
let timer;
|
|
1224
|
+
const closed = worker.close();
|
|
1225
|
+
closed.catch(() => void 0);
|
|
1226
|
+
try {
|
|
1227
|
+
await Promise.race([
|
|
1228
|
+
closed,
|
|
1229
|
+
new Promise((_resolve, reject) => {
|
|
1230
|
+
timer = setTimeout(() => {
|
|
1231
|
+
reject(new Error("drain timeout"));
|
|
1232
|
+
}, timeoutMs);
|
|
1233
|
+
})
|
|
1234
|
+
]);
|
|
1235
|
+
} finally {
|
|
1236
|
+
clearTimeout(timer);
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
/** Close every cached `QueueEvents`, swallowing individual failures. */
|
|
1240
|
+
async closeQueueEvents() {
|
|
1241
|
+
for (const [, queueEvents] of this.events.getAll()) {
|
|
1242
|
+
await queueEvents.close().catch(() => void 0);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Close the duplicated worker/queue-events connections the library opened.
|
|
1247
|
+
* BullMQ treats a passed-in client as shared and never closes it, so the
|
|
1248
|
+
* library must release these explicitly. In Mode A the consumer's shared
|
|
1249
|
+
* client is never among them.
|
|
1250
|
+
*/
|
|
1251
|
+
async closeDuplicatedConnections() {
|
|
1252
|
+
const connections = [
|
|
1253
|
+
...this.workers.getConnections().values(),
|
|
1254
|
+
...this.events.getConnections().values()
|
|
1255
|
+
];
|
|
1256
|
+
for (const connection of connections) {
|
|
1257
|
+
try {
|
|
1258
|
+
await connection.quit();
|
|
1259
|
+
} catch {
|
|
1260
|
+
connection.disconnect();
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
/**
|
|
1265
|
+
* Drain every cached queue when `drainOnShutdown` is enabled (dev/test only).
|
|
1266
|
+
* Removes waiting and delayed jobs; off by default to stay production-safe.
|
|
1267
|
+
*/
|
|
1268
|
+
async drainQueuesIfRequested() {
|
|
1269
|
+
if (!this.resolved.shutdown.drainOnShutdown) return;
|
|
1270
|
+
for (const [, queue] of this.queues.getCachedQueues()) {
|
|
1271
|
+
await queue.drain().catch(() => void 0);
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
/** Close the flow producer (idempotent), swallowing any failure. */
|
|
1275
|
+
async closeFlowProducer() {
|
|
1276
|
+
await this.flow.close().catch(() => void 0);
|
|
1277
|
+
}
|
|
1278
|
+
/** Close every cached queue and clear the cache, swallowing individual failures. */
|
|
1279
|
+
async closeQueues() {
|
|
1280
|
+
await this.queues.closeAll().catch(() => void 0);
|
|
1281
|
+
}
|
|
1282
|
+
/**
|
|
1283
|
+
* Tear down the Queue-role connection. Mode B quits the library-owned client;
|
|
1284
|
+
* Mode A is a no-op for the consumer's shared client.
|
|
1285
|
+
*/
|
|
1286
|
+
async teardownConnection() {
|
|
1287
|
+
await this.connection.teardown().catch(() => void 0);
|
|
1288
|
+
}
|
|
1289
|
+
};
|
|
1290
|
+
QueueLifecycle = __decorateClass([
|
|
1291
|
+
Injectable(),
|
|
1292
|
+
__decorateParam(0, Inject(WorkerRegistry)),
|
|
1293
|
+
__decorateParam(1, Inject(QueueEventsRegistry)),
|
|
1294
|
+
__decorateParam(2, Inject(QueueService)),
|
|
1295
|
+
__decorateParam(3, Inject(FlowService)),
|
|
1296
|
+
__decorateParam(4, Inject(ConnectionResolver)),
|
|
1297
|
+
__decorateParam(5, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
|
|
1298
|
+
], QueueLifecycle);
|
|
1299
|
+
|
|
1300
|
+
// src/server/bymax-queue.module.ts
|
|
1301
|
+
var { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = new ConfigurableModuleBuilder({ moduleName: "BymaxQueue" }).setClassMethodName("forRoot").setFactoryMethodName("createQueueOptions").setExtras({ isGlobal: true }, (definition, extras) => ({
|
|
1302
|
+
...definition,
|
|
1303
|
+
global: extras.isGlobal
|
|
1304
|
+
})).build();
|
|
1305
|
+
var BymaxQueueModule = class extends ConfigurableModuleClass {
|
|
1306
|
+
/**
|
|
1307
|
+
* Synchronous registration. Validates options, resolves defaults, and wires the
|
|
1308
|
+
* connection resolver (initialized via async factory), the queue service, the
|
|
1309
|
+
* worker registry, the queue-events registry, processor discovery, and the
|
|
1310
|
+
* graceful-shutdown lifecycle.
|
|
1311
|
+
*
|
|
1312
|
+
* @param options - Static module options.
|
|
1313
|
+
* @returns The configured dynamic module.
|
|
1314
|
+
*/
|
|
1315
|
+
static forRoot(options) {
|
|
1316
|
+
validateOptions(options);
|
|
1317
|
+
const resolved = applyDefaults(options);
|
|
1318
|
+
const base = super.forRoot(options);
|
|
1319
|
+
return {
|
|
1320
|
+
...base,
|
|
1321
|
+
imports: [DiscoveryModule],
|
|
1322
|
+
providers: [
|
|
1323
|
+
...base.providers ?? [],
|
|
1324
|
+
{ provide: BYMAX_QUEUE_RESOLVED_OPTIONS, useValue: resolved },
|
|
1325
|
+
{
|
|
1326
|
+
provide: FlowService,
|
|
1327
|
+
useFactory: (conn) => new FlowService(conn, resolved.flows.enabled, resolved.prefix, resolved.telemetry),
|
|
1328
|
+
inject: [ConnectionResolver]
|
|
1329
|
+
},
|
|
1330
|
+
{
|
|
1331
|
+
provide: MetricsService,
|
|
1332
|
+
useFactory: (qs) => new MetricsService(qs, resolved.metrics.enabled, resolved.metrics.cacheTtlMs),
|
|
1333
|
+
inject: [QueueService]
|
|
1334
|
+
},
|
|
1335
|
+
...BymaxQueueModule.buildCoreProviders()
|
|
1336
|
+
],
|
|
1337
|
+
exports: BymaxQueueModule.buildExports()
|
|
1338
|
+
};
|
|
1339
|
+
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Asynchronous registration. Use when options depend on other modules
|
|
1342
|
+
* (ConfigService, BymaxCacheModule, etc.). Mirrors the NestJS standard async
|
|
1343
|
+
* dynamic-module pattern (`useFactory` | `useClass` | `useExisting` + `inject`)
|
|
1344
|
+
* and registers the same provider graph and exports as {@link forRoot}, with
|
|
1345
|
+
* `global` applied from the `isGlobal` extra. The resolved options and the
|
|
1346
|
+
* initialized connection are derived from the async-resolved options at runtime.
|
|
1347
|
+
*
|
|
1348
|
+
* @param options - Asynchronous module options.
|
|
1349
|
+
* @returns The configured dynamic module.
|
|
1350
|
+
*/
|
|
1351
|
+
static forRootAsync(options) {
|
|
1352
|
+
const base = super.forRootAsync(options);
|
|
1353
|
+
return {
|
|
1354
|
+
...base,
|
|
1355
|
+
imports: [DiscoveryModule, ...base.imports ?? []],
|
|
1356
|
+
providers: [
|
|
1357
|
+
...base.providers ?? [],
|
|
1358
|
+
{
|
|
1359
|
+
provide: BYMAX_QUEUE_RESOLVED_OPTIONS,
|
|
1360
|
+
useFactory: (opts) => {
|
|
1361
|
+
validateOptions(opts);
|
|
1362
|
+
return applyDefaults(opts);
|
|
1363
|
+
},
|
|
1364
|
+
inject: [MODULE_OPTIONS_TOKEN]
|
|
1365
|
+
},
|
|
1366
|
+
{
|
|
1367
|
+
provide: FlowService,
|
|
1368
|
+
useFactory: (conn, resolved) => new FlowService(conn, resolved.flows.enabled, resolved.prefix, resolved.telemetry),
|
|
1369
|
+
inject: [ConnectionResolver, BYMAX_QUEUE_RESOLVED_OPTIONS]
|
|
1370
|
+
},
|
|
1371
|
+
{
|
|
1372
|
+
provide: MetricsService,
|
|
1373
|
+
useFactory: (qs, resolved) => new MetricsService(qs, resolved.metrics.enabled, resolved.metrics.cacheTtlMs),
|
|
1374
|
+
inject: [QueueService, BYMAX_QUEUE_RESOLVED_OPTIONS]
|
|
1375
|
+
},
|
|
1376
|
+
...BymaxQueueModule.buildCoreProviders()
|
|
1377
|
+
],
|
|
1378
|
+
exports: BymaxQueueModule.buildExports()
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* Providers shared verbatim by both registration paths: the options alias, the
|
|
1383
|
+
* initialized connection resolver, the resolved client/mode tokens, and the
|
|
1384
|
+
* singleton services. `BYMAX_QUEUE_RESOLVED_OPTIONS`, `FlowService`, and
|
|
1385
|
+
* `MetricsService` are intentionally omitted — they differ per path.
|
|
1386
|
+
*
|
|
1387
|
+
* @returns The shared provider list.
|
|
1388
|
+
*/
|
|
1389
|
+
static buildCoreProviders() {
|
|
1390
|
+
return [
|
|
1391
|
+
{ provide: BYMAX_QUEUE_OPTIONS, useExisting: MODULE_OPTIONS_TOKEN },
|
|
1392
|
+
{
|
|
1393
|
+
provide: ConnectionResolver,
|
|
1394
|
+
useFactory: async (opts) => {
|
|
1395
|
+
const resolver = new ConnectionResolver(opts);
|
|
1396
|
+
await resolver.init();
|
|
1397
|
+
return resolver;
|
|
1398
|
+
},
|
|
1399
|
+
inject: [BYMAX_QUEUE_OPTIONS]
|
|
1400
|
+
},
|
|
1401
|
+
{
|
|
1402
|
+
provide: BYMAX_QUEUE_REDIS_CLIENT,
|
|
1403
|
+
useFactory: (resolver) => resolver.getClient(),
|
|
1404
|
+
inject: [ConnectionResolver]
|
|
1405
|
+
},
|
|
1406
|
+
{
|
|
1407
|
+
provide: BYMAX_QUEUE_CONNECTION_MODE,
|
|
1408
|
+
useFactory: (resolver) => resolver.getMode(),
|
|
1409
|
+
inject: [ConnectionResolver]
|
|
1410
|
+
},
|
|
1411
|
+
QueueService,
|
|
1412
|
+
WorkerRegistry,
|
|
1413
|
+
QueueEventsRegistry,
|
|
1414
|
+
ProcessorDiscoveryService,
|
|
1415
|
+
QueueLifecycle
|
|
1416
|
+
];
|
|
1417
|
+
}
|
|
1418
|
+
/**
|
|
1419
|
+
* The public exports shared by both registration paths.
|
|
1420
|
+
*
|
|
1421
|
+
* @returns A fresh exports list (services and injection tokens).
|
|
1422
|
+
*/
|
|
1423
|
+
static buildExports() {
|
|
1424
|
+
return [
|
|
1425
|
+
QueueService,
|
|
1426
|
+
FlowService,
|
|
1427
|
+
MetricsService,
|
|
1428
|
+
ConnectionResolver,
|
|
1429
|
+
WorkerRegistry,
|
|
1430
|
+
QueueEventsRegistry,
|
|
1431
|
+
BYMAX_QUEUE_OPTIONS,
|
|
1432
|
+
BYMAX_QUEUE_RESOLVED_OPTIONS,
|
|
1433
|
+
BYMAX_QUEUE_REDIS_CLIENT,
|
|
1434
|
+
BYMAX_QUEUE_CONNECTION_MODE
|
|
1435
|
+
];
|
|
1436
|
+
}
|
|
1437
|
+
};
|
|
1438
|
+
BymaxQueueModule = __decorateClass([
|
|
1439
|
+
Module({})
|
|
1440
|
+
], BymaxQueueModule);
|
|
1441
|
+
function Processor(queueName, options) {
|
|
1442
|
+
return (target) => {
|
|
1443
|
+
const concurrencyOmitted = options?.concurrency === void 0;
|
|
1444
|
+
const metadata = {
|
|
1445
|
+
queueName,
|
|
1446
|
+
workerOptions: {
|
|
1447
|
+
concurrency: DEFAULT_WORKER_CONCURRENCY,
|
|
1448
|
+
autorun: true,
|
|
1449
|
+
...options
|
|
1450
|
+
},
|
|
1451
|
+
...concurrencyOmitted ? { _warnedNoConcurrency: true } : {}
|
|
1452
|
+
};
|
|
1453
|
+
Reflect.defineMetadata(PROCESSOR_METADATA_KEY, metadata, target);
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
function Process(jobName) {
|
|
1457
|
+
return (target, propertyKey) => {
|
|
1458
|
+
const existing = Reflect.getOwnMetadata(PROCESS_HANDLERS_METADATA_KEY, target.constructor) ?? [];
|
|
1459
|
+
const entry = jobName !== void 0 ? { jobName, methodKey: propertyKey } : { methodKey: propertyKey };
|
|
1460
|
+
Reflect.defineMetadata(PROCESS_HANDLERS_METADATA_KEY, [...existing, entry], target.constructor);
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1463
|
+
function OnWorkerEvent(eventName) {
|
|
1464
|
+
return (target, propertyKey) => {
|
|
1465
|
+
const existing = Reflect.getOwnMetadata(
|
|
1466
|
+
WORKER_EVENT_LISTENERS_METADATA_KEY,
|
|
1467
|
+
target.constructor
|
|
1468
|
+
) ?? [];
|
|
1469
|
+
Reflect.defineMetadata(
|
|
1470
|
+
WORKER_EVENT_LISTENERS_METADATA_KEY,
|
|
1471
|
+
[...existing, { eventName, methodKey: propertyKey }],
|
|
1472
|
+
target.constructor
|
|
1473
|
+
);
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
function OnQueueEvent(eventName) {
|
|
1477
|
+
return (target, propertyKey) => {
|
|
1478
|
+
const existing = Reflect.getOwnMetadata(
|
|
1479
|
+
QUEUE_EVENT_LISTENERS_METADATA_KEY,
|
|
1480
|
+
target.constructor
|
|
1481
|
+
) ?? [];
|
|
1482
|
+
Reflect.defineMetadata(
|
|
1483
|
+
QUEUE_EVENT_LISTENERS_METADATA_KEY,
|
|
1484
|
+
[...existing, { eventName, methodKey: propertyKey }],
|
|
1485
|
+
target.constructor
|
|
1486
|
+
);
|
|
1487
|
+
};
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
// src/shared/constants/job-status.ts
|
|
1491
|
+
var JOB_STATUS = {
|
|
1492
|
+
WAITING: "waiting",
|
|
1493
|
+
ACTIVE: "active",
|
|
1494
|
+
COMPLETED: "completed",
|
|
1495
|
+
FAILED: "failed",
|
|
1496
|
+
DELAYED: "delayed",
|
|
1497
|
+
PAUSED: "paused"
|
|
1498
|
+
};
|
|
1499
|
+
|
|
1500
|
+
export { BYMAX_QUEUE_CONNECTION_MODE, BYMAX_QUEUE_OPTIONS, BYMAX_QUEUE_REDIS_CLIENT, BYMAX_QUEUE_RESOLVED_OPTIONS, BymaxQueueModule, ConnectionResolver, DEFAULT_JOB_OPTIONS, DEFAULT_METRICS_CACHE_TTL_MS, DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS, DEFAULT_WORKER_CONCURRENCY, FlowService, JOB_STATUS, MAX_WORKER_CONCURRENCY, MetricsService, OnQueueEvent, OnWorkerEvent, Process, Processor, QUEUE_ERROR_CODES, QUEUE_ERROR_MESSAGES, QueueEventsRegistry, QueueException, QueueService, WorkerRegistry };
|