@bymax-one/nest-queue 1.2.13 → 1.2.15

package/dist/server/index.cjs

@@ -78,6 +78,12 @@ var QueueException = class extends common.HttpException {
   }
 };
 
+// src/server/utils/timer-delay.ts
+var MAX_TIMER_DELAY_MS = 2147483647;
+function isTimerDelayMs(value) {
+  return Number.isInteger(value) && value >= 1 && value <= MAX_TIMER_DELAY_MS;
+}
+
 // src/server/config/validate-options.ts
 function invalid(reason) {
   throw new QueueException(QUEUE_ERROR_CODES.INVALID_OPTIONS, common.HttpStatus.INTERNAL_SERVER_ERROR, {
@@ -128,6 +134,12 @@ function validateOptions(opts) {
   if (cacheTtlMs !== void 0 && cacheTtlMs < 0) {
     invalid("metrics.cacheTtlMs must be >= 0");
   }
+  for (const key of ["commandTimeoutMs", "enqueueTimeoutMs"]) {
+    const value = opts[key];
+    if (value !== void 0 && !isTimerDelayMs(value)) {
+      invalid(`${key} must be a whole number from 1 to ${String(MAX_TIMER_DELAY_MS)}`);
+    }
+  }
 }
 
 // src/server/constants/default-options.ts
@@ -135,11 +147,14 @@ var DEFAULT_WORKER_CONCURRENCY = 2;
 var MAX_WORKER_CONCURRENCY = 1e3;
 var DEFAULT_JOB_OPTIONS = {
   attempts: 3,
-  backoff: { type: "exponential", delay: 2e3 },
+  backoff: { type: "exponential", delay: 2e3, jitter: 0.5 },
   removeOnComplete: { age: 24 * 3600, count: 1e3 },
   removeOnFail: { age: 7 * 24 * 3600, count: 5e3 }
 };
 var DEFAULT_CONNECTION_READY_TIMEOUT_MS = 1e4;
+var DEFAULT_COMMAND_TIMEOUT_MS = 5e3;
+var DEFAULT_ENQUEUE_TIMEOUT_MS = 1e4;
+var MAX_JOBS_PAGE_SIZE = 1e3;
 var DEFAULT_METRICS_CACHE_TTL_MS = 5e3;
 var DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS = 3e4;
 
@@ -159,7 +174,9 @@ function applyDefaults(opts) {
       drainTimeoutMs: opts.shutdown?.drainTimeoutMs ?? DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS,
       drainOnShutdown: opts.shutdown?.drainOnShutdown ?? false
     },
-    connectionReadyTimeoutMs: opts.connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
+    connectionReadyTimeoutMs: opts.connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS,
+    commandTimeoutMs: opts.commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS,
+    enqueueTimeoutMs: opts.enqueueTimeoutMs ?? DEFAULT_ENQUEUE_TIMEOUT_MS
   };
   if (opts.telemetry !== void 0) base.telemetry = opts.telemetry;
   Object.defineProperty(base, "connection", {
@@ -224,17 +241,23 @@ function attachClientErrorListener(client, logger, role) {
 
 // src/server/utils/duplicate-connection.ts
 function duplicateConnection(client) {
-  return client.duplicate({ maxRetriesPerRequest: null });
+  const override = {
+    maxRetriesPerRequest: null,
+    enableOfflineQueue: true,
+    commandTimeout: void 0
+  };
+  return client.duplicate(override);
 }
 
 // src/server/services/connection-resolver.service.ts
-var _logger, _client, _options;
+var _logger, _client, _options, _pendingReady;
 exports.ConnectionResolver = class ConnectionResolver {
   constructor(options) {
     __privateAdd(this, _logger, new common.Logger(exports.ConnectionResolver.name));
     __privateAdd(this, _client);
     __privateAdd(this, _options);
     __publicField(this, "mode");
+    __privateAdd(this, _pendingReady);
     __privateSet(this, _options, options);
   }
   async init() {
@@ -246,12 +269,22 @@ exports.ConnectionResolver = class ConnectionResolver {
     this.mode = "mode-b-owned";
     __privateSet(this, _client, "url" in cfg ? new ioredis.Redis(cfg.url, this.ownedOptions(cfg.options)) : new ioredis.Redis(this.ownedOptions(cfg.options)));
     attachClientErrorListener(__privateGet(this, _client), __privateGet(this, _logger), "owned");
-    await this.waitReady(
-      __privateGet(this, _options).connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
-    );
+    try {
+      await this.waitReady(
+        __privateGet(this, _options).connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
+      );
+    } catch (cause) {
+      __privateGet(this, _client).disconnect();
+      throw cause;
+    }
   }
   ownedOptions(options) {
-    return { ...options ?? {}, lazyConnect: false };
+    return {
+      commandTimeout: __privateGet(this, _options).commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS,
+      enableOfflineQueue: false,
+      ...options ?? {},
+      lazyConnect: false
+    };
   }
   getClient() {
     if (!__privateGet(this, _client)) {
@@ -280,6 +313,17 @@ exports.ConnectionResolver = class ConnectionResolver {
   isOwned() {
     return this.mode === "mode-b-owned";
   }
+  awaitReady() {
+    const client = this.getClient();
+    if (!WAITING_STATUSES.has(client.status)) return Promise.resolve();
+    __privateGet(this, _pendingReady) ?? __privateSet(this, _pendingReady, new Promise((resolve) => {
+      client.once("ready", () => {
+        __privateSet(this, _pendingReady, void 0);
+        resolve();
+      });
+    }));
+    return __privateGet(this, _pendingReady);
+  }
   async teardown() {
     if (this.isOwned() && __privateGet(this, _client)) {
       const client = __privateGet(this, _client);
@@ -298,6 +342,11 @@ exports.ConnectionResolver = class ConnectionResolver {
         { status: client.status }
       );
     }
+    if (client.options.maxRetriesPerRequest === null) {
+      __privateGet(this, _logger).warn(
+        "The shared client has maxRetriesPerRequest: null, so a command issued while Redis is unreachable stays queued in the client for as long as the outage lasts. QueueService calls are bounded by enqueueTimeoutMs and the caller is released, but the command is not cancelled. Give the client a finite limit to fail fast."
+      );
+    }
     const probe = duplicateConnection(client);
     attachClientErrorListener(probe, __privateGet(this, _logger), "probe");
     try {
@@ -310,41 +359,62 @@ exports.ConnectionResolver = class ConnectionResolver {
     const client = this.getClient();
     if (client.status === "ready") return;
     await new Promise((resolve, reject) => {
+      let lastErrorCode;
+      const details = (extra) => lastErrorCode === void 0 ? extra : { ...extra, lastErrorCode };
       const cleanup = () => {
         clearTimeout(timer);
         client.off("ready", onReady);
         client.off("error", onError);
+        client.off("end", onEnd);
+      };
+      const fail = (code, extra) => {
+        cleanup();
+        reject(new QueueException(code, common.HttpStatus.INTERNAL_SERVER_ERROR, details(extra)));
       };
       const onReady = () => {
         cleanup();
         resolve();
       };
       const onError = (err) => {
-        cleanup();
-        reject(err);
+        lastErrorCode = startupErrorCode(err);
+        if (isReplyError(err)) fail(QUEUE_ERROR_CODES.CONNECTION_INVALID, {});
+      };
+      const onEnd = () => {
+        fail(QUEUE_ERROR_CODES.CONNECTION_INVALID, {});
       };
       const timer = setTimeout(() => {
-        cleanup();
-        reject(
-          new QueueException(
-            QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
-            common.HttpStatus.INTERNAL_SERVER_ERROR,
-            { timeoutMs }
-          )
-        );
+        fail(QUEUE_ERROR_CODES.CONNECTION_TIMEOUT, { timeoutMs });
       }, timeoutMs);
       client.once("ready", onReady);
-      client.once("error", onError);
+      client.on("error", onError);
+      client.once("end", onEnd);
     });
   }
 };
 _logger = new WeakMap();
 _client = new WeakMap();
 _options = new WeakMap();
+_pendingReady = new WeakMap();
 exports.ConnectionResolver = __decorateClass([
   common.Injectable(),
   __decorateParam(0, common.Inject(BYMAX_QUEUE_OPTIONS))
 ], exports.ConnectionResolver);
+var WAITING_STATUSES = /* @__PURE__ */ new Set([
+  "connecting",
+  "connect",
+  "reconnecting",
+  "close"
+]);
+function isReplyError(err) {
+  return typeof err === "object" && err !== null && err.name === "ReplyError";
+}
+function startupErrorCode(err) {
+  if (typeof err !== "object" || err === null) return void 0;
+  const { code, message } = err;
+  if (typeof code === "string") return code;
+  if (isReplyError(err) && typeof message === "string") return /^[A-Z][A-Z0-9_]*/.exec(message)?.[0];
+  return void 0;
+}
 function invalid2(reason) {
   throw new QueueException(QUEUE_ERROR_CODES.INVALID_REPEAT_OPTIONS, common.HttpStatus.BAD_REQUEST, {
     reason
@@ -384,6 +454,51 @@ function attachDefaultErrorListener(emitter, logger, kind, queueName) {
   });
 }
 
+// src/server/utils/with-deadline.ts
+function withDeadline(work, timeoutMs, onTimeout) {
+  let timer;
+  const deadline = new Promise((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(onTimeout());
+    }, timeoutMs);
+  });
+  return Promise.race([work, deadline]).finally(() => {
+    clearTimeout(timer);
+  });
+}
+function timeoutException(operation, timeoutMs, cause) {
+  const exception = new QueueException(
+    QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
+    common.HttpStatus.INTERNAL_SERVER_ERROR,
+    { operation, timeoutMs }
+  );
+  if (cause !== void 0) exception.cause = cause;
+  return exception;
+}
+function unavailableException(operation, cause) {
+  const exception = new QueueException(
+    QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
+    common.HttpStatus.INTERNAL_SERVER_ERROR,
+    { operation, reason: "offline" }
+  );
+  exception.cause = cause;
+  return exception;
+}
+function isOfflineRejection(err) {
+  return err instanceof Error && err.message === "Stream isn't writeable and enableOfflineQueue options is false";
+}
+function isCommandTimeout(err) {
+  return err instanceof Error && err.message === "Command timed out";
+}
+async function withCommandTimeoutCode(operation, commandTimeoutMs, work) {
+  try {
+    return await work;
+  } catch (err) {
+    if (isCommandTimeout(err)) throw timeoutException(operation, commandTimeoutMs(), err);
+    throw isOfflineRejection(err) ? unavailableException(operation, err) : err;
+  }
+}
+
 // src/server/services/queue.service.ts
 var MAX_BULK_SIZE = 1e3;
 function assertValidCron(repeat, queueOptions) {
@@ -396,13 +511,14 @@ function assertValidCron(repeat, queueOptions) {
     });
   }
 }
-var _queues;
+var _queues, _warm;
 exports.QueueService = class QueueService {
   constructor(connection, options) {
     this.connection = connection;
     this.options = options;
     __publicField(this, "logger", new common.Logger(exports.QueueService.name));
     __privateAdd(this, _queues, /* @__PURE__ */ new Map());
+    __privateAdd(this, _warm, /* @__PURE__ */ new Set());
   }
   connection;
   options;
@@ -421,11 +537,39 @@ exports.QueueService = class QueueService {
     __privateGet(this, _queues).set(queueName, queue);
     return queue;
   }
+  call(operation, queueName, run) {
+    let expired = false;
+    const guarded = withDeadline(
+      this.afterReady(queueName).then(() => {
+        if (expired) throw new Error("the call had already timed out");
+        return run();
+      }),
+      this.options.enqueueTimeoutMs,
+      () => {
+        expired = true;
+        return timeoutException(operation, this.options.enqueueTimeoutMs);
+      }
+    );
+    return withCommandTimeoutCode(
+      operation,
+      () => this.connection.getClient().options.commandTimeout ?? this.options.commandTimeoutMs,
+      guarded
+    );
+  }
+  async afterReady(queueName) {
+    if (__privateGet(this, _warm).has(queueName)) return;
+    await this.connection.awaitReady();
+    __privateGet(this, _warm).add(queueName);
+  }
   queueFor(queueName) {
     return this.getOrCreateQueue(queueName);
   }
   async enqueue(queueName, jobName, data, options) {
-    const job = await this.queueFor(queueName).add(jobName, data, options);
+    const job = await this.call(
+      "enqueue",
+      queueName,
+      () => this.queueFor(queueName).add(jobName, data, options)
+    );
     return job;
   }
   async enqueueBulk(queueName, jobs) {
@@ -441,15 +585,22 @@ exports.QueueService = class QueueService {
       );
     }
     try {
-      const created = await this.queueFor(queueName).addBulk(
-        jobs.map((job) => ({
-          name: job.name,
-          data: job.data,
-          ...job.options === void 0 ? {} : { opts: job.options }
-        }))
+      const created = await this.call(
+        "enqueueBulk",
+        queueName,
+        () => this.queueFor(queueName).addBulk(
+          jobs.map((job) => ({
+            name: job.name,
+            data: job.data,
+            ...job.options === void 0 ? {} : { opts: job.options }
+          }))
+        )
       );
       return created;
     } catch (err) {
+      if (err instanceof QueueException && errorCodeOf(err) === QUEUE_ERROR_CODES.CONNECTION_TIMEOUT) {
+        throw err;
+      }
       throw new QueueException(
         QUEUE_ERROR_CODES.BULK_ENQUEUE_FAILED,
         common.HttpStatus.INTERNAL_SERVER_ERROR,
@@ -460,20 +611,27 @@ exports.QueueService = class QueueService {
     }
   }
   async getJob(queueName, jobId) {
-    const job = await this.queueFor(queueName).getJob(jobId);
+    const job = await this.call("getJob", queueName, () => this.queueFor(queueName).getJob(jobId));
     return job ?? null;
   }
   async getJobs(queueName, status, start = 0, end = 50) {
-    const jobs = await this.queueFor(queueName).getJobs([status], start, end);
+    if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start || end - start >= MAX_JOBS_PAGE_SIZE) {
+      throw new QueueException(QUEUE_ERROR_CODES.INVALID_OPTIONS, common.HttpStatus.BAD_REQUEST, {
+        reason: `getJobs needs integers with 0 <= start <= end and at most ${String(MAX_JOBS_PAGE_SIZE)} jobs per call`
+      });
+    }
+    const jobs = await this.call(
+      "getJobs",
+      queueName,
+      () => this.queueFor(queueName).getJobs([status], start, end)
+    );
     return jobs;
   }
   async getMetrics(queueName) {
-    const counts = await this.queueFor(queueName).getJobCounts(
-      "waiting",
-      "active",
-      "completed",
-      "failed",
-      "delayed"
+    const counts = await this.call(
+      "getMetrics",
+      queueName,
+      () => this.queueFor(queueName).getJobCounts("waiting", "active", "completed", "failed", "delayed")
     );
     return {
       queue: queueName,
@@ -490,23 +648,39 @@ exports.QueueService = class QueueService {
       ...template?.opts === void 0 ? {} : { opts: template.opts }
     };
     if ("pattern" in repeat) assertValidCron(repeat, this.options.queueOptions);
-    const job = await queue.upsertJobScheduler(schedulerId, repeat, jobTemplate);
+    const job = await this.call(
+      "upsertJobScheduler",
+      queueName,
+      () => queue.upsertJobScheduler(schedulerId, repeat, jobTemplate)
+    );
     return job;
   }
   async removeJobScheduler(queueName, schedulerId) {
-    return this.queueFor(queueName).removeJobScheduler(schedulerId);
+    return this.call(
+      "removeJobScheduler",
+      queueName,
+      () => this.queueFor(queueName).removeJobScheduler(schedulerId)
+    );
   }
   async getJobSchedulers(queueName, start = 0, end = 50, asc = true) {
-    return this.queueFor(queueName).getJobSchedulers(start, end, asc);
+    return this.call(
+      "getJobSchedulers",
+      queueName,
+      () => this.queueFor(queueName).getJobSchedulers(start, end, asc)
+    );
   }
   async pauseQueue(queueName) {
-    await this.queueFor(queueName).pause();
+    await this.call("pauseQueue", queueName, () => this.queueFor(queueName).pause());
   }
   async resumeQueue(queueName) {
-    await this.queueFor(queueName).resume();
+    await this.call("resumeQueue", queueName, () => this.queueFor(queueName).resume());
   }
   async cleanQueue(queueName, gracePeriodMs, limit, status) {
-    return this.queueFor(queueName).clean(gracePeriodMs, limit, status);
+    return this.call(
+      "cleanQueue",
+      queueName,
+      () => this.queueFor(queueName).clean(gracePeriodMs, limit, status)
+    );
   }
   getCachedQueues() {
     return __privateGet(this, _queues);
@@ -516,18 +690,26 @@ exports.QueueService = class QueueService {
       await queue.close().catch(() => void 0);
     }
     __privateGet(this, _queues).clear();
+    __privateGet(this, _warm).clear();
   }
 };
 _queues = new WeakMap();
+_warm = new WeakMap();
 exports.QueueService = __decorateClass([
   common.Injectable(),
   __decorateParam(0, common.Inject(exports.ConnectionResolver)),
   __decorateParam(1, common.Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
 ], exports.QueueService);
+function errorCodeOf(err) {
+  return err.getResponse().error.code;
+}
+var _connection;
 exports.FlowService = class FlowService {
-  logger = new common.Logger(exports.FlowService.name);
-  producer;
   constructor(connection, enabled, prefix, telemetry) {
+    __publicField(this, "logger", new common.Logger(exports.FlowService.name));
+    __publicField(this, "producer");
+    __privateAdd(this, _connection);
+    __privateSet(this, _connection, connection);
     if (enabled) {
       this.producer = new bullmq.FlowProducer({
         connection: connection.getClient(),
@@ -538,10 +720,21 @@ exports.FlowService = class FlowService {
     }
   }
   async add(flow) {
-    return this.requireProducer().add(flow);
+    return withCommandTimeoutCode(
+      "addFlow",
+      () => this.commandTimeout(),
+      this.requireProducer().add(flow)
+    );
   }
   async addBulk(flows) {
-    return this.requireProducer().addBulk([...flows]);
+    return withCommandTimeoutCode(
+      "addFlowBulk",
+      () => this.commandTimeout(),
+      this.requireProducer().addBulk([...flows])
+    );
+  }
+  commandTimeout() {
+    return __privateGet(this, _connection).getClient().options.commandTimeout ?? 0;
   }
   getProducer() {
     return this.requireProducer();
@@ -558,6 +751,7 @@ exports.FlowService = class FlowService {
     return this.producer;
   }
 };
+_connection = new WeakMap();
 exports.FlowService = __decorateClass([
   common.Injectable()
 ], exports.FlowService);
@@ -603,6 +797,17 @@ exports.MetricsService = class MetricsService {
 exports.MetricsService = __decorateClass([
   common.Injectable()
 ], exports.MetricsService);
+
+// src/server/utils/abandoned-workers.ts
+var abandoned = /* @__PURE__ */ new WeakSet();
+function abandonWorker(worker) {
+  abandoned.add(worker);
+}
+function isAbandoned(worker) {
+  return abandoned.has(worker);
+}
+
+// src/server/services/worker-registry.service.ts
 var _entries;
 exports.WorkerRegistry = class WorkerRegistry {
   constructor(connection, options) {
@@ -621,7 +826,11 @@ exports.WorkerRegistry = class WorkerRegistry {
     const bullOpts = this.buildBullOptions(options, conn);
     let worker;
     try {
-      worker = new bullmq.Worker(queueName, handler, bullOpts);
+      worker = new bullmq.Worker(
+        queueName,
+        toProcessor(handler, options?.jobTimeoutMs, () => isAbandoned(worker)),
+        bullOpts
+      );
     } catch (cause) {
       conn.disconnect();
       throw new QueueException(
@@ -639,6 +848,13 @@ exports.WorkerRegistry = class WorkerRegistry {
   }
   registerSandboxed(config) {
     const { queueName, processorFile, options } = config;
+    if (options?.jobTimeoutMs !== void 0) {
+      throw new QueueException(
+        QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
+        common.HttpStatus.INTERNAL_SERVER_ERROR,
+        { reason: "jobTimeoutMs is not available for sandboxed workers" }
+      );
+    }
     this.validateProcessorFile(processorFile);
     this.guardDuplicate(queueName);
     this.validateOptions(options);
@@ -668,9 +884,12 @@ exports.WorkerRegistry = class WorkerRegistry {
   async unregister(queueName) {
     const entry = __privateGet(this, _entries).get(queueName);
     if (!entry) return;
-    await entry.worker.close();
-    await this.closeConnection(entry.connection);
-    __privateGet(this, _entries).delete(queueName);
+    try {
+      await entry.worker.close();
+    } finally {
+      await this.closeConnection(entry.connection);
+      __privateGet(this, _entries).delete(queueName);
+    }
   }
   list() {
     return Array.from(__privateGet(this, _entries).keys());
@@ -724,6 +943,7 @@ exports.WorkerRegistry = class WorkerRegistry {
         );
       }
     }
+    this.validateJobTimeout(opts);
     if (opts?.limiter !== void 0) {
       if (opts.limiter.max < 1) {
         throw new QueueException(
@@ -747,6 +967,18 @@ exports.WorkerRegistry = class WorkerRegistry {
       }
     }
   }
+  validateJobTimeout(opts) {
+    if (opts?.jobTimeoutMs !== void 0 && !isTimerDelayMs(opts.jobTimeoutMs)) {
+      throw new QueueException(
+        QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
+        common.HttpStatus.INTERNAL_SERVER_ERROR,
+        {
+          reason: "jobTimeoutMs must be a whole number from 1 to 2147483647",
+          actual: opts.jobTimeoutMs
+        }
+      );
+    }
+  }
   validateProcessorFile(processorFile) {
     if (processorFile instanceof URL) {
       if (processorFile.protocol !== "file:") {
@@ -804,6 +1036,44 @@ exports.WorkerRegistry = __decorateClass([
   __decorateParam(0, common.Inject(exports.ConnectionResolver)),
   __decorateParam(1, common.Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
 ], exports.WorkerRegistry);
+var JobTimeoutError = class extends Error {
+  constructor(timeoutMs) {
+    super(`queue.job_timeout: the handler did not settle within ${String(timeoutMs)}ms`);
+    this.name = "JobTimeoutError";
+  }
+};
+function adaptHandler(handler, timeoutMs) {
+  if (timeoutMs === void 0) return (job, _token, signal) => handler(job, signal);
+  return (job, _token, signal) => {
+    const own = new AbortController();
+    const relay = () => {
+      own.abort(signal?.reason);
+    };
+    if (signal?.aborted === true) relay();
+    signal?.addEventListener("abort", relay);
+    return withDeadline(
+      Promise.resolve().then(() => handler(job, own.signal)),
+      timeoutMs,
+      () => {
+        const error = new JobTimeoutError(timeoutMs);
+        own.abort(error);
+        return error;
+      }
+    ).finally(() => {
+      signal?.removeEventListener("abort", relay);
+    });
+  };
+}
+function toProcessor(handler, timeoutMs, abandoned2) {
+  const run = adaptHandler(handler, timeoutMs);
+  return (job, token, signal) => run(job, token, signal).then(
+    (value) => abandoned2() ? new Promise(() => void 0) : value,
+    (error) => {
+      if (abandoned2()) return new Promise(() => void 0);
+      throw error;
+    }
+  );
+}
 var _events, _connections;
 exports.QueueEventsRegistry = class QueueEventsRegistry {
   constructor(connection, options) {
@@ -893,10 +1163,10 @@ var ProcessorDiscoveryService = class {
         catchAll = fn;
       }
     }
-    return (job) => {
+    return (job, signal) => {
       const specific = namedHandlers.get(job.name);
-      if (specific) return specific(job);
-      if (catchAll) return catchAll(job);
+      if (specific) return specific(job, signal);
+      if (catchAll) return catchAll(job, signal);
       return Promise.reject(
         new Error(
           `No handler registered for job name "${job.name}" on queue \u2014 add a matching @Process("name") or a catch-all @Process() method.`
@@ -982,6 +1252,7 @@ ProcessorDiscoveryService = __decorateClass([
   __decorateParam(1, common.Inject(exports.WorkerRegistry)),
   __decorateParam(2, common.Inject(exports.QueueEventsRegistry))
 ], ProcessorDiscoveryService);
+var FORCE_CLOSE_GRACE_MS = 1e3;
 var DrainTimeoutError = class extends Error {
 };
 var QueueLifecycle = class {
@@ -1004,9 +1275,10 @@ var QueueLifecycle = class {
   forcedWorkers = 0;
   async onModuleDestroy() {
     this.startedAt = Date.now();
-    this.forcedWorkers = await this.closeWorkers();
-    await this.closeQueueEvents();
-    await this.closeDuplicatedConnections();
+    const deadline = this.startedAt + this.resolved.shutdown.drainTimeoutMs;
+    this.forcedWorkers = await this.closeWorkers(deadline);
+    await this.closeQueueEvents(deadline);
+    await this.closeDuplicatedConnections(deadline);
   }
   async onApplicationShutdown() {
     await this.drainQueuesIfRequested();
@@ -1014,59 +1286,53 @@ var QueueLifecycle = class {
     await this.closeQueues();
     await this.teardownConnection();
     this.logger.log(
-      `Queue shutdown complete in ${String(Date.now() - (this.startedAt ?? Date.now()))}ms (forced ${String(this.forcedWorkers)} worker(s))`
+      `Queue shutdown complete in ${String(Date.now() - (this.startedAt ?? Date.now()))}ms (${String(this.forcedWorkers)} worker(s) stopped without draining)`
     );
   }
-  async closeWorkers() {
-    let forced = 0;
-    const drainTimeoutMs = this.resolved.shutdown.drainTimeoutMs;
-    for (const [name, worker] of this.workers.getAll()) {
-      try {
-        await this.closeWorkerWithTimeout(worker, drainTimeoutMs);
-      } catch (cause) {
-        forced++;
-        this.logger.warn(
-          cause instanceof DrainTimeoutError ? `Worker "${name}" exceeded the ${String(drainTimeoutMs)}ms drain budget \u2014 forcing close (${QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED}); the in-flight job becomes stalled and is retried` : `Worker "${name}" failed to close gracefully (${describeError(cause)}) \u2014 forcing close; the in-flight job becomes stalled and is retried`
-        );
-        void worker.close(true).catch(() => void 0);
-      }
-    }
-    return forced;
-  }
-  async closeWorkerWithTimeout(worker, timeoutMs) {
-    let timer;
-    const closed = worker.close();
-    closed.catch(() => void 0);
-    try {
-      await Promise.race([
-        closed,
-        new Promise((_resolve, reject) => {
-          timer = setTimeout(() => {
-            reject(new DrainTimeoutError());
-          }, timeoutMs);
-        })
-      ]);
-    } finally {
-      clearTimeout(timer);
-    }
+  async closeWorkers(deadline) {
+    let undrained = 0;
+    await Promise.all(
+      Array.from(this.workers.getAll(), async ([name, worker]) => {
+        try {
+          await bounded(
+            worker.pause().then(() => worker.close()),
+            deadline
+          );
+        } catch (cause) {
+          undrained++;
+          abandonWorker(worker);
+          const stopped = "it is stopped without waiting for its in-flight job, which becomes stalled and is retried";
+          this.logger.warn(
+            cause instanceof DrainTimeoutError ? `Worker "${name}" exceeded the ${String(this.resolved.shutdown.drainTimeoutMs)}ms drain budget (${QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED}); ${stopped}` : `Worker "${name}" failed to close gracefully (${describeError(cause)}); ${stopped}`
+          );
+          const deadlineError = () => new DrainTimeoutError();
+          await withDeadline(worker.close(true), FORCE_CLOSE_GRACE_MS, deadlineError).catch(
+            () => void 0
+          );
+        }
+      })
+    );
+    return undrained;
   }
-  async closeQueueEvents() {
-    for (const [, queueEvents] of this.events.getAll()) {
-      await queueEvents.close().catch(() => void 0);
-    }
+  async closeQueueEvents(deadline) {
+    await Promise.all(
+      Array.from(this.events.getAll().values(), async (queueEvents) => {
+        await bounded(queueEvents.close(), deadline).catch(() => void 0);
+      })
+    );
   }
-  async closeDuplicatedConnections() {
+  async closeDuplicatedConnections(deadline) {
     const connections = [
       ...this.workers.getConnections().values(),
       ...this.events.getConnections().values()
     ];
-    for (const connection of connections) {
-      try {
-        await connection.quit();
-      } catch {
-        connection.disconnect();
-      }
-    }
+    await Promise.all(
+      connections.map(async (connection) => {
+        await bounded(connection.quit(), deadline).catch(() => {
+          connection.disconnect();
+        });
+      })
+    );
   }
   async drainQueuesIfRequested() {
     if (!this.resolved.shutdown.drainOnShutdown) return;
@@ -1097,6 +1363,9 @@ QueueLifecycle = __decorateClass([
   __decorateParam(4, common.Inject(exports.ConnectionResolver)),
   __decorateParam(5, common.Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
 ], QueueLifecycle);
+function bounded(work, deadline) {
+  return withDeadline(work, Math.max(0, deadline - Date.now()), () => new DrainTimeoutError());
+}
 
 // src/server/bymax-queue.module.ts
 var { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = (