@bymax-one/nest-queue 1.2.13 → 1.2.15

package/dist/server/index.mjs

@@ -76,6 +76,12 @@ var QueueException = class extends HttpException {
   }
 };
 
+// src/server/utils/timer-delay.ts
+var MAX_TIMER_DELAY_MS = 2147483647;
+function isTimerDelayMs(value) {
+  return Number.isInteger(value) && value >= 1 && value <= MAX_TIMER_DELAY_MS;
+}
+
 // src/server/config/validate-options.ts
 function invalid(reason) {
   throw new QueueException(QUEUE_ERROR_CODES.INVALID_OPTIONS, HttpStatus.INTERNAL_SERVER_ERROR, {
@@ -126,6 +132,12 @@ function validateOptions(opts) {
   if (cacheTtlMs !== void 0 && cacheTtlMs < 0) {
     invalid("metrics.cacheTtlMs must be >= 0");
   }
+  for (const key of ["commandTimeoutMs", "enqueueTimeoutMs"]) {
+    const value = opts[key];
+    if (value !== void 0 && !isTimerDelayMs(value)) {
+      invalid(`${key} must be a whole number from 1 to ${String(MAX_TIMER_DELAY_MS)}`);
+    }
+  }
 }
 
 // src/server/constants/default-options.ts
@@ -133,11 +145,14 @@ var DEFAULT_WORKER_CONCURRENCY = 2;
 var MAX_WORKER_CONCURRENCY = 1e3;
 var DEFAULT_JOB_OPTIONS = {
   attempts: 3,
-  backoff: { type: "exponential", delay: 2e3 },
+  backoff: { type: "exponential", delay: 2e3, jitter: 0.5 },
   removeOnComplete: { age: 24 * 3600, count: 1e3 },
   removeOnFail: { age: 7 * 24 * 3600, count: 5e3 }
 };
 var DEFAULT_CONNECTION_READY_TIMEOUT_MS = 1e4;
+var DEFAULT_COMMAND_TIMEOUT_MS = 5e3;
+var DEFAULT_ENQUEUE_TIMEOUT_MS = 1e4;
+var MAX_JOBS_PAGE_SIZE = 1e3;
 var DEFAULT_METRICS_CACHE_TTL_MS = 5e3;
 var DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS = 3e4;
 
@@ -157,7 +172,9 @@ function applyDefaults(opts) {
       drainTimeoutMs: opts.shutdown?.drainTimeoutMs ?? DEFAULT_SHUTDOWN_DRAIN_TIMEOUT_MS,
       drainOnShutdown: opts.shutdown?.drainOnShutdown ?? false
     },
-    connectionReadyTimeoutMs: opts.connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
+    connectionReadyTimeoutMs: opts.connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS,
+    commandTimeoutMs: opts.commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS,
+    enqueueTimeoutMs: opts.enqueueTimeoutMs ?? DEFAULT_ENQUEUE_TIMEOUT_MS
   };
   if (opts.telemetry !== void 0) base.telemetry = opts.telemetry;
   Object.defineProperty(base, "connection", {
@@ -222,17 +239,23 @@ function attachClientErrorListener(client, logger, role) {
 
 // src/server/utils/duplicate-connection.ts
 function duplicateConnection(client) {
-  return client.duplicate({ maxRetriesPerRequest: null });
+  const override = {
+    maxRetriesPerRequest: null,
+    enableOfflineQueue: true,
+    commandTimeout: void 0
+  };
+  return client.duplicate(override);
 }
 
 // src/server/services/connection-resolver.service.ts
-var _logger, _client, _options;
+var _logger, _client, _options, _pendingReady;
 var ConnectionResolver = class {
   constructor(options) {
     __privateAdd(this, _logger, new Logger(ConnectionResolver.name));
     __privateAdd(this, _client);
     __privateAdd(this, _options);
     __publicField(this, "mode");
+    __privateAdd(this, _pendingReady);
     __privateSet(this, _options, options);
   }
   async init() {
@@ -244,12 +267,22 @@ var ConnectionResolver = class {
     this.mode = "mode-b-owned";
     __privateSet(this, _client, "url" in cfg ? new Redis(cfg.url, this.ownedOptions(cfg.options)) : new Redis(this.ownedOptions(cfg.options)));
     attachClientErrorListener(__privateGet(this, _client), __privateGet(this, _logger), "owned");
-    await this.waitReady(
-      __privateGet(this, _options).connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
-    );
+    try {
+      await this.waitReady(
+        __privateGet(this, _options).connectionReadyTimeoutMs ?? DEFAULT_CONNECTION_READY_TIMEOUT_MS
+      );
+    } catch (cause) {
+      __privateGet(this, _client).disconnect();
+      throw cause;
+    }
   }
   ownedOptions(options) {
-    return { ...options ?? {}, lazyConnect: false };
+    return {
+      commandTimeout: __privateGet(this, _options).commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS,
+      enableOfflineQueue: false,
+      ...options ?? {},
+      lazyConnect: false
+    };
   }
   getClient() {
     if (!__privateGet(this, _client)) {
@@ -278,6 +311,17 @@ var ConnectionResolver = class {
   isOwned() {
     return this.mode === "mode-b-owned";
   }
+  awaitReady() {
+    const client = this.getClient();
+    if (!WAITING_STATUSES.has(client.status)) return Promise.resolve();
+    __privateGet(this, _pendingReady) ?? __privateSet(this, _pendingReady, new Promise((resolve) => {
+      client.once("ready", () => {
+        __privateSet(this, _pendingReady, void 0);
+        resolve();
+      });
+    }));
+    return __privateGet(this, _pendingReady);
+  }
   async teardown() {
     if (this.isOwned() && __privateGet(this, _client)) {
       const client = __privateGet(this, _client);
@@ -296,6 +340,11 @@ var ConnectionResolver = class {
         { status: client.status }
       );
     }
+    if (client.options.maxRetriesPerRequest === null) {
+      __privateGet(this, _logger).warn(
+        "The shared client has maxRetriesPerRequest: null, so a command issued while Redis is unreachable stays queued in the client for as long as the outage lasts. QueueService calls are bounded by enqueueTimeoutMs and the caller is released, but the command is not cancelled. Give the client a finite limit to fail fast."
+      );
+    }
     const probe = duplicateConnection(client);
     attachClientErrorListener(probe, __privateGet(this, _logger), "probe");
     try {
@@ -308,41 +357,62 @@ var ConnectionResolver = class {
     const client = this.getClient();
     if (client.status === "ready") return;
     await new Promise((resolve, reject) => {
+      let lastErrorCode;
+      const details = (extra) => lastErrorCode === void 0 ? extra : { ...extra, lastErrorCode };
       const cleanup = () => {
         clearTimeout(timer);
         client.off("ready", onReady);
         client.off("error", onError);
+        client.off("end", onEnd);
+      };
+      const fail = (code, extra) => {
+        cleanup();
+        reject(new QueueException(code, HttpStatus.INTERNAL_SERVER_ERROR, details(extra)));
       };
       const onReady = () => {
         cleanup();
         resolve();
       };
       const onError = (err) => {
-        cleanup();
-        reject(err);
+        lastErrorCode = startupErrorCode(err);
+        if (isReplyError(err)) fail(QUEUE_ERROR_CODES.CONNECTION_INVALID, {});
+      };
+      const onEnd = () => {
+        fail(QUEUE_ERROR_CODES.CONNECTION_INVALID, {});
       };
       const timer = setTimeout(() => {
-        cleanup();
-        reject(
-          new QueueException(
-            QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
-            HttpStatus.INTERNAL_SERVER_ERROR,
-            { timeoutMs }
-          )
-        );
+        fail(QUEUE_ERROR_CODES.CONNECTION_TIMEOUT, { timeoutMs });
       }, timeoutMs);
       client.once("ready", onReady);
-      client.once("error", onError);
+      client.on("error", onError);
+      client.once("end", onEnd);
     });
   }
 };
 _logger = new WeakMap();
 _client = new WeakMap();
 _options = new WeakMap();
+_pendingReady = new WeakMap();
 ConnectionResolver = __decorateClass([
   Injectable(),
   __decorateParam(0, Inject(BYMAX_QUEUE_OPTIONS))
 ], ConnectionResolver);
+var WAITING_STATUSES = /* @__PURE__ */ new Set([
+  "connecting",
+  "connect",
+  "reconnecting",
+  "close"
+]);
+function isReplyError(err) {
+  return typeof err === "object" && err !== null && err.name === "ReplyError";
+}
+function startupErrorCode(err) {
+  if (typeof err !== "object" || err === null) return void 0;
+  const { code, message } = err;
+  if (typeof code === "string") return code;
+  if (isReplyError(err) && typeof message === "string") return /^[A-Z][A-Z0-9_]*/.exec(message)?.[0];
+  return void 0;
+}
 function invalid2(reason) {
   throw new QueueException(QUEUE_ERROR_CODES.INVALID_REPEAT_OPTIONS, HttpStatus.BAD_REQUEST, {
     reason
@@ -382,6 +452,51 @@ function attachDefaultErrorListener(emitter, logger, kind, queueName) {
   });
 }
 
+// src/server/utils/with-deadline.ts
+function withDeadline(work, timeoutMs, onTimeout) {
+  let timer;
+  const deadline = new Promise((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(onTimeout());
+    }, timeoutMs);
+  });
+  return Promise.race([work, deadline]).finally(() => {
+    clearTimeout(timer);
+  });
+}
+function timeoutException(operation, timeoutMs, cause) {
+  const exception = new QueueException(
+    QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
+    HttpStatus.INTERNAL_SERVER_ERROR,
+    { operation, timeoutMs }
+  );
+  if (cause !== void 0) exception.cause = cause;
+  return exception;
+}
+function unavailableException(operation, cause) {
+  const exception = new QueueException(
+    QUEUE_ERROR_CODES.CONNECTION_TIMEOUT,
+    HttpStatus.INTERNAL_SERVER_ERROR,
+    { operation, reason: "offline" }
+  );
+  exception.cause = cause;
+  return exception;
+}
+function isOfflineRejection(err) {
+  return err instanceof Error && err.message === "Stream isn't writeable and enableOfflineQueue options is false";
+}
+function isCommandTimeout(err) {
+  return err instanceof Error && err.message === "Command timed out";
+}
+async function withCommandTimeoutCode(operation, commandTimeoutMs, work) {
+  try {
+    return await work;
+  } catch (err) {
+    if (isCommandTimeout(err)) throw timeoutException(operation, commandTimeoutMs(), err);
+    throw isOfflineRejection(err) ? unavailableException(operation, err) : err;
+  }
+}
+
 // src/server/services/queue.service.ts
 var MAX_BULK_SIZE = 1e3;
 function assertValidCron(repeat, queueOptions) {
@@ -394,13 +509,14 @@ function assertValidCron(repeat, queueOptions) {
     });
   }
 }
-var _queues;
+var _queues, _warm;
 var QueueService = class {
   constructor(connection, options) {
     this.connection = connection;
     this.options = options;
     __publicField(this, "logger", new Logger(QueueService.name));
     __privateAdd(this, _queues, /* @__PURE__ */ new Map());
+    __privateAdd(this, _warm, /* @__PURE__ */ new Set());
   }
   connection;
   options;
@@ -419,11 +535,39 @@ var QueueService = class {
     __privateGet(this, _queues).set(queueName, queue);
     return queue;
   }
+  call(operation, queueName, run) {
+    let expired = false;
+    const guarded = withDeadline(
+      this.afterReady(queueName).then(() => {
+        if (expired) throw new Error("the call had already timed out");
+        return run();
+      }),
+      this.options.enqueueTimeoutMs,
+      () => {
+        expired = true;
+        return timeoutException(operation, this.options.enqueueTimeoutMs);
+      }
+    );
+    return withCommandTimeoutCode(
+      operation,
+      () => this.connection.getClient().options.commandTimeout ?? this.options.commandTimeoutMs,
+      guarded
+    );
+  }
+  async afterReady(queueName) {
+    if (__privateGet(this, _warm).has(queueName)) return;
+    await this.connection.awaitReady();
+    __privateGet(this, _warm).add(queueName);
+  }
   queueFor(queueName) {
     return this.getOrCreateQueue(queueName);
   }
   async enqueue(queueName, jobName, data, options) {
-    const job = await this.queueFor(queueName).add(jobName, data, options);
+    const job = await this.call(
+      "enqueue",
+      queueName,
+      () => this.queueFor(queueName).add(jobName, data, options)
+    );
     return job;
   }
   async enqueueBulk(queueName, jobs) {
@@ -439,15 +583,22 @@ var QueueService = class {
       );
     }
     try {
-      const created = await this.queueFor(queueName).addBulk(
-        jobs.map((job) => ({
-          name: job.name,
-          data: job.data,
-          ...job.options === void 0 ? {} : { opts: job.options }
-        }))
+      const created = await this.call(
+        "enqueueBulk",
+        queueName,
+        () => this.queueFor(queueName).addBulk(
+          jobs.map((job) => ({
+            name: job.name,
+            data: job.data,
+            ...job.options === void 0 ? {} : { opts: job.options }
+          }))
+        )
       );
       return created;
     } catch (err) {
+      if (err instanceof QueueException && errorCodeOf(err) === QUEUE_ERROR_CODES.CONNECTION_TIMEOUT) {
+        throw err;
+      }
       throw new QueueException(
         QUEUE_ERROR_CODES.BULK_ENQUEUE_FAILED,
         HttpStatus.INTERNAL_SERVER_ERROR,
@@ -458,20 +609,27 @@ var QueueService = class {
     }
   }
   async getJob(queueName, jobId) {
-    const job = await this.queueFor(queueName).getJob(jobId);
+    const job = await this.call("getJob", queueName, () => this.queueFor(queueName).getJob(jobId));
     return job ?? null;
   }
   async getJobs(queueName, status, start = 0, end = 50) {
-    const jobs = await this.queueFor(queueName).getJobs([status], start, end);
+    if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start || end - start >= MAX_JOBS_PAGE_SIZE) {
+      throw new QueueException(QUEUE_ERROR_CODES.INVALID_OPTIONS, HttpStatus.BAD_REQUEST, {
+        reason: `getJobs needs integers with 0 <= start <= end and at most ${String(MAX_JOBS_PAGE_SIZE)} jobs per call`
+      });
+    }
+    const jobs = await this.call(
+      "getJobs",
+      queueName,
+      () => this.queueFor(queueName).getJobs([status], start, end)
+    );
     return jobs;
   }
   async getMetrics(queueName) {
-    const counts = await this.queueFor(queueName).getJobCounts(
-      "waiting",
-      "active",
-      "completed",
-      "failed",
-      "delayed"
+    const counts = await this.call(
+      "getMetrics",
+      queueName,
+      () => this.queueFor(queueName).getJobCounts("waiting", "active", "completed", "failed", "delayed")
     );
     return {
       queue: queueName,
@@ -488,23 +646,39 @@ var QueueService = class {
       ...template?.opts === void 0 ? {} : { opts: template.opts }
     };
     if ("pattern" in repeat) assertValidCron(repeat, this.options.queueOptions);
-    const job = await queue.upsertJobScheduler(schedulerId, repeat, jobTemplate);
+    const job = await this.call(
+      "upsertJobScheduler",
+      queueName,
+      () => queue.upsertJobScheduler(schedulerId, repeat, jobTemplate)
+    );
     return job;
   }
   async removeJobScheduler(queueName, schedulerId) {
-    return this.queueFor(queueName).removeJobScheduler(schedulerId);
+    return this.call(
+      "removeJobScheduler",
+      queueName,
+      () => this.queueFor(queueName).removeJobScheduler(schedulerId)
+    );
   }
   async getJobSchedulers(queueName, start = 0, end = 50, asc = true) {
-    return this.queueFor(queueName).getJobSchedulers(start, end, asc);
+    return this.call(
+      "getJobSchedulers",
+      queueName,
+      () => this.queueFor(queueName).getJobSchedulers(start, end, asc)
+    );
   }
   async pauseQueue(queueName) {
-    await this.queueFor(queueName).pause();
+    await this.call("pauseQueue", queueName, () => this.queueFor(queueName).pause());
   }
   async resumeQueue(queueName) {
-    await this.queueFor(queueName).resume();
+    await this.call("resumeQueue", queueName, () => this.queueFor(queueName).resume());
   }
   async cleanQueue(queueName, gracePeriodMs, limit, status) {
-    return this.queueFor(queueName).clean(gracePeriodMs, limit, status);
+    return this.call(
+      "cleanQueue",
+      queueName,
+      () => this.queueFor(queueName).clean(gracePeriodMs, limit, status)
+    );
   }
   getCachedQueues() {
     return __privateGet(this, _queues);
@@ -514,18 +688,26 @@ var QueueService = class {
       await queue.close().catch(() => void 0);
     }
     __privateGet(this, _queues).clear();
+    __privateGet(this, _warm).clear();
   }
 };
 _queues = new WeakMap();
+_warm = new WeakMap();
 QueueService = __decorateClass([
   Injectable(),
   __decorateParam(0, Inject(ConnectionResolver)),
   __decorateParam(1, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
 ], QueueService);
+function errorCodeOf(err) {
+  return err.getResponse().error.code;
+}
+var _connection;
 var FlowService = class {
-  logger = new Logger(FlowService.name);
-  producer;
   constructor(connection, enabled, prefix, telemetry) {
+    __publicField(this, "logger", new Logger(FlowService.name));
+    __publicField(this, "producer");
+    __privateAdd(this, _connection);
+    __privateSet(this, _connection, connection);
     if (enabled) {
       this.producer = new FlowProducer({
         connection: connection.getClient(),
@@ -536,10 +718,21 @@ var FlowService = class {
     }
   }
   async add(flow) {
-    return this.requireProducer().add(flow);
+    return withCommandTimeoutCode(
+      "addFlow",
+      () => this.commandTimeout(),
+      this.requireProducer().add(flow)
+    );
   }
   async addBulk(flows) {
-    return this.requireProducer().addBulk([...flows]);
+    return withCommandTimeoutCode(
+      "addFlowBulk",
+      () => this.commandTimeout(),
+      this.requireProducer().addBulk([...flows])
+    );
+  }
+  commandTimeout() {
+    return __privateGet(this, _connection).getClient().options.commandTimeout ?? 0;
   }
   getProducer() {
     return this.requireProducer();
@@ -556,6 +749,7 @@ var FlowService = class {
     return this.producer;
   }
 };
+_connection = new WeakMap();
 FlowService = __decorateClass([
   Injectable()
 ], FlowService);
@@ -601,6 +795,17 @@ var MetricsService = class {
 MetricsService = __decorateClass([
   Injectable()
 ], MetricsService);
+
+// src/server/utils/abandoned-workers.ts
+var abandoned = /* @__PURE__ */ new WeakSet();
+function abandonWorker(worker) {
+  abandoned.add(worker);
+}
+function isAbandoned(worker) {
+  return abandoned.has(worker);
+}
+
+// src/server/services/worker-registry.service.ts
 var _entries;
 var WorkerRegistry = class {
   constructor(connection, options) {
@@ -619,7 +824,11 @@ var WorkerRegistry = class {
     const bullOpts = this.buildBullOptions(options, conn);
     let worker;
     try {
-      worker = new Worker(queueName, handler, bullOpts);
+      worker = new Worker(
+        queueName,
+        toProcessor(handler, options?.jobTimeoutMs, () => isAbandoned(worker)),
+        bullOpts
+      );
     } catch (cause) {
       conn.disconnect();
       throw new QueueException(
@@ -637,6 +846,13 @@ var WorkerRegistry = class {
   }
   registerSandboxed(config) {
     const { queueName, processorFile, options } = config;
+    if (options?.jobTimeoutMs !== void 0) {
+      throw new QueueException(
+        QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
+        HttpStatus.INTERNAL_SERVER_ERROR,
+        { reason: "jobTimeoutMs is not available for sandboxed workers" }
+      );
+    }
     this.validateProcessorFile(processorFile);
     this.guardDuplicate(queueName);
     this.validateOptions(options);
@@ -666,9 +882,12 @@ var WorkerRegistry = class {
   async unregister(queueName) {
     const entry = __privateGet(this, _entries).get(queueName);
     if (!entry) return;
-    await entry.worker.close();
-    await this.closeConnection(entry.connection);
-    __privateGet(this, _entries).delete(queueName);
+    try {
+      await entry.worker.close();
+    } finally {
+      await this.closeConnection(entry.connection);
+      __privateGet(this, _entries).delete(queueName);
+    }
   }
   list() {
     return Array.from(__privateGet(this, _entries).keys());
@@ -722,6 +941,7 @@ var WorkerRegistry = class {
         );
       }
     }
+    this.validateJobTimeout(opts);
     if (opts?.limiter !== void 0) {
       if (opts.limiter.max < 1) {
         throw new QueueException(
@@ -745,6 +965,18 @@ var WorkerRegistry = class {
       }
     }
   }
+  validateJobTimeout(opts) {
+    if (opts?.jobTimeoutMs !== void 0 && !isTimerDelayMs(opts.jobTimeoutMs)) {
+      throw new QueueException(
+        QUEUE_ERROR_CODES.WORKER_REGISTRATION_FAILED,
+        HttpStatus.INTERNAL_SERVER_ERROR,
+        {
+          reason: "jobTimeoutMs must be a whole number from 1 to 2147483647",
+          actual: opts.jobTimeoutMs
+        }
+      );
+    }
+  }
   validateProcessorFile(processorFile) {
     if (processorFile instanceof URL) {
       if (processorFile.protocol !== "file:") {
@@ -802,6 +1034,44 @@ WorkerRegistry = __decorateClass([
   __decorateParam(0, Inject(ConnectionResolver)),
   __decorateParam(1, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
 ], WorkerRegistry);
+var JobTimeoutError = class extends Error {
+  constructor(timeoutMs) {
+    super(`queue.job_timeout: the handler did not settle within ${String(timeoutMs)}ms`);
+    this.name = "JobTimeoutError";
+  }
+};
+function adaptHandler(handler, timeoutMs) {
+  if (timeoutMs === void 0) return (job, _token, signal) => handler(job, signal);
+  return (job, _token, signal) => {
+    const own = new AbortController();
+    const relay = () => {
+      own.abort(signal?.reason);
+    };
+    if (signal?.aborted === true) relay();
+    signal?.addEventListener("abort", relay);
+    return withDeadline(
+      Promise.resolve().then(() => handler(job, own.signal)),
+      timeoutMs,
+      () => {
+        const error = new JobTimeoutError(timeoutMs);
+        own.abort(error);
+        return error;
+      }
+    ).finally(() => {
+      signal?.removeEventListener("abort", relay);
+    });
+  };
+}
+function toProcessor(handler, timeoutMs, abandoned2) {
+  const run = adaptHandler(handler, timeoutMs);
+  return (job, token, signal) => run(job, token, signal).then(
+    (value) => abandoned2() ? new Promise(() => void 0) : value,
+    (error) => {
+      if (abandoned2()) return new Promise(() => void 0);
+      throw error;
+    }
+  );
+}
 var _events, _connections;
 var QueueEventsRegistry = class {
   constructor(connection, options) {
@@ -891,10 +1161,10 @@ var ProcessorDiscoveryService = class {
         catchAll = fn;
       }
     }
-    return (job) => {
+    return (job, signal) => {
       const specific = namedHandlers.get(job.name);
-      if (specific) return specific(job);
-      if (catchAll) return catchAll(job);
+      if (specific) return specific(job, signal);
+      if (catchAll) return catchAll(job, signal);
       return Promise.reject(
         new Error(
           `No handler registered for job name "${job.name}" on queue \u2014 add a matching @Process("name") or a catch-all @Process() method.`
@@ -980,6 +1250,7 @@ ProcessorDiscoveryService = __decorateClass([
   __decorateParam(1, Inject(WorkerRegistry)),
   __decorateParam(2, Inject(QueueEventsRegistry))
 ], ProcessorDiscoveryService);
+var FORCE_CLOSE_GRACE_MS = 1e3;
 var DrainTimeoutError = class extends Error {
 };
 var QueueLifecycle = class {
@@ -1002,9 +1273,10 @@ var QueueLifecycle = class {
   forcedWorkers = 0;
   async onModuleDestroy() {
     this.startedAt = Date.now();
-    this.forcedWorkers = await this.closeWorkers();
-    await this.closeQueueEvents();
-    await this.closeDuplicatedConnections();
+    const deadline = this.startedAt + this.resolved.shutdown.drainTimeoutMs;
+    this.forcedWorkers = await this.closeWorkers(deadline);
+    await this.closeQueueEvents(deadline);
+    await this.closeDuplicatedConnections(deadline);
   }
   async onApplicationShutdown() {
     await this.drainQueuesIfRequested();
@@ -1012,59 +1284,53 @@ var QueueLifecycle = class {
     await this.closeQueues();
     await this.teardownConnection();
     this.logger.log(
-      `Queue shutdown complete in ${String(Date.now() - (this.startedAt ?? Date.now()))}ms (forced ${String(this.forcedWorkers)} worker(s))`
+      `Queue shutdown complete in ${String(Date.now() - (this.startedAt ?? Date.now()))}ms (${String(this.forcedWorkers)} worker(s) stopped without draining)`
     );
   }
-  async closeWorkers() {
-    let forced = 0;
-    const drainTimeoutMs = this.resolved.shutdown.drainTimeoutMs;
-    for (const [name, worker] of this.workers.getAll()) {
-      try {
-        await this.closeWorkerWithTimeout(worker, drainTimeoutMs);
-      } catch (cause) {
-        forced++;
-        this.logger.warn(
-          cause instanceof DrainTimeoutError ? `Worker "${name}" exceeded the ${String(drainTimeoutMs)}ms drain budget \u2014 forcing close (${QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED}); the in-flight job becomes stalled and is retried` : `Worker "${name}" failed to close gracefully (${describeError(cause)}) \u2014 forcing close; the in-flight job becomes stalled and is retried`
-        );
-        void worker.close(true).catch(() => void 0);
-      }
-    }
-    return forced;
-  }
-  async closeWorkerWithTimeout(worker, timeoutMs) {
-    let timer;
-    const closed = worker.close();
-    closed.catch(() => void 0);
-    try {
-      await Promise.race([
-        closed,
-        new Promise((_resolve, reject) => {
-          timer = setTimeout(() => {
-            reject(new DrainTimeoutError());
-          }, timeoutMs);
-        })
-      ]);
-    } finally {
-      clearTimeout(timer);
-    }
+  async closeWorkers(deadline) {
+    let undrained = 0;
+    await Promise.all(
+      Array.from(this.workers.getAll(), async ([name, worker]) => {
+        try {
+          await bounded(
+            worker.pause().then(() => worker.close()),
+            deadline
+          );
+        } catch (cause) {
+          undrained++;
+          abandonWorker(worker);
+          const stopped = "it is stopped without waiting for its in-flight job, which becomes stalled and is retried";
+          this.logger.warn(
+            cause instanceof DrainTimeoutError ? `Worker "${name}" exceeded the ${String(this.resolved.shutdown.drainTimeoutMs)}ms drain budget (${QUEUE_ERROR_CODES.SHUTDOWN_TIMEOUT_EXCEEDED}); ${stopped}` : `Worker "${name}" failed to close gracefully (${describeError(cause)}); ${stopped}`
+          );
+          const deadlineError = () => new DrainTimeoutError();
+          await withDeadline(worker.close(true), FORCE_CLOSE_GRACE_MS, deadlineError).catch(
+            () => void 0
+          );
+        }
+      })
+    );
+    return undrained;
   }
-  async closeQueueEvents() {
-    for (const [, queueEvents] of this.events.getAll()) {
-      await queueEvents.close().catch(() => void 0);
-    }
+  async closeQueueEvents(deadline) {
+    await Promise.all(
+      Array.from(this.events.getAll().values(), async (queueEvents) => {
+        await bounded(queueEvents.close(), deadline).catch(() => void 0);
+      })
+    );
   }
-  async closeDuplicatedConnections() {
+  async closeDuplicatedConnections(deadline) {
     const connections = [
       ...this.workers.getConnections().values(),
       ...this.events.getConnections().values()
     ];
-    for (const connection of connections) {
-      try {
-        await connection.quit();
-      } catch {
-        connection.disconnect();
-      }
-    }
+    await Promise.all(
+      connections.map(async (connection) => {
+        await bounded(connection.quit(), deadline).catch(() => {
+          connection.disconnect();
+        });
+      })
+    );
   }
   async drainQueuesIfRequested() {
     if (!this.resolved.shutdown.drainOnShutdown) return;
@@ -1095,6 +1361,9 @@ QueueLifecycle = __decorateClass([
   __decorateParam(4, Inject(ConnectionResolver)),
   __decorateParam(5, Inject(BYMAX_QUEUE_RESOLVED_OPTIONS))
 ], QueueLifecycle);
+function bounded(work, deadline) {
+  return withDeadline(work, Math.max(0, deadline - Date.now()), () => new DrainTimeoutError());
+}
 
 // src/server/bymax-queue.module.ts
 var { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN, OPTIONS_TYPE, ASYNC_OPTIONS_TYPE } = (