@zucker-framework/queue 1.0.2 → 1.0.5

package/README.md

@@ -0,0 +1,17 @@
+# @zucker-framework/queue
+
+`BullMqQueueProvider` owns producers; `BullMqWorkerHost` owns workers. Consumers retain outboxes, transaction boundaries, authorization, quota decisions and cancellation policy. Neither transport makes a business transaction atomic.
+
+## Producer lifecycle
+
+Construct with explicit `connection` and optional `prefix`, `onError(queueName, error)`. Call `waitUntilReady(['queue-name'])` to prewarm producers without adding a job; the no-argument form only waits for already-created queues. Close producers **after** gracefully closing workers that may still publish work. Error callbacks should redact connection errors before logging.
+
+`addBulk(queueName, entries)` on the BullMQ adapter preserves native bulk submission, validating all job IDs before creating the queue; empty batches create no connection. `add` preserves stable transport IDs. `ensure` may replace terminal records: do not use it when a terminal job must remain evidence for outbox failure reconciliation. `getJob` exposes normalized status and failed reason. `getStats` combines waiting/prioritized with a read-only legacy paused-list count; BullMQ 6 does not accept `paused` as a getJobCounts type. This does not resume or migrate legacy queues and is not proof of v5→v6 compatibility. `getJobCounts(queueName)` retains BullMQ's native default count keys (including `waiting-children`) for backend-specific monitoring contracts without merging counts.
+
+## Workers
+
+`register({ queueName, handler, concurrency, lockDurationMs, stalledIntervalMs, autorun })` requires positive safe-integer settings. Handler jobs include `attempt` (one-based) and `maxAttempts` (configured budget, default 1), allowing product terminal-attempt policy without reaching into raw BullMQ jobs. BullMQ unrecoverable errors can still be thrown directly by a handler.
+
+`close(false)` drains workers; concurrent calls share the same drain promise. Every close is awaited even if another close throws/rejects; failures are reported together as `AggregateError`, so a failed drain cannot signal completion while another worker is still active. Closed hosts/providers cannot register workers or reopen producers. Force closing has different recovery implications. `probeHealth` is a bounded read-only connection probe, not a task execution probe. Test with fake factories or explicitly isolated disposable Redis; never use existing queues for compatibility experiments.
+
+New prewarm/error/retry-budget/stalled-interval contracts are local source additions pending a new formal release; 1.0.2 consumers must not import sibling source to obtain them.

package/dist/index.d.mts

@@ -93,14 +93,21 @@ interface BullMqQueueProviderOptions {
     prefix?: string;
     defaultJobName?: string;
     queueFactory?: QueueFactory;
+    onError?: (queueName: string, error: Error) => void;
 }
 declare class BullMqQueueProvider implements QueueProvider {
     private readonly options;
     private readonly queues;
+    private closing?;
     private readonly defaultJobName;
     private readonly queueFactory;
     constructor(options: BullMqQueueProviderOptions);
     add<T>(queueName: string, data: T, options?: JobOptions): Promise<Job<T>>;
+    /** Preserve BullMQ native bulk submission semantics rather than looping add(). */
+    addBulk<T>(queueName: string, entries: readonly {
+        data: T;
+        options?: JobOptions;
+    }[]): Promise<Job<T>[]>;
     ensure<T>(queueName: string, data: T, options: JobOptions & {
         jobId: string;
     }): Promise<Job<T>>;
@@ -110,8 +117,11 @@ declare class BullMqQueueProvider implements QueueProvider {
     removeJob(queueName: string, jobId: string): Promise<void>;
     pause(queueName: string): Promise<void>;
     resume(queueName: string): Promise<void>;
+    /** Native count keys for existing BullMQ monitoring contracts. Unlike
+     * getStats(), this does not merge prioritized/legacy-paused into waiting. */
+    getJobCounts(queueName: string): Promise<Record<string, number>>;
     getStats(queueName: string): Promise<QueueStats>;
-    waitUntilReady(): Promise<void>;
+    waitUntilReady(queueNames?: readonly string[]): Promise<void>;
     close(): Promise<void>;
     onModuleDestroy(): Promise<void>;
     private getQueue;
@@ -133,6 +143,8 @@ interface QueueWorkerJob<TData> {
     name: string;
     data: TData;
     attempt: number;
+    /** Configured retry budget, including this attempt. */
+    maxAttempts: number;
     updateProgress(progress: JobProgress): Promise<void>;
     log(message: string): Promise<number>;
 }
@@ -142,6 +154,7 @@ interface QueueWorkerDefinition<TData = unknown, TResult = unknown> {
     handler: QueueJobHandler<TData, TResult>;
     concurrency?: number;
     lockDurationMs?: number;
+    stalledIntervalMs?: number;
     autorun?: boolean;
 }
 type WorkerFactory = (queueName: string, processor: (job: Job$1) => Promise<unknown>, options: WorkerOptions) => Worker;
@@ -160,6 +173,7 @@ declare class BullMqWorkerHost {
     private readonly options;
     private readonly workers;
     private readonly readyQueues;
+    private closing?;
     private readonly workerFactory;
     constructor(options: BullMqWorkerHostOptions);
     register<TData, TResult>(definition: QueueWorkerDefinition<TData, TResult>): void;

package/dist/index.d.ts

@@ -93,14 +93,21 @@ interface BullMqQueueProviderOptions {
     prefix?: string;
     defaultJobName?: string;
     queueFactory?: QueueFactory;
+    onError?: (queueName: string, error: Error) => void;
 }
 declare class BullMqQueueProvider implements QueueProvider {
     private readonly options;
     private readonly queues;
+    private closing?;
     private readonly defaultJobName;
     private readonly queueFactory;
     constructor(options: BullMqQueueProviderOptions);
     add<T>(queueName: string, data: T, options?: JobOptions): Promise<Job<T>>;
+    /** Preserve BullMQ native bulk submission semantics rather than looping add(). */
+    addBulk<T>(queueName: string, entries: readonly {
+        data: T;
+        options?: JobOptions;
+    }[]): Promise<Job<T>[]>;
     ensure<T>(queueName: string, data: T, options: JobOptions & {
         jobId: string;
     }): Promise<Job<T>>;
@@ -110,8 +117,11 @@ declare class BullMqQueueProvider implements QueueProvider {
     removeJob(queueName: string, jobId: string): Promise<void>;
     pause(queueName: string): Promise<void>;
     resume(queueName: string): Promise<void>;
+    /** Native count keys for existing BullMQ monitoring contracts. Unlike
+     * getStats(), this does not merge prioritized/legacy-paused into waiting. */
+    getJobCounts(queueName: string): Promise<Record<string, number>>;
     getStats(queueName: string): Promise<QueueStats>;
-    waitUntilReady(): Promise<void>;
+    waitUntilReady(queueNames?: readonly string[]): Promise<void>;
     close(): Promise<void>;
     onModuleDestroy(): Promise<void>;
     private getQueue;
@@ -133,6 +143,8 @@ interface QueueWorkerJob<TData> {
     name: string;
     data: TData;
     attempt: number;
+    /** Configured retry budget, including this attempt. */
+    maxAttempts: number;
     updateProgress(progress: JobProgress): Promise<void>;
     log(message: string): Promise<number>;
 }
@@ -142,6 +154,7 @@ interface QueueWorkerDefinition<TData = unknown, TResult = unknown> {
     handler: QueueJobHandler<TData, TResult>;
     concurrency?: number;
     lockDurationMs?: number;
+    stalledIntervalMs?: number;
     autorun?: boolean;
 }
 type WorkerFactory = (queueName: string, processor: (job: Job$1) => Promise<unknown>, options: WorkerOptions) => Worker;
@@ -160,6 +173,7 @@ declare class BullMqWorkerHost {
     private readonly options;
     private readonly workers;
     private readonly readyQueues;
+    private closing?;
     private readonly workerFactory;
     constructor(options: BullMqWorkerHostOptions);
     register<TData, TResult>(definition: QueueWorkerDefinition<TData, TResult>): void;

package/dist/index.js

@@ -161,6 +161,7 @@ var BullMqQueueProvider = class {
   }
   options;
   queues = /* @__PURE__ */ new Map();
+  closing;
   defaultJobName;
   queueFactory;
   constructor(options) {
@@ -173,6 +174,17 @@ var BullMqQueueProvider = class {
     const job = await this.getQueue(queueName).add(options?.name ?? this.defaultJobName, data, this.toBullMqOptions(options));
     return this.toJob(job);
   }
+  /** Preserve BullMQ native bulk submission semantics rather than looping add(). */
+  async addBulk(queueName, entries) {
+    for (const entry of entries) this.assertValidJobId(entry.options?.jobId);
+    if (entries.length === 0) return [];
+    const jobs = await this.getQueue(queueName).addBulk(entries.map(({ data, options }) => ({
+      name: options?.name ?? this.defaultJobName,
+      data,
+      opts: this.toBullMqOptions(options)
+    })));
+    return Promise.all(jobs.map((job) => this.toJob(job)));
+  }
   async ensure(queueName, data, options) {
     this.assertValidJobId(options.jobId);
     const queue = this.getQueue(queueName);
@@ -222,39 +234,54 @@ var BullMqQueueProvider = class {
   async resume(queueName) {
     await this.getQueue(queueName).resume();
   }
+  /** Native count keys for existing BullMQ monitoring contracts. Unlike
+  * getStats(), this does not merge prioritized/legacy-paused into waiting. */
+  async getJobCounts(queueName) {
+    return this.getQueue(queueName).getJobCounts();
+  }
   async getStats(queueName) {
-    const pausedStateForStats = "paused";
-    const counts = await this.getQueue(queueName).getJobCounts("wait", "active", "completed", "failed", "delayed", "prioritized", pausedStateForStats);
+    const queue = this.getQueue(queueName);
+    const counts = await queue.getJobCounts("wait", "active", "completed", "failed", "delayed", "prioritized");
+    const client = await queue.backend.client;
+    const paused = await client.llen(queue.toKey("paused"));
     return {
-      waiting: (counts.wait ?? 0) + (counts.prioritized ?? 0) + (counts.paused ?? 0),
+      waiting: (counts.wait ?? 0) + (counts.prioritized ?? 0) + paused,
       active: counts.active ?? 0,
       completed: counts.completed ?? 0,
       failed: counts.failed ?? 0,
       delayed: counts.delayed ?? 0
     };
   }
-  async waitUntilReady() {
+  async waitUntilReady(queueNames = []) {
+    for (const name of queueNames) this.getQueue(name);
     await Promise.all([
       ...this.queues.values()
     ].map((queue) => queue.waitUntilReady()));
   }
-  async close() {
+  close() {
+    if (this.closing) return this.closing;
     const queues = [
       ...this.queues.values()
     ];
     this.queues.clear();
-    await Promise.all(queues.map((queue) => queue.close()));
+    this.closing = Promise.allSettled(queues.map(async (queue) => queue.close())).then((results) => {
+      const failures = results.filter((result) => result.status === "rejected");
+      if (failures.length) throw new AggregateError(failures.map((result) => result.reason), "Queue shutdown failed");
+    });
+    return this.closing;
   }
   async onModuleDestroy() {
     await this.close();
   }
   getQueue(name) {
+    if (this.closing) throw new Error("Queue provider is closing");
     const existing = this.queues.get(name);
     if (existing) return existing;
     const queue = this.queueFactory(name, {
       connection: this.options.connection,
       prefix: this.options.prefix
     });
+    if (this.options.onError) queue.on("error", (error) => this.options.onError(name, error));
     this.queues.set(name, queue);
     return queue;
   }
@@ -344,24 +371,30 @@ var BullMqWorkerHost = class {
   options;
   workers = /* @__PURE__ */ new Map();
   readyQueues = /* @__PURE__ */ new Set();
+  closing;
   workerFactory;
   constructor(options) {
     this.options = options;
     this.workerFactory = options.workerFactory ?? ((queueName, processor, workerOptions) => new import_bullmq2.Worker(queueName, processor, workerOptions));
   }
   register(definition) {
+    if (this.closing) throw new Error("Worker host is closing");
     if (this.workers.has(definition.queueName)) {
       throw new Error(`Worker already registered for queue: ${definition.queueName}`);
     }
-    if ((definition.concurrency ?? 1) < 1) throw new RangeError("concurrency must be at least 1");
-    if ((definition.lockDurationMs ?? 3e4) < 1) {
-      throw new RangeError("lockDurationMs must be at least 1");
+    for (const [name, value] of Object.entries({
+      concurrency: definition.concurrency ?? 1,
+      lockDurationMs: definition.lockDurationMs ?? 3e4,
+      stalledIntervalMs: definition.stalledIntervalMs ?? 3e4
+    })) {
+      if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
     }
     const worker = this.workerFactory(definition.queueName, async (job) => definition.handler({
       id: String(job.id),
       name: job.name,
       data: job.data,
       attempt: job.attemptsMade + 1,
+      maxAttempts: job.opts?.attempts ?? 1,
       updateProgress: /* @__PURE__ */ __name((progress) => job.updateProgress(progress), "updateProgress"),
       log: /* @__PURE__ */ __name((message) => job.log(message), "log")
     }), {
@@ -369,9 +402,14 @@ var BullMqWorkerHost = class {
       prefix: this.options.prefix,
       concurrency: definition.concurrency ?? 1,
       lockDuration: definition.lockDurationMs ?? 3e4,
-      autorun: definition.autorun ?? true
+      autorun: definition.autorun ?? true,
+      ...definition.stalledIntervalMs === void 0 ? {} : {
+        stalledInterval: definition.stalledIntervalMs
+      }
+    });
+    worker.on("ready", () => {
+      if (!this.closing) this.readyQueues.add(definition.queueName);
     });
-    worker.on("ready", () => this.readyQueues.add(definition.queueName));
     worker.on("error", (error) => {
       this.readyQueues.delete(definition.queueName);
       this.options.onError?.(definition.queueName, error);
@@ -447,13 +485,18 @@ var BullMqWorkerHost = class {
       ...this.workers.values()
     ].map((worker) => worker.resume()));
   }
-  async close(force = false) {
+  close(force = false) {
+    if (this.closing) return this.closing;
     const workers = [
       ...this.workers.values()
     ];
     this.workers.clear();
     this.readyQueues.clear();
-    await Promise.all(workers.map((worker) => worker.close(force)));
+    this.closing = Promise.allSettled(workers.map(async (worker) => worker.close(force))).then((results) => {
+      const failures = results.filter((result) => result.status === "rejected");
+      if (failures.length) throw new AggregateError(failures.map((result) => result.reason), "Worker shutdown failed");
+    });
+    return this.closing;
   }
   async onModuleDestroy() {
     await this.close();

package/dist/index.mjs

@@ -127,6 +127,7 @@ var BullMqQueueProvider = class {
   }
   options;
   queues = /* @__PURE__ */ new Map();
+  closing;
   defaultJobName;
   queueFactory;
   constructor(options) {
@@ -139,6 +140,17 @@ var BullMqQueueProvider = class {
     const job = await this.getQueue(queueName).add(options?.name ?? this.defaultJobName, data, this.toBullMqOptions(options));
     return this.toJob(job);
   }
+  /** Preserve BullMQ native bulk submission semantics rather than looping add(). */
+  async addBulk(queueName, entries) {
+    for (const entry of entries) this.assertValidJobId(entry.options?.jobId);
+    if (entries.length === 0) return [];
+    const jobs = await this.getQueue(queueName).addBulk(entries.map(({ data, options }) => ({
+      name: options?.name ?? this.defaultJobName,
+      data,
+      opts: this.toBullMqOptions(options)
+    })));
+    return Promise.all(jobs.map((job) => this.toJob(job)));
+  }
   async ensure(queueName, data, options) {
     this.assertValidJobId(options.jobId);
     const queue = this.getQueue(queueName);
@@ -188,39 +200,54 @@ var BullMqQueueProvider = class {
   async resume(queueName) {
     await this.getQueue(queueName).resume();
   }
+  /** Native count keys for existing BullMQ monitoring contracts. Unlike
+  * getStats(), this does not merge prioritized/legacy-paused into waiting. */
+  async getJobCounts(queueName) {
+    return this.getQueue(queueName).getJobCounts();
+  }
   async getStats(queueName) {
-    const pausedStateForStats = "paused";
-    const counts = await this.getQueue(queueName).getJobCounts("wait", "active", "completed", "failed", "delayed", "prioritized", pausedStateForStats);
+    const queue = this.getQueue(queueName);
+    const counts = await queue.getJobCounts("wait", "active", "completed", "failed", "delayed", "prioritized");
+    const client = await queue.backend.client;
+    const paused = await client.llen(queue.toKey("paused"));
     return {
-      waiting: (counts.wait ?? 0) + (counts.prioritized ?? 0) + (counts.paused ?? 0),
+      waiting: (counts.wait ?? 0) + (counts.prioritized ?? 0) + paused,
       active: counts.active ?? 0,
       completed: counts.completed ?? 0,
       failed: counts.failed ?? 0,
       delayed: counts.delayed ?? 0
     };
   }
-  async waitUntilReady() {
+  async waitUntilReady(queueNames = []) {
+    for (const name of queueNames) this.getQueue(name);
     await Promise.all([
       ...this.queues.values()
     ].map((queue) => queue.waitUntilReady()));
   }
-  async close() {
+  close() {
+    if (this.closing) return this.closing;
     const queues = [
       ...this.queues.values()
     ];
     this.queues.clear();
-    await Promise.all(queues.map((queue) => queue.close()));
+    this.closing = Promise.allSettled(queues.map(async (queue) => queue.close())).then((results) => {
+      const failures = results.filter((result) => result.status === "rejected");
+      if (failures.length) throw new AggregateError(failures.map((result) => result.reason), "Queue shutdown failed");
+    });
+    return this.closing;
   }
   async onModuleDestroy() {
     await this.close();
   }
   getQueue(name) {
+    if (this.closing) throw new Error("Queue provider is closing");
     const existing = this.queues.get(name);
     if (existing) return existing;
     const queue = this.queueFactory(name, {
       connection: this.options.connection,
       prefix: this.options.prefix
     });
+    if (this.options.onError) queue.on("error", (error) => this.options.onError(name, error));
     this.queues.set(name, queue);
     return queue;
   }
@@ -310,24 +337,30 @@ var BullMqWorkerHost = class {
   options;
   workers = /* @__PURE__ */ new Map();
   readyQueues = /* @__PURE__ */ new Set();
+  closing;
   workerFactory;
   constructor(options) {
     this.options = options;
     this.workerFactory = options.workerFactory ?? ((queueName, processor, workerOptions) => new Worker(queueName, processor, workerOptions));
   }
   register(definition) {
+    if (this.closing) throw new Error("Worker host is closing");
     if (this.workers.has(definition.queueName)) {
       throw new Error(`Worker already registered for queue: ${definition.queueName}`);
     }
-    if ((definition.concurrency ?? 1) < 1) throw new RangeError("concurrency must be at least 1");
-    if ((definition.lockDurationMs ?? 3e4) < 1) {
-      throw new RangeError("lockDurationMs must be at least 1");
+    for (const [name, value] of Object.entries({
+      concurrency: definition.concurrency ?? 1,
+      lockDurationMs: definition.lockDurationMs ?? 3e4,
+      stalledIntervalMs: definition.stalledIntervalMs ?? 3e4
+    })) {
+      if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`);
     }
     const worker = this.workerFactory(definition.queueName, async (job) => definition.handler({
       id: String(job.id),
       name: job.name,
       data: job.data,
       attempt: job.attemptsMade + 1,
+      maxAttempts: job.opts?.attempts ?? 1,
       updateProgress: /* @__PURE__ */ __name((progress) => job.updateProgress(progress), "updateProgress"),
       log: /* @__PURE__ */ __name((message) => job.log(message), "log")
     }), {
@@ -335,9 +368,14 @@ var BullMqWorkerHost = class {
       prefix: this.options.prefix,
       concurrency: definition.concurrency ?? 1,
       lockDuration: definition.lockDurationMs ?? 3e4,
-      autorun: definition.autorun ?? true
+      autorun: definition.autorun ?? true,
+      ...definition.stalledIntervalMs === void 0 ? {} : {
+        stalledInterval: definition.stalledIntervalMs
+      }
+    });
+    worker.on("ready", () => {
+      if (!this.closing) this.readyQueues.add(definition.queueName);
     });
-    worker.on("ready", () => this.readyQueues.add(definition.queueName));
     worker.on("error", (error) => {
       this.readyQueues.delete(definition.queueName);
       this.options.onError?.(definition.queueName, error);
@@ -413,13 +451,18 @@ var BullMqWorkerHost = class {
       ...this.workers.values()
     ].map((worker) => worker.resume()));
   }
-  async close(force = false) {
+  close(force = false) {
+    if (this.closing) return this.closing;
     const workers = [
       ...this.workers.values()
     ];
     this.workers.clear();
     this.readyQueues.clear();
-    await Promise.all(workers.map((worker) => worker.close(force)));
+    this.closing = Promise.allSettled(workers.map(async (worker) => worker.close(force))).then((results) => {
+      const failures = results.filter((result) => result.status === "rejected");
+      if (failures.length) throw new AggregateError(failures.map((result) => result.reason), "Worker shutdown failed");
+    });
+    return this.closing;
   }
   async onModuleDestroy() {
     await this.close();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zucker-framework/queue",
-  "version": "1.0.2",
+  "version": "1.0.5",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
   "types": "dist/index.d.ts",