@tsed/bullmq 7.47.1 → 7.48.0

package/lib/types/config/config.d.ts

@@ -1,10 +1,49 @@
-import { type ConnectionOptions, type DefaultJobOptions } from "bullmq";
+import type { ConnectionOptions, DefaultJobOptions, QueueOptions, WorkerOptions } from "bullmq";
 export type BullMQConfig = {
+    /**
+     * Specify queue name's to create
+     */
     queues: string[];
+    /**
+     * Default connection to use for queue's and worker's
+     */
     connection: ConnectionOptions;
+    /**
+     * @deprecated Use defaultQueueOptions instead. Will be removed in the next major release
+     */
     defaultJobOptions?: DefaultJobOptions;
+    /**
+     * Default queue options which are applied to every queue
+     *
+     * Can be extended/overridden by `queueOptions`
+     */
+    defaultQueueOptions?: QueueOptions;
+    /**
+     * Specify additional queue options by queue name
+     */
+    queueOptions?: Record<string, QueueOptions>;
+    /**
+     * Disable the creation of any worker.
+     *
+     * All other worker configuration will be ignored
+     */
     disableWorker?: boolean;
+    /**
+     * Specify for which queues to start a worker for.
+     *
+     * Defaultly for every queue added in the `queues` parameter
+     */
     workerQueues?: string[];
+    /**
+     * Default worker options which are applied to every worker
+     *
+     * Can be extended/overridden by `workerOptions`
+     */
+    defaultWorkerOptions?: WorkerOptions;
+    /**
+     * Specify additional worker options by queue name
+     */
+    workerOptions?: Record<string, WorkerOptions>;
 };
 declare global {
     namespace TsED {

package/lib/types/contracts/JobMethods.d.ts

@@ -1,3 +1,4 @@
-export interface JobMethods {
-    handle(payload: unknown): unknown;
+import { Job } from "bullmq";
+export interface JobMethods<DataType = unknown, ReturnType = unknown> {
+    handle(payload: DataType, job: Job<DataType, ReturnType>): ReturnType | Promise<ReturnType>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tsed/bullmq",
-  "version": "7.47.1",
+  "version": "7.48.0",
   "private": false,
   "description": "BullMQ integration for Ts.ED",
   "keywords": [
@@ -29,11 +29,11 @@
     "tslib": "2.5.0"
   },
   "devDependencies": {
-    "@tsed/common": "7.47.1",
-    "@tsed/core": "7.47.1",
-    "@tsed/di": "7.47.1",
+    "@tsed/common": "7.48.0",
+    "@tsed/core": "7.48.0",
+    "@tsed/di": "7.48.0",
     "@tsed/eslint": "7.44.1",
-    "@tsed/schema": "7.47.1",
+    "@tsed/schema": "7.48.0",
     "@tsed/typescript": "7.44.1",
     "bullmq": "^4.12.3",
     "eslint": "^8.12.0",

package/src/BullMQModule.spec.ts

@@ -1,7 +1,7 @@
-import {InjectorService, PlatformTest} from "@tsed/common";
+import {PlatformTest} from "@tsed/common";
 import {catchAsyncError} from "@tsed/core";
 import {Queue, Worker} from "bullmq";
-import {anyString, anything, instance, mock, verify, when} from "ts-mockito";
+import {anything, instance, mock, verify, when} from "ts-mockito";
 
 import "./BullMQModule";
 import {BullMQModule} from "./BullMQModule";
@@ -10,33 +10,26 @@ import {JobMethods} from "./contracts";
 import {JobController} from "./decorators";
 import {JobDispatcher} from "./dispatchers";
 
+const queueConstructorSpy = jest.fn();
+const workerConstructorSpy = jest.fn();
+
 jest.mock("bullmq", () => {
   return {
     Queue: class {
-      public name: string;
-
-      constructor(name: string) {
-        this.name = name;
+      constructor(...args: any[]) {
+        queueConstructorSpy(...args);
       }
       close() {}
     },
     Worker: class {
-      public name: string;
-
-      constructor(name: string) {
-        this.name = name;
+      constructor(...args: any[]) {
+        workerConstructorSpy(...args);
       }
       close() {}
     }
   };
 });
 
-const bullmq = {
-  queues: ["default", "foo", "bar"],
-  connection: {},
-  workerQueues: ["default", "foo"]
-} as BullMQConfig;
-
 @JobController("cron", "default", {
   repeat: {
     pattern: "* * * * *"
@@ -51,154 +44,279 @@ class RegularJob {
   handle() {}
 }
 
-describe("module", () => {
+describe("BullMQModule", () => {
   let dispatcher: JobDispatcher;
-  beforeEach(async () => {
+
+  beforeEach(() => {
     dispatcher = mock(JobDispatcher);
     when(dispatcher.dispatch(CustomCronJob)).thenResolve();
-
-    await PlatformTest.create({
-      bullmq,
-      imports: [
-        {
-          token: JobDispatcher,
-          use: instance(dispatcher)
-        }
-      ]
-    });
   });
-  beforeEach(() => {});
+
   afterEach(PlatformTest.reset);
 
-  describe("cronjobs", () => {
-    it("should dispatch cron jobs", () => {
-      verify(dispatcher.dispatch(CustomCronJob)).once();
+  describe("configuration", () => {
+    beforeEach(() => {
+      queueConstructorSpy.mockClear();
+      workerConstructorSpy.mockClear();
     });
-  });
 
-  describe("queues", () => {
-    it("should get default", () => {
-      const instance = PlatformTest.get<Queue>("bullmq.queue.default");
+    describe("merges config correctly", () => {
+      beforeEach(async () => {
+        await PlatformTest.create({
+          bullmq: {
+            queues: ["default", "special"],
+            connection: {
+              connectionName: "defaultConnectionName"
+            },
+            defaultQueueOptions: {
+              defaultJobOptions: {
+                delay: 100
+              },
+              blockingConnection: true
+            },
+            queueOptions: {
+              special: {
+                connection: {
+                  connectionName: "specialConnectionName"
+                },
+                defaultJobOptions: {
+                  attempts: 9
+                }
+              }
+            },
+            defaultWorkerOptions: {
+              connection: {
+                connectTimeout: 123
+              },
+              concurrency: 50
+            },
+            workerOptions: {
+              special: {
+                concurrency: 1,
+                lockDuration: 2
+              }
+            }
+          },
+          imports: [
+            {
+              token: JobDispatcher,
+              use: instance(dispatcher)
+            }
+          ]
+        });
+      });
 
-      expect(instance).toBeInstanceOf(Queue);
-      expect(instance.name).toBe("default");
+      it("queue", () => {
+        expect(queueConstructorSpy).toHaveBeenCalledTimes(2);
+
+        expect(queueConstructorSpy).toHaveBeenNthCalledWith(1, "default", {
+          connection: {
+            connectionName: "defaultConnectionName"
+          },
+          defaultJobOptions: {
+            delay: 100
+          },
+          blockingConnection: true
+        });
+
+        expect(queueConstructorSpy).toHaveBeenNthCalledWith(2, "special", {
+          connection: {
+            connectionName: "specialConnectionName"
+          },
+          defaultJobOptions: {
+            attempts: 9,
+            delay: 100
+          },
+          blockingConnection: true
+        });
+      });
+
+      it("worker", () => {
+        expect(workerConstructorSpy).toHaveBeenCalledTimes(2);
+
+        expect(workerConstructorSpy).toHaveBeenNthCalledWith(1, "default", expect.any(Function), {
+          connection: {
+            connectTimeout: 123
+          },
+          concurrency: 50
+        });
+
+        expect(workerConstructorSpy).toHaveBeenNthCalledWith(2, "special", expect.any(Function), {
+          connection: {
+            connectTimeout: 123
+          },
+          concurrency: 1,
+          lockDuration: 2
+        });
+      });
     });
 
-    it.each(bullmq.queues)("should register queue(%s)", (queue) => {
-      const instance = PlatformTest.get<Queue>(`bullmq.queue.${queue}`);
+    describe("disableWorker", () => {
+      const config = {
+        queues: ["default", "foo", "bar"],
+        connection: {},
+        disableWorker: true
+      } as BullMQConfig;
+
+      beforeEach(async () => {
+        await PlatformTest.create({
+          bullmq: config,
+          imports: [
+            {
+              token: JobDispatcher,
+              use: instance(dispatcher)
+            }
+          ]
+        });
+      });
 
-      expect(instance).toBeInstanceOf(Queue);
-      expect(instance.name).toBe(queue);
+      it("should not create any workers", () => {
+        expect(workerConstructorSpy).toHaveBeenCalledTimes(0);
+      });
     });
 
-    it("should not allow direct injection of the queue", () => {
-      expect(PlatformTest.get(Queue)).not.toBeInstanceOf(Queue);
+    describe("without", () => {
+      it("skips initialization", async () => {
+        await PlatformTest.create({
+          imports: [
+            {
+              token: JobDispatcher,
+              use: instance(dispatcher)
+            }
+          ]
+        });
+
+        expect(queueConstructorSpy).not.toHaveBeenCalled();
+        verify(dispatcher.dispatch(anything())).never();
+      });
     });
   });
 
-  describe("workers", () => {
-    it("should get default", () => {
-      const instance = PlatformTest.get<Worker>("bullmq.worker.default");
+  describe("functionality", () => {
+    const config = {
+      queues: ["default", "foo", "bar"],
+      connection: {},
+      workerQueues: ["default", "foo"]
+    } as BullMQConfig;
+
+    beforeEach(async () => {
+      await PlatformTest.create({
+        bullmq: config,
+        imports: [
+          {
+            token: JobDispatcher,
+            use: instance(dispatcher)
+          }
+        ]
+      });
+    });
 
-      expect(instance).toBeInstanceOf(Worker);
-      expect(instance.name).toBe("default");
+    describe("cronjobs", () => {
+      it("should dispatch cron jobs automatically", () => {
+        verify(dispatcher.dispatch(CustomCronJob)).once();
+      });
     });
 
-    it.each(bullmq.workerQueues)("should register worker(%s)", (queue) => {
-      const instance = PlatformTest.get<Worker>(`bullmq.worker.${queue}`);
+    describe("queues", () => {
+      it("should get default", () => {
+        const instance = PlatformTest.get<Queue>("bullmq.queue.default");
 
-      expect(instance).toBeInstanceOf(Worker);
-      expect(instance.name).toBe(queue);
-    });
+        expect(instance).toBeInstanceOf(Queue);
+      });
 
-    it("should not register unspecified worker queue", () => {
-      expect(PlatformTest.get("bullmq.worker.bar")).toBeUndefined();
-    });
+      it.each(config.queues)("should register queue(%s)", (queue) => {
+        const instance = PlatformTest.get<Queue>(`bullmq.queue.${queue}`);
+
+        expect(instance).toBeInstanceOf(Queue);
+      });
 
-    it("should not allow direct injection of the worker", () => {
-      expect(PlatformTest.get(Worker)).not.toBeInstanceOf(Worker);
+      it("should not allow direct injection of the queue", () => {
+        expect(PlatformTest.get(Queue)).not.toBeInstanceOf(Queue);
+      });
     });
 
-    it("should run worker and execute processor", async () => {
-      const bullMQModule = PlatformTest.get<BullMQModule>(BullMQModule);
-      const worker = PlatformTest.get<JobMethods>("bullmq.job.default.regular");
-      const job = {
-        name: "regular",
-        queueName: "default",
-        data: {test: "test"}
-      };
+    describe("workers", () => {
+      it("should get default", () => {
+        const instance = PlatformTest.get<Worker>("bullmq.worker.default");
 
-      jest.spyOn(worker, "handle").mockResolvedValueOnce(undefined as never);
+        expect(instance).toBeInstanceOf(Worker);
+      });
 
-      await (bullMQModule as any).onProcess(job);
+      it.each(config.workerQueues)("should register worker(%s)", (queue) => {
+        const instance = PlatformTest.get<Worker>(`bullmq.worker.${queue}`);
 
-      expect(worker.handle).toHaveBeenCalledWith({test: "test"}, job);
-    });
+        expect(instance).toBeInstanceOf(Worker);
+      });
 
-    it("should log warning when the worker doesn't exists", async () => {
-      const bullMQModule = PlatformTest.get<BullMQModule>(BullMQModule);
+      it("should not register unspecified worker queue", () => {
+        expect(PlatformTest.get("bullmq.worker.bar")).toBeUndefined();
+      });
 
-      jest.spyOn(PlatformTest.injector.logger, "warn");
+      it("should not allow direct injection of the worker", () => {
+        expect(PlatformTest.get(Worker)).not.toBeInstanceOf(Worker);
+      });
+
+      it("should run worker and execute processor", async () => {
+        const bullMQModule = PlatformTest.get<BullMQModule>(BullMQModule);
+        const worker = PlatformTest.get<JobMethods>("bullmq.job.default.regular");
+        const job = {
+          name: "regular",
+          queueName: "default",
+          data: {test: "test"}
+        };
 
-      const job = {
-        name: "regular",
-        queueName: "toto",
-        data: {test: "test"}
-      };
+        jest.spyOn(worker, "handle").mockResolvedValueOnce(undefined as never);
 
-      await (bullMQModule as any).onProcess(job);
+        await (bullMQModule as any).onProcess(job);
 
-      expect(PlatformTest.injector.logger.warn).toHaveBeenCalledWith({
-        event: "BULLMQ_JOB_NOT_FOUND",
-        message: "Job regular toto not found"
+        expect(worker.handle).toHaveBeenCalledWith({test: "test"}, job);
       });
-    });
 
-    it("should run worker, execute processor and handle error", async () => {
-      const bullMQModule = PlatformTest.get<BullMQModule>(BullMQModule);
-      const worker = PlatformTest.get<JobMethods>("bullmq.job.default.regular");
-      const job = {
-        name: "regular",
-        queueName: "default",
-        data: {test: "test"}
-      };
+      it("should log warning when the worker doesn't exists", async () => {
+        const bullMQModule = PlatformTest.get<BullMQModule>(BullMQModule);
 
-      jest.spyOn(PlatformTest.injector.logger, "error");
+        jest.spyOn(PlatformTest.injector.logger, "warn");
 
-      jest.spyOn(worker, "handle").mockRejectedValue(new Error("error") as never);
+        const job = {
+          name: "regular",
+          queueName: "toto",
+          data: {test: "test"}
+        };
 
-      const error = await catchAsyncError(() => (bullMQModule as any).onProcess(job));
+        await (bullMQModule as any).onProcess(job);
+
+        expect(PlatformTest.injector.logger.warn).toHaveBeenCalledWith({
+          event: "BULLMQ_JOB_NOT_FOUND",
+          message: "Job regular toto not found"
+        });
+      });
 
-      expect(worker.handle).toHaveBeenCalledWith({test: "test"}, job);
-      expect(PlatformTest.injector.logger.error).toHaveBeenCalledWith({
-        duration: expect.any(Number),
-        event: "BULLMQ_JOB_ERROR",
-        message: "error",
-        reqId: expect.any(String),
-        stack: expect.any(String),
-        time: expect.any(Object)
+      it("should run worker, execute processor and handle error", async () => {
+        const bullMQModule = PlatformTest.get<BullMQModule>(BullMQModule);
+        const worker = PlatformTest.get<JobMethods>("bullmq.job.default.regular");
+        const job = {
+          name: "regular",
+          queueName: "default",
+          data: {test: "test"}
+        };
+
+        jest.spyOn(PlatformTest.injector.logger, "error");
+
+        jest.spyOn(worker, "handle").mockRejectedValue(new Error("error") as never);
+
+        const error = await catchAsyncError(() => (bullMQModule as any).onProcess(job));
+
+        expect(worker.handle).toHaveBeenCalledWith({test: "test"}, job);
+        expect(PlatformTest.injector.logger.error).toHaveBeenCalledWith({
+          duration: expect.any(Number),
+          event: "BULLMQ_JOB_ERROR",
+          message: "error",
+          reqId: expect.any(String),
+          stack: expect.any(String),
+          time: expect.any(Object)
+        });
+        expect(error?.message).toEqual("error");
       });
-      expect(error?.message).toEqual("error");
     });
   });
 });
-
-it('skips initialization when no config is provided', async () => {
-  const injector = mock(InjectorService)
-  const dispatcher = mock(JobDispatcher);
-  await PlatformTest.create({
-    imports: [
-      {
-        token: InjectorService,
-        use: instance(injector),
-      },
-      {
-        token: JobDispatcher,
-        use: instance(dispatcher)
-      }
-    ]
-  })
-
-  verify(injector.add(anyString(), anything())).never();
-  verify(dispatcher.dispatch(anything())).never()
-});

package/src/BullMQModule.ts

@@ -1,7 +1,8 @@
 import {BeforeInit, DIContext, runInContext} from "@tsed/common";
 import {Constant, InjectorService, Module} from "@tsed/di";
+import {deepMerge} from "@tsed/core";
 import {getComputedType} from "@tsed/schema";
-import {Job, Queue, Worker} from "bullmq";
+import {Job, Queue, type QueueOptions, Worker, type WorkerOptions} from "bullmq";
 import {v4} from "uuid";
 import {BullMQConfig} from "./config/config";
 import {JobMethods} from "./contracts";
@@ -19,7 +20,9 @@ export class BullMQModule implements BeforeInit {
       return;
     }
 
-    this.buildWorkers();
+    if (!this.bullmq.disableWorker) {
+      this.buildWorkers();
+    }
     this.buildQueues();
 
     this.injector.getMany<JobMethods>("bullmq:cron").map((job) => this.dispatcher.dispatch(getComputedType(job)));
@@ -27,13 +30,19 @@ export class BullMQModule implements BeforeInit {
 
   private buildQueues() {
     this.bullmq.queues.forEach((queue) => {
+      const ops = deepMerge<QueueOptions, QueueOptions>(
+        {
+          connection: this.bullmq.connection,
+          defaultJobOptions: this.bullmq.defaultJobOptions,
+          ...this.bullmq.defaultQueueOptions
+        },
+        this.bullmq.queueOptions?.[queue]
+      )!;
+
       this.injector
         .add(`bullmq.queue.${queue}`, {
           type: "bullmq:queue",
-          useValue: new Queue(queue, {
-            connection: this.bullmq.connection,
-            defaultJobOptions: this.bullmq.defaultJobOptions
-          }),
+          useValue: new Queue(queue, ops),
           hooks: {
             $onDestroy: (queue) => queue.close()
           }
@@ -44,12 +53,18 @@ export class BullMQModule implements BeforeInit {
 
   private buildWorkers() {
     (this.bullmq.workerQueues ?? this.bullmq.queues).forEach((queue) => {
+      const ops = deepMerge<WorkerOptions, WorkerOptions>(
+        {
+          connection: this.bullmq.connection,
+          ...this.bullmq.defaultWorkerOptions
+        },
+        this.bullmq.workerOptions?.[queue]
+      )!;
+
       this.injector
         .add(`bullmq.worker.${queue}`, {
           type: "bullmq:worker",
-          useValue: new Worker(queue, this.onProcess.bind(this), {
-            connection: this.bullmq.connection
-          }),
+          useValue: new Worker(queue, this.onProcess.bind(this), ops),
           hooks: {
             $onDestroy: (worker) => worker.close()
           }
@@ -59,7 +74,7 @@ export class BullMQModule implements BeforeInit {
   }
 
   private getJob(name: string, queueName: string) {
-    return this.injector.get(`bullmq.job.${queueName}.${name}`);
+    return this.injector.get<JobMethods>(`bullmq.job.${queueName}.${name}`);
   }
 
   private async onProcess(job: Job) {

package/src/config/config.ts

@@ -1,13 +1,58 @@
-import {type ConnectionOptions, type DefaultJobOptions} from "bullmq";
+import type {ConnectionOptions, DefaultJobOptions, QueueOptions, WorkerOptions} from "bullmq";
 
 export type BullMQConfig = {
+  /**
+   * Specify queue name's to create
+   */
   queues: string[];
+
+  /**
+   * Default connection to use for queue's and worker's
+   */
   connection: ConnectionOptions;
+
+  /**
+   * @deprecated Use defaultQueueOptions instead. Will be removed in the next major release
+   */
   defaultJobOptions?: DefaultJobOptions;
+
+  /**
+   * Default queue options which are applied to every queue
+   *
+   * Can be extended/overridden by `queueOptions`
+   */
+  defaultQueueOptions?: QueueOptions;
+
+  /**
+   * Specify additional queue options by queue name
+   */
+  queueOptions?: Record<string, QueueOptions>;
+
+  /**
+   * Disable the creation of any worker.
+   *
+   * All other worker configuration will be ignored
+   */
   disableWorker?: boolean;
-  // optionally specify for which queues to start a worker for
-  // in case not all queues should be worked on
+
+  /**
+   * Specify for which queues to start a worker for.
+   *
+   * Defaultly for every queue added in the `queues` parameter
+   */
   workerQueues?: string[];
+
+  /**
+   * Default worker options which are applied to every worker
+   *
+   * Can be extended/overridden by `workerOptions`
+   */
+  defaultWorkerOptions?: WorkerOptions;
+
+  /**
+   * Specify additional worker options by queue name
+   */
+  workerOptions?: Record<string, WorkerOptions>;
 };
 
 declare global {

package/src/contracts/JobMethods.ts

@@ -1,3 +1,5 @@
-export interface JobMethods {
-  handle(payload: unknown): unknown;
+import {Job} from "bullmq";
+
+export interface JobMethods<DataType = unknown, ReturnType = unknown> {
+  handle(payload: DataType, job: Job<DataType, ReturnType>): ReturnType | Promise<ReturnType>;
 }