@midwayjs/bullmq 0.0.1 → 0.0.2

package/dist/config/config.default.d.ts

@@ -1,18 +1,5 @@
-export declare const bullmq: {
-    prefix: string;
-    defaultQueueOptions: {
-        defaultJobOptions: {
-            removeOnComplete: number;
-            removeOnFail: number;
-        };
-    };
-    defaultWorkerOptions: {
-        concurrency: number;
-    };
-    clearRepeatJobWhenStart: boolean;
-    contextLoggerApplyLogger: string;
-    contextLoggerFormat: (info: any) => string;
-};
+import { BullMQConfig } from '../interface';
+export declare const bullmq: BullMQConfig;
 export declare const midwayLogger: {
     clients: {
         bullMQLogger: {

package/dist/config/config.default.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.midwayLogger = exports.bullmq = void 0;
 exports.bullmq = {
-    prefix: '{midway-bullmq}',
+    defaultPrefix: '{midway-bullmq}',
     defaultQueueOptions: {
         defaultJobOptions: {
             removeOnComplete: 3,

package/dist/configuration.js

@@ -19,6 +19,12 @@ let BullConfiguration = class BullConfiguration {
         this.decoratorService.registerPropertyHandler(constants_1.BULLMQ_QUEUE_KEY, (propertyName, meta) => {
             return this.framework.getQueue(meta.queueName);
         });
+        this.decoratorService.registerPropertyHandler(constants_1.BULLMQ_WORKER_KEY, (propertyName, meta) => {
+            return this.framework.getWorker(meta.queueName);
+        });
+        this.decoratorService.registerPropertyHandler(constants_1.BULLMQ_FLOW_PRODUCER_KEY, (propertyName, meta) => {
+            return this.framework.getFlowProducer(meta.producerName);
+        });
     }
     async onReady() {
         this.framework.loadConfig();

package/dist/constants.d.ts

@@ -1,3 +1,5 @@
 export declare const BULLMQ_QUEUE_KEY = "bullmq:queue";
 export declare const BULLMQ_PROCESSOR_KEY = "bullmq:processor";
+export declare const BULLMQ_WORKER_KEY = "bullmq:worker";
+export declare const BULLMQ_FLOW_PRODUCER_KEY = "bullmq:flow-producer";
 //# sourceMappingURL=constants.d.ts.map

package/dist/constants.js

@@ -1,7 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BULLMQ_PROCESSOR_KEY = exports.BULLMQ_QUEUE_KEY = void 0;
+exports.BULLMQ_FLOW_PRODUCER_KEY = exports.BULLMQ_WORKER_KEY = exports.BULLMQ_PROCESSOR_KEY = exports.BULLMQ_QUEUE_KEY = void 0;
 // task
 exports.BULLMQ_QUEUE_KEY = 'bullmq:queue';
 exports.BULLMQ_PROCESSOR_KEY = 'bullmq:processor';
+exports.BULLMQ_WORKER_KEY = 'bullmq:worker';
+exports.BULLMQ_FLOW_PRODUCER_KEY = 'bullmq:flow-producer';
 //# sourceMappingURL=constants.js.map

package/dist/decorator.d.ts

@@ -1,5 +1,6 @@
-import { IQueueOptions, IWorkerOptions } from './interface';
-import { JobsOptions } from 'bullmq';
-export declare function Processor(queueName: string, jobOptions?: JobsOptions, workerOptions?: IWorkerOptions, queueOptions?: IQueueOptions): ClassDecorator;
+import { QueueOptions, WorkerOptions, JobsOptions } from 'bullmq';
+export declare function Processor(queueName: string, jobOptions?: JobsOptions, workerOptions?: Partial<WorkerOptions>, queueOptions?: Partial<QueueOptions>): ClassDecorator;
 export declare function InjectQueue(queueName: string): PropertyDecorator;
+export declare function InjectWorker(queueName: string): PropertyDecorator;
+export declare function InjectFlowProducer(producerName: string): PropertyDecorator;
 //# sourceMappingURL=decorator.d.ts.map

package/dist/decorator.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.InjectQueue = exports.Processor = void 0;
+exports.InjectFlowProducer = exports.InjectWorker = exports.InjectQueue = exports.Processor = void 0;
 const core_1 = require("@midwayjs/core");
 const constants_1 = require("./constants");
 function Processor(queueName, jobOptions, workerOptions, queueOptions) {
@@ -9,8 +9,8 @@ function Processor(queueName, jobOptions, workerOptions, queueOptions) {
         (0, core_1.saveClassMetadata)(constants_1.BULLMQ_PROCESSOR_KEY, {
             queueName,
             jobOptions,
-            queueOptions,
             workerOptions,
+            queueOptions,
         }, target);
         (0, core_1.Provide)()(target);
         (0, core_1.Scope)(core_1.ScopeEnum.Request)(target);
@@ -23,4 +23,16 @@ function InjectQueue(queueName) {
     });
 }
 exports.InjectQueue = InjectQueue;
+function InjectWorker(queueName) {
+    return (0, core_1.createCustomPropertyDecorator)(constants_1.BULLMQ_WORKER_KEY, {
+        queueName,
+    });
+}
+exports.InjectWorker = InjectWorker;
+function InjectFlowProducer(producerName) {
+    return (0, core_1.createCustomPropertyDecorator)(constants_1.BULLMQ_FLOW_PRODUCER_KEY, {
+        producerName,
+    });
+}
+exports.InjectFlowProducer = InjectFlowProducer;
 //# sourceMappingURL=decorator.js.map

package/dist/framework.d.ts

@@ -1,21 +1,27 @@
 import { BaseFramework, IMidwayBootstrapOptions, ILogger } from '@midwayjs/core';
-import { Application, Context, IProcessor, IQueue, IQueueManager, IQueueOptions, IWorkerOptions } from './interface';
-import { Job, JobsOptions, Queue, Worker, QueueOptions } from 'bullmq';
-export declare class BullMQQueue extends Queue implements IQueue<Job> {
+import { Application, Context, IProcessor } from './interface';
+import { Job, JobsOptions, Queue, Worker, QueueOptions, WorkerOptions, QueueEvents, QueueEventsProducer, QueueBaseOptions, QueueEventsOptions, FlowProducer } from 'bullmq';
+export declare class BullMQQueue extends Queue {
+    protected queueName: string;
+    protected queueOptions: QueueOptions;
+    private queueEventsList;
+    private queueEventsProducerList;
     constructor(queueName: string, queueOptions: QueueOptions);
-    getJob(name: string): Promise<Job>;
     runJob(data: any, options?: JobsOptions): Promise<Job>;
     getQueueName(): string;
+    createQueueEvents(options?: QueueEventsOptions): QueueEvents;
+    getQueueEventsList(): QueueEvents[];
+    createQueueEventsProducer(options?: QueueBaseOptions): QueueEventsProducer;
+    getQueueEventsProducerList(): QueueEventsProducer[];
 }
-export declare class BullMQFramework extends BaseFramework<Application, Context, any> implements IQueueManager<BullMQQueue, Job> {
-    private connection;
-    private prefix?;
+export declare class BullMQFramework extends BaseFramework<Application, Context, any> {
+    private defaultConnection;
     private defaultQueueConfig;
     private defaultWorkerConfig;
     private clearRepeatJobWhenStart;
     private queueMap;
-    /** keep a map of workers for gracefully shutdown  */
     private workerMap;
+    private flowProducerMap;
     protected bullMQLogger: ILogger;
     applicationInitialize(options: IMidwayBootstrapOptions): Promise<void>;
     loadConfig(): void;
@@ -23,13 +29,42 @@ export declare class BullMQFramework extends BaseFramework<Application, Context,
     getFrameworkName(): string;
     run(): Promise<void>;
     protected beforeStop(): Promise<void>;
-    createQueue(name: string, queueOptions: IQueueOptions): BullMQQueue;
+    /**
+     * Create a queue with name and queueOptions
+     */
+    createQueue(name: string, queueOptions?: Partial<QueueOptions>): BullMQQueue;
+    /**
+     * Get a queue by name
+     */
     getQueue(name: string): BullMQQueue;
-    ensureQueue(name: string, queueOptions: IQueueOptions): BullMQQueue;
+    /**
+     * Ensure a queue by name and queueOptions
+     */
+    protected ensureQueue(name: string, queueOptions?: Partial<QueueOptions>): BullMQQueue;
     getQueueList(): BullMQQueue[];
-    getWorker(name: string): Worker<any, any, string>;
-    addProcessor(processor: new (...args: any[]) => IProcessor, queueName: string, workerOptions?: IWorkerOptions): Promise<void>;
-    runJob(queueName: string, jobData: any, options?: JobsOptions): Promise<Job | undefined>;
-    getJob(queueName: string, jobName: string): Promise<Job | undefined>;
+    /**
+     * Get the first worker by queueName
+     */
+    getWorker(queueName: string): Worker;
+    /**
+     * Get all workers by queueName
+     */
+    getWorkers(queueName: string): Worker[];
+    /**
+     * Create a worker
+     */
+    createWorker(queueName: string, processor: (job: Job, token?: string) => Promise<any>, workerOptions?: Partial<WorkerOptions>): Worker;
+    /**
+     * Add a processor class and init a worker
+     */
+    addProcessor(processor: new (...args: any[]) => IProcessor, queueName: string, workerOptions?: WorkerOptions): Promise<Worker<any, any, string>>;
+    /**
+     * Create a flow producer, if producerName is provided, it will be store.
+     */
+    createFlowProducer(options?: Partial<QueueBaseOptions>, producerName?: string): FlowProducer;
+    /**
+     * Get a flow producer by name
+     */
+    getFlowProducer(producerName: string): FlowProducer | undefined;
 }
 //# sourceMappingURL=framework.d.ts.map

package/dist/framework.js

@@ -16,11 +16,11 @@ const constants_1 = require("./constants");
 class BullMQQueue extends bullmq_1.Queue {
     constructor(queueName, queueOptions) {
         super(queueName, queueOptions);
+        this.queueName = queueName;
+        this.queueOptions = queueOptions;
+        this.queueEventsList = [];
+        this.queueEventsProducerList = [];
     }
-    getJob(name) {
-        throw new Error('Method not implemented.');
-    }
-    // bullmq 在 queue.add 新增第一个参数 jobName
     // runJob 与 @midwayjs/bull 保持一致，如果想要使用 jobName 则可以直接调用 queue.add
     async runJob(data, options) {
         const { repeat, ...OtherOptions } = options !== null && options !== void 0 ? options : {};
@@ -34,7 +34,31 @@ class BullMQQueue extends bullmq_1.Queue {
         return this.add('jobName', data || {}, options);
     }
     getQueueName() {
-        return this.name;
+        return this.queueName;
+    }
+    createQueueEvents(options) {
+        const evt = new bullmq_1.QueueEvents(this.name, {
+            connection: this.queueOptions.connection,
+            prefix: this.queueOptions.prefix,
+            ...options,
+        });
+        this.queueEventsList.push(evt);
+        return evt;
+    }
+    getQueueEventsList() {
+        return this.queueEventsList;
+    }
+    createQueueEventsProducer(options) {
+        const producer = new bullmq_1.QueueEventsProducer(this.name, {
+            connection: this.queueOptions.connection,
+            prefix: this.queueOptions.prefix,
+            ...options,
+        });
+        this.queueEventsProducerList.push(producer);
+        return producer;
+    }
+    getQueueEventsProducerList() {
+        return this.queueEventsProducerList;
     }
 }
 exports.BullMQQueue = BullMQQueue;
@@ -42,15 +66,19 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
     constructor() {
         super(...arguments);
         this.queueMap = new Map();
-        /** keep a map of workers for gracefully shutdown  */
         this.workerMap = new Map();
+        this.flowProducerMap = new Map();
     }
     async applicationInitialize(options) {
         this.app = {};
     }
     loadConfig() {
-        this.connection = this.configService.getConfiguration('bullmq.connection');
-        this.prefix = this.configService.getConfiguration('bullmq.prefix');
+        const defaultConnection = this.configService.getConfiguration('bullmq.defaultConnection');
+        const defaultPrefix = this.configService.getConfiguration('bullmq.defaultPrefix');
+        this.defaultConnection = {
+            connection: defaultConnection,
+            prefix: defaultPrefix,
+        };
         this.defaultQueueConfig = this.configService.getConfiguration('bullmq.defaultQueueOptions');
         this.defaultWorkerConfig = this.configService.getConfiguration('bullmq.defaultWorkerOptions');
         this.clearRepeatJobWhenStart = this.configService.getConfiguration('bullmq.clearRepeatJobWhenStart');
@@ -62,7 +90,7 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
         return 'bullmq';
     }
     async run() {
-        var _a, _b;
+        var _a, _b, _c;
         const processorModules = (0, core_1.listModule)(constants_1.BULLMQ_PROCESSOR_KEY);
         for (const mod of processorModules) {
             const options = (0, core_1.getClassMetadata)(constants_1.BULLMQ_PROCESSOR_KEY, mod);
@@ -72,8 +100,9 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
                 ...queueOptions,
                 defaultJobOptions: otherOptions,
             });
-            if (!currentQueue)
-                throw Error('ensureQueue failed');
+            if (!currentQueue) {
+                throw new core_1.MidwayCommonError(`[midway:bullmq] Queue ${options.queueName} not found`);
+            }
             // clear old repeat job when start
             if (this.clearRepeatJobWhenStart) {
                 const jobs = await currentQueue.getJobSchedulers();
@@ -88,7 +117,8 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
             }
             await this.addProcessor(mod, options.queueName, options.workerOptions);
             if (repeat) {
-                await this.runJob(options.queueName, {}, options.jobOptions);
+                // add repeatable job
+                await ((_c = this.getQueue(options.queueName)) === null || _c === void 0 ? void 0 : _c.runJob({}, options.jobOptions));
             }
         }
     }
@@ -98,24 +128,40 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
             await queue.close();
         }
         for (const worker of this.workerMap.values()) {
-            await worker.close();
+            for (const w of worker) {
+                await w.close();
+            }
+        }
+        for (const producer of this.flowProducerMap.values()) {
+            await producer.close();
         }
     }
-    createQueue(name, queueOptions) {
-        const queue = new BullMQQueue(name, (0, core_1.extend)(true, {}, this.defaultQueueConfig, queueOptions, {
-            connection: this.connection,
-            prefix: this.prefix,
-        }));
+    /**
+     * Create a queue with name and queueOptions
+     */
+    createQueue(name, queueOptions = {}) {
+        const mergedOptions = {
+            ...this.defaultQueueConfig,
+            ...this.defaultConnection,
+            ...queueOptions,
+        };
+        const queue = new BullMQQueue(name, mergedOptions);
         this.queueMap.set(name, queue);
         queue.on('error', err => {
             this.bullMQLogger.error(err);
         });
         return queue;
     }
+    /**
+     * Get a queue by name
+     */
     getQueue(name) {
         return this.queueMap.get(name);
     }
-    ensureQueue(name, queueOptions) {
+    /**
+     * Ensure a queue by name and queueOptions
+     */
+    ensureQueue(name, queueOptions = {}) {
         if (!this.queueMap.has(name)) {
             this.createQueue(name, queueOptions);
         }
@@ -124,17 +170,47 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
     getQueueList() {
         return Array.from(this.queueMap.values());
     }
-    getWorker(name) {
-        return this.workerMap.get(name);
+    /**
+     * Get the first worker by queueName
+     */
+    getWorker(queueName) {
+        var _a;
+        return (_a = this.workerMap.get(queueName)) === null || _a === void 0 ? void 0 : _a[0];
+    }
+    /**
+     * Get all workers by queueName
+     */
+    getWorkers(queueName) {
+        return this.workerMap.get(queueName);
+    }
+    /**
+     * Create a worker
+     */
+    createWorker(queueName, processor, workerOptions = {}) {
+        const merged = {
+            ...this.defaultConnection,
+            ...this.defaultWorkerConfig,
+            ...workerOptions,
+        };
+        const worker = new bullmq_1.Worker(queueName, processor, merged);
+        if (!this.workerMap.has(queueName)) {
+            this.workerMap.set(queueName, []);
+        }
+        this.workerMap.get(queueName).push(worker);
+        return worker;
     }
+    /**
+     * Add a processor class and init a worker
+     */
     async addProcessor(processor, queueName, workerOptions) {
         const queue = this.queueMap.get(queueName);
         if (!queue)
             throw Error(`queue not found ${queueName}`);
-        const worker = new bullmq_1.Worker(queueName, async (job) => {
+        return this.createWorker(queueName, async (job, token) => {
             const ctx = this.app.createAnonymousContext({
                 jobId: job.id,
                 job,
+                token,
                 from: processor,
             });
             try {
@@ -147,7 +223,7 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
                 }
                 const service = await ctx.requestContext.getAsync(processor);
                 const fn = await this.applyMiddleware(async (ctx) => {
-                    return await core_1.Utils.toAsyncFunction(service.execute.bind(service))(job.data, job);
+                    return await core_1.Utils.toAsyncFunction(service.execute.bind(service))(job.data, job, token);
                 });
                 const result = await Promise.resolve(await fn(ctx));
                 ctx.logger.info(`complete process job ${job.id} from ${processor.name}`);
@@ -157,23 +233,26 @@ let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
                 ctx.logger.error(err);
                 return Promise.reject(err);
             }
-        }, (0, core_1.extend)(true, {}, this.defaultWorkerConfig, workerOptions, {
-            connection: this.connection,
-            prefix: this.prefix,
-        }));
-        this.workerMap.set(queueName, worker);
+        }, workerOptions);
     }
-    async runJob(queueName, jobData, options) {
-        const queue = this.queueMap.get(queueName);
-        if (queue) {
-            return await queue.runJob(jobData, options);
+    /**
+     * Create a flow producer, if producerName is provided, it will be store.
+     */
+    createFlowProducer(options, producerName) {
+        const producer = new bullmq_1.FlowProducer({
+            ...this.defaultConnection,
+            ...options,
+        });
+        if (producerName) {
+            this.flowProducerMap.set(producerName, producer);
         }
+        return producer;
     }
-    async getJob(queueName, jobName) {
-        const queue = this.queueMap.get(queueName);
-        if (queue) {
-            return queue.getJob(jobName);
-        }
+    /**
+     * Get a flow producer by name
+     */
+    getFlowProducer(producerName) {
+        return this.flowProducerMap.get(producerName);
     }
 };
 __decorate([

package/dist/index.d.ts

@@ -2,4 +2,5 @@ export { BullConfiguration as Configuration } from './configuration';
 export { BullMQFramework as Framework, BullMQQueue } from './framework';
 export * from './decorator';
 export * from './interface';
+export * as BullMQ from 'bullmq';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BullMQQueue = exports.Framework = exports.Configuration = void 0;
+exports.BullMQ = exports.BullMQQueue = exports.Framework = exports.Configuration = void 0;
 var configuration_1 = require("./configuration");
 Object.defineProperty(exports, "Configuration", { enumerable: true, get: function () { return configuration_1.BullConfiguration; } });
 var framework_1 = require("./framework");
@@ -22,4 +22,5 @@ Object.defineProperty(exports, "Framework", { enumerable: true, get: function ()
 Object.defineProperty(exports, "BullMQQueue", { enumerable: true, get: function () { return framework_1.BullMQQueue; } });
 __exportStar(require("./decorator"), exports);
 __exportStar(require("./interface"), exports);
+exports.BullMQ = require("bullmq");
 //# sourceMappingURL=index.js.map

package/dist/interface.d.ts

@@ -1,19 +1,7 @@
 import { IMidwayApplication, IMidwayContext, NextFunction as BaseNextFunction } from '@midwayjs/core';
-import { WorkerOptions, QueueOptions, Job, Worker } from 'bullmq';
+import { WorkerOptions, QueueOptions, Job, ConnectionOptions } from 'bullmq';
 export interface IProcessor {
-    execute(data: any): any;
-}
-export interface IQueue<Job> {
-    runJob(data: Record<string, any>, options?: unknown): Promise<Job>;
-    getJob(name: string): Promise<Job>;
-    getQueueName(): string;
-}
-export interface IQueueManager<Queue extends IQueue<Job>, Job> {
-    runJob(queueName: string, jobData: any, options?: unknown): Promise<Job | undefined>;
-    getJob(queueName: string, jobName: string): Promise<Job | undefined>;
-    createQueue(queueName: string, queueOptions?: unknown): Queue;
-    getQueue(queueName: string): Queue | undefined;
-    getWorker(queueName: string): Worker | undefined;
+    execute(data: any, job: Job, token?: string): Promise<void>;
 }
 export interface Application extends IMidwayApplication<Context> {
 }
@@ -23,6 +11,13 @@ export interface Context extends IMidwayContext {
     job: Job;
     from: new (...args: any[]) => IProcessor;
 }
-export type IWorkerOptions = Omit<WorkerOptions, 'connection' | 'prefix'>;
-export type IQueueOptions = Omit<QueueOptions, 'connection' | 'prefix'>;
+export interface BullMQConfig {
+    defaultConnection?: ConnectionOptions;
+    defaultPrefix?: string;
+    defaultQueueOptions?: Partial<QueueOptions>;
+    defaultWorkerOptions?: Partial<WorkerOptions>;
+    clearRepeatJobWhenStart?: boolean;
+    contextLoggerApplyLogger?: string;
+    contextLoggerFormat?: (info: any) => string;
+}
 //# sourceMappingURL=interface.d.ts.map

package/index.d.ts

@@ -1,20 +1,8 @@
-import { ConnectionOptions } from 'bullmq';
-import { IQueueOptions, IWorkerOptions } from './dist/index';
+import { BullMQConfig } from './dist/index';
 export * from './dist/index';
-export { Job } from 'bullmq';
 
 declare module '@midwayjs/core/dist/interface' {
-  // bullmq 新引入了 worker 作为执行任务的实例，一个队列 queue 和 worker 中 connection， prefix 必须一致才能正常执行
-  // 所以 config 中 connection， prefix 单独配置
-  // eslint-disable-next-line
   interface MidwayConfig {
-    bullmq?: {
-      connection: ConnectionOptions;
-      prefix?: string;
-      defaultQueueOptions?: IQueueOptions;
-      defaultWorkerOptions?: IWorkerOptions;
-      clearRepeatJobWhenStart?: boolean;
-      contextLoggerFormat?: (info: any) => string;
-    };
+    bullmq?: BullMQConfig;
   }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midwayjs/bullmq",
-  "version": "0.0.1",
-  "description": "midway component for bullmq",
+  "version": "0.0.2",
+  "description": "midway component for bullMQ",
   "main": "dist/index.js",
   "typings": "index.d.ts",
   "scripts": {