@midwayjs/bullmq 0.0.1

package/README.md

@@ -0,0 +1,9 @@
+# @midwayjs/bullmq
+
+this is a sub package for midway.
+
+Document: [https://midwayjs.org](https://midwayjs.org)
+
+## License
+
+[MIT]((https://github.com/midwayjs/midway/blob/master/LICENSE))

package/dist/config/config.default.d.ts

@@ -0,0 +1,23 @@
+export declare const bullmq: {
+    prefix: string;
+    defaultQueueOptions: {
+        defaultJobOptions: {
+            removeOnComplete: number;
+            removeOnFail: number;
+        };
+    };
+    defaultWorkerOptions: {
+        concurrency: number;
+    };
+    clearRepeatJobWhenStart: boolean;
+    contextLoggerApplyLogger: string;
+    contextLoggerFormat: (info: any) => string;
+};
+export declare const midwayLogger: {
+    clients: {
+        bullMQLogger: {
+            fileLogName: string;
+        };
+    };
+};
+//# sourceMappingURL=config.default.d.ts.map

package/dist/config/config.default.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.midwayLogger = exports.bullmq = void 0;
+exports.bullmq = {
+    prefix: '{midway-bullmq}',
+    defaultQueueOptions: {
+        defaultJobOptions: {
+            removeOnComplete: 3,
+            removeOnFail: 10,
+        },
+    },
+    defaultWorkerOptions: {
+        concurrency: 1,
+    },
+    clearRepeatJobWhenStart: true,
+    contextLoggerApplyLogger: 'bullMQLogger',
+    contextLoggerFormat: info => {
+        const { jobId, from } = info.ctx;
+        return `${info.timestamp} ${info.LEVEL} ${info.pid} [${jobId} ${from.name}] ${info.message}`;
+    },
+};
+exports.midwayLogger = {
+    clients: {
+        bullMQLogger: {
+            fileLogName: 'midway-bullmq.log',
+        },
+    },
+};
+//# sourceMappingURL=config.default.js.map

package/dist/configuration.d.ts

@@ -0,0 +1,9 @@
+import { MidwayDecoratorService } from '@midwayjs/core';
+import { BullMQFramework } from './framework';
+export declare class BullConfiguration {
+    framework: BullMQFramework;
+    decoratorService: MidwayDecoratorService;
+    init(): Promise<void>;
+    onReady(): Promise<void>;
+}
+//# sourceMappingURL=configuration.d.ts.map

package/dist/configuration.js

@@ -0,0 +1,52 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BullConfiguration = void 0;
+const core_1 = require("@midwayjs/core");
+const DefaultConfig = require("./config/config.default");
+const framework_1 = require("./framework");
+const constants_1 = require("./constants");
+let BullConfiguration = class BullConfiguration {
+    async init() {
+        this.decoratorService.registerPropertyHandler(constants_1.BULLMQ_QUEUE_KEY, (propertyName, meta) => {
+            return this.framework.getQueue(meta.queueName);
+        });
+    }
+    async onReady() {
+        this.framework.loadConfig();
+    }
+};
+__decorate([
+    (0, core_1.Inject)(),
+    __metadata("design:type", framework_1.BullMQFramework)
+], BullConfiguration.prototype, "framework", void 0);
+__decorate([
+    (0, core_1.Inject)(),
+    __metadata("design:type", core_1.MidwayDecoratorService)
+], BullConfiguration.prototype, "decoratorService", void 0);
+__decorate([
+    (0, core_1.Init)(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], BullConfiguration.prototype, "init", null);
+BullConfiguration = __decorate([
+    (0, core_1.Configuration)({
+        namespace: 'bullmq',
+        importConfigs: [
+            {
+                default: DefaultConfig,
+            },
+        ],
+    })
+], BullConfiguration);
+exports.BullConfiguration = BullConfiguration;
+//# sourceMappingURL=configuration.js.map

package/dist/constants.d.ts

@@ -0,0 +1,3 @@
+export declare const BULLMQ_QUEUE_KEY = "bullmq:queue";
+export declare const BULLMQ_PROCESSOR_KEY = "bullmq:processor";
+//# sourceMappingURL=constants.d.ts.map

package/dist/constants.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BULLMQ_PROCESSOR_KEY = exports.BULLMQ_QUEUE_KEY = void 0;
+// task
+exports.BULLMQ_QUEUE_KEY = 'bullmq:queue';
+exports.BULLMQ_PROCESSOR_KEY = 'bullmq:processor';
+//# sourceMappingURL=constants.js.map

package/dist/decorator.d.ts

@@ -0,0 +1,5 @@
+import { IQueueOptions, IWorkerOptions } from './interface';
+import { JobsOptions } from 'bullmq';
+export declare function Processor(queueName: string, jobOptions?: JobsOptions, workerOptions?: IWorkerOptions, queueOptions?: IQueueOptions): ClassDecorator;
+export declare function InjectQueue(queueName: string): PropertyDecorator;
+//# sourceMappingURL=decorator.d.ts.map

package/dist/decorator.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InjectQueue = exports.Processor = void 0;
+const core_1 = require("@midwayjs/core");
+const constants_1 = require("./constants");
+function Processor(queueName, jobOptions, workerOptions, queueOptions) {
+    return function (target) {
+        (0, core_1.saveModule)(constants_1.BULLMQ_PROCESSOR_KEY, target);
+        (0, core_1.saveClassMetadata)(constants_1.BULLMQ_PROCESSOR_KEY, {
+            queueName,
+            jobOptions,
+            queueOptions,
+            workerOptions,
+        }, target);
+        (0, core_1.Provide)()(target);
+        (0, core_1.Scope)(core_1.ScopeEnum.Request)(target);
+    };
+}
+exports.Processor = Processor;
+function InjectQueue(queueName) {
+    return (0, core_1.createCustomPropertyDecorator)(constants_1.BULLMQ_QUEUE_KEY, {
+        queueName,
+    });
+}
+exports.InjectQueue = InjectQueue;
+//# sourceMappingURL=decorator.js.map

package/dist/framework.d.ts

@@ -0,0 +1,35 @@
+import { BaseFramework, IMidwayBootstrapOptions, ILogger } from '@midwayjs/core';
+import { Application, Context, IProcessor, IQueue, IQueueManager, IQueueOptions, IWorkerOptions } from './interface';
+import { Job, JobsOptions, Queue, Worker, QueueOptions } from 'bullmq';
+export declare class BullMQQueue extends Queue implements IQueue<Job> {
+    constructor(queueName: string, queueOptions: QueueOptions);
+    getJob(name: string): Promise<Job>;
+    runJob(data: any, options?: JobsOptions): Promise<Job>;
+    getQueueName(): string;
+}
+export declare class BullMQFramework extends BaseFramework<Application, Context, any> implements IQueueManager<BullMQQueue, Job> {
+    private connection;
+    private prefix?;
+    private defaultQueueConfig;
+    private defaultWorkerConfig;
+    private clearRepeatJobWhenStart;
+    private queueMap;
+    /** keep a map of workers for gracefully shutdown  */
+    private workerMap;
+    protected bullMQLogger: ILogger;
+    applicationInitialize(options: IMidwayBootstrapOptions): Promise<void>;
+    loadConfig(): void;
+    configure(): any;
+    getFrameworkName(): string;
+    run(): Promise<void>;
+    protected beforeStop(): Promise<void>;
+    createQueue(name: string, queueOptions: IQueueOptions): BullMQQueue;
+    getQueue(name: string): BullMQQueue;
+    ensureQueue(name: string, queueOptions: IQueueOptions): BullMQQueue;
+    getQueueList(): BullMQQueue[];
+    getWorker(name: string): Worker<any, any, string>;
+    addProcessor(processor: new (...args: any[]) => IProcessor, queueName: string, workerOptions?: IWorkerOptions): Promise<void>;
+    runJob(queueName: string, jobData: any, options?: JobsOptions): Promise<Job | undefined>;
+    getJob(queueName: string, jobName: string): Promise<Job | undefined>;
+}
+//# sourceMappingURL=framework.d.ts.map

package/dist/framework.js

@@ -0,0 +1,187 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BullMQFramework = exports.BullMQQueue = void 0;
+const core_1 = require("@midwayjs/core");
+const bullmq_1 = require("bullmq");
+const constants_1 = require("./constants");
+class BullMQQueue extends bullmq_1.Queue {
+    constructor(queueName, queueOptions) {
+        super(queueName, queueOptions);
+    }
+    getJob(name) {
+        throw new Error('Method not implemented.');
+    }
+    // bullmq 在 queue.add 新增第一个参数 jobName
+    // runJob 与 @midwayjs/bull 保持一致，如果想要使用 jobName 则可以直接调用 queue.add
+    async runJob(data, options) {
+        const { repeat, ...OtherOptions } = options !== null && options !== void 0 ? options : {};
+        if (repeat) {
+            return this.upsertJobScheduler(this.name, repeat, {
+                name: 'jobName',
+                data,
+                opts: OtherOptions, // additional job options
+            });
+        }
+        return this.add('jobName', data || {}, options);
+    }
+    getQueueName() {
+        return this.name;
+    }
+}
+exports.BullMQQueue = BullMQQueue;
+let BullMQFramework = class BullMQFramework extends core_1.BaseFramework {
+    constructor() {
+        super(...arguments);
+        this.queueMap = new Map();
+        /** keep a map of workers for gracefully shutdown  */
+        this.workerMap = new Map();
+    }
+    async applicationInitialize(options) {
+        this.app = {};
+    }
+    loadConfig() {
+        this.connection = this.configService.getConfiguration('bullmq.connection');
+        this.prefix = this.configService.getConfiguration('bullmq.prefix');
+        this.defaultQueueConfig = this.configService.getConfiguration('bullmq.defaultQueueOptions');
+        this.defaultWorkerConfig = this.configService.getConfiguration('bullmq.defaultWorkerOptions');
+        this.clearRepeatJobWhenStart = this.configService.getConfiguration('bullmq.clearRepeatJobWhenStart');
+    }
+    configure() {
+        return this.configService.getConfiguration('bullmq');
+    }
+    getFrameworkName() {
+        return 'bullmq';
+    }
+    async run() {
+        var _a, _b;
+        const processorModules = (0, core_1.listModule)(constants_1.BULLMQ_PROCESSOR_KEY);
+        for (const mod of processorModules) {
+            const options = (0, core_1.getClassMetadata)(constants_1.BULLMQ_PROCESSOR_KEY, mod);
+            const { repeat, ...otherOptions } = (_a = options.jobOptions) !== null && _a !== void 0 ? _a : {};
+            const queueOptions = (_b = options.queueOptions) !== null && _b !== void 0 ? _b : {};
+            const currentQueue = this.ensureQueue(options.queueName, {
+                ...queueOptions,
+                defaultJobOptions: otherOptions,
+            });
+            if (!currentQueue)
+                throw Error('ensureQueue failed');
+            // clear old repeat job when start
+            if (this.clearRepeatJobWhenStart) {
+                const jobs = await currentQueue.getJobSchedulers();
+                for (const job of jobs) {
+                    await currentQueue.removeJobScheduler(job.key);
+                }
+                // Repeatable in jobOptions is depecrate
+                const repeatableJobs = await currentQueue.getRepeatableJobs();
+                for (const job of repeatableJobs) {
+                    await currentQueue.removeRepeatableByKey(job.key);
+                }
+            }
+            await this.addProcessor(mod, options.queueName, options.workerOptions);
+            if (repeat) {
+                await this.runJob(options.queueName, {}, options.jobOptions);
+            }
+        }
+    }
+    async beforeStop() {
+        // loop queueMap and stop all queue
+        for (const queue of this.queueMap.values()) {
+            await queue.close();
+        }
+        for (const worker of this.workerMap.values()) {
+            await worker.close();
+        }
+    }
+    createQueue(name, queueOptions) {
+        const queue = new BullMQQueue(name, (0, core_1.extend)(true, {}, this.defaultQueueConfig, queueOptions, {
+            connection: this.connection,
+            prefix: this.prefix,
+        }));
+        this.queueMap.set(name, queue);
+        queue.on('error', err => {
+            this.bullMQLogger.error(err);
+        });
+        return queue;
+    }
+    getQueue(name) {
+        return this.queueMap.get(name);
+    }
+    ensureQueue(name, queueOptions) {
+        if (!this.queueMap.has(name)) {
+            this.createQueue(name, queueOptions);
+        }
+        return this.queueMap.get(name);
+    }
+    getQueueList() {
+        return Array.from(this.queueMap.values());
+    }
+    getWorker(name) {
+        return this.workerMap.get(name);
+    }
+    async addProcessor(processor, queueName, workerOptions) {
+        const queue = this.queueMap.get(queueName);
+        if (!queue)
+            throw Error(`queue not found ${queueName}`);
+        const worker = new bullmq_1.Worker(queueName, async (job) => {
+            const ctx = this.app.createAnonymousContext({
+                jobId: job.id,
+                job,
+                from: processor,
+            });
+            try {
+                ctx.logger.info(`start process job ${job.id} from ${processor.name}`);
+                const isPassed = await this.app
+                    .getFramework()
+                    .runGuard(ctx, processor, 'execute');
+                if (!isPassed) {
+                    throw new core_1.MidwayInvokeForbiddenError('execute', processor);
+                }
+                const service = await ctx.requestContext.getAsync(processor);
+                const fn = await this.applyMiddleware(async (ctx) => {
+                    return await core_1.Utils.toAsyncFunction(service.execute.bind(service))(job.data, job);
+                });
+                const result = await Promise.resolve(await fn(ctx));
+                ctx.logger.info(`complete process job ${job.id} from ${processor.name}`);
+                return result;
+            }
+            catch (err) {
+                ctx.logger.error(err);
+                return Promise.reject(err);
+            }
+        }, (0, core_1.extend)(true, {}, this.defaultWorkerConfig, workerOptions, {
+            connection: this.connection,
+            prefix: this.prefix,
+        }));
+        this.workerMap.set(queueName, worker);
+    }
+    async runJob(queueName, jobData, options) {
+        const queue = this.queueMap.get(queueName);
+        if (queue) {
+            return await queue.runJob(jobData, options);
+        }
+    }
+    async getJob(queueName, jobName) {
+        const queue = this.queueMap.get(queueName);
+        if (queue) {
+            return queue.getJob(jobName);
+        }
+    }
+};
+__decorate([
+    (0, core_1.Logger)('bullMQLogger'),
+    __metadata("design:type", Object)
+], BullMQFramework.prototype, "bullMQLogger", void 0);
+BullMQFramework = __decorate([
+    (0, core_1.Framework)()
+], BullMQFramework);
+exports.BullMQFramework = BullMQFramework;
+//# sourceMappingURL=framework.js.map

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { BullConfiguration as Configuration } from './configuration';
+export { BullMQFramework as Framework, BullMQQueue } from './framework';
+export * from './decorator';
+export * from './interface';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,25 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BullMQQueue = exports.Framework = exports.Configuration = void 0;
+var configuration_1 = require("./configuration");
+Object.defineProperty(exports, "Configuration", { enumerable: true, get: function () { return configuration_1.BullConfiguration; } });
+var framework_1 = require("./framework");
+Object.defineProperty(exports, "Framework", { enumerable: true, get: function () { return framework_1.BullMQFramework; } });
+Object.defineProperty(exports, "BullMQQueue", { enumerable: true, get: function () { return framework_1.BullMQQueue; } });
+__exportStar(require("./decorator"), exports);
+__exportStar(require("./interface"), exports);
+//# sourceMappingURL=index.js.map

package/dist/interface.d.ts

@@ -0,0 +1,28 @@
+import { IMidwayApplication, IMidwayContext, NextFunction as BaseNextFunction } from '@midwayjs/core';
+import { WorkerOptions, QueueOptions, Job, Worker } from 'bullmq';
+export interface IProcessor {
+    execute(data: any): any;
+}
+export interface IQueue<Job> {
+    runJob(data: Record<string, any>, options?: unknown): Promise<Job>;
+    getJob(name: string): Promise<Job>;
+    getQueueName(): string;
+}
+export interface IQueueManager<Queue extends IQueue<Job>, Job> {
+    runJob(queueName: string, jobData: any, options?: unknown): Promise<Job | undefined>;
+    getJob(queueName: string, jobName: string): Promise<Job | undefined>;
+    createQueue(queueName: string, queueOptions?: unknown): Queue;
+    getQueue(queueName: string): Queue | undefined;
+    getWorker(queueName: string): Worker | undefined;
+}
+export interface Application extends IMidwayApplication<Context> {
+}
+export type NextFunction = BaseNextFunction;
+export interface Context extends IMidwayContext {
+    jobId: string;
+    job: Job;
+    from: new (...args: any[]) => IProcessor;
+}
+export type IWorkerOptions = Omit<WorkerOptions, 'connection' | 'prefix'>;
+export type IQueueOptions = Omit<QueueOptions, 'connection' | 'prefix'>;
+//# sourceMappingURL=interface.d.ts.map

package/dist/interface.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=interface.js.map

package/index.d.ts

@@ -0,0 +1,20 @@
+import { ConnectionOptions } from 'bullmq';
+import { IQueueOptions, IWorkerOptions } from './dist/index';
+export * from './dist/index';
+export { Job } from 'bullmq';
+
+declare module '@midwayjs/core/dist/interface' {
+  // bullmq 新引入了 worker 作为执行任务的实例，一个队列 queue 和 worker 中 connection， prefix 必须一致才能正常执行
+  // 所以 config 中 connection， prefix 单独配置
+  // eslint-disable-next-line
+  interface MidwayConfig {
+    bullmq?: {
+      connection: ConnectionOptions;
+      prefix?: string;
+      defaultQueueOptions?: IQueueOptions;
+      defaultWorkerOptions?: IWorkerOptions;
+      clearRepeatJobWhenStart?: boolean;
+      contextLoggerFormat?: (info: any) => string;
+    };
+  }
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@midwayjs/bullmq",
+  "version": "0.0.1",
+  "description": "midway component for bullmq",
+  "main": "dist/index.js",
+  "typings": "index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "node --require=ts-node/register ../../node_modules/.bin/jest --runInBand",
+    "cov": "node --require=ts-node/register ../../node_modules/.bin/jest --runInBand --coverage --forceExit"
+  },
+  "keywords": [
+    "midway",
+    "IoC",
+    "task",
+    "bullmq",
+    "plugin"
+  ],
+  "author": "guo qicong",
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "index.d.ts"
+  ],
+  "license": "MIT",
+  "devDependencies": {
+    "@midwayjs/core": "^3.19.0",
+    "@midwayjs/mock": "^3.19.2"
+  },
+  "dependencies": {
+    "bullmq": "5.34.6"
+  },
+  "engines": {
+    "node": ">=12"
+  },
+  "repository": "https://github.com/midwayjs/midway.git"
+}