@deenruv/job-queue-plugin 1.0.0

package/package/bullmq/plugin.d.ts

@@ -0,0 +1,133 @@
+import { BullMQPluginOptions } from "./types";
+/**
+ * @description
+ * This plugin is a drop-in replacement of the DefaultJobQueuePlugin, which implements a push-based
+ * job queue strategy built on top of the popular [BullMQ](https://github.com/taskforcesh/bullmq) library.
+ *
+ * ## Advantages over the DefaultJobQueuePlugin
+ *
+ * The advantage of this approach is that jobs are stored in Redis rather than in the database. For more complex
+ * applications with many job queues and/or multiple worker instances, this can massively reduce the load on the
+ * DB server. The reason for this is that the DefaultJobQueuePlugin uses polling to check for new jobs. By default
+ * it will poll every 200ms. A typical Deenruv instance uses at least 3 queues (handling emails, collections, search index),
+ * so even with a single worker instance this results in 15 queries per second to the DB constantly. Adding more
+ * custom queues and multiple worker instances can easily result in 50 or 100 queries per second. At this point
+ * performance may be impacted.
+ *
+ * Using this plugin, no polling is needed, as BullMQ will _push_ jobs to the worker(s) as and when they are added
+ * to the queue. This results in significantly more scalable performance characteristics, as well as lower latency
+ * in processing jobs.
+ *
+ * ## Installation
+ *
+ * `yarn add \@deenruv/job-queue-plugin bullmq`
+ *
+ * or
+ *
+ * `npm install \@deenruv/job-queue-plugin bullmq`
+ *
+ * **Note:** The v1.x version of this plugin is designed to work with bullmq v1.x, etc.
+ *
+ * @example
+ * ```ts
+ * import { BullMQJobQueuePlugin } from '\@deenruv/job-queue-plugin/package/bullmq';
+ *
+ * const config: DeenruvConfig = {
+ *   // Add an instance of the plugin to the plugins array
+ *   plugins: [
+ *     // DefaultJobQueuePlugin should be removed from the plugins array
+ *     BullMQJobQueuePlugin.init({
+ *       connection: {
+ *         port: 6379
+ *       }
+ *     }),
+ *   ],
+ * };
+ * ```
+ *
+ * ### Running Redis locally
+ *
+ * To develop with this plugin, you'll need an instance of Redis to connect to. Here's a docker-compose config
+ * that will set up [Redis](https://redis.io/) as well as [Redis Commander](https://github.com/joeferner/redis-commander),
+ * which is a web-based UI for interacting with Redis:
+ *
+ * ```YAML
+ * version: "3"
+ * services:
+ *   redis:
+ *     image: bitnami/redis:6.2
+ *     hostname: redis
+ *     container_name: redis
+ *     environment:
+ *       - ALLOW_EMPTY_PASSWORD=yes
+ *     ports:
+ *       - "6379:6379"
+ *   redis-commander:
+ *     container_name: redis-commander
+ *     hostname: redis-commander
+ *     image: rediscommander/redis-commander:latest
+ *     environment:
+ *       - REDIS_HOSTS=local:redis:6379
+ *     ports:
+ *       - "8085:8081"
+ * ```
+ *
+ * ## Concurrency
+ *
+ * The default concurrency of a single worker is 3, i.e. up to 3 jobs will be processed at the same time.
+ * You can change the concurrency in the `workerOptions` passed to the `init()` method:
+ *
+ * @example
+ * ```ts
+ * const config: DeenruvConfig = {
+ *   plugins: [
+ *     BullMQJobQueuePlugin.init({
+ *       workerOptions: {
+ *         concurrency: 10,
+ *       },
+ *     }),
+ *   ],
+ * };
+ * ```
+ *
+ * ## Removing old jobs
+ *
+ * By default, BullMQ will keep completed jobs in the `completed` set and failed jobs in the `failed` set. Over time,
+ * these sets can grow very large. Since Deenruv v2.1, the default behaviour is to remove jobs from these sets after
+ * 30 days or after a maximum of 5,000 completed or failed jobs.
+ *
+ * This can be configured using the `removeOnComplete` and `removeOnFail` options:
+ *
+ * @example
+ * ```ts
+ * const config: DeenruvConfig = {
+ *   plugins: [
+ *     BullMQJobQueuePlugin.init({
+ *       workerOptions: {
+ *         removeOnComplete: {
+ *           count: 500,
+ *         },
+ *         removeOnFail: {
+ *           age: 60 * 60 * 24 * 7, // 7 days
+ *           count: 1000,
+ *         },
+ *       }
+ *     }),
+ *   ],
+ * };
+ * ```
+ *
+ * The `count` option specifies the maximum number of jobs to keep in the set, while the `age` option specifies the
+ * maximum age of a job in seconds. If both options are specified, then the jobs kept will be the ones that satisfy
+ * both properties.
+ *
+ * @docsCategory core plugins/JobQueuePlugin
+ */
+export declare class BullMQJobQueuePlugin {
+    static options: BullMQPluginOptions;
+    /**
+     * @description
+     * Configures the plugin.
+     */
+    static init(options: BullMQPluginOptions): typeof BullMQJobQueuePlugin;
+}

package/package/bullmq/plugin.js

@@ -0,0 +1,171 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BullMQJobQueuePlugin = void 0;
+const core_1 = require("@deenruv/core");
+const bullmq_job_queue_strategy_1 = require("./bullmq-job-queue-strategy");
+const constants_1 = require("./constants");
+const redis_health_check_strategy_1 = require("./redis-health-check-strategy");
+const redis_health_indicator_1 = require("./redis-health-indicator");
+const redis_job_buffer_storage_strategy_1 = require("./redis-job-buffer-storage-strategy");
+/**
+ * @description
+ * This plugin is a drop-in replacement of the DefaultJobQueuePlugin, which implements a push-based
+ * job queue strategy built on top of the popular [BullMQ](https://github.com/taskforcesh/bullmq) library.
+ *
+ * ## Advantages over the DefaultJobQueuePlugin
+ *
+ * The advantage of this approach is that jobs are stored in Redis rather than in the database. For more complex
+ * applications with many job queues and/or multiple worker instances, this can massively reduce the load on the
+ * DB server. The reason for this is that the DefaultJobQueuePlugin uses polling to check for new jobs. By default
+ * it will poll every 200ms. A typical Deenruv instance uses at least 3 queues (handling emails, collections, search index),
+ * so even with a single worker instance this results in 15 queries per second to the DB constantly. Adding more
+ * custom queues and multiple worker instances can easily result in 50 or 100 queries per second. At this point
+ * performance may be impacted.
+ *
+ * Using this plugin, no polling is needed, as BullMQ will _push_ jobs to the worker(s) as and when they are added
+ * to the queue. This results in significantly more scalable performance characteristics, as well as lower latency
+ * in processing jobs.
+ *
+ * ## Installation
+ *
+ * `yarn add \@deenruv/job-queue-plugin bullmq`
+ *
+ * or
+ *
+ * `npm install \@deenruv/job-queue-plugin bullmq`
+ *
+ * **Note:** The v1.x version of this plugin is designed to work with bullmq v1.x, etc.
+ *
+ * @example
+ * ```ts
+ * import { BullMQJobQueuePlugin } from '\@deenruv/job-queue-plugin/package/bullmq';
+ *
+ * const config: DeenruvConfig = {
+ *   // Add an instance of the plugin to the plugins array
+ *   plugins: [
+ *     // DefaultJobQueuePlugin should be removed from the plugins array
+ *     BullMQJobQueuePlugin.init({
+ *       connection: {
+ *         port: 6379
+ *       }
+ *     }),
+ *   ],
+ * };
+ * ```
+ *
+ * ### Running Redis locally
+ *
+ * To develop with this plugin, you'll need an instance of Redis to connect to. Here's a docker-compose config
+ * that will set up [Redis](https://redis.io/) as well as [Redis Commander](https://github.com/joeferner/redis-commander),
+ * which is a web-based UI for interacting with Redis:
+ *
+ * ```YAML
+ * version: "3"
+ * services:
+ *   redis:
+ *     image: bitnami/redis:6.2
+ *     hostname: redis
+ *     container_name: redis
+ *     environment:
+ *       - ALLOW_EMPTY_PASSWORD=yes
+ *     ports:
+ *       - "6379:6379"
+ *   redis-commander:
+ *     container_name: redis-commander
+ *     hostname: redis-commander
+ *     image: rediscommander/redis-commander:latest
+ *     environment:
+ *       - REDIS_HOSTS=local:redis:6379
+ *     ports:
+ *       - "8085:8081"
+ * ```
+ *
+ * ## Concurrency
+ *
+ * The default concurrency of a single worker is 3, i.e. up to 3 jobs will be processed at the same time.
+ * You can change the concurrency in the `workerOptions` passed to the `init()` method:
+ *
+ * @example
+ * ```ts
+ * const config: DeenruvConfig = {
+ *   plugins: [
+ *     BullMQJobQueuePlugin.init({
+ *       workerOptions: {
+ *         concurrency: 10,
+ *       },
+ *     }),
+ *   ],
+ * };
+ * ```
+ *
+ * ## Removing old jobs
+ *
+ * By default, BullMQ will keep completed jobs in the `completed` set and failed jobs in the `failed` set. Over time,
+ * these sets can grow very large. Since Deenruv v2.1, the default behaviour is to remove jobs from these sets after
+ * 30 days or after a maximum of 5,000 completed or failed jobs.
+ *
+ * This can be configured using the `removeOnComplete` and `removeOnFail` options:
+ *
+ * @example
+ * ```ts
+ * const config: DeenruvConfig = {
+ *   plugins: [
+ *     BullMQJobQueuePlugin.init({
+ *       workerOptions: {
+ *         removeOnComplete: {
+ *           count: 500,
+ *         },
+ *         removeOnFail: {
+ *           age: 60 * 60 * 24 * 7, // 7 days
+ *           count: 1000,
+ *         },
+ *       }
+ *     }),
+ *   ],
+ * };
+ * ```
+ *
+ * The `count` option specifies the maximum number of jobs to keep in the set, while the `age` option specifies the
+ * maximum age of a job in seconds. If both options are specified, then the jobs kept will be the ones that satisfy
+ * both properties.
+ *
+ * @docsCategory core plugins/JobQueuePlugin
+ */
+let BullMQJobQueuePlugin = class BullMQJobQueuePlugin {
+    /**
+     * @description
+     * Configures the plugin.
+     */
+    static init(options) {
+        this.options = options;
+        return this;
+    }
+};
+exports.BullMQJobQueuePlugin = BullMQJobQueuePlugin;
+exports.BullMQJobQueuePlugin = BullMQJobQueuePlugin = __decorate([
+    (0, core_1.DeenruvPlugin)({
+        imports: [core_1.PluginCommonModule],
+        configuration: (config) => {
+            config.jobQueueOptions.jobQueueStrategy = new bullmq_job_queue_strategy_1.BullMQJobQueueStrategy();
+            config.jobQueueOptions.jobBufferStorageStrategy =
+                new redis_job_buffer_storage_strategy_1.RedisJobBufferStorageStrategy();
+            config.systemOptions.healthChecks.push(new redis_health_check_strategy_1.RedisHealthCheckStrategy());
+            return config;
+        },
+        providers: [
+            {
+                provide: constants_1.BULLMQ_PLUGIN_OPTIONS,
+                useFactory: () => BullMQJobQueuePlugin.options,
+            },
+            redis_health_indicator_1.RedisHealthIndicator,
+        ],
+        compatibility: "^0.0.0",
+    })
+], BullMQJobQueuePlugin);
+//# sourceMappingURL=plugin.js.map

package/package/bullmq/plugin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../src/bullmq/plugin.ts"],"names":[],"mappings":";;;;;;;;;AAAA,wCAAkE;AAElE,2EAAqE;AACrE,2CAAoD;AACpD,+EAAyE;AACzE,qEAAgE;AAChE,2FAAoF;AAGpF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2HG;AAmBI,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAG/B;;;OAGG;IACH,MAAM,CAAC,IAAI,CAAC,OAA4B;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAA;AAXY,oDAAoB;+BAApB,oBAAoB;IAlBhC,IAAA,oBAAa,EAAC;QACb,OAAO,EAAE,CAAC,yBAAkB,CAAC;QAC7B,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,MAAM,CAAC,eAAe,CAAC,gBAAgB,GAAG,IAAI,kDAAsB,EAAE,CAAC;YACvE,MAAM,CAAC,eAAe,CAAC,wBAAwB;gBAC7C,IAAI,iEAA6B,EAAE,CAAC;YACtC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,sDAAwB,EAAE,CAAC,CAAC;YACvE,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,SAAS,EAAE;YACT;gBACE,OAAO,EAAE,iCAAqB;gBAC9B,UAAU,EAAE,GAAG,EAAE,CAAC,oBAAoB,CAAC,OAAO;aAC/C;YACD,6CAAoB;SACrB;QACD,aAAa,EAAE,QAAQ;KACxB,CAAC;GACW,oBAAoB,CAWhC"}

package/package/bullmq/redis-health-check-strategy.d.ts

@@ -0,0 +1,6 @@
+import { HealthIndicatorFunction } from "@nestjs/terminus";
+import { HealthCheckStrategy, Injector } from "@deenruv/core";
+export declare class RedisHealthCheckStrategy implements HealthCheckStrategy {
+    init(injector: Injector): void;
+    getHealthIndicator(): HealthIndicatorFunction;
+}

package/package/bullmq/redis-health-check-strategy.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisHealthCheckStrategy = void 0;
+const redis_health_indicator_1 = require("./redis-health-indicator");
+let indicator;
+class RedisHealthCheckStrategy {
+    init(injector) {
+        indicator = injector.get(redis_health_indicator_1.RedisHealthIndicator);
+    }
+    getHealthIndicator() {
+        return () => indicator.isHealthy("redis (job queue)");
+    }
+}
+exports.RedisHealthCheckStrategy = RedisHealthCheckStrategy;
+//# sourceMappingURL=redis-health-check-strategy.js.map

package/package/bullmq/redis-health-check-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-health-check-strategy.js","sourceRoot":"","sources":["../../src/bullmq/redis-health-check-strategy.ts"],"names":[],"mappings":";;;AAGA,qEAAgE;AAEhE,IAAI,SAA+B,CAAC;AAEpC,MAAa,wBAAwB;IACnC,IAAI,CAAC,QAAkB;QACrB,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,6CAAoB,CAAC,CAAC;IACjD,CAAC;IACD,kBAAkB;QAChB,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IACxD,CAAC;CACF;AAPD,4DAOC"}

package/package/bullmq/redis-health-indicator.d.ts

@@ -0,0 +1,8 @@
+import { HealthIndicator, HealthIndicatorResult } from "@nestjs/terminus";
+import { BullMQPluginOptions } from "./types";
+export declare class RedisHealthIndicator extends HealthIndicator {
+    private options;
+    private timeoutTimer;
+    constructor(options: BullMQPluginOptions);
+    isHealthy(key: string, timeoutMs?: number): Promise<HealthIndicatorResult>;
+}

package/package/bullmq/redis-health-indicator.js

@@ -0,0 +1,80 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisHealthIndicator = void 0;
+const common_1 = require("@nestjs/common");
+const terminus_1 = require("@nestjs/terminus");
+const core_1 = require("@deenruv/core");
+const bullmq_1 = require("bullmq");
+const constants_1 = require("./constants");
+let RedisHealthIndicator = class RedisHealthIndicator extends terminus_1.HealthIndicator {
+    constructor(options) {
+        super();
+        this.options = options;
+    }
+    async isHealthy(key, timeoutMs = 5000) {
+        if (!this.options.connection) {
+            throw new Error("Redis connection options are not defined");
+        }
+        const connection = new bullmq_1.RedisConnection(this.options.connection);
+        const pingResult = await new Promise(async (resolve, reject) => {
+            try {
+                connection.on("error", (err) => {
+                    core_1.Logger.error(`Redis health check error: ${JSON.stringify(err.message)}`, constants_1.loggerCtx, err.stack);
+                    resolve(err);
+                });
+                if (this.timeoutTimer) {
+                    clearTimeout(this.timeoutTimer);
+                }
+                const timeout = new Promise((_resolve) => (this.timeoutTimer = setTimeout(_resolve, timeoutMs)));
+                const client = await Promise.race([connection.client, timeout]);
+                clearTimeout(this.timeoutTimer);
+                if (!client) {
+                    resolve("timeout");
+                    return;
+                }
+                void client.ping((err, res) => {
+                    if (err) {
+                        resolve(err);
+                    }
+                    else {
+                        resolve(res);
+                    }
+                });
+            }
+            catch (e) {
+                resolve(e);
+            }
+        });
+        try {
+            await connection.close();
+            // await connection.disconnect();
+        }
+        catch (e) {
+            core_1.Logger.error(`Redis health check error closing connection: ${JSON.stringify(e.message)}`, constants_1.loggerCtx, e.stack);
+        }
+        const result = this.getStatus(key, pingResult === "PONG");
+        if (pingResult === "PONG") {
+            return result;
+        }
+        throw new terminus_1.HealthCheckError("Redis failed", result);
+    }
+};
+exports.RedisHealthIndicator = RedisHealthIndicator;
+exports.RedisHealthIndicator = RedisHealthIndicator = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)(constants_1.BULLMQ_PLUGIN_OPTIONS)),
+    __metadata("design:paramtypes", [Object])
+], RedisHealthIndicator);
+//# sourceMappingURL=redis-health-indicator.js.map

package/package/bullmq/redis-health-indicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-health-indicator.js","sourceRoot":"","sources":["../../src/bullmq/redis-health-indicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAoD;AACpD,+CAI0B;AAC1B,wCAAuC;AACvC,mCAAyC;AAGzC,2CAA+D;AAIxD,IAAM,oBAAoB,GAA1B,MAAM,oBAAqB,SAAQ,0BAAe;IAEvD,YACyC,OAA4B;QAEnE,KAAK,EAAE,CAAC;QAF+B,YAAO,GAAP,OAAO,CAAqB;IAGrE,CAAC;IACD,KAAK,CAAC,SAAS,CACb,GAAW,EACX,SAAS,GAAG,IAAI;QAEhB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,wBAAe,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7D,IAAI,CAAC;gBACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBAC7B,aAAM,CAAC,KAAK,CACV,6BAA6B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAC1D,qBAAS,EACT,GAAG,CAAC,KAAK,CACV,CAAC;oBACF,OAAO,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC,CAAC,CAAC;gBACH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAClC,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,OAAO,CACzB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CACpE,CAAC;gBACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAChE,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAChC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,CAAC,SAAS,CAAC,CAAC;oBACnB,OAAO;gBACT,CAAC;gBACD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,GAAQ,EAAE,GAAQ,EAAE,EAAE;oBACtC,IAAI,GAAG,EAAE,CAAC;wBACR,OAAO,CAAC,GAAG,CAAC,CAAC;oBACf,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,GAAG,CAAC,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,OAAO,CAAC,CAAC,CAAC,CAAC;YACb,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;YACzB,iCAAiC;QACnC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,aAAM,CAAC,KAAK,CACV,gDAAgD,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAC3E,qBAAS,EACT,CAAC,CAAC,KAAK,CACR,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,KAAK,MAAM,CAAC,CAAC;QAE1D,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;YAC1B,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,2BAAgB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;CACF,CAAA;AAnEY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;IAIR,WAAA,IAAA,eAAM,EAAC,iCAAqB,CAAC,CAAA;;GAHrB,oBAAoB,CAmEhC"}

package/package/bullmq/redis-job-buffer-storage-strategy.d.ts

@@ -0,0 +1,16 @@
+import { Injector, Job, JobBufferStorageStrategy } from "@deenruv/core";
+export declare class RedisJobBufferStorageStrategy implements JobBufferStorageStrategy {
+    private redis;
+    init(injector: Injector): void;
+    add(bufferId: string, job: Job<any>): Promise<Job<any>>;
+    bufferSize(bufferIds?: string[]): Promise<{
+        [bufferId: string]: number;
+    }>;
+    flush(bufferIds?: string[]): Promise<{
+        [bufferId: string]: Job[];
+    }>;
+    private keyName;
+    private toJobConfigString;
+    private toJob;
+    private getAllBufferIds;
+}

package/package/bullmq/redis-job-buffer-storage-strategy.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisJobBufferStorageStrategy = void 0;
+const core_1 = require("@deenruv/core");
+const ioredis_1 = require("ioredis");
+const constants_1 = require("./constants");
+const BUFFER_LIST_PREFIX = "deenruv-job-buffer";
+class RedisJobBufferStorageStrategy {
+    init(injector) {
+        const options = injector.get(constants_1.BULLMQ_PLUGIN_OPTIONS);
+        if (options.connection instanceof ioredis_1.Redis) {
+            this.redis = options.connection;
+        }
+        else if (options.connection instanceof ioredis_1.Cluster) {
+            this.redis = options.connection;
+        }
+        else {
+            this.redis = new ioredis_1.Redis(options.connection);
+        }
+    }
+    async add(bufferId, job) {
+        const result = await this.redis.lpush(this.keyName(bufferId), this.toJobConfigString(job));
+        return job;
+    }
+    async bufferSize(bufferIds) {
+        const ids = (bufferIds === null || bufferIds === void 0 ? void 0 : bufferIds.length) ? bufferIds : await this.getAllBufferIds();
+        const result = {};
+        for (const id of bufferIds || []) {
+            const key = this.keyName(id);
+            const count = await this.redis.llen(key);
+            result[id] = count;
+        }
+        return result;
+    }
+    async flush(bufferIds) {
+        const ids = (bufferIds === null || bufferIds === void 0 ? void 0 : bufferIds.length) ? bufferIds : await this.getAllBufferIds();
+        const result = {};
+        for (const id of bufferIds || []) {
+            const key = this.keyName(id);
+            const items = await this.redis.lrange(key, 0, -1);
+            await this.redis.del(key);
+            result[id] = items.map((item) => this.toJob(item));
+        }
+        return result;
+    }
+    keyName(bufferId) {
+        return `${BUFFER_LIST_PREFIX}:${bufferId}`;
+    }
+    toJobConfigString(job) {
+        var _a;
+        const jobConfig = Object.assign(Object.assign({}, job), { data: job.data, id: (_a = job.id) !== null && _a !== void 0 ? _a : undefined });
+        return JSON.stringify(jobConfig);
+    }
+    toJob(jobConfigString) {
+        try {
+            const jobConfig = JSON.parse(jobConfigString);
+            return new core_1.Job(jobConfig);
+        }
+        catch (e) {
+            core_1.Logger.error(`Could not parse buffered job:\n${JSON.stringify(e.message)}`, constants_1.loggerCtx, e.stack);
+            throw e;
+        }
+    }
+    async getAllBufferIds() {
+        const stream = this.redis instanceof ioredis_1.Redis
+            ? this.redis.scanStream({
+                match: `${BUFFER_LIST_PREFIX}:*`,
+            })
+            : this.redis.nodes()[0].scanStream({
+                match: `${BUFFER_LIST_PREFIX}:*`,
+            });
+        const keys = await new Promise((resolve, reject) => {
+            const allKeys = [];
+            stream.on("data", (_keys) => allKeys.push(..._keys));
+            stream.on("end", () => resolve(allKeys));
+            stream.on("error", (err) => reject(err));
+        });
+        return keys.map((key) => key.replace(`${BUFFER_LIST_PREFIX}:`, ""));
+    }
+}
+exports.RedisJobBufferStorageStrategy = RedisJobBufferStorageStrategy;
+//# sourceMappingURL=redis-job-buffer-storage-strategy.js.map

package/package/bullmq/redis-job-buffer-storage-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-job-buffer-storage-strategy.js","sourceRoot":"","sources":["../../src/bullmq/redis-job-buffer-storage-strategy.ts"],"names":[],"mappings":";;;AAAA,wCAMuB;AACvB,qCAAuD;AAEvD,2CAA+D;AAG/D,MAAM,kBAAkB,GAAG,oBAAoB,CAAC;AAEhD,MAAa,6BAA6B;IAGxC,IAAI,CAAC,QAAkB;QACrB,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAsB,iCAAqB,CAAC,CAAC;QACzE,IAAI,OAAO,CAAC,UAAU,YAAY,eAAK,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC;QAClC,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,YAAY,iBAAO,EAAE,CAAC;YACjD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC,OAAO,CAAC,UAA0B,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,GAAa;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CACnC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EACtB,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAC5B,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,UAAU,CACd,SAAoB;QAEpB,MAAM,GAAG,GAAG,CAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,EAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QACzE,MAAM,MAAM,GAAmC,EAAE,CAAC;QAClD,KAAK,MAAM,EAAE,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;QACrB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAoB;QAC9B,MAAM,GAAG,GAAG,CAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,EAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QACzE,MAAM,MAAM,GAAkC,EAAE,CAAC;QACjD,KAAK,MAAM,EAAE,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,OAAO,CAAC,QAAgB;QAC9B,OAAO,GAAG,kBAAkB,IAAI,QAAQ,EAAE,CAAC;IAC7C,CAAC;IAEO,iBAAiB,CAAC,GAAa;;QACrC,MAAM,SAAS,mCACV,GAAG,KACN,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,EAAE,EAAE,MAAA,GAAG,CAAC,EAAE,mCAAI,SAAS,GACxB,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,eAAuB;QACnC,IAAI,CAAC;YACH,MAAM,SAAS,GAAmB,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9D,OAAO,IAAI,UAAG,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,aAAM,CAAC,KAAK,CACV,kCAAkC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAC7D,qBAAS,EACT,CAAC,CAAC,KAAK,CACR,CAAC;YACF,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,MAAM,MAAM,GACV,IAAI,CAAC,KAAK,YAAY,eAAK;YACzB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;gBACpB,KAAK,EAAE,GAAG,kBAAkB,IAAI;aACjC,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;gBAC/B,KAAK,EAAE,GAAG,kBAAkB,IAAI;aACjC,CAAC,CAAC;QACT,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3D,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;YACrD,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,kBAAkB,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;IACtE,CAAC;CACF;AA5FD,sEA4FC"}

package/package/bullmq/scripts/get-jobs-by-type.d.ts

@@ -0,0 +1,10 @@
+import { CustomScriptDefinition } from "../types";
+export declare const getJobsByType: CustomScriptDefinition<[
+    totalItems: number,
+    jobIds: string[]
+], [
+    rangeStart: number,
+    rangeEnd: number,
+    queueName: string | undefined,
+    ...states: string[]
+]>;

package/package/bullmq/scripts/get-jobs-by-type.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getJobsByType = void 0;
+const script = `--[[
+  Get job ids per provided states and filter by name
+    Input:
+      KEYS[1]    'prefix'
+      ARGV[1]    start
+      ARGV[2]    end
+      ARGV[3]    filterName
+      ARGV[4...] types
+]]
+local rcall = redis.call
+local prefix = KEYS[1]
+local rangeStart = tonumber(ARGV[1])
+local rangeEnd = tonumber(ARGV[2])
+local filterName = ARGV[3]
+local results = {}
+
+local targetSets = {}
+
+-- Initialize an empty array to hold the sets to unionize. The "completed" and "failed" lists
+-- are sorted sets
+local setsToUnionize = {}
+local typesInUnion = {}
+
+-- Initialize an empty array to hold lists to include. The "active" and "wait" lists are
+-- regular lists
+local listsToInclude = {}
+
+
+-- Iterate through ARGV starting from the first element (ARGV[1]) up to the end
+for i = 4, #ARGV do
+    local setKey = prefix .. ARGV[i]
+
+    -- Check if the setKey is valid (e.g., it exists and is a sorted set)
+    local targetExists = redis.call('EXISTS', setKey)
+    local listType = redis.call('TYPE', setKey).ok
+
+    if targetExists == 1 and listType == 'zset' then
+        -- Add the valid set to the array
+        table.insert(setsToUnionize, setKey)
+        table.insert(typesInUnion, ARGV[i])
+    end
+    if targetExists == 1 and listType == 'list' then
+        -- Add the valid set to the array
+        table.insert(listsToInclude, setKey)
+        table.insert(typesInUnion, ARGV[i])
+    end
+end
+
+-- Define the destination key for the concatenated sorted set
+local tempSortedSetUnionKey = prefix .. 'union:' .. table.concat(typesInUnion, ':');
+
+if #listsToInclude  == 0 and #setsToUnionize == 0 then
+    return {0, {}}
+end
+
+-- Check if there are valid sets to unionize
+if #setsToUnionize > 0 then
+    -- Use ZUNIONSTORE to concatenate the valid sorted sets into the destination key
+    local numSets = #setsToUnionize
+    redis.call('ZUNIONSTORE', tempSortedSetUnionKey, numSets, unpack(setsToUnionize))
+end
+
+local originalResults = rcall("ZREVRANGE", tempSortedSetUnionKey, 0, -1)
+
+
+if #listsToInclude > 0 then
+    for _, listKey in ipairs(listsToInclude) do
+        local list = rcall("LRANGE", listKey, 0, -1)
+        for _, jobId in ipairs(list) do
+            table.insert(originalResults, jobId)
+        end
+    end
+end
+
+
+-- Define a custom comparison function for sorting in descending order
+local function compareDescending(a, b)
+    return tonumber(a) > tonumber(b)
+end
+
+-- Sort the table in descending order
+table.sort(originalResults, compareDescending)
+
+local filteredResults = {}
+local totalResults = 0
+
+for _, job in ipairs(originalResults) do
+  local jobName = rcall("HGET", prefix .. job, "name");
+  if filterName ~= "" and jobName == filterName then
+    if rangeStart <= totalResults and #filteredResults < rangeEnd then
+      table.insert(filteredResults, job)
+    end
+    totalResults = totalResults + 1
+  elseif filterName == "" then
+    if rangeStart <= totalResults and #filteredResults < rangeEnd then
+      table.insert(filteredResults, job)
+    end
+    totalResults = totalResults + 1
+  end
+end
+
+rcall("DEL", tempSortedSetUnionKey)
+
+return {totalResults, filteredResults}
+`;
+exports.getJobsByType = {
+    script,
+    numberOfKeys: 1,
+    name: "getJobsByType",
+};
+//# sourceMappingURL=get-jobs-by-type.js.map

package/package/bullmq/scripts/get-jobs-by-type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-jobs-by-type.js","sourceRoot":"","sources":["../../../src/bullmq/scripts/get-jobs-by-type.ts"],"names":[],"mappings":";;;AAGA,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwGd,CAAC;AAEW,QAAA,aAAa,GAQtB;IACF,MAAM;IACN,YAAY,EAAE,CAAC;IACf,IAAI,EAAE,eAAe;CACtB,CAAC"}