@yrpri/api 9.0.193 → 9.0.195

package/models/audio.cjs

@@ -5,9 +5,24 @@ const _ = require('lodash');
 const queue = require('../services/workers/queue.cjs');
 let bullAudioQueue;
 if (process.env.USE_YOUR_PRIORITIES_ENCODER) {
-    const Queue = require('bull');
-    const redisUrl = process.env.REDIS_URL ? process.env.REDIS_URL : undefined;
-    bullAudioQueue = new Queue('AudioEncoding', redisUrl);
+    const { Queue } = require('bullmq');
+    let redisUrl = process.env.REDIS_URL ? process.env.REDIS_URL : "redis://localhost:6379";
+    if (redisUrl.startsWith("redis://h:")) {
+        redisUrl = redisUrl.replace("redis://h:", "redis://:");
+    }
+    const parsedRedisUrl = new URL(redisUrl);
+    const redisConnection = {
+        host: parsedRedisUrl.hostname,
+        port: parseInt(parsedRedisUrl.port) || 6379,
+        password: parsedRedisUrl.password || undefined,
+        username: parsedRedisUrl.username || undefined,
+    };
+    if (redisUrl.startsWith("rediss://")) {
+        redisConnection.tls = {
+            rejectUnauthorized: false,
+        };
+    }
+    bullAudioQueue = new Queue('AudioEncoding', { connection: redisConnection });
 }
 module.exports = (sequelize, DataTypes) => {
     const Audio = sequelize.define("Audio", {
@@ -393,7 +408,7 @@ module.exports = (sequelize, DataTypes) => {
                 jobPackage = _.merge(jobPackage, {
                     acBackgroundJobId: jobId,
                 });
-                await bullAudioQueue.add(jobPackage);
+                await bullAudioQueue.add('audio-encoding', jobPackage);
                 callback(null, { Job: { Id: jobId } });
             }
         });

package/models/video.cjs

@@ -6,9 +6,24 @@ const _ = require("lodash");
 const queue = require("../services/workers/queue.cjs");
 let bullVideoQueue;
 if (process.env.USE_YOUR_PRIORITIES_ENCODER) {
-    const Queue = require("bull");
-    const redisUrl = process.env.REDIS_URL ? process.env.REDIS_URL : undefined;
-    bullVideoQueue = new Queue("VideoEncoding", redisUrl);
+    const { Queue } = require("bullmq");
+    let redisUrl = process.env.REDIS_URL ? process.env.REDIS_URL : "redis://localhost:6379";
+    if (redisUrl.startsWith("redis://h:")) {
+        redisUrl = redisUrl.replace("redis://h:", "redis://:");
+    }
+    const parsedRedisUrl = new URL(redisUrl);
+    const redisConnection = {
+        host: parsedRedisUrl.hostname,
+        port: parseInt(parsedRedisUrl.port) || 6379,
+        password: parsedRedisUrl.password || undefined,
+        username: parsedRedisUrl.username || undefined,
+    };
+    if (redisUrl.startsWith("rediss://")) {
+        redisConnection.tls = {
+            rejectUnauthorized: false,
+        };
+    }
+    bullVideoQueue = new Queue("VideoEncoding", { connection: redisConnection });
 }
 module.exports = (sequelize, DataTypes) => {
     const Video = sequelize.define("Video", {
@@ -1077,7 +1092,7 @@ module.exports = (sequelize, DataTypes) => {
                     jobPackage = _.merge(jobPackage, {
                         acBackgroundJobId: jobId,
                     });
-                    await bullVideoQueue.add(jobPackage);
+                    await bullVideoQueue.add('video-encoding', jobPackage);
                     callback(null, { Job: { Id: jobId } });
                 }
             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yrpri/api",
-  "version": "9.0.193",
+  "version": "9.0.195",
   "license": "MIT",
   "author": "Robert Bjarnason & Citizens Foundation",
   "repository": {
@@ -32,7 +32,7 @@
     "axios": "^1.9.0",
     "bcrypt": "^5.1.1",
     "body-parser": "^2.2.0",
-    "bull": "^4.16.5",
+    "bullmq": "^5.31.0",
     "bunyan": "^1.8.15",
     "bunyan-prettystream": "git+https://github.com/rbjarnason/node-bunyan-prettystream.git",
     "cheerio": "^1.0.0",

package/services/workers/queue.cjs

@@ -5,39 +5,80 @@ var airbrake = null;
 if (process.env.AIRBRAKE_PROJECT_ID) {
     airbrake = require('../utils/airbrake.cjs');
 }
-const BullQueue = require('bull');
+const { Queue, Worker } = require('bullmq');
 let redisUrl = process.env.REDIS_URL ? process.env.REDIS_URL : "redis://localhost:6379";
 if (redisUrl.startsWith("redis://h:")) {
     redisUrl = redisUrl.replace("redis://h:", "redis://:");
 }
 const EventEmitter = require('events');
 EventEmitter.defaultMaxListeners = 100;
-const redissOptions = {
-    tls: {
-        rejectUnauthorized: false,
-    },
+// Parse redis URL to get connection options
+const parsedRedisUrl = new URL(redisUrl);
+const redisConnection = {
+    host: parsedRedisUrl.hostname,
+    port: parseInt(parsedRedisUrl.port) || 6379,
+    password: parsedRedisUrl.password || undefined,
+    username: parsedRedisUrl.username || undefined,
 };
-log.info("Starting app access to Bull Queue", { redis_url: redisUrl });
+// Add TLS options for rediss:// URLs
+if (redisUrl.startsWith("rediss://")) {
+    redisConnection.tls = {
+        rejectUnauthorized: false,
+    };
+}
+log.info("Starting app access to BullMQ Queue", { redis_url: redisUrl });
+MAIN_WORKER_QUEUE_NAME = process.env.MAIN_WORKER_QUEUE_NAME || 'mainYpQueue';
 class YpQueue {
     constructor() {
         log.info("Create YpQueue");
-        this.createQueues();
+        this.processors = new Map(); // Store processors per job name
+        this.workers = new Map(); // Store workers per job name
+        this.queues = new Map(); // Store queues per job name
+        this.queuePrefix = MAIN_WORKER_QUEUE_NAME;
     }
-    get defaultQueueOptions() {
+    get defaultJobOptions() {
         return {
-            redis: redisUrl.includes("rediss://") ? redissOptions : undefined,
-            defaultJobOptions: {
-                attempts: 1,
-                removeOnComplete: true,
-                removeOnFail: true
-            }
+            attempts: 1,
+            removeOnComplete: true,
+            removeOnFail: true
+        };
+    }
+    // Wrap callback-style processor to work with BullMQ's promise-based API
+    wrapProcessor(processor) {
+        return (job) => {
+            return new Promise((resolve, reject) => {
+                // Call the processor with job and a done callback
+                processor(job, (error, result) => {
+                    if (error) {
+                        reject(error);
+                    }
+                    else {
+                        resolve(result);
+                    }
+                });
+            });
         };
     }
     process(name, concurrency, processor) {
-        this.mainQueue.process(name, concurrency, processor);
+        if (this.workers.has(name)) {
+            log.warn('Processor already registered, skipping re-registration', { name });
+            return;
+        }
+        const wrappedProcessor = this.wrapProcessor(processor);
+        this.processors.set(name, wrappedProcessor);
+        const queue = this.getQueue(name);
+        log.info('Registered processor', { name, concurrency, queue: queue.name });
+        const worker = new Worker(queue.name, wrappedProcessor, {
+            connection: redisConnection,
+            concurrency,
+            name
+        });
+        this.workers.set(name, worker);
+        this.setupWorkerEvents(worker);
     }
     add(name, workPackage, priority, options) {
-        const jobOptions = options || {};
+        const queue = this.getQueue(name);
+        const jobOptions = { ...this.defaultJobOptions, ...(options || {}) };
         let priorityNumber = 1000;
         switch (priority) {
             case 'now':
@@ -57,32 +98,51 @@ class YpQueue {
                 break;
         }
         jobOptions.priority = priorityNumber;
-        this.mainQueue.add(name, workPackage, jobOptions);
+        queue.add(name, workPackage, jobOptions);
+    }
+    getQueue(name) {
+        if (this.queues.has(name)) {
+            return this.queues.get(name);
+        }
+        const queueName = `${this.queuePrefix}-${name}`;
+        const queue = new Queue(queueName, {
+            connection: redisConnection,
+            defaultJobOptions: this.defaultJobOptions
+        });
+        queue.on('error', (error) => {
+            log.error('Queue Error', { error });
+            if (airbrake) {
+                if (!(error instanceof Error)) {
+                    error = new Error(error);
+                }
+                airbrake.notify(error).then((airbrakeErr) => {
+                    if (airbrakeErr.error) {
+                        log.error("AirBrake Error", { context: 'airbrake', err: airbrakeErr.error, errorStatus: 500 });
+                    }
+                });
+            }
+        });
+        this.queues.set(name, queue);
+        return queue;
     }
-    createQueues() {
-        this.mainQueue = new BullQueue('mainYpQueue', redisUrl, this.defaultQueueOptions);
-        this.mainQueue.on('active', function (job) {
+    setupWorkerEvents(worker) {
+        worker.on('active', (job) => {
             log.info('JQ', { id: job.id, name: job.name });
-        }).on('completed', function (job, result) {
+        });
+        worker.on('completed', (job) => {
             log.info('JC', { id: job.id, name: job.name });
-        }).on('failed', function (job, error) {
+        });
+        worker.on('failed', (job, error) => {
             log.error('Job Failed', { id: job ? job.id : null, name: job ? job.name : null, data: job ? job.data : null, error });
-        }).on('resumed', function (job) {
-            log.info('Job Removed', { id: job.id });
-        }).on('waiting', function (jobId) {
-            log.info('Job Waiting', { id: jobId });
-        }).on('stalled', function (job) {
-            log.info('Job Stalled', { id: job.id, name: job.name, data: job.data, });
-        }).on('progress', function (job, process) {
-            log.info('Job Progress', { id: job.id, process });
-        }).on('paused', function () {
-            log.info('Queue Paused');
-        }).on('cleaned', function (jobs, type) {
-            log.info('Job Cleaned', { jobs, type });
-        }).on('drained', function () {
-            log.info('Queue Drained');
-        }).on('error', function (error) {
-            log.error('Job Error', { error });
+        });
+        worker.on('stalled', (jobId) => {
+            log.info('Job Stalled', { id: jobId });
+        });
+        worker.on('progress', (job, progress) => {
+            log.info('Job Progress', { id: job.id, progress });
+        });
+        worker.on('error', (error) => {
+            log.error('Worker Error', { error });
             if (airbrake) {
                 if (!(error instanceof Error)) {
                     error = new Error(error);

package/services/workers/queue.d.cts

@@ -1,21 +1,18 @@
 declare const _exports: YpQueue;
 export = _exports;
 declare class YpQueue {
-    get defaultQueueOptions(): {
-        redis: {
-            tls: {
-                rejectUnauthorized: boolean;
-            };
-        } | undefined;
-        defaultJobOptions: {
-            attempts: number;
-            removeOnComplete: boolean;
-            removeOnFail: boolean;
-        };
+    processors: Map<any, any>;
+    workers: Map<any, any>;
+    queues: Map<any, any>;
+    queuePrefix: any;
+    get defaultJobOptions(): {
+        attempts: number;
+        removeOnComplete: boolean;
+        removeOnFail: boolean;
     };
+    wrapProcessor(processor: any): (job: any) => Promise<any>;
     process(name: any, concurrency: any, processor: any): void;
     add(name: any, workPackage: any, priority: any, options: any): void;
-    createQueues(): void;
-    mainQueue: BullQueue.Queue<any> | undefined;
+    getQueue(name: any): any;
+    setupWorkerEvents(worker: any): void;
 }
-import BullQueue = require("bull");