@vida-global/core 1.4.6 → 2.0.0

package/lib/jobQueue/queue.js

@@ -0,0 +1,161 @@
+const { AbstractJobComponent } = require('./abstractJobComponent');
+const   BullMQ                 = require('bullmq');
+const { logger }               = require('../logger');
+const { randomUUID }           = require('crypto');
+
+
+logger.addScope('queue');
+
+
+class Queue extends AbstractJobComponent {
+    #_queue;
+    #queueName;
+
+
+    constructor(queueName) {
+        super();
+        this.#queueName = queueName;
+    }
+
+
+    get queueName() { return this.#queueName   }
+    get logger()    { return logger.queue.child(this.fullQueueName) }
+
+
+    async queueJob(job, args, settings) {
+        const id = randomUUID();
+        await this.#queue.add(job.name, { id, args }, settings);
+    }
+
+
+    /***********************************************************************************************
+    * QUEUE SETUP
+    ***********************************************************************************************/
+    get #queue() {
+        if (this.#_queue) return this.#_queue;
+
+        this.#_queue            = new BullMQ.Queue(this.fullQueueName, this.queueOptions);
+        this.#_queue.on('error', this.#handleError.bind(this));
+
+        return this.#_queue
+    }
+
+
+    get queueOptions() {
+        return { connection: this.redisConnectionDetails };
+    }
+
+
+    /***********************************************************************************************
+    * QUEUE STATUS
+    ***********************************************************************************************/
+    async close() {
+        await this.#queue.close();
+    }
+
+
+    async pause() {
+        await this.#queue.pause();
+    }
+
+
+    async resume() {
+        await this.#queue.resume();
+    }
+
+
+    /***********************************************************************************************
+    * JOBS
+    ***********************************************************************************************/
+    async numQueuedJobs() {
+        return await this.#queue.count();
+    }
+
+
+    async getQueuedJobs() {
+        const jobs = await this.#queue.getJobs(["paused", "wait", "prioritized", "delayed"]);
+        return this.#_formatJobsData(jobs);
+    }
+
+
+    async numActiveJobs() {
+        return await this.#queue.getActiveCount();
+    }
+    
+
+    async getActiveJobs() {
+        const jobs = await this.#queue.getActive();
+        return this.#_formatJobsData(jobs);
+    }
+
+
+    async numFailedJobs() {
+        return await this.#queue.getFailedCount();
+    }
+    
+
+    async getFailedJobs() {
+        const jobs = await this.#queue.getFailed();
+        return this.#_formatJobsData(jobs);
+    }
+
+
+    async clearFailedJobs() {
+        await this.#queue.clean(0, null, "failed");
+    }
+
+
+    async clearCompletedJobs() {
+        await this.#queue.clean(0, null, "completed");
+    }
+
+
+    async clearQueuedJobs() {
+        return await this.#queue.drain();
+    }
+
+
+    async numWorkers() {
+        const workerData = await this.#queue.getWorkers();
+        return workerData.length;
+    }
+
+
+    #_formatJobsData(jobs) {
+        return jobs.map((job) => this.#_formatJobData(job));
+    }
+
+
+    #_formatJobData(job) {
+        return {
+            queueName:       job.queue.name,
+            name:            job.name,
+            args:            job.data,
+            id:              job.id,
+            attemptsMade:    job.attemptsMade,
+            attemptsStarted: job.attemptsStarted,
+            progress:        job.progress,
+            failedReason:    job.failedReason
+        }
+    }
+
+
+    /***********************************************************************************************
+    * ERROR HANDLING
+    ***********************************************************************************************/
+    #handleError() {
+        this.logger.error(...arguments);
+    }
+}
+
+
+const queues = {};
+function getQueue(name) {
+    const fullName            = Queue.fullQueueName(name);
+    return queues[fullName] ||= new Queue(name);
+}
+
+
+module.exports = {
+    getQueue
+}

package/lib/jobQueue/worker.js

@@ -0,0 +1,273 @@
+const   APM                    = require('../apm');
+const { AbstractJobComponent } = require('./abstractJobComponent');
+const   BullMQ                 = require('bullmq');
+const { JobImporter }          = require('./jobImporter');
+const { logger }               = require('../logger');
+const { randomUUID }           = require('crypto');
+
+
+const JOB_COMPLETED_EVENT = 'JobCompleted';
+const JOB_FAILED_EVENT    = 'JobFailed';
+
+
+function durationMetricName(queueName, jobName) {
+    return `Custom/Worker/${queueName}/${jobName}/duration`;
+}
+
+
+logger.addScope('worker');
+let registeredJobs;
+
+
+class Worker extends AbstractJobComponent {
+    #defaultJob;
+    #queueName;
+    #_worker;
+
+
+    constructor(queueName) {
+        super();
+        this.#queueName = queueName || process.env.WORKER_QUEUE_NAME;
+    }
+
+
+    async listen() {
+        await this.#registerJobs();
+        this.logger.info(`Worker is listening on queue ${this.fullQueueName}`);
+        await this.#worker.run();
+    }
+
+
+    #registerJobs() {
+        if (registeredJobs) return;
+
+        const importer   = new JobImporter(this.jobDirectories);
+        const jobs       = importer.jobs;
+        registeredJobs   = Object.fromEntries(jobs.map(job => [job.name, job]));
+        this.#defaultJob = jobs.find(job => job.defaultFor == this.queueName) || null;
+
+        if (process.env.NODE_ENV != 'test') {
+            for (const job of jobs) {
+                this.logger.verbose(`JOB: ${job.name}`);
+            }
+        }
+    }
+
+
+    get logger() {
+        return logger.worker;
+    }
+
+
+    get jobDirectories() {
+        return [`${process.cwd()}/jobs`];
+    }
+
+
+    /***********************************************************************************************
+    * WORKER SETUP
+    ***********************************************************************************************/
+    get #worker() {
+        if (this.#_worker) return this.#_worker;
+
+        this.#_worker = new BullMQ.Worker(this.fullQueueName,
+                                          this.processJob.bind(this),
+                                          this.workerOptions);
+        this.#setupEventHandling();
+
+        return this.#_worker;
+    }
+
+
+    get workerOptions() {
+        const options = {
+            autorun:     false,
+            concurrency: this.concurrency,
+            connection:  this.redisConnectionDetails
+        };
+
+        return options;
+    }
+
+
+    get concurrency() {
+      return parseInt(process.env.WORKER_CONCURRENCY || 1);
+    }
+
+
+    get queueName() {
+        return this.#queueName;
+    }
+
+
+    /***********************************************************************************************
+    * PROCESS JOB
+    ***********************************************************************************************/
+    async processJob(bullJob) {
+        const jobClass = this.#getJobClass(bullJob.name);
+        if (!jobClass) throw `Unknown Job: ${bullJob.name}`;
+
+        const context = this.#buildJobContext(bullJob, jobClass);
+        return await APM.startBackgroundTransaction(context.jobName,
+                                                    this.fullQueueName,
+                                                    () => this.#runJob(context));
+    }
+
+
+    #buildJobContext(bullJob, jobClass) {
+        const { id, args } = this.#dataForJob(bullJob);
+        return {
+            bullJob,
+            jobClass,
+            jobName:     jobClass.name,
+            id,
+            args,
+            queueName:   this.queueName,
+            attempt:     bullJob.attemptsMade + 1,
+            maxAttempts: bullJob.opts.attempts,
+        };
+    }
+
+
+    async #runJob(context) {
+        this.#addJobAttributes(context);
+
+        const startedAt = Date.now();
+        try {
+            await this.#executeJob(context);
+        } catch(err) {
+            this.#reportJobError(err, context);
+            throw err;
+        }
+
+        const duration = Date.now() - startedAt;
+        this.#recordJobSuccess(context, duration);
+        return duration;
+    }
+
+
+    async #executeJob(context) {
+        const { bullJob, jobClass, id, args, jobName } = context;
+        const job = new jobClass();
+        await APM.startSegment(`job:${jobName}`,
+                               true,
+                               () => job._run(id, args, bullJob.updateProgress.bind(bullJob)));
+    }
+
+
+    #addJobAttributes(context) {
+        APM.addCustomAttribute('jobId',       context.id);
+        APM.addCustomAttribute('jobName',     context.jobName);
+        APM.addCustomAttribute('queueName',   context.queueName);
+        APM.addCustomAttribute('attempt',     context.attempt);
+        APM.addCustomAttribute('maxAttempts', context.maxAttempts);
+    }
+
+
+    #reportJobError(err, context) {
+        const isFinalRetry = context.attempt >= context.maxAttempts;
+
+        APM.reportError(err, {
+            jobId:       context.id,
+            jobName:     context.jobName,
+            queueName:   context.queueName,
+            attempt:     context.attempt,
+            maxAttempts: context.maxAttempts,
+            isFinalRetry,
+        });
+
+        if (isFinalRetry) this.#recordJobFailedEvent(context, err);
+    }
+
+
+    #recordJobFailedEvent(context, err) {
+        APM.recordCustomEvent(JOB_FAILED_EVENT, {
+            jobName:   context.jobName,
+            queueName: context.queueName,
+            attempts:  context.attempt,
+            error:     err.message || String(err),
+        });
+    }
+
+
+    #recordJobSuccess(context, duration) {
+        APM.recordCustomEvent(JOB_COMPLETED_EVENT, {
+            jobName:    context.jobName,
+            queueName:  context.queueName,
+            durationMs: duration,
+            attempts:   context.attempt,
+        });
+        APM.recordMetric(durationMetricName(context.queueName, context.jobName), duration);
+    }
+
+
+    /***********************************************************************************************
+    * EVENT HANDLING
+    ***********************************************************************************************/
+    #setupEventHandling() {
+        this.#_worker.on('completed', this.#handleJobCompletion.bind(this));
+        this.#_worker.on('progress',  this.#handleJobProgress.bind(this));
+        this.#_worker.on('failed',    this.#handleJobFailure.bind(this));
+        this.#_worker.on('error',     this.#handleError.bind(this));
+    }
+
+
+    #handleJobCompletion({ name, data }, duration) {
+        const logger = this.#loggerForJob({ name, data });
+        logger.debug(`Completed in ${duration}ms`);
+    }
+
+
+    #handleJobProgress(job, data) {
+        const logger = this.#loggerForJob(job);
+        logger.debug(data);
+    }
+
+
+    #handleJobFailure(job, err) {
+        const log = `Job Failed (Attempt ${job.attemptsMade}/${job.opts.attempts}): ${job.name} - ${err}`;
+
+        const lastRetry = job.attemptsMade == job.opts.attempts;
+        const logger    = this.#loggerForJob(job);
+        logger.debug(`ERROR (${job.attemptsMade}/${job.opts.attempts}) ${err}`);
+
+        if (lastRetry) {
+            throw err;
+        }
+    }
+
+
+    #handleError(err) {
+        this.logger.error(err);
+        if (process.env.NODE_ENV == 'development') {
+            throw err;
+        }
+    }
+
+
+    #loggerForJob(bullJob) {
+        const { id } = this.#dataForJob(bullJob);
+        return logger.worker.child(id);
+    }
+
+
+    #getJobClass(name) {
+        if (registeredJobs[name]) return registeredJobs[name];
+        return this.#defaultJob;
+    }
+
+
+    #dataForJob({ name, data }) {
+        if (name == '__default__') { // legacy for vida.live. Remove when vida.live has been fully migrated
+            return { args: [data] };
+        } else {
+            const { id, args } = data;
+            return { id, args };
+        }
+    }
+}
+
+
+module.exports = {
+    Worker
+}

package/lib/logger/README.md

@@ -26,7 +26,7 @@ logger.bar.error('my log');
 ```
 logger.addScope('http');
 const requestId     = 123;
-const requestLogger = logger.http.createChild(requestId);
+const requestLogger = logger.http.child(requestId);
 
 requestLogger.debug('my log');
 // [2026-04-30 16:16:12.797 -0700][DEBUG][HTTP][123] my log

package/lib/logger/index.js

@@ -2,21 +2,25 @@ const   winston                           = require('winston');
 const { colorize, combine, printf, timestamp } = winston.format;
 
 
+const loggers = {};
+
+
 class Logger {
     #id;
     #logger;
-    #scope;
+    #scopes;
 
 
-    constructor(scope, id) {
+    constructor(scopes, id) {
         this.#id     = id;
-        this.#scope  = scope || 'API';
+        scopes       = Array.isArray(scopes) ? scopes : [scopes];
+        this.#scopes = scopes.filter(Boolean).map(scope => scope.toLowerCase());
         this.#logger = winston.createLogger(this.loggerOptions);
     }
 
 
-    get scope() { return this.#scope; }
-    get id()    { return this.#id; }
+    get scopes() { return this.#scopes; }
+    get id()     { return this.#id; }
 
 
     get loggerOptions() {
@@ -47,12 +51,13 @@ class Logger {
 
 
     messageFormatter(info) {
-        const level  = info.level.toUpperCase()
+        const level    = info.level.toUpperCase()
+        const id       = this.id || info.meta?.id;
+        const scopes   = this.scopes.map(scope => `[${scope.toUpperCase()}]`).join('');
+        let prefix     = `[${info.timestamp}][${level}]${scopes}`;
+        if (id) prefix = `${prefix}[${id}]`;
+        info.message   = `${prefix} ${info.message}`;
 
-        let prefix = `[${info.timestamp}][${level}][${this.scope.toUpperCase()}]`;
-        if (this.id) prefix = `${prefix}[${this.id}]`;
-
-        info.message =  `${prefix} ${info.message}`;
         return info;
     }
 
@@ -69,18 +74,31 @@ class Logger {
 
 
     addScope(scope) {
+        scope = scope.toLowerCase();
         const existing = this[scope];
-        if (existing && !existing instanceof Logger) {
-            throw `Property already exists at ${scope}`;
+        if (existing) {
+            if (!existing instanceof Logger) throw `Property already exists at ${scope}`;
+            return;
         }
 
-        const logger = new Logger(scope, this.id);
+        const logger = this.#createLogger(this.scopes.concat([scope]), this.id);
         Object.defineProperty(this, scope, { get: () => logger });
     }
 
 
-    createChild(id) {
-        return new Logger(this.scope, id);
+    child(id) {
+        return this.#createLogger(this.scopes, id);
+    }
+
+
+    #createLogger(scopes, id) {
+        scopes    = Array.isArray(scopes) ? scopes : [scopes];
+        scopes    = scopes.map(scope => scope.toLowerCase());
+        const key = `${scopes.join('|')}:${id}`;
+        if (loggers[key]) return loggers[key];
+
+        loggers[key] = new Logger(scopes, id);
+        return loggers[key];
     }
 
 

package/lib/logger/serverMiddleware.js

@@ -1,4 +1,3 @@
-const { logger }     = require('./index');
 const expressWinston = require('express-winston');
 
 
@@ -10,12 +9,9 @@ const msgFormatter = (req, res) => {
         msg = res.statusCode >= 500 ? 'request errored' : 'request completed';
     }
 
-    const headerPcs       = res._header.split("\n").map(h => h.split(/:(.*)/));
-    const headers         = Object.fromEntries(headerPcs);
-    const responseTime    = headers['X-Response-Time'];
-
+    const responseTime    = res.responseTime || '-';
     const requestDetails  = `req: ${req.method} ${req.url.split('?')[0]} for ${req.ip}`;
-    const responseDetails = `res: statusCode=${res.statusCode} responseTime=${responseTime}`;
+    const responseDetails = `res: statusCode=${res.statusCode} responseTime=${responseTime}ms`;
 
     msg = `${msg}\n    ${requestDetails}\n    ${responseDetails}`;
     if (res.error) {
@@ -26,12 +22,18 @@ const msgFormatter = (req, res) => {
 };
 
 
-logger.addScope('http');
-const middleware = expressWinston.logger({
-    level:           'debug',
-    msg:             msgFormatter,
-    winstonInstance: logger.http._winstonLogger,
-});
+let _middleware;
+
+function middleware(logger) {
+    if (_middleware) return _middleware;
+    _middleware = expressWinston.logger({
+        dynamicMeta:     (req, res) => ({id: req.id}),
+        level:           'debug',
+        msg:             msgFormatter,
+        winstonInstance: logger._winstonLogger,
+    });
+    return _middleware;
+};
 
 
 module.exports = {

package/lib/redis/redisClient.js

@@ -3,40 +3,48 @@ const { logger } = require('../logger');
 const   redis    = require('redis');
 
 
-let redisSingleton;
+const CERT_LOCATION = process.env.REDIS_CERT_LOCATION || `${process.cwd()}/redis_ca.pem`;
+const clients       = {};
+logger.addScope('redis');
 
 
-async function initRedisClientSingleton() {
-    if (redisSingleton) return;
+function redisClientFactory({ host, port, password, subscriber=false } = {}) {
+    host      = host || process.env.REDIS_HOST;
+    port      = parseInt(port || process.env.REDIS_PORT);
+    password  = password || process.env.REDIS_PASSWORD
+    const key = `${host}:${port}:${Boolean(subscriber)}`;
 
-    const certLocation = process.env.REDIS_CERT_LOCATION || `${process.cwd()}/redis_ca.pem`;
-    const cert         = fs.readFileSync(certLocation)
+    if (clients[key]) return clients[key];
 
-    redisSingleton = redis.createClient({
-        socket: {
-            host: process.env.REDIS_HOST,
-            port: parseInt(process.env.REDIS_PORT),
-            tls:  true,
-            ca:   [cert]
-        },
-        password: process.env.REDIS_PASSWORD
-    });
+    const client = createClient(host, port, password);
+    clients[key] = client;
+
+    configureClient(client)
+    return client;
+}
 
-    redisSingleton.on('ready', (err) => {
-        logger.info('Redis Client Connected');
+
+function createClient(host, port, password) {
+    const ca     = fs.readFileSync(CERT_LOCATION)
+    const socket = { host, port, ca, tls:  true, };
+    return redis.createClient({ socket, password });
+}
+
+
+function configureClient(client) {
+    client.on('ready', (err) => {
+        logger.redis.info('Connected');
     });
-    redisSingleton .on('error', (err) => {
+    client.on('error', (err) => {
         const errno = err.errno ? ` (${err.errno})` : '';
-        logger.error(`Redis: ${err.message}${errno}`);
+        logger.redis.error(`${err.message}${errno}`);
     });
 
-    await redisSingleton.connect();
-}
-
 
-async function redisClientFactory() {
-    await initRedisClientSingleton();
-    return redisSingleton;
+    const connect = client.connect.bind(client);
+    client.connect = async function() {
+        if (!this.isReady) await connect();
+    }
 }