@vida-global/core 1.4.5 → 2.0.0

package/lib/jobQueue/queue.js

@@ -0,0 +1,161 @@
+const { AbstractJobComponent } = require('./abstractJobComponent');
+const   BullMQ                 = require('bullmq');
+const { logger }               = require('../logger');
+const { randomUUID }           = require('crypto');
+
+
+logger.addScope('queue');
+
+
+class Queue extends AbstractJobComponent {
+    #_queue;
+    #queueName;
+
+
+    constructor(queueName) {
+        super();
+        this.#queueName = queueName;
+    }
+
+
+    get queueName() { return this.#queueName   }
+    get logger()    { return logger.queue.child(this.fullQueueName) }
+
+
+    async queueJob(job, args, settings) {
+        const id = randomUUID();
+        await this.#queue.add(job.name, { id, args }, settings);
+    }
+
+
+    /***********************************************************************************************
+    * QUEUE SETUP
+    ***********************************************************************************************/
+    get #queue() {
+        if (this.#_queue) return this.#_queue;
+
+        this.#_queue            = new BullMQ.Queue(this.fullQueueName, this.queueOptions);
+        this.#_queue.on('error', this.#handleError.bind(this));
+
+        return this.#_queue
+    }
+
+
+    get queueOptions() {
+        return { connection: this.redisConnectionDetails };
+    }
+
+
+    /***********************************************************************************************
+    * QUEUE STATUS
+    ***********************************************************************************************/
+    async close() {
+        await this.#queue.close();
+    }
+
+
+    async pause() {
+        await this.#queue.pause();
+    }
+
+
+    async resume() {
+        await this.#queue.resume();
+    }
+
+
+    /***********************************************************************************************
+    * JOBS
+    ***********************************************************************************************/
+    async numQueuedJobs() {
+        return await this.#queue.count();
+    }
+
+
+    async getQueuedJobs() {
+        const jobs = await this.#queue.getJobs(["paused", "wait", "prioritized", "delayed"]);
+        return this.#_formatJobsData(jobs);
+    }
+
+
+    async numActiveJobs() {
+        return await this.#queue.getActiveCount();
+    }
+    
+
+    async getActiveJobs() {
+        const jobs = await this.#queue.getActive();
+        return this.#_formatJobsData(jobs);
+    }
+
+
+    async numFailedJobs() {
+        return await this.#queue.getFailedCount();
+    }
+    
+
+    async getFailedJobs() {
+        const jobs = await this.#queue.getFailed();
+        return this.#_formatJobsData(jobs);
+    }
+
+
+    async clearFailedJobs() {
+        await this.#queue.clean(0, null, "failed");
+    }
+
+
+    async clearCompletedJobs() {
+        await this.#queue.clean(0, null, "completed");
+    }
+
+
+    async clearQueuedJobs() {
+        return await this.#queue.drain();
+    }
+
+
+    async numWorkers() {
+        const workerData = await this.#queue.getWorkers();
+        return workerData.length;
+    }
+
+
+    #_formatJobsData(jobs) {
+        return jobs.map((job) => this.#_formatJobData(job));
+    }
+
+
+    #_formatJobData(job) {
+        return {
+            queueName:       job.queue.name,
+            name:            job.name,
+            args:            job.data,
+            id:              job.id,
+            attemptsMade:    job.attemptsMade,
+            attemptsStarted: job.attemptsStarted,
+            progress:        job.progress,
+            failedReason:    job.failedReason
+        }
+    }
+
+
+    /***********************************************************************************************
+    * ERROR HANDLING
+    ***********************************************************************************************/
+    #handleError() {
+        this.logger.error(...arguments);
+    }
+}
+
+
+const queues = {};
+function getQueue(name) {
+    const fullName            = Queue.fullQueueName(name);
+    return queues[fullName] ||= new Queue(name);
+}
+
+
+module.exports = {
+    getQueue
+}

package/lib/jobQueue/worker.js

@@ -0,0 +1,273 @@
+const   APM                    = require('../apm');
+const { AbstractJobComponent } = require('./abstractJobComponent');
+const   BullMQ                 = require('bullmq');
+const { JobImporter }          = require('./jobImporter');
+const { logger }               = require('../logger');
+const { randomUUID }           = require('crypto');
+
+
+const JOB_COMPLETED_EVENT = 'JobCompleted';
+const JOB_FAILED_EVENT    = 'JobFailed';
+
+
+function durationMetricName(queueName, jobName) {
+    return `Custom/Worker/${queueName}/${jobName}/duration`;
+}
+
+
+logger.addScope('worker');
+let registeredJobs;
+
+
+class Worker extends AbstractJobComponent {
+    #defaultJob;
+    #queueName;
+    #_worker;
+
+
+    constructor(queueName) {
+        super();
+        this.#queueName = queueName || process.env.WORKER_QUEUE_NAME;
+    }
+
+
+    async listen() {
+        await this.#registerJobs();
+        this.logger.info(`Worker is listening on queue ${this.fullQueueName}`);
+        await this.#worker.run();
+    }
+
+
+    #registerJobs() {
+        if (registeredJobs) return;
+
+        const importer   = new JobImporter(this.jobDirectories);
+        const jobs       = importer.jobs;
+        registeredJobs   = Object.fromEntries(jobs.map(job => [job.name, job]));
+        this.#defaultJob = jobs.find(job => job.defaultFor == this.queueName) || null;
+
+        if (process.env.NODE_ENV != 'test') {
+            for (const job of jobs) {
+                this.logger.verbose(`JOB: ${job.name}`);
+            }
+        }
+    }
+
+
+    get logger() {
+        return logger.worker;
+    }
+
+
+    get jobDirectories() {
+        return [`${process.cwd()}/jobs`];
+    }
+
+
+    /***********************************************************************************************
+    * WORKER SETUP
+    ***********************************************************************************************/
+    get #worker() {
+        if (this.#_worker) return this.#_worker;
+
+        this.#_worker = new BullMQ.Worker(this.fullQueueName,
+                                          this.processJob.bind(this),
+                                          this.workerOptions);
+        this.#setupEventHandling();
+
+        return this.#_worker;
+    }
+
+
+    get workerOptions() {
+        const options = {
+            autorun:     false,
+            concurrency: this.concurrency,
+            connection:  this.redisConnectionDetails
+        };
+
+        return options;
+    }
+
+
+    get concurrency() {
+      return parseInt(process.env.WORKER_CONCURRENCY || 1);
+    }
+
+
+    get queueName() {
+        return this.#queueName;
+    }
+
+
+    /***********************************************************************************************
+    * PROCESS JOB
+    ***********************************************************************************************/
+    async processJob(bullJob) {
+        const jobClass = this.#getJobClass(bullJob.name);
+        if (!jobClass) throw `Unknown Job: ${bullJob.name}`;
+
+        const context = this.#buildJobContext(bullJob, jobClass);
+        return await APM.startBackgroundTransaction(context.jobName,
+                                                    this.fullQueueName,
+                                                    () => this.#runJob(context));
+    }
+
+
+    #buildJobContext(bullJob, jobClass) {
+        const { id, args } = this.#dataForJob(bullJob);
+        return {
+            bullJob,
+            jobClass,
+            jobName:     jobClass.name,
+            id,
+            args,
+            queueName:   this.queueName,
+            attempt:     bullJob.attemptsMade + 1,
+            maxAttempts: bullJob.opts.attempts,
+        };
+    }
+
+
+    async #runJob(context) {
+        this.#addJobAttributes(context);
+
+        const startedAt = Date.now();
+        try {
+            await this.#executeJob(context);
+        } catch(err) {
+            this.#reportJobError(err, context);
+            throw err;
+        }
+
+        const duration = Date.now() - startedAt;
+        this.#recordJobSuccess(context, duration);
+        return duration;
+    }
+
+
+    async #executeJob(context) {
+        const { bullJob, jobClass, id, args, jobName } = context;
+        const job = new jobClass();
+        await APM.startSegment(`job:${jobName}`,
+                               true,
+                               () => job._run(id, args, bullJob.updateProgress.bind(bullJob)));
+    }
+
+
+    #addJobAttributes(context) {
+        APM.addCustomAttribute('jobId',       context.id);
+        APM.addCustomAttribute('jobName',     context.jobName);
+        APM.addCustomAttribute('queueName',   context.queueName);
+        APM.addCustomAttribute('attempt',     context.attempt);
+        APM.addCustomAttribute('maxAttempts', context.maxAttempts);
+    }
+
+
+    #reportJobError(err, context) {
+        const isFinalRetry = context.attempt >= context.maxAttempts;
+
+        APM.reportError(err, {
+            jobId:       context.id,
+            jobName:     context.jobName,
+            queueName:   context.queueName,
+            attempt:     context.attempt,
+            maxAttempts: context.maxAttempts,
+            isFinalRetry,
+        });
+
+        if (isFinalRetry) this.#recordJobFailedEvent(context, err);
+    }
+
+
+    #recordJobFailedEvent(context, err) {
+        APM.recordCustomEvent(JOB_FAILED_EVENT, {
+            jobName:   context.jobName,
+            queueName: context.queueName,
+            attempts:  context.attempt,
+            error:     err.message || String(err),
+        });
+    }
+
+
+    #recordJobSuccess(context, duration) {
+        APM.recordCustomEvent(JOB_COMPLETED_EVENT, {
+            jobName:    context.jobName,
+            queueName:  context.queueName,
+            durationMs: duration,
+            attempts:   context.attempt,
+        });
+        APM.recordMetric(durationMetricName(context.queueName, context.jobName), duration);
+    }
+
+
+    /***********************************************************************************************
+    * EVENT HANDLING
+    ***********************************************************************************************/
+    #setupEventHandling() {
+        this.#_worker.on('completed', this.#handleJobCompletion.bind(this));
+        this.#_worker.on('progress',  this.#handleJobProgress.bind(this));
+        this.#_worker.on('failed',    this.#handleJobFailure.bind(this));
+        this.#_worker.on('error',     this.#handleError.bind(this));
+    }
+
+
+    #handleJobCompletion({ name, data }, duration) {
+        const logger = this.#loggerForJob({ name, data });
+        logger.debug(`Completed in ${duration}ms`);
+    }
+
+
+    #handleJobProgress(job, data) {
+        const logger = this.#loggerForJob(job);
+        logger.debug(data);
+    }
+
+
+    #handleJobFailure(job, err) {
+        const log = `Job Failed (Attempt ${job.attemptsMade}/${job.opts.attempts}): ${job.name} - ${err}`;
+
+        const lastRetry = job.attemptsMade == job.opts.attempts;
+        const logger    = this.#loggerForJob(job);
+        logger.debug(`ERROR (${job.attemptsMade}/${job.opts.attempts}) ${err}`);
+
+        if (lastRetry) {
+            throw err;
+        }
+    }
+
+
+    #handleError(err) {
+        this.logger.error(err);
+        if (process.env.NODE_ENV == 'development') {
+            throw err;
+        }
+    }
+
+
+    #loggerForJob(bullJob) {
+        const { id } = this.#dataForJob(bullJob);
+        return logger.worker.child(id);
+    }
+
+
+    #getJobClass(name) {
+        if (registeredJobs[name]) return registeredJobs[name];
+        return this.#defaultJob;
+    }
+
+
+    #dataForJob({ name, data }) {
+        if (name == '__default__') { // legacy for vida.live. Remove when vida.live has been fully migrated
+            return { args: [data] };
+        } else {
+            const { id, args } = data;
+            return { id, args };
+        }
+    }
+}
+
+
+module.exports = {
+    Worker
+}

package/lib/logger/README.md

@@ -1,2 +1,33 @@
-## Logger ##
-A standard logger to be used across all Vida applications. Currently, a simple implementation of `pino`, supports the full `pino` API.
+# Logger
+A simple logger that support multiple scopes and log levels. Set the `LOG_LEVEL` environment variable to determine what log levels are output.
+
+## Scopes
+```
+logger.addScope('foo');
+logger.addScope('bar');
+
+logger.foo.debug('my log');
+// [2026-04-30 16:16:12.797 -0700][DEBUG][FOO] my log
+
+logger.bar.verbose('my log');
+// [2026-04-30 16:16:12.797 -0700][VERBOSE][BAR] my log
+
+logger.bar.info('my log');
+// [2026-04-30 16:16:12.797 -0700][INFO][BAR] my log
+
+logger.bar.warn('my log');
+// [2026-04-30 16:16:12.797 -0700][WARN][BAR] my log
+
+logger.bar.error('my log');
+// [2026-04-30 16:16:12.797 -0700][ERROR][BAR] my log
+```
+
+## IDs
+```
+logger.addScope('http');
+const requestId     = 123;
+const requestLogger = logger.http.child(requestId);
+
+requestLogger.debug('my log');
+// [2026-04-30 16:16:12.797 -0700][DEBUG][HTTP][123] my log
+``

package/lib/logger/index.js

@@ -1,25 +1,122 @@
-const pino = require('pino');
-
-const config = {
-    level: process.env.LOG_LEVEL || 'info',
-    customLevels: {
-      test: 1000
-    }
-};
-
-if (!process.env.ENV_VERCEL) {
-    config.transport = {
-        target: 'pino-pretty',
-        options: {
-            colorize: true,
-            ignore: 'pid,hostname',
-            translateTime: 'SYS:standard',
-            messageFormat: '{msg}',
+const   winston                           = require('winston');
+const { colorize, combine, printf, timestamp } = winston.format;
+
+
+const loggers = {};
+
+
+class Logger {
+    #id;
+    #logger;
+    #scopes;
+
+
+    constructor(scopes, id) {
+        this.#id     = id;
+        scopes       = Array.isArray(scopes) ? scopes : [scopes];
+        this.#scopes = scopes.filter(Boolean).map(scope => scope.toLowerCase());
+        this.#logger = winston.createLogger(this.loggerOptions);
+    }
+
+
+    get scopes() { return this.#scopes; }
+    get id()     { return this.#id; }
+
+
+    get loggerOptions() {
+        const format     = combine(...this.formatters);
+        const level      = this.level;
+        const transports = this.transports;
+
+        return { format, level, transports };
+    }
+
+
+    get formatters() {
+        const colors = {
+            debug:   'magenta',
+            verbose: 'green',
+            info:    'cyan',
+            warn:    'yellow',
+            error:   'red',
         }
-    };
+        const messageFormatter = winston.format(this.messageFormatter.bind(this));
+        return [
+            timestamp({format: 'YYYY-MM-DD HH:mm:ss.SSS ZZ'}),
+            messageFormatter(),
+            colorize({all: true, colors}),
+            printf(({ message }) => message)
+        ];
+    }
+
+
+    messageFormatter(info) {
+        const level    = info.level.toUpperCase()
+        const id       = this.id || info.meta?.id;
+        const scopes   = this.scopes.map(scope => `[${scope.toUpperCase()}]`).join('');
+        let prefix     = `[${info.timestamp}][${level}]${scopes}`;
+        if (id) prefix = `${prefix}[${id}]`;
+        info.message   = `${prefix} ${info.message}`;
+
+        return info;
+    }
+
+
+    get level() {
+        return process.env.LOG_LEVEL || 'info';
+    }
+
+
+    get transports() {
+        const transports = [ new winston.transports.Console() ];
+        return transports;
+    }
+
+
+    addScope(scope) {
+        scope = scope.toLowerCase();
+        const existing = this[scope];
+        if (existing) {
+            if (!existing instanceof Logger) throw `Property already exists at ${scope}`;
+            return;
+        }
+
+        const logger = this.#createLogger(this.scopes.concat([scope]), this.id);
+        Object.defineProperty(this, scope, { get: () => logger });
+    }
+
+
+    child(id) {
+        return this.#createLogger(this.scopes, id);
+    }
+
+
+    #createLogger(scopes, id) {
+        scopes    = Array.isArray(scopes) ? scopes : [scopes];
+        scopes    = scopes.map(scope => scope.toLowerCase());
+        const key = `${scopes.join('|')}:${id}`;
+        if (loggers[key]) return loggers[key];
+
+        loggers[key] = new Logger(scopes, id);
+        return loggers[key];
+    }
+
+
+    debug(msg)   { this.#logger.debug(msg) }
+    error(msg)   { this.#logger.error(msg) }
+    info(msg)    { this.#logger.info(msg) }
+    silly(msg)   { this.#logger.silly(msg) }
+    verbose(msg) { this.#logger.verbose(msg) }
+    warn(msg)    { this.#logger.warn(msg) }
+
+    get _winstonLogger() { 
+        this.warn('Directly accessing the underlying winston logger should be avoided unless necessary');
+        return this.#logger
+    }
 }
 
-const logger = pino(config);
+
+const logger = new Logger();
 
 
-module.exports = { logger };
+module.exports = { logger, Logger };

package/lib/logger/serverMiddleware.js

@@ -0,0 +1,41 @@
+const expressWinston = require('express-winston');
+
+
+const msgFormatter = (req, res) => {
+    let msg;
+    if (req.controller) {
+        msg = `${req.controller.constructor.name}#${req.action}`;
+    } else {
+        msg = res.statusCode >= 500 ? 'request errored' : 'request completed';
+    }
+
+    const responseTime    = res.responseTime || '-';
+    const requestDetails  = `req: ${req.method} ${req.url.split('?')[0]} for ${req.ip}`;
+    const responseDetails = `res: statusCode=${res.statusCode} responseTime=${responseTime}ms`;
+
+    msg = `${msg}\n    ${requestDetails}\n    ${responseDetails}`;
+    if (res.error) {
+        msg = `${msg}\n${res.error.stack}`;
+    }
+
+    return msg;
+};
+
+
+let _middleware;
+
+function middleware(logger) {
+    if (_middleware) return _middleware;
+    _middleware = expressWinston.logger({
+        dynamicMeta:     (req, res) => ({id: req.id}),
+        level:           'debug',
+        msg:             msgFormatter,
+        winstonInstance: logger._winstonLogger,
+    });
+    return _middleware;
+};
+
+
+module.exports = {
+  middleware
+}