@vida-global/core 2.3.6 → 2.4.1

package/index.js

@@ -1,11 +1,12 @@
-const   ActiveRecord    = require('./lib/activeRecord');
-const   httpLibs        = require('./lib/http/client');
-const   APM              = require('./lib/apm');
-const   JobQueue        = require('./lib/jobQueue');
-const { logger }        = require('./lib/logger');
-const   redisLibs       = require('./lib/redis');
-const   cacheLibs       = require('./lib/cache');
-const   serverLibs      = require('./lib/server');
+const   ActiveRecord      = require('./lib/activeRecord');
+const   httpLibs          = require('./lib/http/client');
+const   APM               = require('./lib/apm');
+const { AppDependencies } = require('./lib/appDependencies');
+const   JobQueue          = require('./lib/jobQueue');
+const { logger }          = require('./lib/logger');
+const   redisLibs         = require('./lib/redis');
+const   cacheLibs         = require('./lib/cache');
+const   serverLibs        = require('./lib/server');
 
 
 const {
@@ -27,6 +28,7 @@ module.exports = {
     ...redisLibs,
     ...serverErrorsToExport,
     ApiDocsGenerator,
+    AppDependencies,
     VidaServer,
     VidaServerController,
 };

package/lib/appDependencies/index.js

@@ -0,0 +1,48 @@
+class AppDependencies {
+    #dependencies    = {};
+    #controllerClass = null;
+
+
+    installOn(controllerClass) {
+        this.#controllerClass = controllerClass;
+        for (const name of Object.keys(this.#dependencies)) this.#defineAccessor(name);
+    }
+
+
+    setDependency(name, dependency) {
+        this.#dependencies[name] = dependency;
+        this.#defineAccessor(name);
+    }
+
+
+    get(name) {
+        return this.#dependencies[name] ?? null;
+    }
+
+
+    // Mainly for tests, which register their own doubles and must not leak them into the next one.
+    reset() {
+        this.#dependencies = {};
+    }
+
+
+    // Registering after `installOn` still defines the accessor, so wiring order does not matter.
+    // An existing property is left alone: a controller that defines the name itself wins.
+    #defineAccessor(name) {
+        if (!this.#controllerClass) return;
+
+        const prototype = this.#controllerClass.prototype;
+        if (Object.getOwnPropertyDescriptor(prototype, name)) return;
+
+        const dependencies = this;
+        Object.defineProperty(prototype, name, {
+            configurable: false,
+            get() { return dependencies.get(name); }
+        });
+    }
+}
+
+
+module.exports = {
+    AppDependencies: new AppDependencies()
+};

package/lib/http/README.md

@@ -54,7 +54,7 @@ const response = await client.getTickets({ page: 1, pageSize: 25 });
 | `put(endpoint, opts)` | `body`, `headers`, `timeout`, `signal` |
 | `delete(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal` |
 
-For `GET`/`DELETE`, `requestParams` becomes the query string. For `POST`/`PUT`, `body` becomes the request body (serialized as JSON by default).
+For `GET`/`DELETE`, `requestParams` becomes the query string. For `POST`/`PUT`, `body` becomes the request body. Object bodies are serialized as JSON; string bodies are sent unchanged.
 
 
 ### Query string encoding

package/lib/http/client.js

@@ -37,7 +37,7 @@ class HttpClient {
 
         endpoint            = `${this.urlRoot}${endpoint}`;
         const url           = new URL(endpoint);
-        const requestData   = structuredClone(requestParams || body || {});
+        const requestData   = structuredClone(requestParams ?? body ?? {});
         const payload       = this.#prepareRequestPayload(url, requestData, headers, method, timeout, signal);
 
         try {
@@ -124,7 +124,7 @@ class HttpClient {
 
     #addRequestData(payload, requestData, method, url, headers) {
         if (method == "POST" || method == "PUT") {
-            payload.body = JSON.stringify(requestData || {});
+            payload.body = typeof requestData == 'string' ? requestData : JSON.stringify(requestData);
         } else {
             this.#addUrlRequestData(url, requestData, headers);
         }

package/lib/jobQueue/README.md

@@ -53,6 +53,9 @@ Inside `run()`:
 - `this.id` — UUID for the job.
 - `this.logger` — child logger tagged with the job's id.
 - `await this.updateProgress(value)` — pushes progress to BullMQ for observation.
+- `this.attempt`, `this.maxAttempts`, `this.isFinalAttempt` — which try this is. Use
+  `isFinalAttempt` when a failure needs cleaning up once the retries are spent, rather than on every
+  attempt. Run outside a worker they read as a single final attempt, since nothing will retry.
 
 Enqueue a job with `queueJob`:
 
@@ -107,12 +110,48 @@ class CatchAllJob extends AbstractJob {
 ```
 
 
-## Concurrency
+## Concurrency, queue maximums and rate limits
 
-A single worker can process multiple jobs in parallel. Set the `WORKER_CONCURRENCY` environment variable to raise the limit (default `1`).
+```js
+const worker = new JobQueue.Worker('myQueue', {
+    concurrency:        5,                          // jobs at once in THIS worker
+    maximumConcurrency: 8,                          // jobs at once across EVERY worker on the queue
+    rateLimit:          { max: 100, duration: 60000 } // jobs per period across every worker
+});
+```
+
+**`concurrency`** is per worker. Two processes at 5 each run 10 at once. It falls back to the
+`WORKER_CONCURRENCY` environment variable, then to 1, so a process running a single worker can be
+configured without touching code.
+
+**`maximumConcurrency`** is the cap across every worker on the queue, enforced by Redis, so it does
+what concurrency alone cannot. It is applied on `listen()` and **persists** — a worker started
+without one clears it, so the value in Redis always matches the code running.
+
+**`rateLimit`** is also enforced queue-wide, through a Redis key on the queue. Every worker on a
+queue therefore shares one limit, and two workers passing different values will fight — the last to
+start wins. A job that needs its own rate belongs on its own queue. There is no per-job or per-group
+rate limiting in the open source BullMQ; that is a Pro feature.
+
+Both `maximumConcurrency` and `rateLimit` are logged when applied, so the value in force is visible
+in the worker's own output.
+
+
+## Shutting down
 
-```sh
-WORKER_CONCURRENCY=10 node bin/worker.js
+`close()` waits for jobs already in flight before resolving, which is what stops a deploy killing a
+job halfway through. Core installs no signal handlers — the application owns the process:
+
+```js
+const worker = new JobQueue.Worker('myQueue');
+worker.listen();
+
+for (const signal of ['SIGTERM', 'SIGINT']) {
+    process.on(signal, async () => {
+        await worker.close();
+        process.exit(0);
+    });
+}
 ```
 
 
@@ -130,27 +169,27 @@ await queue.close();          // close the connection (call before process exit)
 await queue.pause();          // stop dispatching work
 await queue.resume();         // resume after pause
 
-await queue.numQueuedJobs();  // number of jobs waiting
+await queue.getJobCounts();   // { waiting, active, completed, failed, delayed, paused }
+await queue.numWorkers();     // number of workers currently attached
+
 await queue.getQueuedJobs();  // formatted snapshot of waiting jobs
-await queue.numActiveJobs();
 await queue.getActiveJobs();
-await queue.numFailedJobs();
 await queue.getFailedJobs();
+await queue.getJob(id);       // one job, or null
 
-await queue.clearFailedJobs();    // drop failed-job data (do this periodically)
-await queue.clearCompletedJobs(); // drop completed-job data
-await queue.clearQueuedJobs();    // drain all queued jobs
+await queue.clean(grace, limit, type); // remove finished jobs older than `grace` ms
+await queue.clearQueuedJobs();         // drain all queued jobs
 
-await queue.numWorkers();         // number of workers currently attached
+await queue.setMaximumConcurrency(8);  // usually left to the Worker option above
+await queue.maximumConcurrency();
+await queue.removeMaximumConcurrency();
 ```
 
-Each `get*Jobs` result entry has this shape:
+`clean` takes BullMQ's argument order, which is `(grace, limit, type)`. Bull's was
+`(grace, type, limit)` — worth checking when porting a call.
+
+Each `get*Jobs` and `getJob` result has this shape:
 
 ```js
 { queueName, name, args, id, attemptsMade, attemptsStarted, progress, failedReason }
 ```
-
-
-## TODO
-
-- **Rate limiting.** Pre-job rate limiting is not yet wired up.

package/lib/jobQueue/abstractJob.js

@@ -7,9 +7,16 @@ const DEFAULT_NUM_RETRIES         = 3;
 const DEFAULT_RETRY_BACKOFF_TYPE  = 'exponential';
 const DEFAULT_RETRY_BACKOFF_DELAY = 2000;
 
+// BullMQ keeps every finished job record forever unless told otherwise, so without these Redis
+// grows without bound. A bounded window keeps recent history inspectable and lets the rest go.
+const DEFAULT_KEEP_COMPLETED = { age: 86400,  count: 1000 }; // a day, or the last thousand
+const DEFAULT_KEEP_FAILED    = { age: 604800 };              // a week
+
 
 class AbstractJob {
+    #attempt;
     #id;
+    #maxAttempts;
     #updateProgress;
 
 
@@ -36,9 +43,11 @@ class AbstractJob {
     }
 
 
-    async _run(id, args, updateProgress) {
+    async _run(id, args, updateProgress, { attempt = 1, maxAttempts = 1 } = {}) {
         this.#id             = id;
         this.#updateProgress = updateProgress;
+        this.#attempt        = attempt;
+        this.#maxAttempts    = maxAttempts;
         await this.run(...args);
     }
 
@@ -48,6 +57,21 @@ class AbstractJob {
     }
 
 
+    get attempt() {
+        return this.#attempt;
+    }
+
+
+    get maxAttempts() {
+        return this.#maxAttempts;
+    }
+
+
+    get isFinalAttempt() {
+        return this.#attempt >= this.#maxAttempts;
+    }
+
+
     get loggerLibrary() {
         return logger;
     }
@@ -59,7 +83,7 @@ class AbstractJob {
 
 
     get logger() {
-        return this.loggerLibrary[this.constructor.name].child(this.id);
+        return this.loggerLibrary.scope(this.constructor.name).child(this.id);
     }
 
 
@@ -90,7 +114,9 @@ class AbstractJob {
             backoff:  {
                 type:  this.retryBackoffType,
                 delay: this.retryBackoffDelay
-            }
+            },
+            removeOnComplete: this.keepCompleted,
+            removeOnFail:     this.keepFailed
         }
 
         return settings;
@@ -102,6 +128,16 @@ class AbstractJob {
     }
 
 
+    static get keepCompleted() {
+        return DEFAULT_KEEP_COMPLETED;
+    }
+
+
+    static get keepFailed() {
+        return DEFAULT_KEEP_FAILED;
+    }
+
+
     static get retryBackoffType() {
         return DEFAULT_RETRY_BACKOFF_TYPE;
     }

package/lib/jobQueue/queue.js

@@ -1,7 +1,6 @@
 const { AbstractJobComponent } = require('./abstractJobComponent');
 const   BullMQ                 = require('bullmq');
 const { logger }               = require('../logger');
-const { randomUUID }           = require('crypto');
 
 
 logger.addScope('queue');
@@ -22,12 +21,10 @@ class Queue extends AbstractJobComponent {
     get loggerLibrary() { return logger }
     get logger()        { return this.loggerLibrary.queue.child(this.fullQueueName) }
     get queueClass()    { return BullMQ.Queue }
-    get idGenerator()   { return randomUUID }
 
 
     async queueJob(job, args, settings) {
-        const id = this.idGenerator();
-        await this.#queue.add(job.name, { id, args }, settings);
+        await this.#queue.add(job.name, { args }, settings);
     }
 
 
@@ -67,11 +64,29 @@ class Queue extends AbstractJobComponent {
     }
 
 
+    /***********************************************************************************************
+    * LIMITS
+    ***********************************************************************************************/
+    async setMaximumConcurrency(maximum) {
+        await this.#queue.setGlobalConcurrency(maximum);
+    }
+
+
+    async maximumConcurrency() {
+        return await this.#queue.getGlobalConcurrency();
+    }
+
+
+    async removeMaximumConcurrency() {
+        await this.#queue.removeGlobalConcurrency();
+    }
+
+
     /***********************************************************************************************
     * JOBS
     ***********************************************************************************************/
-    async numQueuedJobs() {
-        return await this.#queue.count();
+    async getJobCounts() {
+        return await this.#queue.getJobCounts();
     }
 
 
@@ -81,35 +96,28 @@ class Queue extends AbstractJobComponent {
     }
 
 
-    async numActiveJobs() {
-        return await this.#queue.getActiveCount();
-    }
-    
-
     async getActiveJobs() {
         const jobs = await this.#queue.getActive();
         return this.#_formatJobsData(jobs);
     }
 
 
-    async numFailedJobs() {
-        return await this.#queue.getFailedCount();
-    }
-    
-
     async getFailedJobs() {
         const jobs = await this.#queue.getFailed();
         return this.#_formatJobsData(jobs);
     }
 
 
-    async clearFailedJobs() {
-        await this.#queue.clean(0, null, "failed");
+    async getJob(id) {
+        const job = await this.#queue.getJob(id);
+        return job ? this.#_formatJobData(job) : null;
     }
 
 
-    async clearCompletedJobs() {
-        await this.#queue.clean(0, null, "completed");
+    // `grace` is in milliseconds and only jobs older than that are removed. Note the argument order
+    // differs from bull's `clean(grace, type, limit)`.
+    async clean(grace, limit, type) {
+        return await this.#queue.clean(grace, limit, type);
     }
 
 

package/lib/jobQueue/worker.js

@@ -1,6 +1,7 @@
 const   APM                    = require('../apm');
 const { AbstractJobComponent } = require('./abstractJobComponent');
 const   BullMQ                 = require('bullmq');
+const { getQueue }             = require('./queue');
 const { JobImporter }          = require('./jobImporter');
 const { logger }               = require('../logger');
 const { randomUUID }           = require('crypto');
@@ -21,36 +22,51 @@ let registeredJobs;
 
 class Worker extends AbstractJobComponent {
     #defaultJob;
+    #options;
     #queueName;
     #_worker;
 
 
-    constructor(queueName) {
+    constructor(queueName, options = {}) {
         super();
         this.#queueName = queueName || process.env.WORKER_QUEUE_NAME;
+        this.#options   = options;
     }
 
 
     async listen() {
         await this.#registerJobs();
+        await this.#applyQueueLimits();
         this.logger.info(`Worker is listening on queue ${this.fullQueueName}`);
         await this.#worker.run();
     }
 
 
+    // Waits for jobs already in flight. Core installs no signal handlers of its own — the
+    // application owns the process and decides when this is called.
+    async close() {
+        await this.#worker.close();
+    }
+
+
     #registerJobs() {
-        if (registeredJobs) return;
+        registeredJobs ||= this.#importJobs();
+
+        this.#defaultJob = Object.values(registeredJobs)
+            .find(job => job.defaultFor == this.queueName) || null;
+    }
+
 
-        const importer   = new JobImporter(this.jobDirectories);
-        const jobs       = importer.jobs;
-        registeredJobs   = Object.fromEntries(jobs.map(job => [job.name, job]));
-        this.#defaultJob = jobs.find(job => job.defaultFor == this.queueName) || null;
+    #importJobs() {
+        const jobs = new JobImporter(this.jobDirectories).jobs;
 
         if (process.env.NODE_ENV != 'test') {
             for (const job of jobs) {
                 this.logger.verbose(`JOB: ${job.name}`);
             }
         }
+
+        return Object.fromEntries(jobs.map(job => [job.name, job]));
     }
 
 
@@ -100,13 +116,24 @@ class Worker extends AbstractJobComponent {
             concurrency: this.concurrency,
             connection:  this.redisConnectionDetails
         };
+        if (this.rateLimit) options.limiter = this.rateLimit;
 
         return options;
     }
 
 
+    // How many jobs this one worker runs at once. The constructor option wins so several workers in
+    // a process can differ; the environment variable is the default for any that says nothing.
     get concurrency() {
-      return parseInt(process.env.WORKER_CONCURRENCY || 1);
+        return parseInt(this.#options.concurrency || process.env.WORKER_CONCURRENCY || 1);
+    }
+
+
+    // `{max, duration}`. BullMQ enforces this through a Redis key on the queue, so it is shared by
+    // every worker listening on it — two workers with different values fight and the last to start
+    // wins. A job that needs its own rate gets its own queue.
+    get rateLimit() {
+        return this.#options.rateLimit || null;
     }
 
 
@@ -115,6 +142,28 @@ class Worker extends AbstractJobComponent {
     }
 
 
+    get queue() {
+        return getQueue(this.queueName);
+    }
+
+
+    async #applyQueueLimits() {
+        const maximumConcurrency = this.#options.maximumConcurrency ?? null;
+
+        if (maximumConcurrency === null) {
+            await this.queue.removeMaximumConcurrency();
+        } else {
+            await this.queue.setMaximumConcurrency(maximumConcurrency);
+            this.logger.info(`Queue ${this.fullQueueName} maximum concurrency is ${maximumConcurrency}`);
+        }
+
+        if (this.rateLimit) {
+            const { max, duration } = this.rateLimit;
+            this.logger.info(`Queue ${this.fullQueueName} rate limit is ${max} per ${duration}ms`);
+        }
+    }
+
+
     /***********************************************************************************************
     * PROCESS JOB
     ***********************************************************************************************/
@@ -130,13 +179,12 @@ class Worker extends AbstractJobComponent {
 
 
     #buildJobContext(bullJob, jobClass) {
-        const { id, args } = this.#dataForJob(bullJob);
         return {
             bullJob,
             jobClass,
             jobName:     jobClass.name,
-            id,
-            args,
+            id:          bullJob.id,
+            args:        this.#argsForJob(bullJob),
             queueName:   this.queueName,
             attempt:     bullJob.attemptsMade + 1,
             maxAttempts: bullJob.opts.attempts,
@@ -162,11 +210,14 @@ class Worker extends AbstractJobComponent {
 
 
     async #executeJob(context) {
-        const { bullJob, jobClass, id, args, jobName } = context;
+        const { bullJob, jobClass, id, args, jobName, attempt, maxAttempts } = context;
         const job = new jobClass();
         await this.apm.startSegment(`job:${jobName}`,
                                     true,
-                                    () => job._run(id, args, bullJob.updateProgress.bind(bullJob)));
+                                    () => job._run(id,
+                                                   args,
+                                                   bullJob.updateProgress.bind(bullJob),
+                                                   { attempt, maxAttempts }));
     }
 
 
@@ -227,8 +278,8 @@ class Worker extends AbstractJobComponent {
     }
 
 
-    #handleJobCompletion({ name, data }, duration) {
-        const logger = this.#loggerForJob({ name, data });
+    #handleJobCompletion(job, duration) {
+        const logger = this.#loggerForJob(job);
         logger.debug(`Completed in ${duration}ms`);
     }
 
@@ -240,8 +291,6 @@ class Worker extends AbstractJobComponent {
 
 
     #handleJobFailure(job, err) {
-        const log = `Job Failed (Attempt ${job.attemptsMade}/${job.opts.attempts}): ${job.name} - ${err}`;
-
         const lastRetry = job.attemptsMade == job.opts.attempts;
         const logger    = this.#loggerForJob(job);
         logger.debug(`ERROR (${job.attemptsMade}/${job.opts.attempts}) ${err}`);
@@ -261,8 +310,7 @@ class Worker extends AbstractJobComponent {
 
 
     #loggerForJob(bullJob) {
-        const { id } = this.#dataForJob(bullJob);
-        return this.logger.child(id);
+        return this.logger.child(bullJob.id);
     }
 
 
@@ -272,12 +320,11 @@ class Worker extends AbstractJobComponent {
     }
 
 
-    #dataForJob({ name, data }) {
+    #argsForJob({ name, data }) {
         if (name == '__default__') { // legacy for vida.live. Remove when vida.live has been fully migrated
-            return { args: [data] };
+            return [data];
         } else {
-            const { id, args } = data;
-            return { id, args };
+            return data.args;
         }
     }
 }

package/lib/logger/README.md

@@ -55,6 +55,18 @@ logger.bar.error('my log');
 // [2026-04-30 16:16:12.797 -0700][ERROR][BAR] my log
 ```
 
+`scope(name)` registers on first use and returns the logger, which is how a scope gets named after
+something that is not lowercase, such as a class. The scope answers to both spellings afterwards:
+
+```js
+logger.scope('FinalizeSiprecCall').info('my log');
+// [2026-04-30 16:16:12.797 -0700][INFO][FINALIZESIPRECCALL] my log
+
+logger.FinalizeSiprecCall === logger.finalizesipreccall; // true
+```
+
+The tag is always the lowercased name, so log lines read the same however the scope was spelled.
+
 ### Child loggers with IDs
 
 `child(id)` returns a logger that tags every line with the given id. This is the standard pattern for per-request logging — the server framework already calls `logger.child(requestId)` for every controller instance.

package/lib/logger/index.js

@@ -56,9 +56,8 @@ class Logger {
         const scopes   = this.scopes.map(scope => `[${scope.toUpperCase()}]`).join('');
         let prefix     = `[${info.timestamp}][${level}]${scopes}`;
         if (id) prefix = `${prefix}[${id}]`;
-        info.message   = `${prefix} ${info.message}`;
 
-        return info;
+        return { ...info, message: `${prefix} ${info.message}` };
     }
 
 
@@ -73,16 +72,33 @@ class Logger {
     }
 
 
+    // The scope is reachable both as it was given and lowercased, so a scope named after a class
+    // answers to `logger.FinalizeSiprecCall` and `logger.finalizesiprecall` alike.
     addScope(scope) {
-        scope = scope.toLowerCase();
-        const existing = this[scope];
+        const scopeName = scope.toLowerCase();
+        const existing  = this[scopeName];
         if (existing) {
-            if (!existing instanceof Logger) throw `Property already exists at ${scope}`;
-            return;
+            if (!(existing instanceof Logger)) throw new Error(`Property already exists at ${scopeName}`);
+            return existing;
         }
 
-        const logger = this.#createLogger(this.scopes.concat([scope]), this.id);
-        Object.defineProperty(this, scope, { get: () => logger });
+        const logger = this.#createLogger(this.scopes.concat([scopeName]), this.id);
+        for (const name of [scopeName, scope]) this.#defineScope(name, logger);
+        return logger;
+    }
+
+
+    scope(name) {
+        return this.addScope(name);
+    }
+
+
+    #defineScope(name, logger) {
+        const existing = this[name];
+        if (existing instanceof Logger) return;
+        if (existing) throw new Error(`Property already exists at ${name}`);
+
+        Object.defineProperty(this, name, { get: () => logger });
     }