@vida-global/core 1.4.6 → 2.0.0

package/README.md

@@ -7,3 +7,4 @@ This package is intended to contain core elements used across all Vida Apps. Kee
 - [HttpClient](lib/http)
 - [Logger](lib/logger)
 - [VidaServer](lib/server)
+- [JobQueue](lib/jobQueue)

package/config/newrelic-config.js

@@ -0,0 +1,118 @@
+const path = require('path');
+
+
+/**
+ * Shared New Relic agent configuration for Vida applications.
+ *
+ * Apps consume this via a thin `newrelic.js` shim at the application root:
+ *
+ *   // <app>/newrelic.js
+ *   module.exports = require('@vida-global/core/newrelic-config');
+ *
+ * The New Relic Node.js agent looks for `newrelic.js` in process.cwd() at
+ * startup, so each app needs the shim — but the canonical config lives here.
+ *
+ * Per-app overrides should be expressed via the standard NEW_RELIC_* env vars
+ * (e.g. NEW_RELIC_APP_NAME, NEW_RELIC_LICENSE_KEY, NEW_RELIC_LOG_LEVEL).
+ */
+
+// App name + version are sourced from the consuming app's package.json.
+// process.cwd() is the app root because that is where the agent loads
+// newrelic.js from at startup.
+const appPackage = require(path.join(process.cwd(), 'package.json'));
+const isProd     = process.env.NODE_ENV == 'production'
+const envId      = isProd ? '' : ` [${process.env.NODE_ENV || 'dev'}]`;
+const appName    = process.env.NEW_RELIC_APP_NAME || `${appPackage.name}${envId}`;
+
+
+exports.config = {
+    app_name:    [appName.toLowerCase()],
+    license_key: process.env.NEW_RELIC_LICENSE_KEY,
+
+    // NR has no dedicated "version" field; surface it as a label so it shows
+    // up as a filterable facet in the UI and on entity metadata.
+    labels: `version:${appPackage.version}`,
+
+    /**
+     * Partial approximation of New Relic's High-Security Mode. The HSM toggle
+     * is not available on our account, so each constraint is set explicitly
+     * here. Real HSM also enforces these at the collector; this configuration
+     * only enforces them at the agent, so it is critical that nothing in this
+     * block is loosened without a compliance review.
+     *
+     * Custom events (`recordCustomEvent`) and custom attributes
+     * (`addCustomAttribute`) are intentionally LEFT ENABLED — vida-core relies
+     * on them for job-queue and controller telemetry. Treat those channels as
+     * a trusted egress path: only attach identifier-grade values, never raw
+     * user input, request bodies, or PII.
+     */
+    ssl:               true,
+    allow_all_headers: false,
+
+    transaction_tracer: {
+        enabled:    true,
+        record_sql: 'obfuscated',
+    },
+
+    slow_sql: { enabled: false },
+
+    message_tracer: {
+        segment_parameters: { enabled: false },
+    },
+
+    attributes: {
+        exclude: [
+            // Header redactions (belt-and-suspenders with allow_all_headers: false)
+            'request.headers.cookie',                                                                                                                                                                        
+            'request.headers.authorization',
+            'request.headers.proxyAuthorization',                                                                                                                                                            
+            'request.headers.setCookie*',                                                                                                                                                                    
+            'request.headers.x*',                     
+            'response.headers.cookie',                                                                                                                                                                       
+            'response.headers.authorization',                               
+            'response.headers.proxyAuthorization',    
+            'response.headers.setCookie*',                                                                                                                                                                   
+            'response.headers.x*',
+            // Query string and body parameters
+            'request.parameters.*',
+        ],
+    },
+
+    // Cross-service trace propagation. Required for traces to stitch across
+    // any other instrumented Vida services this app calls.
+    distributed_tracing: { enabled: true },
+
+    // Do not forward app logs through the APM agent. Logging is handled by
+    // the platform's log pipeline.
+    application_logging: { enabled: false },
+
+    /**
+     * Ignore routine, non-business routes so they don't dominate transaction
+     * lists or skew apdex. Patterns are anchored regexes matched against the
+     * request URL path.
+     *
+     *   /                          — vida-core SystemController hello world
+     *   /system/healthCheck        — vida-core SystemController health probe
+     *   /hubspot/workflows/health  — vida.live HubSpot integration health probe
+     *   /pushWorker.js             — vida.live static service-worker asset
+     */
+    rules: {
+        ignore: [
+            '^/system/healthCheck',
+        ],
+    },
+
+    /**
+     * The agent records errors via two independent paths:
+     *   1. exception capture (thrown Error or explicit noticeError call)
+     *   2. HTTP status-code capture (transaction ends with status >= 400)
+     *
+     * Without intervention, a thrown error that results in a 500 is reported
+     * twice. Ignoring the 400-599 range suppresses the status-code derived
+     * error events; the underlying exceptions are still captured via path #1.
+     */
+    error_collector: {
+        enabled:             true,
+        ignore_status_codes: ['400-599'],
+    },
+};

package/index.js

@@ -1,8 +1,10 @@
-const   httpLibs     = require('./lib/http/client');
-const   serverLibs   = require('./lib/server');
-const { logger }     = require('./lib/logger');
-const   ActiveRecord = require('./lib/activeRecord');
-const   redisLibs    = require('./lib/redis');
+const   ActiveRecord    = require('./lib/activeRecord');
+const   httpLibs        = require('./lib/http/client');
+const   APM              = require('./lib/apm');
+const   JobQueue        = require('./lib/jobQueue');
+const { logger }        = require('./lib/logger');
+const   redisLibs       = require('./lib/redis');
+const   serverLibs      = require('./lib/server');
 
 
 const {
@@ -16,7 +18,9 @@ const serverErrorsToExport = Object.values(serverLibs).filter(obj => obj.prototy
 
 module.exports = {
     ActiveRecord,
+    APM,
     ...httpLibs,
+    JobQueue,
     logger,
     ...redisLibs,
     ...serverErrorsToExport,

package/lib/activeRecord/baseRecord.js

@@ -1,7 +1,7 @@
 const { Connection, DEFAULT_DATABASE_ID } = require('./db/connection');
 const { getActiveRecordSchema }           = require('./db/schema');
 const { logger }                          = require('../logger');
-const { Model, Op }                       = require('sequelize');
+const { Model, Op, Sequelize }            = require('sequelize');
 const   nodeUtil                          = require('util');
 const { redisClientFactory }              = require('../redis');
 const { camelize, tableize }              = require('inflection');
@@ -19,7 +19,7 @@ class BaseRecord extends Model {
 
 
     constructor() {
-        if (new.target == BaseRecord) throw new Error("BaseRecord must be subclasses");
+        if (new.target == BaseRecord) throw new Error("Cannot create an instance of BaseRecord");
         super(...arguments);
     }
 
@@ -77,6 +77,7 @@ class BaseRecord extends Model {
         }
 
         this.configureOverridenAccessors(schema);
+        this.configureVirtualAccessors(schema);
 
         return { schema, options };
     }
@@ -96,6 +97,40 @@ class BaseRecord extends Model {
     }
 
 
+    static configureVirtualAccessors(schema) {
+        const seen = new Set();
+        for (const proto of this._prototypeChain()) {
+            this._applyVirtualAccessorsFrom(proto, schema, seen);
+        }
+    }
+
+
+    static *_prototypeChain() {
+        let proto = this.prototype;
+        while (proto && proto !== Model.prototype && proto !== Object.prototype) {
+            yield proto;
+            proto = Object.getPrototypeOf(proto);
+        }
+    }
+
+
+    static _applyVirtualAccessorsFrom(proto, schema, seen) {
+        for (const name of Object.getOwnPropertyNames(proto)) {
+            const m = name.match(/_virtual(?<accessor>Get|Set)(?<propertyName>.*)/);
+            if (!m) continue;
+
+            const propertyName = camelize(m.groups.propertyName, true);
+            const accessor     = m.groups.accessor.toLowerCase();
+            const key          = `${propertyName}:${accessor}`;
+            if (seen.has(key)) continue;
+
+            schema[propertyName]           = schema[propertyName] || {type: Sequelize.DataTypes.VIRTUAL};
+            schema[propertyName][accessor] = proto[name];
+            seen.add(key);
+        }
+    }
+
+
     static initializeHooks() {
         if (this.isCacheable) {
             this.addHook('afterSave',    this._afterSaveCacheHook);
@@ -263,7 +298,9 @@ class BaseRecord extends Model {
 
 
     static async _getRedisClient() {
-      return await redisClientFactory();
+      const client = redisClientFactory();
+      await client.connect();
+      return client;
     }
 
 
@@ -296,6 +333,15 @@ class BaseRecord extends Model {
     static debugLog(tag, log) {
         logger.debug(`[AR:${this.name}][${tag}] ${log}`);
     }
+
+
+    toApiResponse() {
+      const obj = this.dataValues;
+      for (const attr of Object.keys(obj)) {
+        obj[attr] = this[attr]; // Use overridden getters when available
+      }
+      return obj
+    }
 }
 
 

package/lib/apm/agent.js

@@ -0,0 +1,22 @@
+const ENABLED = Boolean(process.env.NEW_RELIC_LICENSE_KEY) && process.env.NODE_ENV !== 'test';
+
+
+function buildStub() {
+    return {
+        addCustomAttribute()                              {},
+        instrumentLoadedModule()                          {},
+        noticeError()                                     {},
+        recordCustomEvent()                               {},
+        recordMetric()                                    {},
+        setControllerName()                               {},
+        setUserID()                                       {},
+        startBackgroundTransaction(name, group, handler)  { return handler(); },
+        startSegment(name, record, handler)               { return handler(); },
+    };
+}
+
+
+module.exports = {
+    enabled: ENABLED,
+    client:  ENABLED ? require('newrelic') : buildStub(),
+};

package/lib/apm/index.js

@@ -0,0 +1,54 @@
+const { logger }          = require('../logger');
+const { enabled, client } = require('./agent');
+const   utils             = require('./utils');
+
+
+let instrumented   = false;
+const toInstrument = [
+  'aws-sdk',
+  'bullmq',
+  'express',
+  'ioredis',
+  'newrelic',
+  'pg',
+  'pg-pool',
+  'redis',
+];
+
+
+function instrumentAPM(packages={}) {
+    if (instrumented) return;
+    instrumented = true;
+
+    if (!enabled) return;
+
+    logger.addScope('apm');
+    for (const pkgName of toInstrument) {
+        instrumentPackageByName(pkgName)
+    }
+
+    for (const [pkgName, pkg] of Object.entries(packages)) {
+        client.instrumentLoadedModule(pkgName, pkg);
+        instrumentPackage(pkgName, pkg)
+    }
+}
+
+
+function instrumentPackageByName(pkgName) {
+    try {
+        const pkg = require(pkgName);
+        instrumentPackage(pkgName, pkg)
+    } catch(err) {}
+}
+
+
+function instrumentPackage(pkgName, pkg) {
+    client.instrumentLoadedModule(pkgName, pkg);
+    if (enabled) logger.apm.info(`Instrumenting ${pkgName}`);
+}
+
+
+module.exports = {
+    instrumentAPM,
+    ...utils
+}

package/lib/apm/utils.js

@@ -0,0 +1,43 @@
+const { client } = require('./agent');
+
+
+module.exports = {
+    addCustomAttribute(key, value) {
+        client.addCustomAttribute(key, value);
+    },
+
+
+    recordCustomEvent(eventType, attributes) {
+        client.recordCustomEvent(eventType, attributes);
+    },
+
+
+    recordMetric(name, value) {
+        client.recordMetric(name, value);
+    },
+
+
+    reportError(err, details={}) {
+        client.noticeError(err, details);
+    },
+
+
+    setControllerName(controllerName, action) {
+        client.setControllerName(controllerName, action);
+    },
+
+
+    setUserId(id) {
+        client.setUserID(id)
+    },
+
+
+    startBackgroundTransaction(name, group, handler) {
+        return client.startBackgroundTransaction(name, group, handler);
+    },
+
+
+    startSegment(name, record, handler) {
+        return client.startSegment(name, record, handler);
+    }
+}

package/lib/http/client.js

@@ -1,7 +1,11 @@
+const   APM                         = require('../apm');
 const { logger }                    = require('../logger');
 const { HttpError, HttpAbortError } = require('./error');
 
 
+logger.addScope('http');
+
+
 class HttpClient {
     async get(endpoint, { requestParams, headers, timeout, signal }={}) {
         return await this.#makeRequest(endpoint, "GET", arguments[1]);
@@ -23,7 +27,12 @@ class HttpClient {
     }
 
 
-    async #makeRequest(endpoint, method, { requestParams, body, headers, timeout, signal }={}) {
+    async #makeRequest(endpoint, method, opts={}) {
+        return await APM.startSegment(`HttpClient:${this.constructor.name}:${method}`, true, () => this.#_makeRequest(endpoint, method, opts));
+    }
+
+
+    async #_makeRequest(endpoint, method, { requestParams, body, headers, timeout, signal }={}) {
         this.#logRequest(method, endpoint);
 
         endpoint            = `${this.urlRoot}${endpoint}`;
@@ -147,7 +156,7 @@ class HttpClient {
 
 
     #logRequest(method, endpoint) {
-        logger.info(`API CALL: ${method} ${endpoint}`);
+        this.logger.debug(`API CALL: ${method} ${endpoint}`);
     }
 
 
@@ -162,6 +171,11 @@ class HttpClient {
     get urlRoot() {
         return "";
     }
+
+
+    get logger() {
+        return logger.http;
+    }
 }
 
 

package/lib/jobQueue/README.md

@@ -0,0 +1,63 @@
+# JobQueue
+The `JobQueue` is a general purpose wrapper around an `BullMQ` job queue.
+
+                                                                                                        
+## USAGE
+The Node Job Queue has 3 key components: `Queues`, `Workers`, and `Jobs`.
+
+                      
+### QUEUES ###                                                                                          
+A `Queue` object is used by a `Job` to add the `Job` and its associated arguments to the Redis queue for a `Worker` in a different process to dequeue and process. The work of a `Queue` is largely encapsulated by the `Job`, so you will rarely need to interact directly with a `Queue`. The exception is when ending a process. In order for your script to exit, you'll need to close any open `Queues`.
+
+Closing a queue looks like this:
+```
+const queue = getQueue(myQueueName);
+await queue.close();
+```
+
+Other less commonly needed `Queue` methods include:
+```
+await queue.numQueuedJobs(); // returns the number of jobs in the queue
+await queue.getQueuedJobs(); // returns an array of job data representing jobs in the queue
+await queue.numActiveJobs(); // returns the number of jobs currently being worked on
+await queue.getActiveJobs(); // returns an array of job data representing jobs currently being worked on
+await queue.numFailedJobs(); // returns the failed jobs since the last failed jobs clear
+await queue.getFailedJobs(); // returns an array of job data representing jobs failed since the last failed jobs clear
+await queue.clearFailedJobs(); // removes all stored data about failed jobs. This should be done periodically to clear memory in Redis
+await queue.clearCompletedJobs(); // removes all stored data about completed jobs. This should be done periodically to clear memory in Redis
+await queue.clearQueuedJobs(); // removes all jobs from the queue
+```
+
+### WORKERS
+A `Worker` is what runs on the server side, listening to Redis for new jobs in its queue. Starting a worker running on a server is simple.
+``` 
+const worker = new Worker('myQueue');
+await worker.listen();
+```
+
+### JOBS
+A `Job` is the object that contains the code that will be run by the worker. All `Jobs` will subclass `AbstractJob` and implement the `run` and `queueName` methods. `Jobs` should be defined in the `jobs` directory at the top level of the repo. This will allow them to be auto registered by a worker.
+```                                                                                                     
+class MyJob extends AbstractJob {                                                                       
+    async run(arg1, arg2...) {                                                                          
+        // do work here                                                                                 
+    }
+                                                                                                     
+    static queueName(arg1, arg2) {                                                                         
+        return 'myQueue';                                                                            
+    }                                                                                                
+}                                                                                                    
+```                                                                                                  
+Once you've defined your job's `run` and `queueName` methods, you can add it to the queue by running `queueJob`
+```                                                                                                  
+await MyJob.queueJob(arg1, arg2...)
+```
+
+## CONCURRENCY
+While there should only be one `Worker` per process, each worker can process multiple jobs in parallel. Concurrency settings are defined through environment variables.                                                                                                     
+
+### SIMPLE CONCURRENCY                                                                          
+In order to limit the number of jobs that a worker can run in parallel, simply set the `WORKER_CONCURRENCY` environment variable. If that variable is not set, concurrency will default to 1.
+
+### RATE LIMITING
+To be added in a future release.

package/lib/jobQueue/abstractJob.js

@@ -0,0 +1,103 @@
+const   APM        = require('../apm');
+const { getQueue } = require('./queue.js');
+const { logger }   = require('../logger');
+
+
+const DEFAULT_NUM_RETRIES         = 3;
+const DEFAULT_RETRY_BACKOFF_TYPE  = 'exponential';
+const DEFAULT_RETRY_BACKOFF_DELAY = 2000;
+
+
+class AbstractJob {
+    #id;
+    #updateProgress;
+
+
+    constructor() {
+        if (new.target === AbstractJob)  throw new Error("Cannot create an instance of AbstractJob");
+        if (!this.constructor.queueName) throw new Error("No queue defined");
+        if (!this.run)                   throw new Error("No run method defined");
+    }
+
+
+    static queue(...jobArgs) {
+        return getQueue(this.queueName(...jobArgs));
+    }
+
+
+    static async queueJob(...jobArgs) {
+      const queue = this.queue(...jobArgs);
+      await queue.queueJob(this, jobArgs, this.queueSettings(...jobArgs));
+    }
+
+
+    async _run(id, args, updateProgress) {
+        this.#id             = id;
+        this.#updateProgress = updateProgress;
+        await this.run(...args);
+    }
+
+
+    get id() {
+        return this.#id;
+    }
+
+
+    get logger() {
+        return logger[this.constructor.name].child(this.id);
+    }
+
+
+    async updateProgress() {
+        await this.#updateProgress(...arguments);
+    }
+
+
+    /***********************************************************************************************
+    * APM
+    ***********************************************************************************************/
+    addApmAttribute(key, value) {
+        APM.addCustomAttribute(key, value);
+    }
+
+
+    startApmSegment(name, record, handler) {
+        return APM.startSegment(name, record, handler);
+    }
+
+
+    /***********************************************************************************************
+    * SETTINGS
+    ***********************************************************************************************/
+    static queueSettings(...jobArgs) {
+        const settings = {
+            attempts: this.numRetries,
+            backoff:  {
+                type:  this.retryBackoffType,
+                delay: this.retryBackoffDelay
+            }
+        }
+
+        return settings;
+    }
+
+
+    static get numRetries() {
+        return DEFAULT_NUM_RETRIES;
+    }
+
+
+    static get retryBackoffType() {
+        return DEFAULT_RETRY_BACKOFF_TYPE;
+    }
+
+
+    static get retryBackoffDelay() {
+        return DEFAULT_RETRY_BACKOFF_DELAY;
+    }
+}
+
+
+module.exports = {
+    AbstractJob
+}

package/lib/jobQueue/abstractJobComponent.js

@@ -0,0 +1,37 @@
+class AbstractJobComponent {
+    get fullQueueName() {
+        return this.constructor.fullQueueName(this.queueName);
+    }
+
+
+    static fullQueueName(queueName) {
+        return `${queueName}-${this.env}`;
+    }
+
+
+    static get env() {
+        return process.env.NODE_ENV;
+    }
+
+
+    /***********************************************************************************************
+    * REDIS
+    ***********************************************************************************************/
+    get redisConnectionDetails() {
+        return {
+            host:     this.host,
+            password: this.password,
+            port:     this.port,
+        }
+    }
+
+      get host()     { return process.env.REDIS_BULL_HOST     }
+      get port()     { return process.env.REDIS_BULL_PORT     }
+      get password() { return process.env.REDIS_BULL_PASSWORD }
+
+}
+
+
+module.exports = {
+    AbstractJobComponent
+}

package/lib/jobQueue/index.js

@@ -0,0 +1,10 @@
+const { AbstractJob } = require('./abstractJob');
+const { getQueue }    = require('./queue');
+const { Worker }      = require('./worker');
+
+
+module.exports = {
+    AbstractJob,
+    getQueue,
+    Worker,
+}

package/lib/jobQueue/jobImporter.js

@@ -0,0 +1,19 @@
+const { AbstractAutoImporter } = require('../utils/abstractAutoImporter');
+const { AbstractJob }          = require('./abstractJob');
+
+
+class JobImporter extends AbstractAutoImporter {
+    get jobs() {
+        return this.imports;
+    }
+
+
+    shouldImport(obj) {
+        return obj.prototype instanceof AbstractJob;
+    }
+}
+
+
+module.exports = {
+    JobImporter
+}