@sera4/essentia 1.1.8 → 1.1.13

package/cache/index.js

@@ -0,0 +1,188 @@
+"use strict"
+
+import { promisify } from "util";
+
+const isEmpty = obj => [Object, Array].includes((obj || {}).constructor) && !Object.entries((obj || {})).length;
+
+class CacheConfigError extends Error {
+  constructor(m) {
+    super(m);
+  }
+}
+
+/* Redis cache helper
+ * Example use: Place the following code in your project to setup redis use
+ *
+ *    import { cache } from "@sera4/essentia";
+ *    const cacheHelper = cache({ password: .., database: .., host: .., port: .. }, "service:"));
+ *
+ */
+
+const convertRedisKeyValueToJson = (v) => {
+  return (v && v.length > 0 && v[0] !== null) ? JSON.parse(v) : null;
+}
+
+const convertRedisHashToJson = (v) => {
+  if (v) {
+    var result = {};
+    for(var k in v) {
+      if (v.hasOwnProperty(k)) {
+        result[k] = JSON.parse(v[k]);
+      }
+    }
+    return result;
+  } else {
+    return null;
+  }
+}
+
+function convertJsonToRedisValue(j) {
+  var result = {};
+  for (var key in j) {
+    if (j.hasOwnProperty(key)) {
+      result[key] = JSON.stringify(j[key]);
+    }
+  }
+  return result;
+}
+
+class Cache {
+  constructor(config, servicePrefix) {
+    if ((config == undefined) || isEmpty(config)) {
+      throw new CacheConfigError("Invalid cache configuration");
+    }
+
+    if (servicePrefix == null) {
+      throw new CacheConfigError("Invalid service prefix");
+    }
+
+    if (config.fake) {
+      // used in testing
+      this.redisClient = require("fakeredis").createClient("fake backend")
+    } else {
+      this.redisClient = require("redis").createClient(config);
+    }
+
+    this.redisClient.on("error", (err) => {
+      logger.error("Cache/Redis error: ", err);
+    });
+
+    this.servicePrefix = servicePrefix;
+  }
+
+  ping() {
+    return this.redisClient.ping();
+  }
+
+  properKey(key, path) {
+    let p = this.servicePrefix + key
+    if (path)
+      p += path
+    return p
+  }
+
+  writeHash(path, key, value) {
+    return this.redisClient.hsetAsync(properKey(key), key, value);
+  }
+
+  writeHashBlocking(path, key, value) {
+    return this.redisClient.hset(properKey(key), key, value);
+  }
+
+  // write multiple hash keys and stringify every value
+  writeMultiHash(path, j) {
+    return this.redisClient.hmsetAsync(properKey(key), convertJsonToRedisValue(j));
+  }
+
+  readHashKey(obj, key, autoConvert=true) {
+    var k = autoConvert;
+
+    return new Promise((resolve, reject) => {
+      this.redisClient.hget(properKey(key, obj), key, (e, r) => {
+        if (e) {
+          logger.error("Redis hmget error:", e);
+          reject(e);
+        } else {
+          resolve(k ? convertRedisKeyValueToJson(r) : r);
+        }
+      });
+    });
+  }
+
+  readHash(path) {
+    return new Promise((resolve, reject) => {
+      this.redisClient.hgetall(properKey(key, path), (e, r) => {
+        if (e) {
+          console.log("Redis hget error:", e);
+          reject(e);
+        } else {
+          resolve(convertRedisHashToJson(r));
+        }
+      });
+    });
+  }
+
+  deleteHashKey(hash, key) {
+    return new Promise((resolve, reject) => {
+      this.redisClient.hdel(properKey(key, hash), key, (err) => {
+        if (err) {
+          logger.error("Redis hdel error:", err);
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+
+  get(key) {
+    return this.redisClient.getAsync(properKey(key));
+  }
+
+  getAsync(key) {
+    return promisify(this.redisClient.get(key)).bind(this.redisClient)
+  }
+
+
+  /* a single key write with optional expiry */
+  // returns bluebird promise
+  writeKey(key, value, expiry) {
+    key = this.servicePrefix + key;
+    if (expiry) {
+      return this.redisClient.setAsync(key, value, "EX", expiry);
+    } else {
+      return this.redisClient.setAsync(key, value);
+    }
+  }
+
+  deleteKey(key) {
+    return redisClient.delAsync(properKey(key));
+  }
+
+  expire(path, expiry) {
+    return this.redisClient.expire(path, expiry);
+  }
+
+  exists(path) {
+    return this.redisClient.exists(path);
+  }
+
+  deleteAllKeys() {
+    return new Promise((resolve, reject) => {
+      if(process.env.NODE_ENV === "production") {
+        throw new Error("Clearing system cache is not permitted in production");
+      }
+      this.redisClient.flushdb((err, res) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(res);
+        }
+      });
+    });
+  }
+}
+
+export default function(config, servicePrefix) {
+  return new Cache(config, servicePrefix)
+}

package/health/health-logger.js

@@ -0,0 +1,40 @@
+import logger from "../logger/s4-logger.js";
+let enabled = false;
+
+const helpers = {
+  debug(...args) {
+    if (enabled) {
+      logger.debug(":: HealthCheck ::", args);
+    }
+  },
+
+  log(...args) {
+    if (enabled) {
+      logger.debug(":: HealthCheck ::", args);
+    }
+  },
+
+  warn(...args) {
+    if (enabled) {
+      logger.warn(":: HealthCheck ::", args);
+    }
+  },
+
+  info(...args) {
+    if (enabled) {
+      logger.info(":: HealthCheck ::", args);
+    }
+  },
+
+  error(...args) {
+    if (enabled) {
+      logger.error(":: HealthCheck ::", args);
+    }
+  },
+
+  setEnabled(e) {
+    enabled = e;
+  }
+}
+
+export default helpers;

package/health/index.js

@@ -1,6 +1,7 @@
 "use strict"
 
 import pathHelper from "path";
+import logger from "./health-logger.js";
 
 class HealthCheckError extends Error {
   constructor(m) {
@@ -15,12 +16,16 @@ class HealthCheckError extends Error {
  *    const router = require('express').Router();
  *
  *      ....
+ *
+ *    const checkRedis = async () => { await cache.ping(); }
+ *    hc.setupCheck({ name: "redis", checkResult: false, critical: true, healthFunction: checkRedis });
  *    app.use(hc.defaultPath(serverConfigs.api_prefix), hc.routes(router));
  *
  */
-
 class HealthCheck {
-  constructor(appendData = {}) {
+  constructor(appendData = {}, enableLogging = true) {
+    logger.setEnabled(enableLogging);
+
     this.defaultResponse = {
       ...
       appendData,
@@ -29,11 +34,13 @@ class HealthCheck {
     };
 
     this.defaultPathRoute = `health_check\(.*\)?`
+    this.suppressLogs = false;
     this.lastCallId = 0; // default start point/integer
     this.healthChecks = [];
   }
 
-  /* call this with an async function, must support wait capability
+  /* Deprecated and replaced by setupCheck
+   *   call this with an async function, must support wait capability
    *
    *   const dbCheck = async () => {
    *     return { name: "db", "result": "success", critical: false };
@@ -60,15 +67,40 @@ class HealthCheck {
     return router
   }
 
-  /* performs health checks and returns 200 if true and 500 if not */
+  // v2 version of health check, replaced addCheck
+  // example:
+  //  const checkRedisFcn = async () => { await cache.ping().then(() => true) }
+  //  setupCheck({ name: "redis", checkResult: false, critical: false, healthFunction: checkRedisFcn });
+  setupCheck(config) {
+    this.healthChecks.push(config);
+  }
+
+  /* performs health checks and returns 200 if true and 500 if not */ 
   async checkHealth(req, res) {
     const checkFormat = pathHelper.extname(req.baseUrl);
 
-    try {
-      const result = await Promise.all(this.healthChecks.map(async healthCallback => {
-        return await healthCallback();
-      }));
+    const result = await Promise.all(this.healthChecks.map(async hc => {
+      if (typeof(hc) === "function") {
+        try {
+          return await hc();
+          return { "result": "success", critical: false };
+        } catch(e) {
+          return  { name: "unknown", result: 'fail', status: e.status }
+        }
+      } else {
+        let res = { name: hc.name, critical: hc.critical }
+        try {
+          await hc.healthFunction();
+          return { ...res, "result": "success" };
+        } catch(e) {
+          const appendStatus = e.status ? ` (${e.status})` : ""
+          logger.warn(`${hc.name} service failed${appendStatus}` )
+          return { ...res, "result": "fail", status: e.status } 
+        }
+      }
+    }));
 
+    try {
       let callId = ++this.lastCallId;
 
       // start the counter over
@@ -78,10 +110,18 @@ class HealthCheck {
 
       let status = this.defaultResponse.status;
 
+      let nonCriticalFail = false;
+      let criticalFail = false;
+
       result.some(e => {
         if (e.result.match(/fail/)) {
-          status = 500;
-          return true; // break
+          if (e.critical == false) {
+            nonCriticalFail = true;
+          } else {
+            criticalFail = true;
+            status = 500;
+            return true; // break
+          }
         }
       })
 
@@ -89,33 +129,36 @@ class HealthCheck {
       res.setHeader('check', callId)
 
       if (checkFormat === ".json") {
-        const payload = { "check": callId }
+        const payload = { "check": callId, healthy: "true", message: "success" }
 
-        if (status === 200) {
+        if (criticalFail) {
           res.status(status).json({
             ... payload,
-            "healthy":true,
-            "message":"success"
+            "healthy": false,
+            "message": "fail",
+            "services": result
           });
-        } else {
+        } else if (nonCriticalFail) {
           res.status(status).json({
             ... payload,
-            "healthy": false,
             "message": "fail",
-            "services": [result]
+            "services": result
           });
+        } else if (status === 200) {
+          res.status(status).json(payload);
         }
       } else {
         let status_message;
-        if (status === 200) {
-          status_message = "success"
-        } else {
+        if (criticalFail) {
           status_message = "fail"
-        }
+        } else if ((status === 200) || nonCriticalFail) {
+          status_message = "success"
+        } 
 
         res.status(status).send(status_message);
       }
     } catch(error) {
+      console.debug("error: ", error)
       res.status(500).send();
     }
   }

package/logger/s4-logger.js

@@ -4,6 +4,30 @@ import jsonStringify from "fast-safe-stringify";
 const colorizer = winston.format.colorize();
 import "winston-logstash";
 
+// IETF RFC5424:
+// Severity of all levels is assumed to be numerically ascending from most important to least important.
+const levels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  debug: 3,
+  trace: 4,
+};
+
+// These colors are not used, however if colors are not defined for custom levels
+// then winston will throw an exception.
+winston.addColors({
+  trace: 'white'
+});
+
+const defaultLevel = process.env.NODE_ENV === 'production' ? 'info' : 'debug';
+
+// These colors are not used, however if colors are not defined for custom levels
+// then winston will throw an exception.
+winston.addColors({
+  trace: 'white'
+});
+
 const devFormat = {
   transform(info) {
     const { message } = info;
@@ -16,11 +40,13 @@ const devFormat = {
 };
 
 const defaultTransport = new winston.transports.Console({
-  format: devFormat
+  format: devFormat,
+  level: 'trace'
 });
 
 const logger = winston.createLogger({
-  level: 'debug',
+  levels: levels,
+  level: defaultLevel,
   defaultMeta: { },
   transports: [defaultTransport]
 });
@@ -31,15 +57,13 @@ class S4Logger {
       return;
     }
 
-    if (options.level && ["debug", "info", "error", "warn"].indexOf(options.level.toLowerCase())) {
-      logger.level = options.level;
-    }
+    this.setLevel(options.level);
 
     logger.defaultMeta.service = options.service;
     switch(options.format.toLowerCase()) {
       case "logstash":
-          defaultTransport.format = winston.format.logstash();
-          break;
+        defaultTransport.format = winston.format.logstash();
+        break;
       case "json":
         defaultTransport.format = winston.format.json();
         break;
@@ -50,7 +74,7 @@ class S4Logger {
   }
 
   setLevel(level) {
-    if (level && ["debug", "info", "error", "warn"].indexOf(level.toLowerCase())) {
+    if (level && ["trace", "debug", "info", "error", "warn"].indexOf(level.toLowerCase())) {
       logger.level = level;
     }
   }
@@ -67,6 +91,10 @@ class S4Logger {
     });
   }
 
+  trace(...args) {
+    logger.trace(util.format(...args));
+  }
+
   debug(...args) {
     logger.debug(util.format(...args));
   }

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@sera4/essentia",
-  "version": "1.1.8",
+  "version": "1.1.13",
   "description": "A library of utilities for Teleporte Web Services",
   "main": "index.js",
   "type": "module",
   "scripts": {
     "test": "NODE_ENV=test ./node_modules/.bin/mocha --forbid-only --reporter mocha-junit-reporter --reporter-options mochaFile=./test-results/result.xml --exit",
-    "test:dev": "NODE_ENV=test ./node_modules/.bin/mocha --watch --forbid-only",
+    "test:dev": "NODE_ENV=test ./node_modules/.bin/mocha --forbid-only",
     "test:only": "NODE_ENV=test ./node_modules/.bin/mocha --watch",
     "test:pretty": "NODE_ENV=test ./node_modules/.bin/mocha --reporter mocha-simple-html-reporter --reporter-options output=./test-results/result.html --exit",
     "last-commit": "node last-commit.js"
@@ -28,13 +28,13 @@
     "sinon": "^5.1.1"
   },
   "dependencies": {
-    "amqplib": "^0.7.1",
+    "amqplib": "^0.8.0",
     "async": "^2.6.3",
     "axios": "^0.21.1",
     "fast-safe-stringify": "^2.0.7",
     "git-rev": "^0.2.1",
     "lodash": "^4.17.21",
-    "url-parse": "^1.5.1",
+    "url-parse": "^1.5.3",
     "uuid": "^3.4.0",
     "winston": "^3.3.3",
     "winston-logstash": "^0.4.0"

package/queue/index.js

@@ -29,6 +29,7 @@ class S4Queue {
    */
   async testConnection() {
     if (!this.isConnected()) {
+      this.connectionTestChannel = null;
       throw("queue service not connected")
     }
 
@@ -72,12 +73,16 @@ class S4Queue {
       amqp.connect(options.connectionUrl, socketOptions, (err, conn) => {
 
         if (err) {
-          return reject(err);
+          logger.error("Error while connecting", err)
+          logger.error("Attempting again in", this.retry, "seconds");
+          return setTimeout(() => this.openConnection(options), this.retry * 1000);
         }
 
         this.connection = conn;
         if (!this.connection) {
           return reject(new Error("Failed to obtain connection"));
+        } else {
+          this.updatePubsAndSubsConnectionRef(conn);
         }
 
         this.connection.on("error", (err) => {
@@ -91,8 +96,8 @@ class S4Queue {
           logger.error("connection closed");
           this.connection = null;
           if (err) {
-            logger.debug("connection closed due to error:", err);
-            logger.debug("Reconnecting in", this.retry, "seconds");
+            logger.error("connection closed due to error:", err);
+            logger.error("Reconnecting in", this.retry, "seconds");
             setTimeout(() => this.openConnection(options), this.retry * 1000);
           } else {
             // If there was no error, it is us who closed
@@ -171,6 +176,22 @@ class S4Queue {
     this.pubs[name] = p;
   }
 
+  /**
+   * Registers a new fanout publisher on the given exchange.
+   * If exchange doesn't exist, one will be created.
+   * @param {String} exchange the name of the exchange on AMQP service.
+   */
+  async registerFanoutPublisher(exchange) {
+    const name = exchange;
+    if (Object.keys(this.pubs).includes(name)) {
+      throw new Error(`Publisher ${name} already registered`);
+    }
+
+    const p = new Publisher(exchange, 'fanout', true, name);
+    await p.init(this.connection);
+    this.pubs[name] = p;
+  }
+
   /**
    * Registers a new direct subscriber on a given exchange and queue.
    * If exchange and queue don't exists, they'll be created
@@ -273,6 +294,26 @@ class S4Queue {
     return await p.publish({key: topic, content});
   }
 
+  /**
+   * Binds an exchange to a queue
+   * @param {String} exchange the name of the exchange.
+   * @param {String} queue the name of the queue.
+   * @param {pattern} (optional) pattern for routing key
+  */
+  async bindQueueToExchange(exchange, queue, pattern) {
+    const p = this.pubs[exchange];
+
+    if (!p) {
+      throw(new Error(`No PUB for ${exchange} found`));
+    }
+
+    if (!queue) {
+      throw(new Error(`Binding a queue requires a queue name`));
+    }
+
+    return await p.bindQueue(queue, pattern)
+  }
+
   /**
    * Adds a listener for a specific key.
    * If a subscriber that was regitered was found
@@ -289,6 +330,29 @@ class S4Queue {
       }
     });
   }
+
+  /**
+   * Updates the connection reference to all the previously
+   * registered publishers and subscribers.
+   * After we successfully establish a connection and register pubs/subs,
+   * it's possible for the connection to break. This object will try
+   * to re-establish a new connection every 3 seconds (or whatever configuration is set)
+   * and if a new connection is established, we need to pass that connection to
+   * all our pubs/subs, otherwise they'll be operating with old/stale references
+   * @param {Object} connRef a new connection reference to the AMQP service
+   */
+  updatePubsAndSubsConnectionRef(connRef) {
+    Object.keys(this.pubs).forEach(k => {
+      logger.debug(`Updating publisher ${k} with a new connection object`);
+      const p = this.pubs[k];
+      p.init(connRef);
+    });
+    Object.keys(this.subs).forEach(k => {
+      logger.debug(`Updating subscriber ${k} with a new connection object`);
+      const s = this.subs[k];
+      s.init(connRef);
+    });
+  }
 }
 
 const queue = new S4Queue();

package/queue/publisher.js

@@ -5,9 +5,7 @@ import uuidv4 from "uuid/v4.js";
  * A publisher for AMQP service
  */
 export class Publisher {
-
   /**
-   *
    * @param {String} exchange AMQP exchange name.
    * @param {String} type direct|fanout| etc (supported types by RabbitMQ).
    * @param {boolean} durable true if the messages should persist service restarts.
@@ -81,4 +79,8 @@ export class Publisher {
 
     return parcel.tracking_id;
   }
+
+  async bindQueue(queue, pattern, args) {
+    await this.channel.bindQueue(queue, this.exchange, pattern, args)
+  }
 }

package/safe_proxy/index.js

@@ -7,9 +7,8 @@ const { omitBy, isNil } = lodash;
 /* A class which safely wraps an axios connection in order to pass the current user through to
  * the next service.  Useful when one of our microservices requires auth as either an admin (sera4tal)
  * or as the end user (example: DELETE /auth/sessions/:id
- * 
- * similar to proxy however this isn't a proxy, it's a full out new call that we can manage
  *
+ * Similar to proxy however this isn't a proxy, it's a full out new call that we can manage
  * each method requires a res.locals from a pre-authd session
  */
 class SafeProxy {