@mojaloop/sdk-scheme-adapter 12.2.3 → 12.3.0

package/CHANGELOG.md

@@ -1,4 +1,12 @@
 # Changelog: [mojaloop/thirdparty-api-svc](https://github.com/mojaloop/thirdparty-api-svc)
+## [12.3.0](https://github.com/mojaloop/sdk-scheme-adapter/compare/v12.2.3...v12.3.0) (2022-05-04)
+
+
+### Features
+
+* port over prom client metrics ([#312](https://github.com/mojaloop/sdk-scheme-adapter/issues/312)) ([8de66d5](https://github.com/mojaloop/sdk-scheme-adapter/commit/8de66d505b94cddb5e3b8e857ae491f85058d395))
+* pull in live reconfiguration logic ([#313](https://github.com/mojaloop/sdk-scheme-adapter/issues/313)) ([ae5648a](https://github.com/mojaloop/sdk-scheme-adapter/commit/ae5648a500eaab80804db0298facc1e352482fb9))
+
 ### [12.2.3](https://github.com/mojaloop/sdk-scheme-adapter/compare/v12.2.2...v12.2.3) (2022-04-26)
 
 

package/audit-resolve.json

@@ -434,8 +434,78 @@
       "decision": "ignore",
       "madeAt": 1650459475376,
       "expiresAt": 1653051469252
+    },
+    "1070245|@mojaloop/central-services-shared>@mojaloop/event-sdk>moment": {
+      "decision": "ignore",
+      "madeAt": 1651072086111,
+      "expiresAt": 1653664079920
+    },
+    "1070245|@mojaloop/event-sdk>moment": {
+      "decision": "ignore",
+      "madeAt": 1651072086111,
+      "expiresAt": 1653664079920
+    },
+    "1070030|@mojaloop/central-services-shared>@mojaloop/event-sdk>moment>shins>markdown-it": {
+      "decision": "ignore",
+      "madeAt": 1651072088351,
+      "expiresAt": 1653664079920
+    },
+    "1070030|widdershins>markdown-it": {
+      "decision": "ignore",
+      "madeAt": 1651072088351,
+      "expiresAt": 1653664079920
+    },
+    "1068154|@mojaloop/central-services-shared>@mojaloop/event-sdk>moment>shins>markdown-it>sanitize-html": {
+      "decision": "ignore",
+      "madeAt": 1651072089481,
+      "expiresAt": 1653664079920
+    },
+    "1068155|@mojaloop/central-services-shared>@mojaloop/event-sdk>moment>shins>markdown-it>sanitize-html": {
+      "decision": "ignore",
+      "madeAt": 1651072090420,
+      "expiresAt": 1653664079920
+    },
+    "1070250|ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1651072091389,
+      "expiresAt": 1653664079920
+    },
+    "1070252|ansi-regex": {
+      "decision": "ignore",
+      "madeAt": 1651072092335,
+      "expiresAt": 1653664079920
+    },
+    "1070256|ejs": {
+      "decision": "ignore",
+      "madeAt": 1651072093304,
+      "expiresAt": 1653664079920
+    },
+    "1067536|json-pointer": {
+      "decision": "ignore",
+      "madeAt": 1651072094695,
+      "expiresAt": 1653664079920
+    },
+    "1067553|swagger2openapi>better-ajv-errors>jsonpointer": {
+      "decision": "ignore",
+      "madeAt": 1651072095680,
+      "expiresAt": 1653664079920
+    },
+    "1067946|swagger2openapi>better-ajv-errors>jsonpointer>oas-validator>ajv": {
+      "decision": "ignore",
+      "madeAt": 1651072096735,
+      "expiresAt": 1653664079920
+    },
+    "1068310|widdershins>markdown-it>yargs>yargs-parser": {
+      "decision": "ignore",
+      "madeAt": 1651072098082,
+      "expiresAt": 1653664079920
+    },
+    "1070260|@mojaloop/central-services-shared>shins>sanitize-html": {
+      "decision": "ignore",
+      "madeAt": 1651249549067,
+      "expiresAt": 1653841542147
     }
   },
   "rules": {},
   "version": 1
-}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mojaloop/sdk-scheme-adapter",
-  "version": "12.2.3",
+  "version": "12.3.0",
   "description": "An adapter for connecting to Mojaloop API enabled switches.",
   "main": "index.js",
   "types": "index.d.ts",
@@ -74,9 +74,11 @@
     "json-schema-ref-parser": "^9.0.9",
     "koa": "^2.13.1",
     "koa-body": "^4.2.0",
+    "lodash": "^4.17.21",
     "module-alias": "^2.2.2",
     "oauth2-server": "^4.0.0-dev.2",
     "openapi-jsonschema-parameters": "^9.3.0",
+    "prom-client": "^12.0.0",
     "promise-timeout": "^1.3.0",
     "random-word-slugs": "^0.1.6",
     "redis": "^3.1.2",

package/src/OutboundServer/handlers.js

@@ -112,6 +112,7 @@ const postTransfers = async (ctx) => {
             cache: ctx.state.cache,
             logger: ctx.state.logger,
             wso2: ctx.state.wso2,
+            metricsClient: ctx.state.metricsClient,
         });
 
         // initialize the transfer model and start it running
@@ -144,6 +145,7 @@ const getTransfers = async (ctx) => {
             cache: ctx.state.cache,
             logger: ctx.state.logger,
             wso2: ctx.state.wso2,
+            metricsClient: ctx.state.metricsClient,
         });
 
         // initialize the transfer model and start it running
@@ -172,6 +174,7 @@ const putTransfers = async (ctx) => {
             cache: ctx.state.cache,
             logger: ctx.state.logger,
             wso2: ctx.state.wso2,
+            metricsClient: ctx.state.metricsClient,
         });
 
         // TODO: check the incoming body to reject party or quote when requested to do so

package/src/OutboundServer/index.js

@@ -28,12 +28,13 @@ const middlewares = require('./middlewares');
 const endpointRegex = /\/.*/g;
 
 class OutboundApi extends EventEmitter {
-    constructor(conf, logger, cache, validator) {
+    constructor(conf, logger, cache, validator, metricsClient) {
         super({ captureExceptions: true });
         this._logger = logger;
         this._api = new Koa();
         this._conf = conf;
         this._cache = cache;
+        this._metricsClient = metricsClient;
 
         this._wso2 = {
             auth: new WSO2Auth({
@@ -54,7 +55,7 @@ class OutboundApi extends EventEmitter {
         this._api.use(middlewares.createErrorHandler(this._logger));
         this._api.use(middlewares.createRequestIdGenerator());
         this._api.use(koaBody()); // outbound always expects application/json
-        this._api.use(middlewares.applyState({ cache, wso2: this._wso2, conf }));
+        this._api.use(middlewares.applyState({ cache, wso2: this._wso2, conf, metricsClient }));
         this._api.use(middlewares.createLogger(this._logger));
 
         //Note that we strip off any path on peerEndpoint config after the origin.
@@ -91,7 +92,7 @@ class OutboundApi extends EventEmitter {
 }
 
 class OutboundServer extends EventEmitter {
-    constructor(conf, logger, cache) {
+    constructor(conf, logger, cache, metricsClient) {
         super({ captureExceptions: true });
         this._validator = new Validate();
         this._conf = conf;
@@ -101,7 +102,8 @@ class OutboundServer extends EventEmitter {
             conf,
             this._logger.push({ component: 'api' }),
             cache,
-            this._validator
+            this._validator,
+            metricsClient
         );
         this._api.on('error', (...args) => {
             this.emit('error', ...args);

package/src/index.js

@@ -10,14 +10,18 @@
 
 'use strict';
 
+const assert = require('assert/strict');
 const { hostname } = require('os');
 const config = require('./config');
 const EventEmitter = require('events');
+const _ = require('lodash');
 
 const InboundServer = require('./InboundServer');
 const OutboundServer = require('./OutboundServer');
 const OAuthTestServer = require('./OAuthTestServer');
 const TestServer = require('./TestServer');
+const { MetricsServer, MetricsClient } = require('./lib/metrics');
+const ControlAgent = require('./ControlAgent');
 
 // import things we want to expose e.g. for unit tests and users who dont want to use the entire
 // scheme adapter as a service
@@ -26,8 +30,19 @@ const OutboundServerMiddleware = require('./OutboundServer/middlewares.js');
 const Router = require('./lib/router');
 const Validate = require('./lib/validate');
 const Cache = require('./lib/cache');
+const check = require('./lib/check');
 const { Logger } = require('@mojaloop/sdk-standard-components');
 
+const LOG_ID = {
+    INBOUND:   { app: 'mojaloop-connector-inbound-api' },
+    OUTBOUND:  { app: 'mojaloop-connector-outbound-api' },
+    TEST:      { app: 'mojaloop-connector-test-api' },
+    OAUTHTEST: { app: 'mojaloop-connector-oauth-test-server' },
+    CONTROL:   { app: 'mojaloop-connector-control-client' },
+    METRICS:   { app: 'mojaloop-connector-metrics' },
+    CACHE:     { component: 'cache' },
+};
+
 /**
  * Class that creates and manages http servers that expose the scheme adapter APIs.
  */
@@ -38,14 +53,22 @@ class Server extends EventEmitter {
         this.logger = logger;
         this.cache = new Cache({
             ...conf.cacheConfig,
-            logger: this.logger.push({ component: 'cache' }),
+            logger: this.logger.push(LOG_ID.CACHE),
             enableTestFeatures: conf.enableTestFeatures,
         });
 
+        this.metricsClient = new MetricsClient();
+
+        this.metricsServer = new MetricsServer({
+            port: this.conf.metrics.port,
+            logger: this.logger.push(LOG_ID.METRICS)
+        });
+
         this.inboundServer = new InboundServer(
             this.conf,
-            this.logger.push({ app: 'mojaloop-sdk-inbound-api' }),
-            this.cache
+            this.logger.push(LOG_ID.INBOUND),
+            this.cache,
+            this.metricsClient
         );
         this.inboundServer.on('error', (...args) => {
             this.logger.push({ args }).log('Unhandled error in Inbound Server');
@@ -54,8 +77,9 @@ class Server extends EventEmitter {
 
         this.outboundServer = new OutboundServer(
             this.conf,
-            this.logger.push({ app: 'mojaloop-sdk-outbound-api' }),
-            this.cache
+            this.logger.push(LOG_ID.OUTBOUND),
+            this.cache,
+            this.metricsClient
         );
         this.outboundServer.on('error', (...args) => {
             this.logger.push({ args }).log('Unhandled error in Outbound Server');
@@ -66,16 +90,140 @@ class Server extends EventEmitter {
             clientKey: this.conf.oauthTestServer.clientKey,
             clientSecret: this.conf.oauthTestServer.clientSecret,
             port: this.conf.oauthTestServer.listenPort,
-            logger: this.logger.push({ app: 'mojaloop-sdk-oauth-test-server' }),
+            logger: this.logger.push(LOG_ID.OAUTHTEST),
         });
 
         this.testServer = new TestServer({
-            port: this.conf.testServerPort,
-            logger: this.logger.push({ app: 'mojaloop-sdk-test-api' }),
+            port: this.conf.test.port,
+            logger: this.logger.push(LOG_ID.TEST),
             cache: this.cache,
         });
     }
 
+    async restart(newConf) {
+        // Figuring out what config is necessary in each server and component is a pretty big job
+        // that we'll have to save for later. For now, when the config changes, we'll restart
+        // more than we might have to.
+        // We'll do this by:
+        // 0. creating a new instance of the logger, if necessary
+        // 1. creating a new instance of the cache, if necessary
+        // 2. calling the async reconfigure method of each of the servers as necessary- this will
+        //    return a synchronous function that we can call to swap over the server events and
+        //    object properties to the new ones. It will:
+        //    1. remove the `request` listener for each of the HTTP servers
+        //    2. add the new appropriate `request` listener
+        //    This results in a completely synchronous listener changeover to the new config and
+        //    therefore hopefully avoids any concurrency issues arising from restarting different
+        //    servers or components concurrently.
+        // TODO: in the sense of being able to reason about the code, it would make some sense to
+        // turn the config items or object passed to each server into an event emitter, or pass an
+        // additional event emitter to the server constructor for the server to listen to and act
+        // on changes. Before this, however, it's probably necessary to ensure each server gets
+        // _only_ the config it needs, not the entire config object.
+        // Further: it might be possible to use Object.observe for this functionality.
+        // TODO: what happens if this is run concurrently? I.e. if it is called twice in rapid
+        // succession. This question probably needs to be asked of the reconfigure message on every
+        // server.
+        // Note that it should be possible to reconfigure ports on a running server by reassigning
+        // servers, e.g.
+        //   this.inboundServer._server = createHttpServer();
+        //   this.inboundServer._server.listen(newPort);
+        // If there are conflicts, for example if the new configuration specifies the new inbound
+        // port to be the same value as the old outbound port, this will require either
+        // 1. some juggling of HTTP servers, e.g.
+        //      const oldInboundServer = this.inboundServer._server;
+        //      this.inboundServer._server = this.outboundServer._server;
+        //    .. etc.
+        // 2. some juggling of sockets between servers, if possible
+        // 3. rearchitecting of the servers, perhaps splitting the .start() method on the servers
+        //    to an .init() and .listen() methods, with the latter optionally taking an HTTP server
+        //    as argument
+        // This _might_ introduce some confusion/complexity for existing websocket clients, but as
+        // the event handlers _should_ not be modified this shouldn't be a problem. A careful
+        // analysis of this will be necessary.
+        assert(newConf.inbound.port === this.conf.inbound.port
+            && newConf.outbound.port === this.conf.outbound.port
+            && newConf.test.port === this.conf.test.port
+            && newConf.oauthTestServer.listenPort === this.conf.oauthTestServer.listenPort
+            && newConf.control.mgmtAPIWsPort === this.conf.control.mgmtAPIWsPort,
+        'Cannot reconfigure ports on running server');
+        const doNothing = () => {};
+        const updateLogger = check.notDeepEqual(newConf.logIndent, this.conf.logIndent);
+        if (updateLogger) {
+            this.logger = new Logger.Logger({
+                context: {
+                    // If we're running from a Mojaloop helm chart deployment, we'll have a SIM_NAME
+                    simulator: process.env['SIM_NAME'],
+                    hostname: hostname(),
+                },
+                stringify: Logger.buildStringify({ space: this.conf.logIndent }),
+            });
+        }
+        let oldCache;
+        const updateCache = (
+            updateLogger ||
+            check.notDeepEqual(this.conf.cacheConfig, newConf.cacheConfig) ||
+            check.notDeepEqual(this.conf.enableTestFeatures, newConf.enableTestFeatures)
+        );
+        if (updateCache) {
+            oldCache = this.cache;
+            this.cache = new Cache({
+                ...newConf.cacheConfig,
+                logger: this.logger.push(LOG_ID.CACHE),
+                enableTestFeatures: newConf.enableTestFeatures,
+            });
+            await this.cache.connect();
+        }
+        const confChanged = !check.deepEqual(newConf, this.conf);
+        // TODO: find better naming than "restart", because that's not really what's happening.
+        const [restartInboundServer, restartOutboundServer, restartControlClient] = confChanged
+            ? await Promise.all([
+                this.inboundServer.reconfigure(newConf, this.logger.push(LOG_ID.INBOUND), this.cache),
+                this.outboundServer.reconfigure(newConf, this.logger.push(LOG_ID.OUTBOUND), this.cache, this.metricsClient),
+                this.controlClient.reconfigure({
+                    logger: this.logger.push(LOG_ID.CONTROL),
+                    port: newConf.control.mgmtAPIWsPort,
+                    appConfig: newConf
+                }),
+            ])
+            : [doNothing, doNothing, doNothing];
+        const updateOAuthTestServer = (
+            updateLogger || check.notDeepEqual(newConf.oauthTestServer, this.conf.oauthTestServer)
+        );
+        const restartOAuthTestServer = updateOAuthTestServer
+            ? await this.oauthTestServer.reconfigure({
+                clientKey: this.conf.oauthTestServer.clientKey,
+                clientSecret: this.conf.oauthTestServer.clientSecret,
+                port: this.conf.oauthTestServer.listenPort,
+                logger: this.logger.push(LOG_ID.OAUTHTEST),
+            })
+            : doNothing;
+        const updateTestServer = (
+            updateLogger || updateCache || check.notDeepEqual(newConf.test.port, this.conf.test.port)
+        );
+        const restartTestServer = updateTestServer
+            ? await this.testServer.reconfigure({
+                port: newConf.test.port,
+                logger: this.logger.push(LOG_ID.TEST),
+                cache: this.cache,
+            })
+            : doNothing;
+        // You may not return an async restart function. Perform any required async activity in the
+        // reconfigure function and return a sync restart function. See the note at the top of this
+        // file.
+        [restartTestServer, restartOAuthTestServer, restartInboundServer, restartOutboundServer, restartControlClient]
+            .map(f => assert(Promise.resolve(f) !== f, 'Restart functions must be synchronous'));
+        restartTestServer();
+        restartOAuthTestServer();
+        restartInboundServer();
+        restartOutboundServer();
+        restartControlClient();
+        this.conf = newConf;
+        await Promise.all([
+            oldCache && oldCache.disconnect(),
+        ]);
+    }
+
     async start() {
         await this.cache.connect();
 
@@ -83,9 +231,25 @@ class Server extends EventEmitter {
         const startOauthTestServer = this.conf.oauthTestServer.enabled
             ?  this.oauthTestServer.start()
             : null;
+
+        // We only start the control client if we're running within Mojaloop Payment Manager.
+        // The control server is the Payment Manager Management API Service.
+        // We only start the client to connect to and listen to the Management API service for
+        // management protocol messages e.g configuration changes, certificate updates etc.
+        if (this.conf.pm4mlEnabled) {
+            this.controlClient = await ControlAgent.Client.Create({
+                address: this.conf.control.mgmtAPIWsUrl,
+                port: this.conf.control.mgmtAPIWsPort,
+                logger: this.logger.push(LOG_ID.CONTROL),
+                appConfig: this.conf,
+            });
+            this.controlClient.on(ControlAgent.EVENT.RECONFIGURE, this.restart.bind(this));
+        }
+
         await Promise.all([
             this.inboundServer.start(),
             this.outboundServer.start(),
+            this.metricsServer.start(),
             startTestServer,
             startOauthTestServer,
         ]);
@@ -97,10 +261,23 @@ class Server extends EventEmitter {
             this.outboundServer.stop(),
             this.oauthTestServer.stop(),
             this.testServer.stop(),
+            this.controlClient.stop(),
+            this.metricsServer.stop(),
         ]);
     }
 }
 
+/*
+* Call the Connector Manager in Management API to get the updated config
+*/
+async function _GetUpdatedConfigFromMgmtAPI(conf, logger, client) {
+    logger.log(`Getting updated config from Management API at ${conf.control.mgmtAPIWsUrl}:${conf.control.mgmtAPIWsPort}...`);
+    const clientSendResponse = await client.send(ControlAgent.build.CONFIGURATION.READ());
+    logger.log('client send returned:: ', clientSendResponse);
+    const responseRead = await client.receive();
+    logger.log('client receive returned:: ', responseRead);
+    return responseRead.data;
+}
 
 if(require.main === module) {
     (async () => {
@@ -114,6 +291,18 @@ if(require.main === module) {
             },
             stringify: Logger.buildStringify({ space: config.logIndent }),
         });
+        if(config.pm4mlEnabled) {
+            const controlClient = await ControlAgent.Client.Create({
+                address: config.control.mgmtAPIWsUrl,
+                port: config.control.mgmtAPIWsPort,
+                logger: logger,
+                appConfig: config,
+            });
+            const updatedConfigFromMgmtAPI = await _GetUpdatedConfigFromMgmtAPI(config, logger, controlClient);
+            logger.log(`updatedConfigFromMgmtAPI: ${JSON.stringify(updatedConfigFromMgmtAPI)}`);
+            _.merge(config, updatedConfigFromMgmtAPI);
+            controlClient.terminate();
+        }
         const svr = new Server(config, logger);
         svr.on('error', (err) => {
             logger.push({ err }).log('Unhandled server error');
@@ -139,6 +328,7 @@ if(require.main === module) {
 // scheme adapter as a service
 module.exports = {
     Cache,
+    ControlAgent,
     InboundServerMiddleware,
     OutboundServerMiddleware,
     Router,

package/src/lib/cache.js

@@ -21,8 +21,8 @@ const CONN_ST = {
 };
 
 /**
- * A shared cache abstraction over a REDIS distributed key/value store
- */
+  * A shared cache abstraction over a REDIS distributed key/value store
+  */
 class Cache {
     constructor(config) {
         this._config = config;
@@ -53,12 +53,12 @@ class Cache {
     }
 
     /**
-     * Connects to a redis server and waits for ready events
-     * Note: We create two connections. One for get, set and publish commands
-     * and another for subscribe commands. We do this as we are not supposed
-     * to issue any non-pub/sub related commands on a connection used for sub
-     * See: https://redis.io/topics/pubsub
-     */
+      * Connects to a redis server and waits for ready events
+      * Note: We create two connections. One for get, set and publish commands
+      * and another for subscribe commands. We do this as we are not supposed
+      * to issue any non-pub/sub related commands on a connection used for sub
+      * See: https://redis.io/topics/pubsub
+      */
     async connect() {
         switch(this._connectionState) {
             case CONN_ST.CONNECTED:
@@ -74,7 +74,6 @@ class Cache {
                 await this._inProgressDisconnection;
                 break;
         }
-
         this._connectionState = CONN_ST.CONNECTING;
         this._inProgressConnection = Promise.all([this._getClient(), this._getClient()]);
         [this._client, this._subscriptionClient] = await this._inProgressConnection;
@@ -82,15 +81,19 @@ class Cache {
         // hook up our sub message handler
         this._subscriptionClient.on('message', this._onMessage.bind(this));
 
+        if (this._config.enableTestFeatures) {
+            this.setTestMode(true);
+        }
+
         this._inProgressConnection = null;
         this._connectionState = CONN_ST.CONNECTED;
     }
 
     /**
-     * Configure Redis to emit keyevent events. This corresponds to the application test mode, and
-     * enables us to listen for changes on callback_* and request_* keys.
-     * Docs: https://redis.io/topics/notifications
-     */
+      * Configure Redis to emit keyevent events. This corresponds to the application test mode, and
+      * enables us to listen for changes on callback_* and request_* keys.
+      * Docs: https://redis.io/topics/notifications
+      */
     async setTestMode(enable) {
         // See for modes: https://redis.io/topics/notifications#configuration
         // This mode, 'Es$' is:
@@ -100,7 +103,7 @@ class Cache {
         const mode = enable ? 'Es$' : '';
         this._logger
             .push({ 'notify-keyspace-events': mode })
-            .log('REDIS client Configured to emit keyspace-events');
+            .log('Configuring Redis to emit keyevent events');
         this._client.config('SET', 'notify-keyspace-events', mode);
     }
 
@@ -133,12 +136,12 @@ class Cache {
 
 
     /**
-     * Subscribes to a channel
-     *
-     * @param channel {string} - The channel name to subscribe to
-     * @param callback {function} - Callback function to be executed when messages arrive on the specified channel
-     * @returns {Promise} - Promise that resolves with an integer callback Id to submit in unsubscribe request
-     */
+      * Subscribes to a channel
+      *
+      * @param channel {string} - The channel name to subscribe to
+      * @param callback {function} - Callback function to be executed when messages arrive on the specified channel
+      * @returns {Promise} - Promise that resolves with an integer callback Id to submit in unsubscribe request
+      */
     async subscribe(channel, callback) {
         return new Promise((resolve, reject) => {
             this._subscriptionClient.subscribe(channel, (err) => {
@@ -168,11 +171,11 @@ class Cache {
 
 
     /**
-     * Unsubscribes a callback from a channel
-     *
-     * @param channel {string} - name of the channel to unsubscribe from
-     * @param callbackId {integer} - id of the callback to remove
-     */
+      * Unsubscribes a callback from a channel
+      *
+      * @param channel {string} - name of the channel to unsubscribe from
+      * @param callbackId {integer} - id of the callback to remove
+      */
     async unsubscribe(channel, callbackId) {
         return new Promise((resolve, reject) => {
             if(this._callbacks[channel] && this._callbacks[channel][callbackId]) {
@@ -196,8 +199,8 @@ class Cache {
 
 
     /**
-     * Handler for published messages
-     */
+      * Handler for published messages
+      */
     async _onMessage(channel, msg) {
         if(this._callbacks[channel]) {
             // we have some callbacks to make
@@ -219,16 +222,16 @@ class Cache {
 
 
     /**
-     * Returns a new redis client
-     *
-     * @returns {object} - a connected REDIS client
-     * */
+      * Returns a new redis client
+      *
+      * @returns {object} - a connected REDIS client
+      * */
     async _getClient() {
         return new Promise((resolve, reject) => {
             const client = redis.createClient(this._config);
 
             client.on('error', (err) => {
-                this._logger.push({ err }).log('REDIS client Error');
+                this._logger.push({ err }).log('Error from REDIS client getting subscriber');
                 return reject(err);
             });
 
@@ -247,25 +250,25 @@ class Cache {
                 }
             });
 
-            client.on('ready', () => {
-                this._logger.log(`REDIS client ready at: ${this._config.host}:${this._config.port}`);
-                return resolve(client);
-            });
-
             client.on('connect', () => {
                 this._logger.log(`REDIS client connected at: ${this._config.host}:${this._config.port}`);
             });
+
+            client.on('ready', () => {
+                this._logger.log(`Connected to REDIS at: ${this._config.host}:${this._config.port}`);
+                return resolve(client);
+            });
         });
     }
 
 
     /**
-     * Publishes the specified message to the specified channel
-     *
-     * @param channelName {string} - channel name to publish to
-     * @param value - any type that will be converted to a JSON string (unless it is already a string) and published as the message
-     * @returns {Promise} - Promise that will resolve with redis replies or reject with an error
-     */
+      * Publishes the specified message to the specified channel
+      *
+      * @param channelName {string} - channel name to publish to
+      * @param value - any type that will be converted to a JSON string (unless it is already a string) and published as the message
+      * @returns {Promise} - Promise that will resolve with redis replies or reject with an error
+      */
     async publish(channelName, value) {
         return new Promise((resolve, reject) => {
             if(typeof(value) !== 'string') {
@@ -288,11 +291,11 @@ class Cache {
 
 
     /**
-     * Sets a value in the cache
-     *
-     * @param key {string} - cache key
-     * @param value {stirng} - cache value
-     */
+      * Sets a value in the cache
+      *
+      * @param key {string} - cache key
+      * @param value {stirng} - cache value
+      */
     async set(key, value) {
         return new Promise((resolve, reject) => {
             //if we are given an object, turn it into a string
@@ -313,10 +316,65 @@ class Cache {
     }
 
     /**
-     * Gets a value from the cache
-     *
-     * @param key {string} - cache key
-     */
+      * Add the specified value to the set stored at key
+      *
+      * @param key {string} - cache key
+      * @param value {string} - cache value
+      */
+    async add(key, value) {
+        return new Promise((resolve, reject) => {
+            //if we are given an object, turn it into a string
+            if(typeof(value) !== 'string') {
+                value = JSON.stringify(value);
+            }
+
+            this._client.sadd(key, value, (err, replies) => {
+                if(err) {
+                    this._logger.push({ key, value, err }).log(`Error setting cache key: ${key}`);
+                    return reject(err);
+                }
+
+                this._logger.push({ key, value, replies }).log(`Add cache key: ${key}`);
+                return resolve(replies);
+            });
+        });
+    }
+
+    /**
+      * Returns all the members of the set value stored at key
+      *
+      * @param key {string} - cache key
+      */
+    async members(key) {
+        return new Promise((resolve, reject) => {
+            this._client.smembers(key, (err, value) => {
+                if(err) {
+                    this._logger.push({ key, err }).log(`Error getting cache key: ${key}`);
+                    return reject(err);
+                }
+
+                this._logger.push({ key, value }).log(`Got cache key: ${key}`);
+
+                if(typeof(value) === 'string') {
+                    try {
+                        value = JSON.parse(value);
+                    }
+                    catch(err) {
+                        this._logger.push({ err }).log('Error parsing JSON cache value');
+                        return reject(err);
+                    }
+                }
+
+                return resolve(value);
+            });
+        });
+    }
+
+    /**
+      * Gets a value from the cache
+      *
+      * @param key {string} - cache key
+      */
     async get(key) {
         return new Promise((resolve, reject) => {
             this._client.get(key, (err, value) => {

package/src/lib/metrics.js

@@ -0,0 +1,148 @@
+/**************************************************************************
+ *  (C) Copyright ModusBox Inc. 2019 - All rights reserved.               *
+ *                                                                        *
+ *  This file is made available under the terms of the license agreement  *
+ *  specified in the corresponding source code repository.                *
+ *                                                                        *
+ *  ORIGINAL AUTHOR:                                                      *
+ *       James Bush - james.bush@modusbox.com                             *
+ **************************************************************************/
+
+'use strict';
+
+const http = require('http');
+const Koa = require('koa');
+const koaBody = require('koa-body');
+const PrometheusClient = require('prom-client');
+
+
+/**
+  * A utility class that abstracts the underlying metrics implementation (Prometheus)
+  * from the consumer. This may be premature if Prometheus is never swapped out...but
+  * who can tell what the Universe will bring us.
+  *
+  * This object exposes methods for getting different types of measurement construct
+  * in order for consuming code to record metrics. The constructs are quite tightly
+  * coupled to Prometheus view of metrics, although that is fairly abstract so the
+  * risk appears low that this will cause conflicts in future.
+  *
+  * The metrics client is intended to be used as a singleton in a process and keeps a
+  * 'per name' cache of metrics to avoid duplicates. Not sure if this is strictly
+  * necessary but I dont have time to dig into the prom-client code to see what
+  * happens if you create the same metric twice.
+  */
+class MetricsClient {
+    constructor() {
+        this._prometheusRegister = PrometheusClient.register;
+        this._metrics = {};
+
+        this._counterPrefix = 'cntr_';
+        this._histogramPrefix = 'hist_';
+        this._gaugePrefix = 'gage_';
+    }
+
+
+    getHistogram(name, description, buckets) {
+        const metricName = `${this._histogramPrefix}${name}`;
+
+        let conf = {
+            name: name,
+            help: description,
+        };
+
+        if(buckets) {
+            conf.buckets = buckets;
+        }
+
+        if(!this._metrics[metricName]) {
+            this._metrics[metricName] = new PrometheusClient.Histogram(conf);
+        }
+
+        return this._metrics[metricName];
+    }
+
+
+    getCounter(name, description) {
+        const metricName = `${this._counterPrefix}${name}`;
+
+        if(!this._metrics[metricName]) {
+            this._metrics[metricName] = new PrometheusClient.Counter({
+                name: name,
+                help: description
+            });
+        }
+
+        return this._metrics[metricName];
+    }
+
+
+    getGauge(name, description) {
+        const metricName = `${this._counterPrefix}${name}`;
+
+        if(!this._metrics[metricName]) {
+            this._metrics[metricName] = new PrometheusClient.Gauge({
+                name: name,
+                help: description
+            });
+        }
+
+        return this._metrics[metricName];
+    }
+}
+
+
+/**
+  * Exposes an HTTP endpoint for metrics to be scraped by some external daemon
+  */
+class MetricsServer {
+    /**
+      * @param {number} port metrics server listen port
+      * @param {Logger} logger Logger
+      * @param {Object} prometheusClient Prometheus client instance
+      */
+    constructor({ port, logger }) {
+        this._port = port;
+        this._logger = logger;
+        this._prometheusClient = PrometheusClient;
+        this._prometheusRegister = PrometheusClient.register;
+        this._api = this.setupApi();
+        this._server = http.createServer(this._api.callback());
+    }
+
+    async start() {
+        if (this._server.listening) {
+            return;
+        }
+        this._prometheusClient.collectDefaultMetrics({
+            prefix: 'mojaloop_connector_default_'
+        });
+
+        await new Promise((resolve) => this._server.listen(this._port, resolve));
+        this._logger.push({ port: this._port }).log('Serving Metrics');
+    }
+
+    async stop() {
+        await new Promise(resolve => this._server.close(resolve));
+        this._logger.log('Metrics Server shut down complete');
+    }
+
+    setupApi() {
+        const result = new Koa();
+
+        result.use(koaBody());
+        result.use(async ctx => {
+            this._logger.log('Metrics request received');
+
+            ctx.response.set('Content-Type', this._prometheusRegister.contentType);
+            ctx.response.body = this._prometheusRegister.metrics();
+        });
+
+        return result;
+    }
+}
+
+
+module.exports = {
+    MetricsServer,
+    MetricsClient
+};

package/src/lib/model/OutboundTransfersModel.js

@@ -66,6 +66,36 @@ class OutboundTransfersModel {
             secret: config.ilpSecret,
             logger: this._logger,
         });
+
+        this.metrics = {
+            partyLookupRequests: config.metricsClient.getCounter(
+                'mojaloop_connector_outbound_party_lookup_request_count',
+                'Count of outbound party lookup requests sent'),
+            partyLookupResponses: config.metricsClient.getCounter(
+                'mojaloop_connector_outbound_party_lookup_response_count',
+                'Count of responses received to outbound party lookups'),
+            quoteRequests: config.metricsClient.getCounter(
+                'mojaloop_connector_outbound_quote_request_count',
+                'Count of outbound quote requests sent'),
+            quoteResponses: config.metricsClient.getCounter(
+                'mojaloop_connector_outbound_quote_response_count',
+                'Count of responses received to outbound quote requests'),
+            transferPrepares: config.metricsClient.getCounter(
+                'mojaloop_connector_outbound_transfer_prepare_count',
+                'Count of outbound transfer prepare requests sent'),
+            transferFulfils: config.metricsClient.getCounter(
+                'mojaloop_connector_outbound_transfer_fulfil_response_count',
+                'Count of responses received to outbound transfer prepares'),
+            partyLookupLatency: config.metricsClient.getHistogram(
+                'mojaloop_connector_outbound_party_lookup_latency',
+                'Time taken for a response to a party lookup request to be received'),
+            quoteRequestLatency: config.metricsClient.getHistogram(
+                'mojaloop_connector_outbound_quote_request_latency',
+                'Time taken for a response to a quote request to be received'),
+            transferLatency: config.metricsClient.getHistogram(
+                'mojaloop_connector_outbound_transfer_latency',
+                'Time taken for a response to a transfer prepare to be received')
+        };
     }
 
 
@@ -185,9 +215,16 @@ class OutboundTransfersModel {
                 subId: this.data.to.idSubValue
             });
 
+            let latencyTimerDone;
+
             // hook up a subscriber to handle response messages
             const subId = await this._cache.subscribe(payeeKey, (cn, msg, subId) => {
                 try {
+                    if(latencyTimerDone) {
+                        latencyTimerDone();
+                    }
+                    this.metrics.partyLookupResponses.inc();
+
                     let payee = JSON.parse(msg);
 
                     if(payee.errorInformation) {
@@ -279,8 +316,10 @@ class OutboundTransfersModel {
             // now we have a timeout handler and a cache subscriber hooked up we can fire off
             // a GET /parties request to the switch
             try {
+                latencyTimerDone = this.metrics.partyLookupLatency.startTimer();
                 const res = await this._requests.getParties(this.data.to.idType, this.data.to.idValue,
                     this.data.to.idSubValue, this.data.to.fspId);
+                this.metrics.partyLookupRequests.inc();
                 this._logger.push({ peer: res }).log('Party lookup sent to peer');
             }
             catch(err) {
@@ -312,10 +351,16 @@ class OutboundTransfersModel {
 
             // listen for events on the quoteId
             const quoteKey = `qt_${quote.quoteId}`;
+            let latencyTimerDone;
 
             // hook up a subscriber to handle response messages
             const subId = await this._cache.subscribe(quoteKey, (cn, msg, subId) => {
                 try {
+                    if(latencyTimerDone) {
+                        latencyTimerDone();
+                    }
+                    this.metrics.quoteResponses.inc();
+
                     let error;
                     let message = JSON.parse(msg);
 
@@ -380,7 +425,9 @@ class OutboundTransfersModel {
             // now we have a timeout handler and a cache subscriber hooked up we can fire off
             // a POST /quotes request to the switch
             try {
+                latencyTimerDone = this.metrics.quoteRequestLatency.startTimer();
                 const res = await this._requests.postQuotes(quote, this.data.to.fspId);
+                this.metrics.quoteRequests.inc();
                 this._logger.push({ res }).log('Quote request sent to peer');
             }
             catch(err) {
@@ -459,12 +506,20 @@ class OutboundTransfersModel {
             // listen for events on the transferId
             const transferKey = `tf_${this.data.transferId}`;
 
+            let latencyTimerDone;
+
             const subId = await this._cache.subscribe(transferKey, async (cn, msg, subId) => {
                 try {
                     let error;
                     let message = JSON.parse(msg);
 
+                    if(latencyTimerDone) {
+                        latencyTimerDone();
+                    }
+
                     if (message.type === 'transferFulfil') {
+                        this.metrics.transferFulfils.inc();
+
                         if (this._rejectExpiredTransferFulfils) {
                             const now = new Date().toISOString();
                             if (now > prepare.expiration) {
@@ -523,7 +578,9 @@ class OutboundTransfersModel {
             // now we have a timeout handler and a cache subscriber hooked up we can fire off
             // a POST /transfers request to the switch
             try {
+                latencyTimerDone = this.metrics.transferLatency.startTimer();
                 const res = await this._requests.postTransfers(prepare, this.data.quoteResponseSource);
+                this.metrics.transferPrepares.inc();
                 this._logger.push({ res }).log('Transfer prepare sent to peer');
             }
             catch(err) {

package/test/__mocks__/redis.js

@@ -26,6 +26,10 @@ class RedisClient extends redisMock.RedisClient {
         }
     }
 
+    config(...args) {
+        this._executeCallback(...args);
+    }
+
     subscribe(...args) {
         super.subscribe(...args);
         this._executeCallback(...args);

package/test/unit/api/utils.js

@@ -6,6 +6,7 @@ const Validate = require('~/lib/validate');
 
 const InboundServer = require('~/InboundServer');
 const OutboundServer = require('~/OutboundServer');
+const { MetricsClient } = require('~/lib/metrics');
 const { Logger } = require('@mojaloop/sdk-standard-components');
 const Cache = require('~/lib/cache');
 
@@ -47,7 +48,9 @@ const createTestServers = async (config) => {
     });
     await cache.connect();
     defConfig.requestProcessingTimeoutSeconds = 2;
-    const serverOutbound = new OutboundServer(defConfig, logger, cache);
+    const metricsClient = new MetricsClient();
+    metricsClient._prometheusRegister.clear();
+    const serverOutbound = new OutboundServer(defConfig, logger, cache, metricsClient);
     await serverOutbound.start();
     const reqOutbound = supertest(serverOutbound._server);
 

package/test/unit/index.test.js

@@ -10,13 +10,18 @@
 
 'use strict';
 
-const { Logger } = require('@mojaloop/sdk-standard-components');
-const defaultConfig = require('./data/defaultConfig');
-
 jest.mock('dotenv', () => ({
     config: jest.fn()
 }));
 
+const promClient = require('prom-client');
+const defaultConfig = require('./data/defaultConfig.json');
+const { Logger } = require('@mojaloop/sdk-standard-components');
+const { MetricsClient } = require('~/lib/metrics');
+
+const TestControlServer = require('./ControlServer');
+
+
 process.env.PEER_ENDPOINT = '172.17.0.3:4000';
 process.env.BACKEND_ENDPOINT = '172.17.0.5:4000';
 process.env.CACHE_HOST = '172.17.0.2';
@@ -26,6 +31,10 @@ process.env.MGMT_API_WS_URL = '0.0.0.0';
 const index = require('~/index.js');
 
 describe('index.js', () => {
+    beforeEach(() => {
+        promClient.register.clear();
+    });
+
     test('WSO2 error events in OutboundServer propagate to top-level server', () => {
         const logger = new Logger.Logger({ stringify: () => '' });
         const svr = new index.Server(defaultConfig, logger);
@@ -53,3 +62,86 @@ describe('index.js', () => {
         expect(typeof(index.Cache)).toBe('function');
     });
 });
+
+describe('Server', () => {
+    let server, controlServer, conf, logger;
+
+    beforeEach(async () => {
+        promClient.register.clear();
+        logger = new Logger.Logger({ stringify: () => '' });
+        conf = JSON.parse(JSON.stringify(defaultConfig));
+        conf.enableTestFeatures = true;
+        conf.pm4mlEnabled = true;
+        conf.control.mgmtAPIWsUrl = 'localhost';
+        conf.control.mgmtAPIWsPort = 4005;
+        conf.control.port = conf.control.mgmtAPIWsPort;
+        controlServer = new TestControlServer.Server({ logger, appConfig: conf });
+        server = new index.Server(conf, logger);
+        await server.start();
+    });
+
+    afterEach(async () => {
+        await controlServer.stop();
+        await server.stop();
+    });
+
+    describe('is reconfigured correctly by the control client', () => {
+        let newConf;
+        beforeEach(async () => {
+            // not every server restarts on every config change, we'll make sure they all restart
+            newConf = { ...conf, logIndent: conf.logIndent + 1, control: { ...conf.control, rubbish: 'data' }, test: { trash: 'data' } };
+            // Just in case, we'll assert the new configuration is different to the old one
+            expect(newConf).not.toEqual(conf);
+        });
+
+        it('reconfigures and restarts constituent servers when triggered by control client', async () => {
+            const [restartInbound, restartOutbound, restartControl, restartOAuthTest, restartTest] =
+                 Array.from({ length: 5 }).map(() => jest.fn());
+            server.inboundServer.reconfigure = jest.fn(() => restartInbound);
+            server.outboundServer.reconfigure = jest.fn(() => restartOutbound);
+            server.testServer.reconfigure = jest.fn(() => restartTest);
+            server.oauthTestServer.reconfigure = jest.fn(() => restartOAuthTest);
+            server.controlClient.reconfigure = jest.fn(() => restartControl);
+
+            await controlServer.broadcastConfigChange(newConf);
+
+            // We wait for the servers to get restarted
+            await new Promise((wait) => setTimeout(wait, 1000));
+
+            expect(server.inboundServer.reconfigure).toHaveBeenCalledTimes(1);
+            expect(server.inboundServer.reconfigure).toHaveBeenCalledWith(
+                newConf, expect.any(Logger.Logger), expect.any(index.Cache)
+            );
+            expect(server.outboundServer.reconfigure).toHaveBeenCalledTimes(1);
+            const metricsClient = new MetricsClient();
+            expect(server.outboundServer.reconfigure).toHaveBeenCalledWith(
+                newConf, expect.any(Logger.Logger), expect.any(index.Cache), metricsClient
+            );
+            expect(server.controlClient.reconfigure).toHaveBeenCalledTimes(1);
+            expect(server.controlClient.reconfigure).toHaveBeenCalledWith({
+                logger: expect.any(Logger.Logger),
+                port: newConf.control.port,
+                appConfig: newConf
+            });
+            expect(server.testServer.reconfigure).toHaveBeenCalledTimes(1);
+            expect(server.testServer.reconfigure).toHaveBeenCalledWith({
+                logger: expect.any(Logger.Logger),
+                cache: expect.any(index.Cache),
+                port: newConf.test.port
+            });
+            expect(server.oauthTestServer.reconfigure).toHaveBeenCalledTimes(1);
+            expect(server.oauthTestServer.reconfigure).toHaveBeenCalledWith({
+                logger: expect.any(Logger.Logger),
+                clientKey: newConf.oauthTestServer.clientKey,
+                clientSecret: newConf.oauthTestServer.clientSecret,
+                port: newConf.oauthTestServer.listenPort,
+            });
+
+            expect(restartInbound).toHaveBeenCalledTimes(1);
+            expect(restartOutbound).toHaveBeenCalledTimes(1);
+            expect(restartTest).toHaveBeenCalledTimes(1);
+            expect(restartOAuthTest).toHaveBeenCalledTimes(1);
+            expect(restartControl).toHaveBeenCalledTimes(1);
+        });
+    });
+});

package/test/unit/lib/model/OutboundTransfersModel.test.js

@@ -15,6 +15,7 @@ jest.mock('@mojaloop/sdk-standard-components');
 jest.mock('redis');
 
 const Cache = require('~/lib/cache');
+const { MetricsClient } = require('~/lib/metrics');
 const Model = require('~/lib/model').OutboundTransfersModel;
 const PartiesModel = require('~/lib/model').PartiesModel;
 
@@ -50,6 +51,7 @@ describe('outboundModel', () => {
     let config;
     let logger;
     let cache;
+    let metricsClient;
 
     /**
      *
@@ -91,6 +93,7 @@ describe('outboundModel', () => {
             ...config,
             cache,
             logger,
+            metricsClient,
         });
 
         await model.initialize(JSON.parse(JSON.stringify(transferRequest)));
@@ -113,6 +116,7 @@ describe('outboundModel', () => {
     beforeAll(async () => {
         logger = new Logger.Logger({ context: { app: 'outbound-model-unit-tests-cache' }, stringify: () => '' });
         quoteResponse = JSON.parse(JSON.stringify(quoteResponseTemplate));
+        metricsClient = new MetricsClient();
     });
 
     beforeEach(async () => {
@@ -140,6 +144,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -194,6 +199,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -274,6 +280,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -303,6 +310,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -333,6 +341,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -360,6 +369,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -395,6 +405,7 @@ describe('outboundModel', () => {
         let model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -421,6 +432,7 @@ describe('outboundModel', () => {
         model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -451,6 +463,7 @@ describe('outboundModel', () => {
         let model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -477,6 +490,7 @@ describe('outboundModel', () => {
         model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -502,6 +516,7 @@ describe('outboundModel', () => {
         model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -557,6 +572,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -608,6 +624,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -701,6 +718,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -763,6 +781,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -823,6 +842,7 @@ describe('outboundModel', () => {
         const model = new Model({
             cache,
             logger,
+            metricsClient,
             ...config,
         });
 
@@ -854,6 +874,7 @@ describe('outboundModel', () => {
         new Model({
             cache,
             logger,
+            metricsClient,
             ...config
         });
 
@@ -866,4 +887,19 @@ describe('outboundModel', () => {
 
     test('Outbound server should use HTTP if outbound mTLS disabled', () =>
         testTlsServer(false));
+
+    test('Outbound transfers model should record metrics', async () => {
+        const metrics = metricsClient._prometheusRegister.metrics();
+        expect(metrics).toBeTruthy();
+
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_party_lookup_request_count'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_party_lookup_response_count'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_quote_request_count'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_quote_response_count'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_transfer_prepare_count'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_transfer_fulfil_response_count'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_quote_request_latency'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_transfer_latency'));
+        expect(metrics).toEqual(expect.stringContaining('mojaloop_connector_outbound_party_lookup_latency'));
+    });
 });