@pager/minion-army 2.0.0 → 2.2.0

package/.eslintrc.js

@@ -1,7 +1,7 @@
 module.exports = {
   extends: '@hapi/eslint-config-hapi',
   parserOptions: {
-    ecmaVersion: 9
+    ecmaVersion: 12
   },
   rules: {
     'no-console': 2

package/.github/workflows/release.yml

@@ -9,13 +9,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - name: Setup Node.js
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v2
         with:
-          node-version: 12
+          node-version: 18
       - name: Cache
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ~/.npm
           key: ${{ runner.os }}-npm-${{ hashFiles('**/package.json') }}

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [2.1.0](https://github.com/pagerinc/minion-army/compare/v2.0.0...v2.1.0) (2023-05-08)
+
+
+### Features
+
+* add requeue to the schema ([#453](https://github.com/pagerinc/minion-army/issues/453)) ([74a5c41](https://github.com/pagerinc/minion-army/commit/74a5c4127aee8513500edfe04760bf03ffee057f))
+
 # [2.0.0](https://github.com/pagerinc/minion-army/compare/v1.3.0...v2.0.0) (2020-08-19)
 
 

package/cloudbuild.yaml

@@ -0,0 +1,39 @@
+steps:
+  - id: npm-install
+    name: node:18
+    secretEnv: ['NPM_TOKEN']
+    entrypoint: npm
+    args: ['i', '--quiet', '--package-lock-only']
+
+  - id: npm-ci
+    name: node:18
+    secretEnv: ['NPM_TOKEN']
+    entrypoint: npm
+    args: ['ci', '--quiet']
+
+  - id: test-unit
+    name: node:18
+    secretEnv: ['NPM_TOKEN']
+    entrypoint: npm
+    args: ['test']
+
+  - id: npm-publish
+    name: 'gcr.io/$PROJECT_ID/cloudbuilders/npm:6.13.4'
+    secretEnv: ['NPM_TOKEN']
+    env:
+      - 'TAG_NAME=$TAG_NAME'
+      - '_PR_NUMBER=$_PR_NUMBER'
+
+timeout: 10m
+
+logsBucket: 'gs://$PROJECT_ID-primary-cloudbuild-logs'
+
+tags:
+  - 'backend'
+  - 'npm'
+  - 'nodejs'
+
+secrets:
+  - kmsKeyName: projects/production-197117/locations/global/keyRings/gcb/cryptoKeys/main
+    secretEnv:
+      NPM_TOKEN: 'CiUA/4lqmXwRPPaGHe+X7TS7mwqARNCw5QFq7yfq7ESHaJrf+tzeElEADvOwrLQvnxCLG2wy+H2vD+DWHMosEgIfzpKNBJAVHX1u4FSwIF5utaN6tMIrLuZB18HnK2SKpsXTPvB/+0Eoz1acnj6WO+slz+GUGUnxefU='

package/lib/index.js

@@ -5,8 +5,9 @@ const { EventEmitter } = require('events');
 const Jackrabbit = require('@pager/jackrabbit');
 const Joi = require('joi');
 const Schema = require('./schema');
+const { createLoggerContext } = require('./loggingUtils');
 
-const validatorFactory = (handler, schema) => (payload, metadata) => { // eslint-disable-line
+const validatorFactory = (handler, schema) => (payload, ...rest) => { // eslint-disable-line
 
     const { error, value } = schema.validate(payload, { stripUnknown: true });
 
@@ -14,7 +15,7 @@ const validatorFactory = (handler, schema) => (payload, metadata) => { // eslint
         throw error;
     }
 
-    return handler(value, metadata);
+    return handler(value, ...rest);
 };
 
 const createExchange = (connection, workerConfig, exchangeMap) => {
@@ -83,9 +84,15 @@ module.exports = (_manifest) => {
             ...manifest.defaults,
             ...worker.config
         };
+
+        let handlerWithLoggerContext = handlerWithValidation;
+        if (manifest.logger) {
+            handlerWithLoggerContext = createLoggerContext(manifest.logger, worker.config.name, handlerWithValidation);
+        }
+
         const exchange = createExchange(manifest.connection, workerConfig, exchangeMap);
 
-        const minion = Minion(handlerWithValidation, { ...workerConfig, exchange });
+        const minion = Minion(handlerWithLoggerContext, { ...workerConfig, exchange });
 
         minion.on('ready', checkReady);
         minion.on('message', (m, meta) => eventEmitter.emit('message', worker.config.name, m, meta));

package/lib/loggingUtils.js

@@ -0,0 +1,112 @@
+'use strict';
+
+const { v4: Uuid } = require('uuid');
+
+const Lodash = require('lodash');
+
+/**
+ * This is a wrapper around the handler that creates a pino style logger and
+ * adds it to a context object as a third argument of the handler
+ *
+ * @param {*} logger a Pino style logger
+ * @param {*} minionWorkerName the name of the minion army worker
+ * @param {*} handler the handler to be wrapped
+ * @returns
+ */
+exports.createLoggerContext = (logger, minionWorkerName, handler) => {
+
+    return (value, metadata, context = {}) => {
+
+        let eventId;
+        try {
+            eventId = metadata?.properties?.headers?.eventId || Uuid();
+            const routingKey = metadata?.fields?.routingKey;
+
+            context.logger = (context.logger || logger).child({ eventId, routingKey, minionWorkerName });
+        }
+        catch (error) {
+            // eslint-disable-next-line no-console
+            console.log('There was an error while creating a child logger in %s for eventId(%s): %o', minionWorkerName, eventId, error);
+        }
+
+        return handler(value, metadata, context);
+    };
+};
+
+/**
+ * This utility crteates a wrapper that adds a field from the event into the logger.
+ *
+ * Usage:
+ *
+ * const Logger = require('@pager/logger');
+ * const Army = require('@pager/minion-army');
+ *
+ * const injectEncounterIdFromEvent = injectFieldFromEventAs(Logger, 'encounterId', 'triageId');
+ *
+ * const handler = (message, metadata, context) {
+ *     context.logger.info('handling message');
+ * }
+ *
+ * const army = Army({
+ *   workers: [
+ *    {
+ *      handler: injectEncounterIdFromEvent(handler),
+ *      config: {
+ *        name: `events.foo.encounter.state.updated`,
+ *        key: '#.encounter.state.updated'
+ *      },
+ *      validate: Schemas.states
+ *    },
+ * });
+ *
+ * @param {*} logger a Pino style logger
+ * @param {*} loggedFieldName what is the name of the field in the log line
+ * @param {*} eventFieldName what is the name of the field in the event payload
+ * @returns
+ */
+exports.injectFieldFromEventAs = (logger, loggedFieldName, eventFieldName) => (handler) => { // eslint-disable-line @hapi/hapi/scope-start, @hapi/hapi/no-arrowception
+
+    return (value, metadata, context = {}) => {
+
+        context.logger = (context.logger || logger).child({ [loggedFieldName]: value[eventFieldName] });
+
+        return handler(value, metadata, context);
+    };
+};
+
+/**
+ * This utility function sets up default event handlers that log
+ *
+ * Usage:
+ *
+ * const Logger = require('@pager/logger');
+ * const Army = require('@pager/minion-army');
+ *
+ * const army = Army({ ... });
+ * addDefaultLoggingEventHandlers(army, 'UpdatedUserArmy', Logger);
+ *
+ * @param {*} army: the Army instance
+ * @param {*} armyName: a unique identifier for this instance of Army
+ * @param {*} parentLogger: a Pino style logger
+ */
+exports.addDefaultLoggingEventHandlers = (parentLogger, army, armyName) => {
+
+    const logger = parentLogger.child({ armyName });
+
+    army.on('error', (error) => {
+
+        logger.error({ error }, 'Handler error in %s', armyName);
+    });
+
+    army.on('message', (queue, event, metadata) => {
+
+        const pickedMetadata = Lodash.pick(metadata, ['properties', 'fields']);
+
+        logger.info({ queue, event, metadata: pickedMetadata }, 'Got event in %s', armyName);
+    });
+
+    army.on('ready', (queue) => {
+
+        logger.info({ queue }, 'Ready to consume on %s by %s', queue.name, armyName);
+    });
+};

package/lib/schema.js

@@ -19,7 +19,8 @@ const config = Joi.object({
     rabbit,
     rabbitUrl: Joi.string(),
     prefetch: Joi.number(),
-    queueMode: Joi.string()
+    queueMode: Joi.string(),
+    requeue: Joi.boolean()
 });
 
 const worker = {
@@ -35,5 +36,8 @@ module.exports = Joi.object({
         rabbitUrl: Joi.string()
     }),
     defaults: config,
-    workers: Joi.array().items(worker)
+    workers: Joi.array().items(worker),
+    logger: Joi.object({
+        child: Joi.func().required()
+    }).unknown(true)
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pager/minion-army",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "description": "Microservice Framework for RabbitMQ Workers",
   "main": "lib/index.js",
   "scripts": {
@@ -25,17 +25,20 @@
   "homepage": "https://github.com/pagerinc/minion#readme",
   "dependencies": {
     "@pager/jackrabbit": "5.x",
-    "@pager/minion": "3.x"
+    "@pager/minion": "3.x",
+    "lodash": "^4.17.21"
   },
   "devDependencies": {
+    "@faker-js/faker": "^8.0.1",
     "@hapi/eslint-config-hapi": "13.x",
     "@hapi/eslint-plugin-hapi": "4.x",
-    "@pager/semantic-release-config": "1.x",
+    "@pager/semantic-release-config": "2.x",
     "ava": "3.x",
     "eslint": "7.x",
     "joi": "^17.2.1",
     "nyc": "15.x",
-    "semantic-release": "17.x"
+    "semantic-release": "21.x",
+    "sinon": "^15.0.4"
   },
   "peerDependencies": {
     "joi": "^17.2.0"

package/test/index.js

@@ -4,8 +4,41 @@ const { EventEmitter } = require('events');
 const Test = require('ava');
 const Joi = require('joi');
 
+const Sinon = require('sinon');
+const { faker } = require('@faker-js/faker');
+
 const Army = require('../lib/index');
 
+const sandbox = Sinon.createSandbox();
+
+
+Test.beforeEach((t) => {
+
+    t.context.logger = {
+        child: sandbox.stub(),
+        info: sandbox.stub()
+    };
+    t.context.rabbit = {
+        topic: () => ({
+            publish: sandbox.spy()
+        }),
+        direct: () => ({
+            publish: sandbox.spy()
+        })
+    };
+    t.context.metadata = {
+        properties: {
+            headers: {
+                eventId: faker.string.uuid()
+            }
+        },
+        fields: {
+            routingKey: faker.string.uuid()
+        }
+    };
+
+});
+
 Test('Creates army from manifest and workers work', async (t) => {
 
     const manifest = {
@@ -399,3 +432,49 @@ Test('Army uses worker exchange overrides', (t) => {
     const army = Army(manifest);
     t.deepEqual(Object.keys(army.exchangeMap), ['direct.events.something.happened', 'topic.my-other-exchange']);
 });
+
+
+Test('Army starts with logger context', async (t) => {
+
+    const { context: { rabbit, logger, metadata } } = t;
+
+    const minionWorkerName = faker.word.sample();
+
+    const handler = sandbox.spy();
+
+    const manifest = {
+        connection: {
+            rabbit
+        },
+        defaults: {
+            exchangeName: faker.word.sample()
+        },
+        workers: [
+            {
+                handler,
+                config: {
+                    name: minionWorkerName,
+                    key: faker.word.sample()
+                }
+            }
+        ],
+        logger
+    };
+
+    const army = Army(manifest);
+    t.truthy(army);
+
+    const event = {
+        data: faker.word.sample()
+    };
+
+    const childLogger = sandbox.spy();
+
+    t.context.logger.child.returns(childLogger);
+
+    await army.minions[minionWorkerName].handle(event, metadata);
+
+    Sinon.assert.calledOnceWithExactly(logger.child, { eventId: metadata.properties.headers.eventId, routingKey: metadata.fields.routingKey, minionWorkerName });
+
+    Sinon.assert.calledOnceWithExactly(handler, event, metadata, { logger: childLogger });
+});

package/test/loggingUtils.js

@@ -0,0 +1,178 @@
+'use strict';
+
+const Test = require('ava');
+
+const Sinon = require('sinon');
+const { faker } = require('@faker-js/faker');
+
+const Army = require('../lib/index');
+const { createLoggerContext, injectFieldFromEventAs, addDefaultLoggingEventHandlers } = require('../lib/loggingUtils');
+
+const sandbox = Sinon.createSandbox();
+
+
+Test.beforeEach((t) => {
+
+    t.context.logger = {
+        child: sandbox.stub()
+    };
+    t.context.metadata = {
+        properties: {
+            headers: {
+                eventId: faker.string.uuid()
+            }
+        },
+        fields: {
+            routingKey: faker.string.uuid()
+        }
+    };
+    t.context.event = sandbox.spy();
+    t.context.minionWorkerName = faker.word.sample();
+    t.context.handler = sandbox.spy();
+    t.context.rabbit = {
+        topic: () => ({
+            publish: sandbox.spy()
+        })
+    };
+});
+
+Test.afterEach((t) => {
+
+    sandbox.restore();
+});
+
+Test('createLoggerContext wraps a handler', (t) => {
+
+    const { context: { logger, event, metadata, minionWorkerName, handler } } = t;
+
+    const wrappedHandler = createLoggerContext(logger, minionWorkerName, handler);
+
+    const childLogger = sandbox.spy();
+
+    t.context.logger.child.returns(childLogger);
+
+    wrappedHandler(event, metadata);
+
+    Sinon.assert.calledOnceWithExactly(logger.child, { eventId: metadata.properties.headers.eventId, routingKey: metadata.fields.routingKey, minionWorkerName });
+
+    Sinon.assert.calledOnceWithExactly(handler, event, metadata, { logger: childLogger });
+
+    t.pass();
+});
+
+Test('createLoggerContext does not fail if the logger fails', (t) => {
+
+    const { context: { logger, event, metadata, minionWorkerName, handler } } = t;
+
+    const wrappedHandler = createLoggerContext(logger, minionWorkerName, handler);
+
+    t.context.logger.child.throws(new Error('failed to create logger'));
+
+    wrappedHandler(event, metadata);
+
+    Sinon.assert.calledOnceWithExactly(logger.child, { eventId: metadata.properties.headers.eventId, routingKey: metadata.fields.routingKey, minionWorkerName });
+
+    Sinon.assert.calledOnceWithExactly(handler, event, metadata, {});
+
+    t.pass();
+});
+
+Test('injectFieldFromEventAs adds field to the logger', (t) => {
+
+    const { context: { logger, metadata, handler } } = t;
+
+    const wrappedHandler = injectFieldFromEventAs(logger, 'encounterId', 'triageId')(handler);
+
+    const childLogger = sandbox.spy();
+
+    t.context.logger.child.returns(childLogger);
+
+    const event = {
+        triageId: faker.database.mongodbObjectId()
+    };
+
+    wrappedHandler(event, metadata);
+
+    Sinon.assert.calledOnceWithExactly(logger.child, { encounterId: event.triageId });
+
+    Sinon.assert.calledOnceWithExactly(handler, event, metadata, { logger: childLogger });
+
+    t.pass();
+});
+
+Test('injectFieldFromEventAs does not add the field to the logger if it is missing', (t) => {
+
+    const { context: { logger, metadata, handler } } = t;
+
+    const wrappedHandler = injectFieldFromEventAs(logger, 'encounterId', 'triageId')(handler);
+
+    const childLogger = sandbox.spy();
+
+    t.context.logger.child.returns(childLogger);
+
+    const event = {};
+
+    wrappedHandler(event, metadata);
+
+    Sinon.assert.calledOnceWithExactly(logger.child, { encounterId: undefined });
+
+    Sinon.assert.calledOnceWithExactly(handler, event, metadata, { logger: childLogger });
+
+    t.pass();
+});
+
+
+Test('addDefaultLoggingEventHandlers adds the handlers', (t) => {
+
+    const { context: { logger, rabbit, metadata } } = t;
+
+    const workerName = faker.word.sample();
+    const queueName = faker.word.sample();
+
+    const manifest = {
+        connection: {
+            rabbit
+        },
+        defaults: {
+            exchangeName: faker.word.sample()
+        },
+        workers: [
+            {
+                handler: sandbox.spy(),
+                config: {
+                    name: workerName,
+                    key: queueName
+                }
+            }
+        ]
+    };
+
+    const army = Army(manifest);
+
+    t.truthy(army);
+
+    const childLogger = {
+        info: sandbox.spy()
+    };
+
+    t.context.logger.child.returns(childLogger);
+
+    const armyName = faker.word.sample();
+
+    addDefaultLoggingEventHandlers(logger, army, armyName);
+
+    Sinon.assert.calledOnceWithExactly(logger.child, { armyName });
+
+    const event = {
+        data: faker.word.sample()
+    };
+
+    army.minions[workerName].handle(event, metadata);
+
+    const expectedMetadata = {
+        properties: metadata.properties,
+        fields: metadata.fields
+    };
+
+    Sinon.assert.calledOnceWithExactly(childLogger.info, { queue: workerName, event, metadata: expectedMetadata }, 'Got event in %s', armyName);
+});

package/.circleci/config.yml

@@ -1,61 +0,0 @@
-version: 2
-
-defaults: &defaults
-  working_directory: ~/repo
-  docker:
-    - image: circleci/node:12.18@sha256:cc3bed34d3606a476fcb4ef0f2c03890e7a8f2114c2d81055a7695ddbf2eefc9
-
-jobs:
-  test:
-    <<: *defaults
-    steps:
-      - checkout
-
-      - restore_cache:
-          keys:
-            - dependency-cache-{{ checksum "package.json" }}
-            - dependency-cache-
-
-      - run: npm install
-
-      - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-          paths:
-            - ./node_modules
-
-      - run:
-          name: Run tests
-          command: npm test
-
-      - persist_to_workspace:
-          root: ~/repo
-          paths: .
-
-  deploy:
-    <<: *defaults
-    steps:
-      - attach_workspace:
-          at: ~/repo
-      - run:
-          name: Authenticate with registry
-          command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/repo/.npmrc
-      - run:
-          name: Publish package
-          command: npm publish
-
-workflows:
-  version: 2
-  test-deploy:
-    jobs:
-      - test:
-          filters:
-            tags:
-              only: /^v.*/
-      - deploy:
-          requires:
-            - test
-          filters:
-            tags:
-              only: /^v.*/
-            branches:
-              ignore: /.*/