@itentialopensource/adapter-kafkav2 0.5.0 → 0.6.1

package/CHANGELOG.md

@@ -1,4 +1,20 @@
 
+## 0.6.1 [05-04-2023]
+
+* added support for separate topics files per adapter
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!5
+
+---
+
+## 0.6.0 [04-10-2023]
+
+* Turn Off stream if WFE or OpMgr are down
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!4
+
+---
+
 ## 0.5.0 [03-13-2023]
 
 * minor/adapt-2595

package/README.md

@@ -87,6 +87,10 @@ This section defines **all** the properties that are available for the adapter,
     "stub": false,
     "parseMessage": true,
     "wrapMessage": "myKey",
+    "check_iap_apps": true,
+    "check_wfe_status": true,
+    "check_ops_manager_status": false,
+    "iap_apps_check_interval": 15000,
     "client": {
       "connectTimeout": 10000,
       "requestTimeout": 30000,
@@ -272,6 +276,12 @@ The following properties are used to define the Kafka Consumer. These properties
 
 The `parseMessage` property allows the user to define how they want the Kafka message to be published to IAP's event system. If `parseMessage` is set to true or omitted, the value of the Kafka message will be parsed as either an object or string and wrapped in an outer object. The wrapper object's key can be defined with the property `wrapMessage`, or the default value `payload` can be used if omitted. If `parseMessage` is set to false, the entire kafka payload, including metadata, would be returned and the message itself would need to be transformed at a later point.
 
+## Turning off stream if WorkflowEngine or OperationsManager is down
+
+`check_iap_apps` (boolean) and `iap_apps_check_interval` (integer) are used when the user wants to turn off stream if WorkflowEngine or OperationsManager is down. If `check_iap_apps` is set to true, by default, the adapter will check the status of both WorkflowEngine and OperationsManager at a defined interval. If any IAP apps are down, the consumer will be paused until the apps are active again. If the user does not want to healthcheck OperationsManager, set `check_ops_manager_status` to false. Similarly, if the user does not want to check WorkflowEngine status, set `check_wfe_status` to false.
+
+`iap_apps_check_interval` (default 30000ms - 30s) allows the user to set the frequency in which to run IAP app healtcheck.
+
 ## Testing an Itential Product Adapter
 
 Mocha is generally used to test all Itential Product Adapters. There are unit tests as well as integration tests performed. Integration tests can generally be run as standalone using mock data and running the adapter in stub mode, or as integrated. When running integrated, every effort is made to prevent environmental failures, however there is still a possibility.

package/adapter.js

@@ -2,6 +2,7 @@
 
 // Set globals
 /* global eventSystem log */
+/* global cogs */
 /* eslint no-underscore-dangle: warn  */
 /* eslint no-loop-func: warn */
 /* eslint no-cond-assign: warn */
@@ -16,7 +17,8 @@ const fs = require('fs-extra');
 const path = require('path');
 const util = require('util');
 /* Fetch in the other needed components for the this Adaptor */
-const EventEmitterCl = require('events').EventEmitter;
+// const EventEmitterCl = require('events').EventEmitter;
+const AdapterBaseCl = require(path.join(__dirname, 'adapterBase.js'));
 const request = require('request');
 
 function consoleLoggerProvider(name) {
@@ -160,13 +162,13 @@ function formatErrorObject(origin, type, variables, sysCode, sysRes, stack) {
 /**
  * This is the adapter/interface into Kafka
  */
-class Kafkav2 extends EventEmitterCl {
+class Kafkav2 extends AdapterBaseCl {
   /**
    * Kafka Adapter
    * @constructor
    */
   constructor(prongid, properties) {
-    super();
+    super(prongid, properties);
 
     this.props = properties;
     this.alive = false;
@@ -176,11 +178,15 @@ class Kafkav2 extends EventEmitterCl {
 
     // put topics from file into memory
     this.topicsEvents = [];
-    const topicsFile = path.join(__dirname, '/.topics.json');
+    let topicsFile = path.join(__dirname, `/.topics-${this.id}.json`);
 
     // if the file does not exist - error
     if (fs.existsSync(topicsFile)) {
       this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
+    } else if (fs.existsSync(path.join(__dirname, '/.topics.json'))) {
+      log.debug('Found old .topics.json file.');
+      topicsFile = path.join(__dirname, '/.topics.json');
+      this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
     }
 
     // get the topics into the right format --- fix older .topics.json files so they have the right format
@@ -393,7 +399,8 @@ class Kafkav2 extends EventEmitterCl {
       const intTime = this.props.interval_time || 30000;
       setInterval(() => {
         try {
-          log.debug('Update .topics.json file');
+          const split = topicsFile.split('/');
+          log.debug(`Update ${split[split.length - 1]} file.`);
           fs.writeFileSync(topicsFile, JSON.stringify(this.topicsEvents, null, 2));
         } catch (err) {
           log.error(err);
@@ -466,6 +473,36 @@ class Kafkav2 extends EventEmitterCl {
         }
         await this.consumer.connect();
         await this.consumer.subscribe({ topics, fromBeginning: true });
+
+        let isAppActive = null;
+        const topicPartitions = topics.map((topic) => ({ topic }));
+
+        // if any IAP apps are down, pause consumer before it consumes any messages
+        if (this.props && this.props.check_iap_apps) {
+          isAppActive = await super.checkIapAppsStatus();
+          if (!isAppActive) {
+            log.debug('Pause consumer since IAP apps are down');
+            // wait for the consumer to connect to Kafka before pausing it
+            this.consumer.on(this.consumer.events.CONNECT, async () => {
+              this.consumer.pause(topicPartitions);
+            });
+          }
+        }
+
+        // keep checking IAP apps status
+        if (this.props && this.props.check_iap_apps) {
+          const intTime = this.props.iap_apps_check_interval || 30000;
+          setInterval(async () => {
+            isAppActive = await super.checkIapAppsStatus();
+            if (isAppActive) {
+              log.debug('Resume consumer since IAP apps are active');
+              this.consumer.resume(topicPartitions);
+            } else {
+              log.debug('Keep pausing consumer until IAP apps are active');
+              this.consumer.pause(topicPartitions);
+            }
+          }, intTime);
+        }
         await this.consumer.run({
           eachMessage: async ({ topic, partition, message }) => {
             log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
@@ -610,67 +647,23 @@ class Kafkav2 extends EventEmitterCl {
     return callback(retstatus);
   }
 
-  /**
-   * getAllFunctions is used to get all of the exposed function in the adapter
-   *
-   * @function getAllFunctions
-   */
-  getAllFunctions() {
-    let myfunctions = [];
-    let obj = this;
-
-    // find the functions in this class
-    do {
-      const l = Object.getOwnPropertyNames(obj)
-        .concat(Object.getOwnPropertySymbols(obj).map((s) => s.toString()))
-        .sort()
-        .filter((p, i, arr) => typeof obj[p] === 'function' && p !== 'constructor' && (i === 0 || p !== arr[i - 1]) && myfunctions.indexOf(p) === -1);
-      myfunctions = myfunctions.concat(l);
-    }
-    while (
-      (obj = Object.getPrototypeOf(obj)) && Object.getPrototypeOf(obj)
-    );
-
-    return myfunctions;
-  }
-
   /**
    * getWorkflowFunctions is used to get all of the workflow function in the adapter
    *
    * @function getWorkflowFunctions
    */
-  getWorkflowFunctions() {
-    const myIgnore = [
+  getWorkflowFunctions(inIgnore) {
+    let myIgnore = [
       'subscribeAvroWithSubscriber',
       'subscribeWithSubscriber'
     ];
-
-    const myfunctions = this.getAllFunctions();
-    const wffunctions = [];
-
-    // remove the functions that should not be in a Workflow
-    for (let m = 0; m < myfunctions.length; m += 1) {
-      if (myfunctions[m] === 'addListener') {
-        // got to the second tier (adapterBase)
-        break;
-      }
-      if (myfunctions[m] !== 'connect' && myfunctions[m] !== 'healthCheck'
-        && myfunctions[m] !== 'getAllFunctions' && myfunctions[m] !== 'getWorkflowFunctions') {
-        let found = false;
-        if (myIgnore && Array.isArray(myIgnore)) {
-          for (let i = 0; i < myIgnore.length; i += 1) {
-            if (myfunctions[m].toUpperCase() === myIgnore[i].toUpperCase()) {
-              found = true;
-            }
-          }
-        }
-        if (!found) {
-          wffunctions.push(myfunctions[m]);
-        }
-      }
+    if (!inIgnore && Array.isArray(inIgnore)) {
+      myIgnore = inIgnore;
+    } else if (!inIgnore && typeof inIgnore === 'string') {
+      myIgnore = [inIgnore];
     }
 
-    return wffunctions;
+    return super.getWorkflowFunctions(myIgnore);
   }
 
   /**

package/adapterBase.js

@@ -0,0 +1,129 @@
+/* @copyright Itential, LLC 2019 (pre-modifications) */
+
+// Set globals
+/* global eventSystem log */
+/* global cogs */
+/* eslint no-underscore-dangle: warn  */
+/* eslint no-loop-func: warn */
+/* eslint no-cond-assign: warn */
+/* eslint no-unused-vars: warn */
+/* eslint consistent-return: warn */
+/* eslint import/no-dynamic-require: warn */
+/* eslint global-require: warn */
+/* eslint no-await-in-loop: warn */
+
+/* Required libraries.  */
+const EventEmitterCl = require('events').EventEmitter;
+
+class AdapterBase extends EventEmitterCl {
+  constructor(prongid, properties) {
+    super();
+
+    this.props = properties;
+    this.alive = false;
+    this.healthy = false;
+    this.id = prongid;
+  }
+
+  /**
+   * getAllFunctions is used to get all of the exposed function in the adapter
+   *
+   * @function getAllFunctions
+   */
+  getAllFunctions() {
+    let myfunctions = [];
+    let obj = this;
+
+    // find the functions in this class
+    do {
+      const l = Object.getOwnPropertyNames(obj)
+        .concat(Object.getOwnPropertySymbols(obj).map((s) => s.toString()))
+        .sort()
+        .filter((p, i, arr) => typeof obj[p] === 'function' && p !== 'constructor' && (i === 0 || p !== arr[i - 1]) && myfunctions.indexOf(p) === -1);
+      myfunctions = myfunctions.concat(l);
+    }
+    while (
+      (obj = Object.getPrototypeOf(obj)) && Object.getPrototypeOf(obj)
+    );
+    return myfunctions;
+  }
+
+  /**
+   * getWorkflowFunctions is used to get all of the workflow function in the adapter
+   *
+   * @function getWorkflowFunctions
+   */
+  getWorkflowFunctions(ignoreThes) {
+    const myfunctions = this.getAllFunctions();
+    const wffunctions = [];
+    // remove the functions that should not be in a Workflow
+    for (let m = 0; m < myfunctions.length; m += 1) {
+      if (myfunctions[m] === 'checkIapAppsStatus') {
+        // got to the second tier (adapterBase)
+        break;
+      }
+      if (myfunctions[m] !== 'connect' && myfunctions[m] !== 'healthCheck'
+        && myfunctions[m] !== 'getAllFunctions' && myfunctions[m] !== 'getWorkflowFunctions') {
+        let found = false;
+        if (ignoreThes && Array.isArray(ignoreThes)) {
+          for (let i = 0; i < ignoreThes.length; i += 1) {
+            if (myfunctions[m].toUpperCase() === ignoreThes[i].toUpperCase()) {
+              found = true;
+            }
+          }
+        }
+        if (!found) {
+          wffunctions.push(myfunctions[m]);
+        }
+      }
+    }
+    return wffunctions;
+  }
+
+  /**
+   * @summary checkIapAppsStatus is used to check if any IAP apps are down.
+   *
+   * @function checkIapAppsStatus
+   */
+  async checkIapAppsStatus() {
+    const origin = `${this.id}-adapterBase-checkIapAppsStatus`;
+    log.trace(origin);
+
+    let wfEngineActive = true;
+    let opManagerActive = true;
+    if (this.props && (this.props.check_wfe_status || this.props.check_wfe_status === undefined)
+      && cogs.WorkFlowEngine && cogs.WorkFlowEngine.isActive) {
+      wfEngineActive = await new Promise((resolve, reject) => {
+        cogs.WorkFlowEngine.isActive((data, error) => {
+          if (error || !data) {
+            reject(error);
+          } else {
+            resolve(true);
+          }
+        });
+      }).catch((error) => {
+        log.error(error);
+        return false;
+      });
+    }
+
+    if (this.props && (this.props.check_ops_manager_status || this.props.check_ops_manager_status === undefined)
+      && cogs.OperationsManager && cogs.OperationsManager.getTriggers) {
+      opManagerActive = await new Promise((resolve, reject) => {
+        cogs.OperationsManager.getTriggers({}, (data, error) => {
+          if (error || !data) {
+            reject(error);
+          } else {
+            resolve(true);
+          }
+        });
+      }).catch((error) => {
+        log.error(error);
+        return false;
+      });
+    }
+    return wfEngineActive && opManagerActive;
+  }
+}
+
+module.exports = AdapterBase;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-kafkav2",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "description": "Itential adapter to connect to kafka",
   "main": "adapter.js",
   "wizardVersion": "2.35.0",
@@ -10,6 +10,7 @@
     "preinstall": "node utils/setup.js && npm install --package-lock-only --ignore-scripts && npx npm-force-resolutions",
     "lint": "eslint . --ext .json --ext .js",
     "lint:errors": "node --max_old_space_size=4096 ./node_modules/eslint/bin/eslint.js . --ext .json --ext .js --quiet",
+    "test:baseunit": "mocha test/unit/adapterBaseTestUnit.js --LOG=error",
     "test:unit": "mocha test/unit/adapterTestUnit.js --LOG=error",
     "test:integration": "mocha test/integration/adapterTestIntegration.js --LOG=error",
     "test:cover": "nyc --reporter html --reporter text mocha --reporter dot test/*",

package/test/unit/adapterBaseTestUnit.js

@@ -0,0 +1,239 @@
+// Set globals
+/* global describe it log pronghornProps */
+/* eslint global-require:warn */
+/* eslint no-unused-vars: warn */
+
+// include required items for testing & logging
+const assert = require('assert');
+const fs = require('fs-extra');
+const mocha = require('mocha');
+const path = require('path');
+const util = require('util');
+const winston = require('winston');
+const execute = require('child_process').execSync;
+const { expect } = require('chai');
+const { use } = require('chai');
+const td = require('testdouble');
+
+const anything = td.matchers.anything();
+
+// stub and attemptTimeout are used throughout the code so set them here
+let logLevel = 'none';
+const stub = true;
+const isRapidFail = false;
+const attemptTimeout = 120000;
+
+// these variables can be changed to run in integrated mode so easier to set them here
+// always check these in with bogus data!!!
+const host = 'replace.hostorip.here';
+const username = 'username';
+const password = 'password';
+const port = 80;
+const sslenable = false;
+const sslinvalid = false;
+
+// these are the adapter properties. You generally should not need to alter
+// any of these after they are initially set up
+global.pronghornProps = {
+  pathProps: {
+    encrypted: false
+  },
+  adapterProps: {
+    adapters: [{
+      id: 'Test-kafkav2',
+      type: 'Kafkav2',
+      interval_time: 5000,
+      stub: true,
+      properties: {
+        topics: []
+      }
+    }]
+  }
+};
+
+global.$HOME = `${__dirname}/../..`;
+
+// set the log levels that Pronghorn uses, spam and trace are not defaulted in so without
+// this you may error on log.trace calls.
+const myCustomLevels = {
+  levels: {
+    spam: 6,
+    trace: 5,
+    debug: 4,
+    info: 3,
+    warn: 2,
+    error: 1,
+    none: 0
+  }
+};
+
+// need to see if there is a log level passed in
+process.argv.forEach((val) => {
+  // is there a log level defined to be passed in?
+  if (val.indexOf('--LOG') === 0) {
+    // get the desired log level
+    const inputVal = val.split('=')[1];
+
+    // validate the log level is supported, if so set it
+    if (Object.hasOwnProperty.call(myCustomLevels.levels, inputVal)) {
+      logLevel = inputVal;
+    }
+  }
+});
+
+// need to set global logging
+global.log = winston.createLogger({
+  level: logLevel,
+  levels: myCustomLevels.levels,
+  transports: [
+    new winston.transports.Console()
+  ]
+});
+
+/**
+ * Runs the error asserts for the test
+ */
+function runErrorAsserts(data, error, code, origin, displayStr) {
+  assert.equal(null, data);
+  assert.notEqual(undefined, error);
+  assert.notEqual(null, error);
+  assert.notEqual(undefined, error.IAPerror);
+  assert.notEqual(null, error.IAPerror);
+  assert.notEqual(undefined, error.IAPerror.displayString);
+  assert.notEqual(null, error.IAPerror.displayString);
+  assert.equal(code, error.icode);
+  assert.equal(origin, error.IAPerror.origin);
+  assert.equal(displayStr, error.IAPerror.displayString);
+}
+
+// require the adapter that we are going to be using
+const AdapterBase = require('../../adapterBase');
+
+// delete the .DS_Store directory in entities -- otherwise this will cause errors
+const dirPath = path.join(__dirname, '../../entities/.DS_Store');
+if (fs.existsSync(dirPath)) {
+  try {
+    fs.removeSync(dirPath);
+    console.log('.DS_Store deleted');
+  } catch (e) {
+    console.log('Error when deleting .DS_Store:', e);
+  }
+}
+
+describe('[unit] Adapter Base Test', () => {
+  describe('Adapter Base Class Tests', () => {
+    const a = new AdapterBase(
+      pronghornProps.adapterProps.adapters[0].id,
+      pronghornProps.adapterProps.adapters[0].properties
+    );
+
+    if (isRapidFail) {
+      const state = {};
+      state.passed = true;
+
+      mocha.afterEach(function x() {
+        state.passed = state.passed
+        && (this.currentTest.state === 'passed');
+      });
+      mocha.beforeEach(function x() {
+        if (!state.passed) {
+          return this.currentTest.skip();
+        }
+        return true;
+      });
+    }
+
+    describe('#class instance created', () => {
+      it('should be a class with properties', (done) => {
+        assert.notEqual(null, a);
+        assert.notEqual(undefined, a);
+        const check = global.pronghornProps.adapterProps.adapters[0].id;
+        assert.equal(check, a.id);
+        done();
+      }).timeout(attemptTimeout);
+    });
+
+    describe('adapterBase.js', () => {
+      it('should have an adapterBase.js', (done) => {
+        try {
+          fs.exists('adapterBase.js', (val) => {
+            assert.equal(true, val);
+            done();
+          });
+        } catch (error) {
+          log.error(`Test Failure: ${error}`);
+          done(error);
+        }
+      });
+    });
+
+    describe('#getAllFunctions', () => {
+      it('should have a getAllFunctions function', (done) => {
+        try {
+          assert.equal(true, typeof a.getAllFunctions === 'function');
+          done();
+        } catch (error) {
+          log.error(`Test Failure: ${error}`);
+          done(error);
+        }
+      });
+      it('should return a list of functions', (done) => {
+        const returnedFunctions = ['checkIapAppsStatus', 'getAllFunctions', 'getWorkflowFunctions', 'addListener', 'emit', 'eventNames', 'getMaxListeners',
+          'listenerCount', 'listeners', 'off', 'on', 'once', 'prependListener', 'prependOnceListener', 'rawListeners', 'removeAllListeners',
+          'removeListener', 'setMaxListeners'];
+        try {
+          const expectedFunctions = a.getAllFunctions();
+          try {
+            assert.equal(JSON.stringify(expectedFunctions), JSON.stringify(returnedFunctions));
+            done();
+          } catch (err) {
+            log.error(`Test Failure: ${err}`);
+            done(err);
+          }
+        } catch (error) {
+          log.error(`Adapter Exception: ${error}`);
+          done(error);
+        }
+      }).timeout(attemptTimeout);
+    });
+
+    describe('#getWorkflowFunctions', () => {
+      it('should have a getWorkflowFunctions function', (done) => {
+        try {
+          assert.equal(true, typeof a.getWorkflowFunctions === 'function');
+          done();
+        } catch (error) {
+          log.error(`Test Failure: ${error}`);
+          done(error);
+        }
+      });
+      it('should retrieve workflow functions', (done) => {
+        try {
+          const expectedFunctions = a.getWorkflowFunctions([]);
+          try {
+            assert.equal(0, expectedFunctions.length);
+            done();
+          } catch (err) {
+            log.error(`Test Failure: ${err}`);
+            done(err);
+          }
+        } catch (error) {
+          log.error(`Adapter Exception: ${error}`);
+          done(error);
+        }
+      }).timeout(attemptTimeout);
+    });
+
+    describe('#checkIapAppsStatus', () => {
+      it('should have a checkIapAppsStatus function', (done) => {
+        try {
+          assert.equal(true, typeof a.checkIapAppsStatus === 'function');
+          done();
+        } catch (error) {
+          log.error(`Test Failure: ${error}`);
+          done(error);
+        }
+      });
+    });
+  });
+});

package/test/unit/adapterTestUnit.js

@@ -156,10 +156,24 @@ describe('[unit] Kafka Adapter Test', () => {
       }).timeout(attemptTimeout);
     });
 
+    describe('adapterBase.js', () => {
+      it('should have an adapterBase.js', (done) => {
+        try {
+          fs.exists('adapterBase.js', (val) => {
+            assert.equal(true, val);
+            done();
+          });
+        } catch (error) {
+          log.error(`Test Failure: ${error}`);
+          done(error);
+        }
+      });
+    });
+
     let wffunctions = [];
     describe('#getWorkflowFunctions', () => {
       it('should retrieve workflow functions', (done) => {
-        wffunctions = a.getWorkflowFunctions();
+        wffunctions = a.getWorkflowFunctions([]);
         assert.notEqual(0, wffunctions.length);
         done();
       }).timeout(attemptTimeout);