@itentialopensource/adapter-kafkav2 0.3.12

package/test/unit/adapterTestUnit.js

@@ -0,0 +1,412 @@
+/* @copyright Itential, LLC 2019 (pre-modifications) */
+
+// Set globals
+/* global describe it log pronghornProps */
+/* eslint global-require:warn */
+/* eslint no-unused-vars: warn */
+
+// include required items for testing & logging
+const assert = require('assert');
+const fs = require('fs-extra');
+const mocha = require('mocha');
+const path = require('path');
+const util = require('util');
+const winston = require('winston');
+const execute = require('child_process').execSync;
+const { expect } = require('chai');
+const { use } = require('chai');
+const td = require('testdouble');
+
+const anything = td.matchers.anything();
+
+// stub and attemptTimeout are used throughout the code so set them here
+let logLevel = 'none';
+const stub = true;
+const isRapidFail = false;
+const attemptTimeout = 120000;
+
+// these variables can be changed to run in integrated mode so easier to set them here
+// always check these in with bogus data!!!
+const host = 'replace.hostorip.here';
+const username = 'username';
+const password = 'password';
+const port = 80;
+const sslenable = false;
+const sslinvalid = false;
+
+// these are the adapter properties. You generally should not need to alter
+// any of these after they are initially set up
+global.pronghornProps = {
+  pathProps: {
+    encrypted: false
+  },
+  adapterProps: {
+    adapters: [{
+      id: 'Test-kafkav2',
+      type: 'Kafkav2',
+      interval_time: 5000,
+      stub: true,
+      properties: {
+        topics: []
+      }
+    }]
+  }
+};
+
+global.$HOME = `${__dirname}/../..`;
+
+// set the log levels that Pronghorn uses, spam and trace are not defaulted in so without
+// this you may error on log.trace calls.
+const myCustomLevels = {
+  levels: {
+    spam: 6,
+    trace: 5,
+    debug: 4,
+    info: 3,
+    warn: 2,
+    error: 1,
+    none: 0
+  }
+};
+
+// need to see if there is a log level passed in
+process.argv.forEach((val) => {
+  // is there a log level defined to be passed in?
+  if (val.indexOf('--LOG') === 0) {
+    // get the desired log level
+    const inputVal = val.split('=')[1];
+
+    // validate the log level is supported, if so set it
+    if (Object.hasOwnProperty.call(myCustomLevels.levels, inputVal)) {
+      logLevel = inputVal;
+    }
+  }
+});
+
+// need to set global logging
+global.log = winston.createLogger({
+  level: logLevel,
+  levels: myCustomLevels.levels,
+  transports: [
+    new winston.transports.Console()
+  ]
+});
+
+/**
+ * Runs the error asserts for the test
+ */
+function runErrorAsserts(data, error, code, origin, displayStr) {
+  assert.equal(null, data);
+  assert.notEqual(undefined, error);
+  assert.notEqual(null, error);
+  assert.notEqual(undefined, error.IAPerror);
+  assert.notEqual(null, error.IAPerror);
+  assert.notEqual(undefined, error.IAPerror.displayString);
+  assert.notEqual(null, error.IAPerror.displayString);
+  assert.equal(code, error.icode);
+  assert.equal(origin, error.IAPerror.origin);
+  assert.equal(displayStr, error.IAPerror.displayString);
+}
+
+// require the adapter that we are going to be using
+const Kafkav2 = require('../../adapter');
+
+// delete the .DS_Store directory in entities -- otherwise this will cause errors
+const dirPath = path.join(__dirname, '../../entities/.DS_Store');
+if (fs.existsSync(dirPath)) {
+  try {
+    fs.removeSync(dirPath);
+    console.log('.DS_Store deleted');
+  } catch (e) {
+    console.log('Error when deleting .DS_Store:', e);
+  }
+}
+
+// begin the testing - these should be pretty well defined between the describe and the it!
+describe('[unit] Kafka Adapter Test', () => {
+  describe('Kafka Class Tests', () => {
+    const a = new Kafkav2(
+      pronghornProps.adapterProps.adapters[0].id,
+      pronghornProps.adapterProps.adapters[0].properties
+    );
+
+    if (isRapidFail) {
+      const state = {};
+      state.passed = true;
+
+      mocha.afterEach(function x() {
+        state.passed = state.passed
+        && (this.currentTest.state === 'passed');
+      });
+      mocha.beforeEach(function x() {
+        if (!state.passed) {
+          return this.currentTest.skip();
+        }
+        return true;
+      });
+    }
+
+    describe('#class instance created', () => {
+      it('should be a class with properties', (done) => {
+        assert.notEqual(null, a);
+        assert.notEqual(undefined, a);
+        const check = global.pronghornProps.adapterProps.adapters[0].id;
+        assert.equal(check, a.id);
+        done();
+      }).timeout(attemptTimeout);
+    });
+
+    let wffunctions = [];
+    describe('#getWorkflowFunctions', () => {
+      it('should retrieve workflow functions', (done) => {
+        wffunctions = a.getWorkflowFunctions();
+        assert.notEqual(0, wffunctions.length);
+        done();
+      }).timeout(attemptTimeout);
+    });
+
+    describe('package.json', () => {
+      it('should have a package.json', (done) => {
+        fs.exists('package.json', (val) => {
+          assert.equal(true, val);
+          done();
+        });
+      });
+      it('package.json should be validated', (done) => {
+        const packageDotJson = require('../../package.json');
+        const { PJV } = require('package-json-validator');
+        const options = {
+          warnings: true, // show warnings
+          recommendations: true // show recommendations
+        };
+        const results = PJV.validate(JSON.stringify(packageDotJson), 'npm', options);
+
+        if (results.valid === false) {
+          log.error('The package.json contains the following errors: ');
+          log.error(util.inspect(results));
+          assert.equal(true, results.valid);
+        } else {
+          assert.equal(true, results.valid);
+        }
+
+        done();
+      });
+      it('package.json should be customized', (done) => {
+        const packageDotJson = require('../../package.json');
+        assert.notEqual(-1, packageDotJson.name.indexOf('kafkav2'));
+        assert.notEqual(undefined, packageDotJson.version);
+        assert.notEqual(null, packageDotJson.version);
+        assert.notEqual('', packageDotJson.version);
+        done();
+      });
+    });
+
+    describe('pronghorn.json', () => {
+      it('should have a pronghorn.json', (done) => {
+        fs.exists('pronghorn.json', (val) => {
+          assert.equal(true, val);
+          done();
+        });
+      });
+      it('pronghorn.json should be customized', (done) => {
+        const pronghornDotJson = require('../../pronghorn.json');
+        assert.notEqual(-1, pronghornDotJson.id.indexOf('kafkav2'));
+        assert.equal('Kafkav2', pronghornDotJson.export);
+        assert.equal('Kafkav2', pronghornDotJson.displayName);
+        assert.equal('Kafkav2', pronghornDotJson.title);
+        done();
+      });
+      it('pronghorn.json should only expose workflow functions', (done) => {
+        const pronghornDotJson = require('../../pronghorn.json');
+
+        for (let m = 0; m < pronghornDotJson.methods.length; m += 1) {
+          let found = false;
+          let paramissue = false;
+
+          for (let w = 0; w < wffunctions.length; w += 1) {
+            if (pronghornDotJson.methods[m].name === wffunctions[w]) {
+              found = true;
+              const methLine = execute(`grep "${wffunctions[w]}(" adapter.js | grep "callback) {"`).toString();
+              let wfparams = [];
+
+              if (methLine.indexOf('(') >= 0 && methLine.indexOf(')') >= 0) {
+                const temp = methLine.substring(methLine.indexOf('(') + 1, methLine.indexOf(')'));
+                wfparams = temp.split(',');
+
+                for (let t = 0; t < wfparams.length; t += 1) {
+                  // remove default value from the parameter name
+                  wfparams[t] = wfparams[t].substring(0, wfparams[t].search(/=/) > 0 ? wfparams[t].search(/#|\?|=/) : wfparams[t].length);
+                  // remove spaces
+                  wfparams[t] = wfparams[t].trim();
+
+                  if (wfparams[t] === 'callback') {
+                    wfparams.splice(t, 1);
+                  }
+                }
+              }
+
+              // if there are inputs defined but not on the method line
+              if (wfparams.length === 0 && (pronghornDotJson.methods[m].input
+                  && pronghornDotJson.methods[m].input.length > 0)) {
+                paramissue = true;
+              } else if (wfparams.length > 0 && (!pronghornDotJson.methods[m].input
+                  || pronghornDotJson.methods[m].input.length === 0)) {
+                // if there are no inputs defined but there are on the method line
+                paramissue = true;
+              } else {
+                for (let p = 0; p < pronghornDotJson.methods[m].input.length; p += 1) {
+                  let pfound = false;
+                  for (let wfp = 0; wfp < wfparams.length; wfp += 1) {
+                    if (pronghornDotJson.methods[m].input[p].name.toUpperCase() === wfparams[wfp].toUpperCase()) {
+                      pfound = true;
+                    }
+                  }
+
+                  if (!pfound) {
+                    paramissue = true;
+                  }
+                }
+                for (let wfp = 0; wfp < wfparams.length; wfp += 1) {
+                  let pfound = false;
+                  for (let p = 0; p < pronghornDotJson.methods[m].input.length; p += 1) {
+                    if (pronghornDotJson.methods[m].input[p].name.toUpperCase() === wfparams[wfp].toUpperCase()) {
+                      pfound = true;
+                    }
+                  }
+
+                  if (!pfound) {
+                    paramissue = true;
+                  }
+                }
+              }
+
+              break;
+            }
+          }
+
+          if (!found) {
+            // this is the reason to go through both loops - log which ones are not found so
+            // they can be worked
+            log.error(`${pronghornDotJson.methods[m].name} not found in workflow functions`);
+          }
+          if (paramissue) {
+            // this is the reason to go through both loops - log which ones are not found so
+            // they can be worked
+            log.error(`${pronghornDotJson.methods[m].name} has a parameter mismatch`);
+          }
+          assert.equal(true, found);
+          assert.equal(false, paramissue);
+        }
+        done();
+      }).timeout(attemptTimeout);
+      it('pronghorn.json should expose all workflow functions', (done) => {
+        const pronghornDotJson = require('../../pronghorn.json');
+        for (let w = 0; w < wffunctions.length; w += 1) {
+          let found = false;
+
+          for (let m = 0; m < pronghornDotJson.methods.length; m += 1) {
+            if (pronghornDotJson.methods[m].name === wffunctions[w]) {
+              found = true;
+              break;
+            }
+          }
+
+          if (!found) {
+            // this is the reason to go through both loops - log which ones are not found so
+            // they can be worked
+            log.error(`${wffunctions[w]} not found in pronghorn.json`);
+          }
+          assert.equal(true, found);
+        }
+        done();
+      });
+    });
+
+    describe('propertiesSchema.json', () => {
+      it('should have a propertiesSchema.json', (done) => {
+        fs.exists('propertiesSchema.json', (val) => {
+          assert.equal(true, val);
+          done();
+        });
+      });
+      it('propertiesSchema.json should be customized', (done) => {
+        const propertiesDotJson = require('../../propertiesSchema.json');
+        assert.equal('adapter-kafkav2', propertiesDotJson.$id);
+        done();
+      });
+    });
+
+    describe('error.json', () => {
+      it('should have an error.json', (done) => {
+        fs.exists('error.json', (val) => {
+          assert.equal(true, val);
+          done();
+        });
+      });
+    });
+
+    describe('README.md', () => {
+      it('should have a README', (done) => {
+        fs.exists('README.md', (val) => {
+          assert.equal(true, val);
+          done();
+        });
+      });
+      it('README.md should be customized', (done) => {
+        fs.readFile('README.md', 'utf8', (err, data) => {
+          assert.equal(-1, data.indexOf('[System]'));
+          assert.equal(-1, data.indexOf('[system]'));
+          assert.equal(-1, data.indexOf('[version]'));
+          assert.equal(-1, data.indexOf('[namespace]'));
+          done();
+        });
+      });
+    });
+
+    describe('#connect', () => {
+      it('should have a connect function', (done) => {
+        assert.equal(true, typeof a.connect === 'function');
+        done();
+      });
+    });
+
+    describe('#healthCheck', () => {
+      it('should have a healthCheck function', (done) => {
+        assert.equal(true, typeof a.healthCheck === 'function');
+        done();
+      });
+    });
+
+    /*
+    -----------------------------------------------------------------------
+    -----------------------------------------------------------------------
+    *** All code above this comment will be replaced during a migration ***
+    ******************* DO NOT REMOVE THIS COMMENT BLOCK ******************
+    -----------------------------------------------------------------------
+    -----------------------------------------------------------------------
+    */
+    describe('#sendMessage - errors', () => {
+      it('should have a sendMessage function', (done) => {
+        assert.equal(true, typeof a.sendMessage === 'function');
+        done();
+      });
+      it('should error on sendMessage - no payloads', (done) => {
+        try {
+          a.sendMessage(null, (data, error) => {
+            try {
+              const displayE = 'payloads is required';
+              runErrorAsserts(data, error, 'AD.300', 'Test-kafkav2-adapter-sendMessage', displayE);
+              done();
+            } catch (ex) {
+              log.error(`Test Failure: ${ex}`);
+              done(ex);
+            }
+          });
+        } catch (exc) {
+          log.error(`Adapter Exception: ${exc}`);
+          done(exc);
+        }
+      }).timeout(attemptTimeout);
+    });
+  });
+});

package/utils/artifactize.js

@@ -0,0 +1,146 @@
+#!/usr/bin/env node
+/* @copyright Itential, LLC 2019 */
+
+const fs = require('fs-extra');
+const path = require('path');
+
+async function createBundle(adapterOldDir) {
+  // set directories
+  const artifactDir = path.join(adapterOldDir, '../artifactTemp');
+  const workflowsDir = path.join(adapterOldDir, 'workflows');
+
+  // read adapter's package and set names
+  const adapterPackage = fs.readJSONSync(path.join(adapterOldDir, 'package.json'));
+  const originalName = adapterPackage.name.substring(adapterPackage.name.lastIndexOf('/') + 1);
+  const shortenedName = originalName.replace('adapter-', '');
+  const artifactName = originalName.replace('adapter', 'bundled-adapter');
+
+  const adapterNewDir = path.join(artifactDir, 'bundles', 'adapters', originalName);
+  fs.ensureDirSync(adapterNewDir);
+
+  const ops = [];
+
+  // copy old adapterDir to bundled hierarchy location
+  ops.push(() => fs.copySync(adapterOldDir, adapterNewDir));
+
+  // copy readme
+  ops.push(() => fs.copySync(path.join(adapterOldDir, 'README.md'), path.join(artifactDir, 'README.md')));
+
+  // copy changelog
+  if (fs.existsSync(path.join(adapterOldDir, 'CHANGELOG.md'))) {
+    ops.push(() => fs.copySync(path.join(adapterOldDir, 'CHANGELOG.md'), path.join(artifactDir, 'CHANGELOG.md')));
+  }
+
+  // copy license
+  if (fs.existsSync(path.join(adapterOldDir, 'LICENSE'))) {
+    ops.push(() => fs.copySync(path.join(adapterOldDir, 'LICENSE'), path.join(artifactDir, 'LICENSE')));
+  }
+
+  // create package
+  const artifactPackage = {
+    name: artifactName,
+    version: adapterPackage.version,
+    description: `A bundled version of the ${originalName} to be used in adapter-artifacts for easy installation`,
+    scripts: {
+      test: 'echo "Error: no test specified" && exit 1',
+      deploy: 'npm publish --registry=http://registry.npmjs.org'
+    },
+    keywords: [
+      'IAP',
+      'artifacts',
+      'Itential',
+      'Pronghorn',
+      'Adapter',
+      'Adapter-Artifacts',
+      shortenedName
+    ],
+    author: 'Itential Artifacts',
+    license: 'Apache-2.0',
+    repository: adapterPackage.repository,
+    private: false,
+    devDependencies: {
+      r2: '^2.0.1',
+      ajv: '6.10.0',
+      'better-ajv-errors': '^0.6.1',
+      'fs-extra': '^7.0.1'
+    }
+  };
+
+  ops.push(() => fs.writeJSONSync(path.join(artifactDir, 'package.json'), artifactPackage, { spaces: 2 }));
+
+  // create manifest
+  const manifest = {
+    bundleName: originalName,
+    version: adapterPackage.version,
+    fingerprint: 'Some verifiable token',
+    createdEpoch: '1554836984020',
+    artifacts: [
+      {
+        id: `${shortenedName}-adapter`,
+        name: `${shortenedName}-adapter`,
+        type: 'adapter',
+        location: `/bundles/adapters/${originalName}`,
+        description: artifactPackage.description,
+        properties: {
+          entryPoint: false
+        }
+      }
+    ]
+  };
+
+  // add workflows into artifact
+  if (fs.existsSync(workflowsDir)) {
+    const workflowFileNames = fs.readdirSync(workflowsDir);
+
+    // if folder isnt empty and only file is not readme
+    if (workflowFileNames.length !== 0 && (!(workflowFileNames.length === 1 && workflowFileNames[0].split('.')[1] === 'md'))) {
+      // add workflows to correct location in bundle
+      ops.push(() => fs.copySync(workflowsDir, path.join(artifactDir, 'bundles', 'workflows')));
+
+      // add workflows to manifest
+      workflowFileNames.forEach((filename) => {
+        const [filenameNoExt, ext] = filename.split('.');
+        if (ext === 'json') {
+          manifest.artifacts.push({
+            id: `workflow-${filenameNoExt}`,
+            name: filenameNoExt,
+            type: 'workflow',
+            location: `/bundles/workflows/${filename}`,
+            description: 'Main entry point to artifact',
+            properties: {
+              entryPoint: false
+            }
+          });
+        }
+      });
+    }
+  }
+
+  ops.push(() => fs.writeJSONSync(path.join(artifactDir, 'manifest.json'), manifest, { spaces: 2 }));
+
+  // Run the commands in parallel
+  try {
+    await Promise.all(ops.map(async (op) => op()));
+  } catch (e) {
+    throw new Error(e);
+  }
+
+  const pathObj = {
+    bundlePath: artifactDir,
+    bundledAdapterPath: path.join(artifactDir, 'bundles', 'adapters', originalName)
+  };
+  return pathObj;
+}
+
+async function artifactize(entryPathToAdapter) {
+  const truePath = path.resolve(entryPathToAdapter);
+  const packagePath = path.join(truePath, 'package');
+  // remove adapter from package and move bundle in
+  const pathObj = await createBundle(packagePath);
+  const { bundlePath } = pathObj;
+  fs.removeSync(packagePath);
+  fs.moveSync(bundlePath, packagePath);
+  return 'Bundle successfully created and old folder system removed';
+}
+
+module.exports = { createBundle, artifactize };

package/utils/packModificationScript.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+/* @copyright Itential, LLC 2019 */
+
+const fs = require('fs-extra');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { createBundle } = require('./artifactize');
+
+const nodeEntryPath = path.resolve('.');
+createBundle(nodeEntryPath).then((pathObj) => {
+  const { bundlePath, bundledAdapterPath } = pathObj;
+  const npmIgnorePath = path.join(bundledAdapterPath, '.npmignore');
+  const adapterPackagePath = path.join(bundledAdapterPath, 'package.json');
+  const artifactPackagePath = path.join(bundlePath, 'package.json');
+
+  // remove node_modules from .npmIgnore so that node_modules are included in the resulting tar from npm pack
+  let npmIgnoreString;
+  if (fs.existsSync(npmIgnorePath)) {
+    npmIgnoreString = fs.readFileSync(npmIgnorePath, 'utf8');
+    npmIgnoreString = npmIgnoreString.replace('node_modules', '');
+    npmIgnoreString = npmIgnoreString.replace('\n\n', '\n');
+    fs.writeFileSync(npmIgnorePath, npmIgnoreString);
+  }
+
+  // add files to package so that node_modules are included in the resulting tar from npm pack
+  const adapterPackage = fs.readJSONSync(adapterPackagePath);
+  adapterPackage.files = ['*'];
+  fs.writeJSONSync(artifactPackagePath, adapterPackage, { spaces: 2 });
+  const npmResult = spawnSync('npm', ['pack', '-q', bundlePath], { cwd: path.resolve(bundlePath, '..') });
+  if (npmResult.status === 0) {
+    fs.removeSync(bundlePath);
+    console.log('Bundle folder removed');
+  }
+  console.log('Script successful');
+});

package/utils/pre-commit.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+# @copyright Itential, LLC 2019
+
+#exit on any failure in the pipeline
+set -e
+
+# --------------------------------------------------
+# pre-commit
+# --------------------------------------------------
+# Contains the standard set of tasks to runbefore
+# committing changes to the repo. If any tasks fail
+# then the commit will be aborted.
+# --------------------------------------------------
+
+printf "%b" "Running pre-commit hooks...\\n"
+
+# verify testing script is stubbed and no credentials
+# node utils/testRunner.js -r
+
+# security audit on the code
+npm audit --registry=https://registry.npmjs.org --audit-level=moderate
+
+# lint the code
+npm run lint
+
+# test the code
+npm run test

package/utils/setup.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+/* @copyright Itential, LLC 2019 */
+
+const fs = require('fs');
+
+/**
+ * This script will execute before an npm install command. The purpose is to
+ * write out some standard git hooks that will enable folks working on this
+ * project to benefit from the protections that the hooks provide.
+ */
+
+const precommit = fs.readFileSync('utils/pre-commit.sh', 'utf8');
+
+fs.stat('.git', (err) => {
+  if (err == null) {
+    // git repo, not an npm repo.
+    // add pre-commit hook if it doesn't exist
+    fs.stat('.git/hooks/pre-commit', (statErr) => {
+      if (statErr == null || statErr.code === 'ENOENT') {
+        fs.writeFile('.git/hooks/pre-commit', precommit, {
+          mode: 0o755
+        }, (writeErr) => {
+          if (writeErr) {
+            return console.log(writeErr.message);
+          }
+          return null;
+        });
+      } else {
+        console.log(statErr.message);
+      }
+    });
+  }
+});