@itentialopensource/adapter-kafkav2 0.3.12

package/adapter.js

@@ -0,0 +1,814 @@
+/* @copyright Itential, LLC 2019 (pre-modifications) */
+
+// Set globals
+/* global eventSystem log */
+/* eslint no-underscore-dangle: warn  */
+/* eslint no-loop-func: warn */
+/* eslint no-cond-assign: warn */
+/* eslint no-unused-vars: warn */
+/* eslint consistent-return: warn */
+/* eslint import/no-dynamic-require: warn */
+/* eslint global-require: warn */
+/* eslint no-await-in-loop: warn */
+
+/* Required libraries.  */
+const fs = require('fs-extra');
+const path = require('path');
+const util = require('util');
+/* Fetch in the other needed components for the this Adaptor */
+const EventEmitterCl = require('events').EventEmitter;
+const request = require('request');
+
+function consoleLoggerProvider(name) {
+  return {
+    debug: log.debug,
+    info: log.info,
+    warn: log.warn,
+    error: log.error
+  };
+}
+
+const { Kafka, logLevel } = require('kafkajs');
+
+const pjson = require(path.resolve(__dirname, 'pronghorn.json'));
+
+let myid = null;
+let errors = [];
+let firstRun = false;
+let firstDone = false;
+const mytopics = Object.keys(pjson.topics);
+const pronghornFile = path.join(__dirname, '/pronghorn.json');
+
+async function fetchSchema(registryUrl, topic, version) {
+  return new Promise((resolve, reject) => {
+    log.debug(`${registryUrl}/subjects/${topic}/versions/${version}`);
+    try {
+      request(
+        `${registryUrl}/subjects/${topic}/versions/${version}`,
+        (err, res, body) => {
+          if (err) {
+            log.debug('schema fetch - evaluating error');
+            try {
+              const error = JSON.parse(err);
+              return reject(
+                new Error(
+                  `Schema registry error: ${error.error_code} - ${error.message}`
+                )
+              );
+            } catch (ex) {
+              return reject(
+                new Error(
+                  `Schema registry error: ${err}`
+                )
+              );
+            }
+          }
+          if (!res || !res.statusCode || res.statusCode !== 200) {
+            log.debug('schema fetch - Evaluating error response');
+            return reject(
+              new Error(
+                'Schema registry error: Did not receive a valid status code'
+              )
+            );
+          }
+          // this is the schema response
+          log.debug('returning schema response');
+          return resolve(JSON.parse(body));
+        }
+      );
+    } catch (e) {
+      log.error(`error sending request: ${e} \n${e.stack}`);
+      return reject(e);
+    }
+  });
+}
+
+/**
+ * @summary Build a standard error object from the data provided
+ *
+ * @function formatErrorObject
+ * @param {String} origin - the originator of the error (optional).
+ * @param {String} type - the internal error type (optional).
+ * @param {String} variables - the variables to put into the error message (optional).
+ * @param {Integer} sysCode - the error code from the other system (optional).
+ * @param {Object} sysRes - the raw response from the other system (optional).
+ * @param {Exception} stack - any available stack trace from the issue (optional).
+ *
+ * @return {Object} - the error object, null if missing pertinent information
+ */
+function formatErrorObject(origin, type, variables, sysCode, sysRes, stack) {
+  log.trace(`${myid}-adapter-formatErrorObject`);
+
+  // add the required fields
+  const errorObject = {
+    icode: 'AD.999',
+    IAPerror: {
+      origin: `${myid}-unidentified`,
+      displayString: 'error not provided',
+      recommendation: 'report this issue to the adapter team!'
+    }
+  };
+
+  if (origin) {
+    errorObject.IAPerror.origin = origin;
+  }
+  if (type) {
+    errorObject.IAPerror.displayString = type;
+  }
+
+  // add the messages from the error.json
+  for (let e = 0; e < errors.length; e += 1) {
+    if (errors[e].key === type) {
+      errorObject.icode = errors[e].icode;
+      errorObject.IAPerror.displayString = errors[e].displayString;
+      errorObject.IAPerror.recommendation = errors[e].recommendation;
+    } else if (errors[e].icode === type) {
+      errorObject.icode = errors[e].icode;
+      errorObject.IAPerror.displayString = errors[e].displayString;
+      errorObject.IAPerror.recommendation = errors[e].recommendation;
+    }
+  }
+
+  // replace the variables
+  let varCnt = 0;
+  while (errorObject.IAPerror.displayString.indexOf('$VARIABLE$') >= 0) {
+    let curVar = '';
+
+    // get the current variable
+    if (variables && Array.isArray(variables) && variables.length >= varCnt + 1) {
+      curVar = variables[varCnt];
+    }
+    varCnt += 1;
+    errorObject.IAPerror.displayString = errorObject.IAPerror.displayString.replace('$VARIABLE$', curVar);
+  }
+
+  // add all of the optional fields
+  if (sysCode) {
+    errorObject.IAPerror.code = sysCode;
+  }
+  if (sysRes) {
+    errorObject.IAPerror.raw_response = sysRes;
+  }
+  if (stack) {
+    errorObject.IAPerror.stack = stack;
+  }
+
+  // return the object
+  return errorObject;
+}
+
+/**
+ * This is the adapter/interface into Kafka
+ */
+class Kafkav2 extends EventEmitterCl {
+  /**
+   * Kafka Adapter
+   * @constructor
+   */
+  constructor(prongid, properties) {
+    super();
+
+    this.props = properties;
+    this.alive = false;
+    this.healthy = false;
+    this.id = prongid;
+    myid = prongid;
+
+    // put topics from file into memory
+    this.topicsEvents = [];
+    const topicsFile = path.join(__dirname, '/.topics.json');
+
+    // if the file does not exist - error
+    if (fs.existsSync(topicsFile)) {
+      this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
+    }
+
+    // get the topics into the right format --- fix older .topics.json files so they have the right format
+    if (!Array.isArray(this.topicsEvents)) {
+      // original format
+      const keys = Object.keys(this.topicsEvents);
+      const tempTops = [];
+      keys.forEach((item) => {
+        tempTops.push({
+          topic: item,
+          partition: this.topicsEvents[item].partitions || this.topicsEvents[item].partition || 0,
+          offset: this.topicsEvents[item].offset || 0,
+          processed: this.topicsEvents[item].offset || 0,
+          subscribers: this.topicsEvents[item].subscribers || 0,
+          avro: this.topicsEvents[item].avro || 'NO',
+          subInfo: [
+            {
+              subname: 'default',
+              filters: [],
+              rabbit: 'kafka',
+              throttle: {}
+            }
+          ]
+        });
+      });
+      this.topicsEvents = tempTops;
+    } else {
+      for (let t = 0; t < this.topicsEvents.length; t += 1) {
+        if (!Object.hasOwnProperty.call(this.topicsEvents[t], 'processed')) {
+          this.topicsEvents[t].processed = this.topicsEvents[t].offset || 0;
+        }
+        if (!Object.hasOwnProperty.call(this.topicsEvents[t], 'subInfo')) {
+          this.topicsEvents[t].subInfo = [
+            {
+              subname: 'default',
+              filters: [],
+              rabbit: 'kafka',
+              throttle: {}
+            }
+          ];
+        }
+      }
+    }
+
+    // If we have a rabbit queue - need to add that to topics.json
+    // if there are topics in the properties
+    if (this.props && this.props.topics) {
+      let needRestart = false;
+      for (let t = 0; t < this.props.topics.length; t += 1) {
+        if (typeof this.props.topics[t] === 'string') {
+          const topName = this.props.topics[t];
+          if (!mytopics.includes(topName)) {
+            pjson.topics[this.props.topics[t]] = {};
+            needRestart = true;
+          }
+          for (let te = 0; te < this.topicsEvents.length; te += 1) {
+            // only set the rabbit in the topic if it is the default (kafka)
+            if (this.topicsEvents[te].topic === topName && this.topicsEvents[te].partition === 0
+              && this.topicsEvents[te].subInfo.rabbit === 'kafka') {
+              this.topicsEvents[te].subInfo.rabbit = topName;
+              break;
+            }
+          }
+        } else if (typeof this.props.topics[t] === 'object') {
+          const topName = this.props.topics[t].name;
+          // if the topic is not in the pronghorn.json
+          if (!mytopics.includes(topName)) {
+            pjson.topics[topName] = {};
+            needRestart = true;
+          }
+          const topPart = this.props.topics[t].partition || 0;
+          let useAvro = 'NO';
+          if (this.props.topics[t].avro !== undefined && this.props.topics[t].avro !== null && this.props.topics[t].avro === true) {
+            useAvro = 'YES';
+          }
+          let subInfo = [
+            {
+              subname: 'default',
+              filters: [],
+              rabbit: 'kafka',
+              throttle: {}
+            }
+          ];
+          if (this.props.topics[t].subscriberInfo) {
+            subInfo = this.props.topics[t].subscriberInfo;
+            for (let r = 0; r < subInfo.length; r += 1) {
+              // if the rabbit topic is not in the pronghorn.json
+              if (!mytopics.includes(subInfo[r].rabbit)) {
+                pjson.topics[subInfo[r].rabbit] = {};
+                needRestart = true;
+              }
+            }
+          }
+          let found = false;
+          for (let te = 0; te < this.topicsEvents.length; te += 1) {
+            // see if we find the topic and partition
+            if (this.topicsEvents[te].topic === topName && this.topicsEvents[te].partition === topPart) {
+              found = true;
+              // only set the rabbit in the topic if it is the default (kafka)
+              if (this.topicsEvents[te].subInfo.rabbit === 'kafka') {
+                this.topicsEvents[te].subInfo.rabbit = topName;
+              }
+              // set the subscribers if always
+              if (this.props.topics[t].always) {
+                this.topicsEvents[te].subscribers = 99999;
+              }
+              // set the avro accordingly
+              this.topicsEvents[te].avro = useAvro;
+
+              // check to see if we need to add a subscriber
+              for (let sub = 0; sub < subInfo.length; sub += 1) {
+                let subFound = false;
+                for (let s = 0; s <= this.topicsEvents[te].subInfo.length; s += 1) {
+                  // if the subscriber is found, no need to add anything
+                  if (subInfo[sub].subname === this.topicsEvents[te].subInfo[s].subname) {
+                    subFound = true;
+                    // set the filters
+                    if (subInfo[sub].filters) {
+                      this.topicsEvents[te].subInfo[s].filters = subInfo[sub].filters;
+                    }
+                    // set the rabbit
+                    if (subInfo[sub].rabbit) {
+                      this.topicsEvents[te].subInfo[s].rabbit = subInfo[sub].rabbit;
+                    }
+                    // set the throttle
+                    if (subInfo[sub].throttle) {
+                      this.topicsEvents[te].subInfo[s].throttle = subInfo[sub].throttle;
+                    }
+                    break;
+                  }
+                }
+                if (!subFound) {
+                  // add the subscriber if it was not found
+                  this.topicsEvents[te].subInfo.push(subInfo[sub]);
+                }
+              }
+              break;
+            }
+          }
+
+          // if the topic was not found but should always be subscribed to, add it
+          if (!found && this.props.topics[t].always) {
+            this.topicsEvents.push({
+              topic: topName,
+              partition: topPart,
+              offset: 0,
+              processed: 0,
+              subscribers: 99999,
+              avro: useAvro,
+              subInfo
+            });
+          }
+        }
+      }
+
+      // if we need to restart the adapter due to changes to pronghorn.json
+      if (needRestart) {
+        fs.writeFileSync(pronghornFile, JSON.stringify(pjson, null, 2));
+        log.error('NEED TO RESTART ADAPTER - EXITING');
+        const errorObj = {
+          origin: `${this.id}-adapter-constructor`,
+          type: 'Restarting to add new topics',
+          vars: []
+        };
+        // log and throw the error
+        log.error(`${errorObj.origin}: ${errorObj.displayString}`);
+        setTimeout(() => {
+          throw new Error(JSON.stringify(errorObj));
+        }, 1000);
+      }
+    }
+
+    this.KafkaClient = null;
+    this.producer = null;
+    this.consumer = null;
+    this.registryUrl = null;
+    this.registry = 'abc';
+    if (this.props && this.props.registry_url) {
+      this.registryUrl = this.props.registry_url;
+      this.registry = require('avro-schema-registry')(this.props.registry_url);
+    }
+
+    // get the path for the specific error file
+    const errorFile = path.join(__dirname, '/error.json');
+
+    // if the file does not exist - error
+    if (!fs.existsSync(errorFile)) {
+      const origin = `${this.id}-adapter-constructor`;
+      log.warn(`${origin}: Could not locate ${errorFile} - errors will be missing details`);
+    }
+
+    // Read the action from the file system
+    const errorData = JSON.parse(fs.readFileSync(errorFile, 'utf-8'));
+    ({ errors } = errorData);
+
+    // rewrite the topics file for persistence
+    if (this.props && this.props.stub === false) {
+      const intTime = this.props.interval_time || 30000;
+      setInterval(() => {
+        try {
+          log.debug('Update .topics.json file');
+          fs.writeFileSync(topicsFile, JSON.stringify(this.topicsEvents, null, 2));
+        } catch (err) {
+          log.error(err);
+        }
+      }, intTime);
+    }
+  }
+
+  /**
+   * @summary Connect function is used during Pronghorn startup to provide instantiation feedback.
+   *
+   * @function connect
+   */
+  connect() {
+    const meth = 'adapter-connect';
+    const origin = `${this.id}-${meth}`;
+    log.trace(origin);
+
+    // initially set as off
+    this.emit('OFFLINE', { id: this.id });
+    this.alive = true;
+
+    try {
+      // Create a kafka client
+      const combinedProps = this.props.client || {};
+      if (this.props.client.ssl.ca) {
+        combinedProps.ssl.ca = [fs.readFileSync(this.props.client.ssl.ca, 'utf-8')];
+      }
+      if (this.props.client.ssl.cert) {
+        combinedProps.ssl.cert = fs.readFileSync(this.props.client.ssl.cert, 'utf-8');
+      }
+      if (this.props.client.ssl.key) {
+        combinedProps.ssl.key = fs.readFileSync(this.props.client.ssl.key, 'utf-8');
+      }
+
+      this.KafkaClient = new Kafka(combinedProps);
+
+      this.producer = this.KafkaClient.producer(this.props.producer || {});
+
+      this.consumer = this.KafkaClient.consumer(this.props.consumer || {});
+
+      const isPassingFiltering = (topic, message) => {
+        const topicConfig = this.topicsEvents.find((e) => e.topic === topic);
+
+        if (!topicConfig) return true;
+
+        if (!topicConfig.subInfo || !Array.isArray(topicConfig.subInfo)) return true;
+
+        const allFilters = topicConfig.subInfo.filter((e) => Array.isArray(e.filters) && e.filters.length > 0);
+        if (allFilters.length === 0) return true;
+        const matched = allFilters.find((s) => {
+          for (let f = 0; f < s.filters.length; f += 1) {
+            const re = new RegExp(s.filters[f]);
+            if (message.value.toString().match(re)) {
+              log.debug('Matched filter: ', re);
+              return true;
+            }
+          }
+          return false;
+        });
+        return matched;
+      };
+
+      const run = async () => {
+        this.emit('ONLINE', {
+          id: this.id
+        });
+        log.info('EMITTED ONLINE');
+        // Start consuming
+        // Iterate through all topics in topicsEvents
+        const topics = [];
+        for (let t = 0; t < this.topicsEvents.length; t += 1) {
+          topics.push(this.topicsEvents[t].topic);
+        }
+        await this.consumer.connect();
+        await this.consumer.subscribe({ topics, fromBeginning: true });
+        await this.consumer.run({
+          eachMessage: async ({ topic, partition, message }) => {
+            log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
+            const newMsg = message;
+            let avroUsed = false;
+            let desiredTopic = 'kafka';
+
+            // find the topic here to change the offset
+            for (let t = 0; t < this.topicsEvents.length; t += 1) {
+              if (this.topicsEvents[t].topic === topic && this.topicsEvents[t].partition === partition) {
+                if (this.topicsEvents[t].offset < message.offset) {
+                  this.topicsEvents[t].offset = message.offset;
+                }
+                // determine if we are using AVRO and set flag
+                if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
+                  avroUsed = true;
+                }
+                // set desired topic
+                if (this.topicsEvents[t].subInfo && this.topicsEvents[t].subInfo.rabbit) {
+                  desiredTopic = this.topicsEvents[t].subInfo.rabbit;
+                } else {
+                  desiredTopic = this.topicsEvents[t].topic;
+                }
+                break;
+              }
+            }
+
+            // if using avro
+            if (this.registry && avroUsed) {
+              // will give us an interval between 5001 and 60000 milliseconds (5 seconds to 1 minute)
+              const fastInt = Math.floor(Math.random() * 55000) + 5001;
+
+              // This interval is to help prevent the adapter from bombarding the registry by sending requests at random intervals
+              const intervalObject = setInterval(async () => {
+                // Prevents running the request while the first one is running
+                if (!firstRun) {
+                  if (!firstDone) {
+                    firstRun = true;
+                  }
+
+                  try {
+                    log.debug('GET AVRO KEY');
+                    newMsg.key = await this.registry.decode(newMsg.key);
+                    log.debug(`AVRO KEY: ${newMsg.key}`);
+                    newMsg.value = await this.registry.decode(newMsg.value);
+                    log.debug(`AVRO MSG: ${newMsg.value}`);
+                    firstDone = true;
+                    firstRun = false;
+                    clearInterval(intervalObject);
+                  } catch (ex) {
+                    log.warn(`Had issue getting registry will try again in ${fastInt} milliseconds`);
+                    firstRun = false;
+                  }
+                }
+              }, fastInt);
+            }
+
+            const goodMsg = isPassingFiltering(topic, newMsg);
+            if (goodMsg) {
+              log.info(`Message: '${message.value.toString()}' being CONSUMED as it does meet the filter condition`);
+              if (mytopics.includes(desiredTopic)) {
+                log.info(`Emitting: ${message.value.toString()} TO ${desiredTopic}`);
+              } else {
+                log.info(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
+                log.info(`Emitting: ${newMsg.value.toString()} TO kafka`);
+                desiredTopic = 'kafka';
+              }
+              if (this.props.parseMessage) {
+                let messageObj;
+                const wrapper = this.props.wrapMessage || 'payload';
+                try {
+                  messageObj = { [wrapper]: JSON.parse(newMsg.value) };
+                } catch (ex) {
+                  messageObj = { [wrapper]: newMsg.value.toString() };
+                }
+                eventSystem.publish(desiredTopic, messageObj);
+              } else {
+                eventSystem.publish(desiredTopic, newMsg.value.toString());
+              }
+            } else {
+              log.info(`Message: '${newMsg.value.toString()}' being DROPPED as it fails to meet any filter condition`);
+            }
+          }
+        });
+      };
+      run().catch((err) => console.error(`[consumer] ${err.message}`, err));
+
+      const errorTypes = ['unhandledRejection', 'uncaughtException'];
+      const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
+
+      errorTypes.forEach((type) => {
+        process.on(type, async (e) => {
+          try {
+            console.log(`process.on ${type}`);
+            console.error(e);
+            await this.consumer.disconnect();
+            process.exit(0);
+          } catch (_) {
+            process.exit(1);
+          }
+        });
+      });
+
+      signalTraps.forEach((type) => {
+        process.once(type, async () => {
+          try {
+            await this.consumer.disconnect();
+          } finally {
+            process.kill(process.pid, type);
+          }
+        });
+      });
+    } catch (ex) {
+      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
+      log.error(JSON.stringify(ex));
+      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+      return (null, errorObj);
+    }
+  }
+
+  /**
+   * @summary HealthCheck function is used to provide Pronghorn the status of this adapter.
+   *
+   * @function healthCheck
+   * @param callback - a callback function to return the result id and status
+   */
+  healthCheck(callback) {
+    // need to work on this later
+    const retstatus = { id: this.id, status: 'success' };
+    return callback(retstatus);
+  }
+
+  /**
+   * getAllFunctions is used to get all of the exposed function in the adapter
+   *
+   * @function getAllFunctions
+   */
+  getAllFunctions() {
+    let myfunctions = [];
+    let obj = this;
+
+    // find the functions in this class
+    do {
+      const l = Object.getOwnPropertyNames(obj)
+        .concat(Object.getOwnPropertySymbols(obj).map((s) => s.toString()))
+        .sort()
+        .filter((p, i, arr) => typeof obj[p] === 'function' && p !== 'constructor' && (i === 0 || p !== arr[i - 1]) && myfunctions.indexOf(p) === -1);
+      myfunctions = myfunctions.concat(l);
+    }
+    while (
+      (obj = Object.getPrototypeOf(obj)) && Object.getPrototypeOf(obj)
+    );
+
+    return myfunctions;
+  }
+
+  /**
+   * getWorkflowFunctions is used to get all of the workflow function in the adapter
+   *
+   * @function getWorkflowFunctions
+   */
+  getWorkflowFunctions() {
+    const myIgnore = [
+      'subscribeAvroWithSubscriber',
+      'subscribeWithSubscriber'
+    ];
+
+    const myfunctions = this.getAllFunctions();
+    const wffunctions = [];
+
+    // remove the functions that should not be in a Workflow
+    for (let m = 0; m < myfunctions.length; m += 1) {
+      if (myfunctions[m] === 'addListener') {
+        // got to the second tier (adapterBase)
+        break;
+      }
+      if (myfunctions[m] !== 'connect' && myfunctions[m] !== 'healthCheck'
+        && myfunctions[m] !== 'getAllFunctions' && myfunctions[m] !== 'getWorkflowFunctions') {
+        let found = false;
+        if (myIgnore && Array.isArray(myIgnore)) {
+          for (let i = 0; i < myIgnore.length; i += 1) {
+            if (myfunctions[m].toUpperCase() === myIgnore[i].toUpperCase()) {
+              found = true;
+            }
+          }
+        }
+        if (!found) {
+          wffunctions.push(myfunctions[m]);
+        }
+      }
+    }
+
+    return wffunctions;
+  }
+
+  /**
+   * Call to send message.
+   * @function send
+   * @param payloads - array of ProduceRequest, ProduceRequest is a JSON object (required)
+   * @param callback - a callback function to return a result
+   */
+  sendMessage(payloads, callback) {
+    const meth = 'adapter-sendMessage';
+    const origin = `${this.id}-${meth}`;
+    log.trace(origin);
+
+    try {
+      if (!payloads) {
+        const errorObj = formatErrorObject(origin, 'Missing Data', ['payloads'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (!Array.isArray(payloads)) {
+        const errorObj = formatErrorObject(origin, 'Invalid payloads format - payloads must be an array', null, null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (!this.producer) {
+        const errorObj = formatErrorObject(origin, 'Producer not created', null, null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      const run = async () => {
+        try {
+          const topicMessages = payloads;
+          // need to go through the payloads that were provided
+          for (let p = 0; p < payloads.length; p += 1) {
+            if (!payloads[p].topic) {
+              const errorObj = formatErrorObject(origin, 'Missing Data', ['payloads[p].topic'], null, null, null);
+              log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+              return callback(null, errorObj);
+            }
+            if (!payloads[p].messages) {
+              const errorObj = formatErrorObject(origin, 'Missing Data', ['payloads[p].messages'], null, null, null);
+              log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+              return callback(null, errorObj);
+            }
+            topicMessages[p].messages = payloads[p].messages.map((message) => {
+              if (typeof message.value === 'object') {
+                return { ...message, value: JSON.stringify(message.value) };
+              }
+              return message;
+            });
+            if (this.registry && topicMessages[p].simple && topicMessages[p].simple.toUpperCase() === 'NO') {
+              log.debug(`Handling AVRO Message: ${JSON.stringify(topicMessages[p])}`);
+              let keyVer = 1;
+              let valVer = 1;
+              if (topicMessages[p].key_version) {
+                keyVer = topicMessages[p].key_version;
+              }
+              if (topicMessages[p].value_version) {
+                valVer = topicMessages[p].value_version;
+              }
+
+              // will give us an interval between 2501 and 10000 milliseconds (2.5 to 10 seconds)
+              const fastInt = Math.floor(Math.random() * 7500) + 2501;
+              let currValue = '';
+              let encodeKey = '';
+              const encodeKeys = [];
+
+              // This interval is to help prevent the adapter from bombarding the registry by sending requests at random intervals
+              const intervalObject = setInterval(async () => {
+                try {
+                  // get the key and value for the topic of this message
+                  log.spam('fetching schema values');
+                  currValue = await fetchSchema(this.registryUrl, `${topicMessages[p].topic}-value`, valVer);
+                  for (let m = 0; m < topicMessages[p].messages.length; m += 1) {
+                    if (topicMessages[p].messages[m].key) {
+                      let schKey = '';
+                      log.spam('fetching schema keys');
+                      schKey = await fetchSchema(this.registryUrl, `${topicMessages[p].topic}-key`, keyVer);
+                      // encode the key
+                      encodeKey = await this.registry.encodeKey(topicMessages[p].topic, schKey.schema, topicMessages[p].messages[m].key);
+                      encodeKeys.push(encodeKey);
+                    } else {
+                      encodeKeys.push(null);
+                    }
+                  }
+
+                  clearInterval(intervalObject);
+                } catch (ex) {
+                  log.warn(`Had issue getting registry will try again in ${fastInt} milliseconds`);
+                  firstRun = false;
+                }
+              }, fastInt);
+
+              log.debug('Encoding message array');
+              for (let m = 0; m < topicMessages[p].messages.length; m += 1) {
+                let msg = '';
+                log.debug(`Attemping to encode message ${m}`);
+                try {
+                  msg = await this.registry.encodeMessage(topicMessages[p].topic, currValue.schema, topicMessages[p].messages[m].value);
+                } catch (encodeErr) {
+                  log.error(encodeErr.stack);
+                  return callback(null, encodeErr);
+                }
+                if (topicMessages[p].messages[m].key) {
+                  log.debug('Replacing key with encoded key');
+                  topicMessages[p].messages[m].key = encodeKeys[m];
+                }
+                log.debug('Replacing message with encoded message');
+                topicMessages[p].messages[m].value = msg;
+              }
+            }
+          }
+          await this.producer.connect();
+          await this.producer.sendBatch({ topicMessages });
+        } catch (err) {
+          const errorObj = formatErrorObject(origin, 'Producer sending message failed', null, null, null, err);
+          log.error('error: '.concat(err));
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+        return callback({
+          status: 'success',
+          code: 200
+        });
+      };
+      run();
+
+      const errorTypes = ['unhandledRejection', 'uncaughtException'];
+      const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
+
+      errorTypes.forEach((type) => {
+        process.on(type, async () => {
+          try {
+            console.log(`process.on ${type}`);
+            await this.producer.disconnect();
+            process.exit(0);
+          } catch (_) {
+            process.exit(1);
+          }
+        });
+      });
+
+      signalTraps.forEach((type) => {
+        process.once(type, async () => {
+          try {
+            await this.producer.disconnect();
+          } finally {
+            process.kill(process.pid, type);
+          }
+        });
+      });
+    } catch (ex) {
+      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
+      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+      return callback(null, errorObj);
+    }
+  }
+}
+
+module.exports = Kafkav2;