@itentialopensource/adapter-kafkav2 0.16.0 → 0.16.2

package/CHANGELOG.md

@@ -1,4 +1,20 @@
 
+## 0.16.2 [08-29-2023]
+
+* Big fixes logLevel and adapter restart
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!15
+
+---
+
+## 0.16.1 [08-24-2023]
+
+* Handle empty topics file
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!13
+
+---
+
 ## 0.16.0 [08-23-2023]
 
 * Add logic to clean topics.json from service config

package/adapter.js

@@ -22,6 +22,8 @@ const util = require('util');
 const AdapterBaseCl = require(path.join(__dirname, 'adapterBase.js'));
 const request = require('request');
 
+let needRestart = false;
+
 function consoleLoggerProvider(name) {
   return {
     debug: log.debug,
@@ -33,6 +35,14 @@ function consoleLoggerProvider(name) {
 
 const { Kafka, logLevel } = require('kafkajs');
 
+const LogLevelEnum = {
+  NOTHING: logLevel.NOTHING,
+  ERROR: logLevel.ERROR,
+  WARN: logLevel.WARN,
+  INFO: logLevel.INFO,
+  DEBUG: logLevel.DEBUG
+};
+
 const pjson = require(path.resolve(__dirname, 'pronghorn.json'));
 
 let myid = null;
@@ -42,6 +52,14 @@ let firstDone = false;
 const mytopics = Object.keys(pjson.topics);
 const pronghornFile = path.join(__dirname, '/pronghorn.json');
 
+// Function to convert a log level string to the corresponding enum value
+function convertLogLevel(logLevelStr) {
+  // Remove 'logLevel.' prefix if it exists
+  const logLevelStrClean = logLevelStr.replace('logLevel.', '');
+  const logLevelEnum = LogLevelEnum[logLevelStrClean];
+  return logLevelEnum !== undefined ? logLevelEnum : logLevel.INFO; // Default to INFO if not found
+}
+
 async function fetchSchema(registryUrl, topic, version) {
   return new Promise((resolve, reject) => {
     log.debug(`${registryUrl}/subjects/${topic}/versions/${version}`);
@@ -181,13 +199,31 @@ class Kafkav2 extends AdapterBaseCl {
     this.topicsEvents = [];
     let topicsFile = path.join(__dirname, `/.topics-${this.id}.json`);
 
-    // if the file does not exist - error
+    // Check if the file exists
     if (fs.existsSync(topicsFile)) {
-      this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
+      const fileContent = fs.readFileSync(topicsFile, 'utf-8');
+
+      // Check if the file is empty
+      if (fileContent.trim() === '') {
+        // If it's empty, write an empty array to the file
+        fs.writeFileSync(topicsFile, '[]');
+      } else {
+        // Parse the existing content
+        this.topicsEvents = JSON.parse(fileContent);
+      }
     } else if (fs.existsSync(path.join(__dirname, '/.topics.json'))) {
       log.debug('Found old .topics.json file.');
       topicsFile = path.join(__dirname, '/.topics.json');
-      this.topicsEvents = JSON.parse(fs.readFileSync(topicsFile, 'utf-8'));
+      const fileContent = fs.readFileSync(topicsFile, 'utf-8');
+
+      // Check if the file is empty
+      if (fileContent.trim() === '') {
+        // If it's empty, write an empty array to the file
+        fs.writeFileSync(topicsFile, '[]');
+      } else {
+        // Parse the existing content
+        this.topicsEvents = JSON.parse(fileContent);
+      }
     }
 
     // get the topics into the right format --- fix older .topics.json files so they have the right format
@@ -233,9 +269,7 @@ class Kafkav2 extends AdapterBaseCl {
     }
 
     // If we have a rabbit queue - need to add that to topics.json
-    // if there are topics in the properties
     if (this.props && this.props.topics) {
-      let needRestart = false;
       for (let t = 0; t < this.props.topics.length; t += 1) {
         if (typeof this.props.topics[t] === 'string') {
           const topName = this.props.topics[t];
@@ -467,229 +501,243 @@ class Kafkav2 extends AdapterBaseCl {
       return;
     }
 
-    try {
-      // Create a kafka client
-      const combinedProps = this.props.client || {};
-      if (this.props.client.ssl && this.props.client.ssl.ca) {
-        combinedProps.ssl.ca = [fs.readFileSync(this.props.client.ssl.ca, 'utf-8')];
-      }
-      if (this.props.client.ssl && this.props.client.ssl.cert) {
-        combinedProps.ssl.cert = fs.readFileSync(this.props.client.ssl.cert, 'utf-8');
-      }
-      if (this.props.client.ssl && this.props.client.ssl.key) {
-        combinedProps.ssl.key = fs.readFileSync(this.props.client.ssl.key, 'utf-8');
-      }
+    if (!needRestart) {
+      try {
+        if (this.props.client.logLevel) {
+          const logLevelEnum = convertLogLevel(this.props.client.logLevel);
+          this.props.client.logLevel = logLevelEnum;
+        }
+        // Create a kafka client
+        const combinedProps = this.props.client || {};
+        if (this.props.client.ssl && this.props.client.ssl.ca) {
+          combinedProps.ssl.ca = [fs.readFileSync(this.props.client.ssl.ca, 'utf-8')];
+        }
+        if (this.props.client.ssl && this.props.client.ssl.cert) {
+          combinedProps.ssl.cert = fs.readFileSync(this.props.client.ssl.cert, 'utf-8');
+        }
+        if (this.props.client.ssl && this.props.client.ssl.key) {
+          combinedProps.ssl.key = fs.readFileSync(this.props.client.ssl.key, 'utf-8');
+        }
 
-      this.KafkaClient = new Kafka(combinedProps);
+        this.KafkaClient = new Kafka(combinedProps);
 
-      if (this.props.client && this.props.client.producersasl) {
-        combinedProps.sasl = this.props.client.producersasl;
-        this.KafkaProducerClient = new Kafka(combinedProps);
-        this.producer = this.KafkaProducerClient.producer(this.props.producer || {});
-      } else {
-        this.producer = this.KafkaClient.producer(this.props.producer || {});
-      }
+        if (this.props.client && this.props.client.producersasl) {
+          combinedProps.sasl = this.props.client.producersasl;
+          this.KafkaProducerClient = new Kafka(combinedProps);
+          this.producer = this.KafkaProducerClient.producer(this.props.producer || {});
+        } else {
+          this.producer = this.KafkaClient.producer(this.props.producer || {});
+        }
 
-      if (this.props.client && this.props.client.consumersasl) {
-        combinedProps.sasl = this.props.client.consumersasl;
-        this.KafkaConsumerClient = new Kafka(combinedProps);
-        this.consumer = this.KafkaConsumerClient.consumer(this.props.consumer || {});
-      } else {
-        this.consumer = this.KafkaClient.consumer(this.props.consumer || {});
-      }
+        if (this.props.client && this.props.client.consumersasl) {
+          combinedProps.sasl = this.props.client.consumersasl;
+          this.KafkaConsumerClient = new Kafka(combinedProps);
+          this.consumer = this.KafkaConsumerClient.consumer(this.props.consumer || {});
+        } else {
+          this.consumer = this.KafkaClient.consumer(this.props.consumer || {});
+        }
 
-      const isPassingFiltering = (topic, message, filters) => {
-        const topicConfig = this.topicsEvents.find((e) => e.topic === topic);
+        const isPassingFiltering = (topic, message, filters) => {
+          const topicConfig = this.topicsEvents.find((e) => e.topic === topic);
 
-        if (!topicConfig) return true;
+          if (!topicConfig) return true;
 
-        if (!topicConfig.subInfo || !Array.isArray(topicConfig.subInfo)) return true;
-        if (filters.length === 0) return true;
-        const matched = filters.find((filter) => {
-          const re = new RegExp(filter);
-          if (message.value.toString().match(re)) {
-            log.debug('Matched filter: ', re);
-            return true;
-          }
-          return false;
-        });
-        return matched;
-      };
+          if (!topicConfig.subInfo || !Array.isArray(topicConfig.subInfo)) return true;
+          if (filters.length === 0) return true;
+          const matched = filters.find((filter) => {
+            const re = new RegExp(filter);
+            if (message.value.toString().match(re)) {
+              log.debug('Matched filter: ', re);
+              return true;
+            }
+            return false;
+          });
+          return matched;
+        };
 
-      const run = async () => {
-        this.emit('ONLINE', {
-          id: this.id
-        });
-        log.info('EMITTED ONLINE');
-        // Start consuming
-        // Iterate through all topics in topicsEvents
-        const topics = [];
-        for (let t = 0; t < this.topicsEvents.length; t += 1) {
-          topics.push(this.topicsEvents[t].topic);
-        }
-        await this.consumer.connect();
-        await this.consumer.subscribe({ topics, fromBeginning: this.props.fromBeginning || false });
-
-        let isAppActive = null;
-        const topicPartitions = topics.map((topic) => ({ topic }));
-
-        // if any IAP apps are down, pause consumer before it consumes any messages
-        if (this.props && this.props.check_iap_apps) {
-          isAppActive = await super.checkIapAppsStatus();
-          if (!isAppActive) {
-            log.debug('Pause consumer since IAP apps are down');
-            // wait for the consumer to connect to Kafka before pausing it
-            this.consumer.on(this.consumer.events.CONNECT, async () => {
-              this.consumer.pause(topicPartitions);
-            });
+        const run = async () => {
+          this.emit('ONLINE', {
+            id: this.id
+          });
+          log.info('EMITTED ONLINE');
+          // Start consuming
+          // Iterate through all topics in topicsEvents
+          const topics = [];
+          for (let t = 0; t < this.topicsEvents.length; t += 1) {
+            topics.push(this.topicsEvents[t].topic);
           }
-        }
+          await this.consumer.connect();
+          await this.consumer.subscribe({ topics, fromBeginning: this.props.fromBeginning || false });
 
-        // keep checking IAP apps status
-        if (this.props && this.props.check_iap_apps) {
-          const intTime = this.props.iap_apps_check_interval || 30000;
-          setInterval(async () => {
+          let isAppActive = null;
+          const topicPartitions = topics.map((topic) => ({ topic }));
+
+          // if any IAP apps are down, pause consumer before it consumes any messages
+          if (this.props && this.props.check_iap_apps) {
             isAppActive = await super.checkIapAppsStatus();
-            if (isAppActive) {
-              log.debug('Resume consumer since IAP apps are active');
-              this.consumer.resume(topicPartitions);
-            } else {
-              log.debug('Keep pausing consumer until IAP apps are active');
-              this.consumer.pause(topicPartitions);
+            if (!isAppActive) {
+              log.debug('Pause consumer since IAP apps are down');
+              // wait for the consumer to connect to Kafka before pausing it
+              this.consumer.on(this.consumer.events.CONNECT, async () => {
+                this.consumer.pause(topicPartitions);
+              });
             }
-          }, intTime);
-        }
-        await this.consumer.run({
-          eachMessage: async ({ topic, partition, message }) => {
-            log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
-            const newMsg = message;
-            let avroUsed = false;
-            let desiredTopic = [];
-            let allowedPartition = false;
-
-            // if using avro
-            if (this.registry && avroUsed) {
-              // will give us an interval between 5001 and 60000 milliseconds (5 seconds to 1 minute)
-              const fastInt = Math.floor(Math.random() * 55000) + 5001;
+          }
 
-              // This interval is to help prevent the adapter from bombarding the registry by sending requests at random intervals
-              const intervalObject = setInterval(async () => {
-                // Prevents running the request while the first one is running
-                if (!firstRun) {
-                  if (!firstDone) {
-                    firstRun = true;
-                  }
+          // keep checking IAP apps status
+          if (this.props && this.props.check_iap_apps) {
+            const intTime = this.props.iap_apps_check_interval || 30000;
+            setInterval(async () => {
+              isAppActive = await super.checkIapAppsStatus();
+              if (isAppActive) {
+                log.debug('Resume consumer since IAP apps are active');
+                // wait for the consumer to connect to Kafka before resuming it
+                this.consumer.on(this.consumer.events.CONNECT, async () => {
+                  this.consumer.resume(topicPartitions);
+                });
+              } else {
+                log.debug('Keep pausing consumer until IAP apps are active');
+                // wait for the consumer to connect to Kafka before pausing it
+                this.consumer.on(this.consumer.events.CONNECT, async () => {
+                  this.consumer.pause(topicPartitions);
+                });
+              }
+            }, intTime);
+          }
+          await this.consumer.run({
+            eachMessage: async ({ topic, partition, message }) => {
+              log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
+              const newMsg = message;
+              let avroUsed = false;
+              let desiredTopic = [];
+              let allowedPartition = false;
+
+              // if using avro
+              if (this.registry && avroUsed) {
+                // will give us an interval between 5001 and 60000 milliseconds (5 seconds to 1 minute)
+                const fastInt = Math.floor(Math.random() * 55000) + 5001;
+
+                // This interval is to help prevent the adapter from bombarding the registry by sending requests at random intervals
+                const intervalObject = setInterval(async () => {
+                  // Prevents running the request while the first one is running
+                  if (!firstRun) {
+                    if (!firstDone) {
+                      firstRun = true;
+                    }
 
-                  try {
-                    log.debug('GET AVRO KEY');
-                    newMsg.key = await this.registry.decode(newMsg.key);
-                    log.debug(`AVRO KEY: ${newMsg.key}`);
-                    newMsg.value = await this.registry.decode(newMsg.value);
-                    log.debug(`AVRO MSG: ${newMsg.value}`);
-                    firstDone = true;
-                    firstRun = false;
-                    clearInterval(intervalObject);
-                  } catch (ex) {
-                    log.warn(`Had issue getting registry will try again in ${fastInt} milliseconds`);
-                    firstRun = false;
-                  }
-                }
-              }, fastInt);
-            }
-            // find the topic here to change the offset
-            for (let t = 0; t < this.topicsEvents.length; t += 1) {
-              if (this.topicsEvents[t].topic === topic && (this.topicsEvents[t].partition === partition || this.topicsEvents[t].partition.includes(partition))) {
-                allowedPartition = true;
-                if (this.topicsEvents[t].offset < message.offset) {
-                  this.topicsEvents[t].offset = message.offset;
-                }
-                // determine if we are using AVRO and set flag
-                if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
-                  avroUsed = true;
-                }
-                // set desired topic for messages passing filtering
-                if (this.topicsEvents[t].subInfo) {
-                  for (let k = 0; k < this.topicsEvents[t].subInfo.length; k += 1) {
-                    let passesFiltering = true;
-                    if (this.topicsEvents[t].subInfo[k].filters) {
-                      passesFiltering = isPassingFiltering(topic, newMsg, this.topicsEvents[t].subInfo[k].filters);
+                    try {
+                      log.debug('GET AVRO KEY');
+                      newMsg.key = await this.registry.decode(newMsg.key);
+                      log.debug(`AVRO KEY: ${newMsg.key}`);
+                      newMsg.value = await this.registry.decode(newMsg.value);
+                      log.debug(`AVRO MSG: ${newMsg.value}`);
+                      firstDone = true;
+                      firstRun = false;
+                      clearInterval(intervalObject);
+                    } catch (ex) {
+                      log.warn(`Had issue getting registry will try again in ${fastInt} milliseconds`);
+                      firstRun = false;
                     }
-                    if (this.topicsEvents[t].subInfo[k].rabbit && passesFiltering) {
-                      desiredTopic.push(this.topicsEvents[t].subInfo[k].rabbit);
-                    } else if (passesFiltering) {
-                      desiredTopic.push(this.topicsEvents[t].topic);
+                  }
+                }, fastInt);
+              }
+              // find the topic here to change the offset
+              for (let t = 0; t < this.topicsEvents.length; t += 1) {
+                if (this.topicsEvents[t].topic === topic && (this.topicsEvents[t].partition === partition || this.topicsEvents[t].partition.includes(partition))) {
+                  allowedPartition = true;
+                  if (this.topicsEvents[t].offset < message.offset) {
+                    this.topicsEvents[t].offset = message.offset;
+                  }
+                  // determine if we are using AVRO and set flag
+                  if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
+                    avroUsed = true;
+                  }
+                  // set desired topic for messages passing filtering
+                  if (this.topicsEvents[t].subInfo) {
+                    for (let k = 0; k < this.topicsEvents[t].subInfo.length; k += 1) {
+                      let passesFiltering = true;
+                      if (this.topicsEvents[t].subInfo[k].filters) {
+                        passesFiltering = isPassingFiltering(topic, newMsg, this.topicsEvents[t].subInfo[k].filters);
+                      }
+                      if (this.topicsEvents[t].subInfo[k].rabbit && passesFiltering) {
+                        desiredTopic.push(this.topicsEvents[t].subInfo[k].rabbit);
+                      } else if (passesFiltering) {
+                        desiredTopic.push(this.topicsEvents[t].topic);
+                      }
                     }
                   }
+                  break;
                 }
-                break;
               }
-            }
 
-            if (desiredTopic.length !== 0 && allowedPartition) {
-              log.info(`Message: '${message.value.toString()}' being CONSUMED as it does meet the filter condition`);
-              if (desiredTopic.every((val) => mytopics.includes(val))) {
-                log.info(`Emitting: ${message.value.toString()} TO ${desiredTopic}`);
-              } else {
-                log.info(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
-                log.info(`Emitting: ${newMsg.value.toString()} TO kafka`);
-                desiredTopic = 'kafka';
-              }
-              if (this.props.parseMessage || this.props.parseMessage === undefined) {
-                let messageObj;
-                const wrapper = this.props.wrapMessage || 'payload';
-                try {
-                  messageObj = { [wrapper]: JSON.parse(newMsg.value) };
-                } catch (ex) {
-                  messageObj = { [wrapper]: newMsg.value.toString() };
+              if (desiredTopic.length !== 0 && allowedPartition) {
+                log.info(`Message: '${message.value.toString()}' being CONSUMED as it does meet the filter condition`);
+                if (desiredTopic.every((val) => mytopics.includes(val))) {
+                  log.info(`Emitting: ${message.value.toString()} TO ${desiredTopic}`);
+                } else {
+                  log.info(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
+                  log.info(`Emitting: ${newMsg.value.toString()} TO kafka`);
+                  desiredTopic = 'kafka';
                 }
-                for (let j = 0; j < desiredTopic.length; j += 1) {
-                  messageObj.topic = topic;
-                  eventSystem.publish(desiredTopic[j], messageObj);
+                if (this.props.parseMessage || this.props.parseMessage === undefined) {
+                  let messageObj;
+                  const wrapper = this.props.wrapMessage || 'payload';
+                  try {
+                    messageObj = { [wrapper]: JSON.parse(newMsg.value) };
+                  } catch (ex) {
+                    messageObj = { [wrapper]: newMsg.value.toString() };
+                  }
+                  for (let j = 0; j < desiredTopic.length; j += 1) {
+                    messageObj.topic = topic;
+                    eventSystem.publish(desiredTopic[j], messageObj);
+                  }
+                } else {
+                  for (let j = 0; j < desiredTopic.length; j += 1) {
+                    newMsg.topic = topic;
+                    eventSystem.publish(desiredTopic[j], newMsg);
+                  }
                 }
               } else {
-                for (let j = 0; j < desiredTopic.length; j += 1) {
-                  newMsg.topic = topic;
-                  eventSystem.publish(desiredTopic[j], newMsg);
-                }
+                log.info(`Message: '${newMsg.value.toString()}' being DROPPED as it fails to meet any filter condition`);
               }
-            } else {
-              log.info(`Message: '${newMsg.value.toString()}' being DROPPED as it fails to meet any filter condition`);
             }
-          }
-        });
-      };
-      run().catch((err) => console.error(`[consumer] ${err.message}`, err));
+          });
+        };
+        run().catch((err) => console.error(`[consumer] ${err.message}`, err));
 
-      const errorTypes = ['unhandledRejection', 'uncaughtException'];
-      const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
+        const errorTypes = ['unhandledRejection', 'uncaughtException'];
+        const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
 
-      errorTypes.forEach((type) => {
-        process.on(type, async (e) => {
-          try {
-            console.log(`process.on ${type}`);
-            console.error(e);
-            await this.consumer.disconnect();
-            process.exit(0);
-          } catch (_) {
-            process.exit(1);
-          }
+        errorTypes.forEach((type) => {
+          process.on(type, async (e) => {
+            try {
+              console.log(`process.on ${type}`);
+              console.error(e);
+              await this.consumer.disconnect();
+              process.exit(0);
+            } catch (_) {
+              process.exit(1);
+            }
+          });
         });
-      });
 
-      signalTraps.forEach((type) => {
-        process.once(type, async () => {
-          try {
-            await this.consumer.disconnect();
-          } finally {
-            process.kill(process.pid, type);
-          }
+        signalTraps.forEach((type) => {
+          process.once(type, async () => {
+            try {
+              await this.consumer.disconnect();
+            } finally {
+              process.kill(process.pid, type);
+            }
+          });
         });
-      });
-    } catch (ex) {
-      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
-      log.error(JSON.stringify(ex));
-      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
-      return (null, errorObj);
+      } catch (ex) {
+        const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
+        log.error(JSON.stringify(ex));
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return (null, errorObj);
+      }
+    } else {
+      log.debug('Waiting for adapter to come back up');
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-kafkav2",
-  "version": "0.16.0",
+  "version": "0.16.2",
   "description": "Itential adapter to connect to kafka",
   "main": "adapter.js",
   "wizardVersion": "2.35.0",