@itentialopensource/adapter-kafkav2 0.26.4 → 0.26.6

package/CHANGELOG.md

@@ -1,4 +1,20 @@
 
+## 0.26.6 [03-17-2026]
+
+* fix console and bad variable
+
+See merge request itentialopensource/adapters/adapter-kafkav2!68
+
+---
+
+## 0.26.5 [03-04-2026]
+
+* Changes made at 2026.03.04_09:07AM
+
+See merge request itentialopensource/adapters/adapter-kafkav2!67
+
+---
+
 ## 0.26.4 [02-23-2026]
 
 * Changes made at 2026.02.22_15:50PM

package/adapter.js

@@ -40,7 +40,18 @@ function consoleLoggerProvider(name) {
   };
 }
 
-const { Kafka, logLevel } = require('kafkajs');
+// add the compression codecs so that compressed messages can be consumed
+// zstd has an issue in the pipeline
+const {
+  Kafka, logLevel, CompressionTypes, CompressionCodecs
+} = require('kafkajs');
+const LZ4 = require('kafka-lz4-lite');
+const SnappyCodec = require('kafkajs-snappy');
+// const ZstdCodec = require('@kafkajs/zstd');
+
+CompressionCodecs[CompressionTypes.LZ4] = LZ4.codec;
+CompressionCodecs[CompressionTypes.Snappy] = SnappyCodec;
+// CompressionCodecs[CompressionTypes.ZSTD] = ZstdCodec();
 
 const LogLevelEnum = {
   NOTHING: logLevel.NOTHING,
@@ -232,12 +243,12 @@ function formatErrorObject(origin, type, variables, sysCode, sysRes, stack) {
  */
 function createSocketFactory(socketOptions, useSSL) {
   return (options) => {
-    console.log('KafkaJS Socket Options:', options); // See what KafkaJS is actually passing
+    log.info('KafkaJS Socket Options:', options); // See what KafkaJS is actually passing
 
     // Destructure properties from the options object
     const { host, port, ssl } = options;
     const brokerAddress = `${host}:${port}`;
-    console.log(
+    log.info(
       `Connecting to Kafka broker: ${brokerAddress} (${host}:${port}) ${useSSL ? 'with SSL' : 'without SSL'
       }`
     );
@@ -259,8 +270,8 @@ function createSocketFactory(socketOptions, useSSL) {
     socket.setKeepAlive(keepAliveEnabled, keepAliveInterval);
     socket.setTimeout(socketTimeout);
 
-    socket.on('error', (err) => console.error(`Socket error with ${brokerAddress}:`, err));
-    socket.on('timeout', () => console.warn(`Socket timeout with ${brokerAddress}`));
+    socket.on('error', (err) => log.error(`Socket error with ${brokerAddress}:`, err));
+    socket.on('timeout', () => log.warn(`Socket timeout with ${brokerAddress}`));
 
     return socket;
   };
@@ -540,6 +551,7 @@ class Kafkav2 extends AdapterBaseCl {
     this.consumer = null;
     this.registryUrl = null;
     this.registry = 'abc';
+    this.consumerDisconnecting = false;
     if (this.props && this.props.registry_url) {
       this.registryUrl = this.props.registry_url;
       this.registry = require('avro-schema-registry')(this.props.registry_url);
@@ -763,6 +775,18 @@ class Kafkav2 extends AdapterBaseCl {
           return matched;
         };
 
+        // Handle multiple disconnect requests
+        const safeConsumerDisconnect = async () => {
+          if (this.consumerDisconnecting) return;
+          this.consumerDisconnecting = true;
+          try {
+            await this.consumer.disconnect();
+            log.error('Consumer disconnected');
+          } catch (_) {
+            // ignore disconnect errors during shutdown
+          }
+        };
+
         const run = async () => {
           this.emit('ONLINE', {
             id: this.id
@@ -821,6 +845,30 @@ class Kafkav2 extends AdapterBaseCl {
             }
           ));
 
+          // Register a listener for the consumer crash event
+          this.consumer.on(this.consumer.events.CRASH, (e) => {
+            log.error(`Consumer crashed: ${JSON.stringify(e)}`, e.payload.error.stack);
+            setTimeout(() => {
+              // process.exit(1); // Or implement a restart mechanism
+              throw new Error(JSON.stringify(e));
+            }, 1000);
+          });
+
+          // Register a listener for the consumer disconnect event
+          this.consumer.on(this.consumer.events.DISCONNECT, (e) => {
+            this.consumerDisconnecting = true;
+            log.error(`Consumer disconnected: ${JSON.stringify(e)}`);
+            setTimeout(() => {
+              // process.exit(1); // Or implement a restart mechanism
+              throw new Error(JSON.stringify(e));
+            }, 1000);
+          });
+
+          // Register a listener for the consumer request_timeout event
+          this.consumer.on(this.consumer.events.REQUEST_TIMEOUT, (e) => {
+            log.error(`Consumer request timed out: ${JSON.stringify(e)}`);
+          });
+
           const runPromise = this.consumer.run({
             eachMessage: async ({ topic, partition, message }) => {
               log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
@@ -892,11 +940,11 @@ class Kafkav2 extends AdapterBaseCl {
               }
 
               if (desiredTopic.length !== 0 && allowedPartition) {
-                log.info(`Message: '${message.value.toString()}' being CONSUMED as it does meet the filter condition`);
+                log.info('Message being CONSUMED as it does meet the filter condition');
                 if (desiredTopic.every((val) => mytopics.includes(val))) {
                   log.info(`Emitting: ${message.value.toString()} TO ${desiredTopic}`);
                 } else {
-                  log.info(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
+                  log.warn(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
                   log.info(`Emitting: ${newMsg.value.toString()} TO kafka`);
                   desiredTopic = 'kafka';
                 }
@@ -906,16 +954,30 @@ class Kafkav2 extends AdapterBaseCl {
                   try {
                     messageObj = { [wrapper]: JSON.parse(newMsg.value) };
                   } catch (ex) {
-                    messageObj = { [wrapper]: newMsg.value.toString() };
+                    try {
+                      messageObj = { [wrapper]: newMsg.value.toString() };
+                    } catch (excep) {
+                      log.error(`Failed to wrap message content: ${excep.message}`, excep);
+                    }
                   }
                   for (let j = 0; j < desiredTopic.length; j += 1) {
-                    messageObj.topic = topic;
-                    eventSystem.publish(desiredTopic[j], messageObj);
+                    try {
+                      messageObj.topic = topic;
+                      eventSystem.publish(desiredTopic[j], messageObj);
+                    } catch (ex) {
+                      log.error(`Failed to publish to eventSystem topic '${desiredTopic[j]}': ${ex.message}`, ex);
+                      log.error(`Failed message content: ${messageObj}`);
+                    }
                   }
                 } else {
                   for (let j = 0; j < desiredTopic.length; j += 1) {
-                    newMsg.topic = topic;
-                    eventSystem.publish(desiredTopic[j], newMsg);
+                    try {
+                      newMsg.topic = topic;
+                      eventSystem.publish(desiredTopic[j], newMsg);
+                    } catch (ex) {
+                      log.error(`Failed to publish to eventSystem topic '${desiredTopic[j]}': ${ex.message}`, ex);
+                      log.error(`Failed message content: ${newMsg}`);
+                    }
                   }
                 }
               } else {
@@ -931,20 +993,27 @@ class Kafkav2 extends AdapterBaseCl {
 
           await runPromise;
         };
-        run().catch((err) => console.error(`[consumer] ${err.message}`, err));
+        run().catch((err) => {
+          log.error(`[consumer] ${err.message}`, err);
+          throw new Error(JSON.stringify(err));
+        });
 
         const errorTypes = ['unhandledRejection', 'uncaughtException'];
         const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
 
         errorTypes.forEach((type) => {
-          process.on(type, async (e) => {
+          process.on(type, async (err) => {
             try {
-              console.log(`process.on ${type}`);
-              console.error(e);
-              await this.consumer.disconnect();
-              process.exit(0);
+              log.error(`Kafka Consumer Error - process.on ${type}`);
+              log.error(err, err.stack);
+              await safeConsumerDisconnect();
+              setTimeout(() => {
+                process.exit(0); // Or implement a restart mechanism
+              }, 1000);
             } catch (_) {
-              process.exit(1);
+              setTimeout(() => {
+                process.exit(1); // Or implement a restart mechanism
+              }, 1000);
             }
           });
         });
@@ -952,7 +1021,7 @@ class Kafkav2 extends AdapterBaseCl {
         signalTraps.forEach((type) => {
           process.once(type, async () => {
             try {
-              await this.consumer.disconnect();
+              await safeConsumerDisconnect();
             } finally {
               process.kill(process.pid, type);
             }
@@ -1030,6 +1099,11 @@ class Kafkav2 extends AdapterBaseCl {
 
       const run = async () => {
         try {
+          // Register a listener for the producer request_timeout event
+          this.producer.on(this.producer.events.REQUEST_TIMEOUT, (e) => {
+            log.error(`Producer request timed out: ${JSON.stringify(e)}`);
+          });
+
           const topicMessages = payloads;
           // need to go through the payloads that were provided
           for (let p = 0; p < payloads.length; p += 1) {
@@ -1043,12 +1117,18 @@ class Kafkav2 extends AdapterBaseCl {
               log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
               return callback(null, errorObj);
             }
+            if (!Array.isArray(payloads[p].messages)) {
+              const errorObj = formatErrorObject(origin, 'Invalid payloads format - messages must be an array', null, null, null, null);
+              log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+              return callback(null, errorObj);
+            }
             topicMessages[p].messages = payloads[p].messages.map((message) => {
               if (typeof message.value === 'object') {
                 return { ...message, value: JSON.stringify(message.value) };
               }
               return message;
             });
+            log.debug(`Sending message(s) to topic: ${payloads[p].topic}`);
             if (this.registry && topicMessages[p].simple && topicMessages[p].simple.toUpperCase() === 'NO') {
               log.debug(`Handling AVRO Message: ${JSON.stringify(topicMessages[p])}`);
               let keyVer = 1;
@@ -1113,6 +1193,7 @@ class Kafkav2 extends AdapterBaseCl {
           }
           await this.producer.connect();
           await this.producer.sendBatch({ topicMessages });
+          log.debug(`Sent message to broker: ${JSON.stringify(topicMessages)}`);
         } catch (err) {
           const errorObj = formatErrorObject(origin, 'Producer sending message failed', null, null, null, err);
           log.error('error: '.concat(err));
@@ -1130,13 +1211,18 @@ class Kafkav2 extends AdapterBaseCl {
       const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
 
       errorTypes.forEach((type) => {
-        process.on(type, async () => {
+        process.on(type, async (err) => {
           try {
-            console.log(`process.on ${type}`);
+            log.error(`Kafka Producer Error - process.on ${type}`);
+            log.error(err, err.stack);
             await this.producer.disconnect();
-            process.exit(0);
+            setTimeout(() => {
+              process.exit(0); // Or implement a restart mechanism
+            }, 1000);
           } catch (_) {
-            process.exit(1);
+            setTimeout(() => {
+              process.exit(1); // Or implement a restart mechanism
+            }, 1000);
           }
         });
       });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-kafkav2",
-  "version": "0.26.4",
+  "version": "0.26.6",
   "description": "Itential adapter to connect to kafka",
   "main": "adapter.js",
   "scripts": {
@@ -40,10 +40,12 @@
     "ajv": "8.18.0",
     "avro-schema-registry": "2.1.0",
     "avsc": "5.4.21",
-    "axios": "1.13.5",
+    "axios": "1.13.6",
     "fs-extra": "11.3.3",
     "json-query": "2.2.2",
-    "kafkajs": "2.2.3",
+    "kafkajs": "2.2.4",
+    "kafka-lz4-lite": "1.0.5",
+    "kafkajs-snappy": "1.1.0",
     "mongodb": "4.17.2",
     "readline-sync": "1.4.10",
     "uuid": "3.0.1",