@itentialopensource/adapter-kafkav2 0.3.13 → 0.3.14

package/CHANGELOG.md

@@ -1,4 +1,12 @@
 
+## 0.3.14 [02-16-2023]
+
+* updated to allow multiple subscribers per topic entry and fixed partition logic
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!1
+
+---
+
 ## 0.3.13 [02-07-2023]
 
 * Bug fixes and performance improvements

package/adapter.js

@@ -217,7 +217,7 @@ class Kafkav2 extends EventEmitterCl {
             {
               subname: 'default',
               filters: [],
-              rabbit: 'kafka',
+              rabbit: this.topicsEvents[t].topic,
               throttle: {}
             }
           ];
@@ -260,7 +260,7 @@ class Kafkav2 extends EventEmitterCl {
             {
               subname: 'default',
               filters: [],
-              rabbit: 'kafka',
+              rabbit: this.props.topics[t].name,
               throttle: {}
             }
           ];
@@ -293,7 +293,7 @@ class Kafkav2 extends EventEmitterCl {
               // check to see if we need to add a subscriber
               for (let sub = 0; sub < subInfo.length; sub += 1) {
                 let subFound = false;
-                for (let s = 0; s <= this.topicsEvents[te].subInfo.length; s += 1) {
+                for (let s = 0; s < this.topicsEvents[te].subInfo.length; s += 1) {
                   // if the subscriber is found, no need to add anything
                   if (subInfo[sub].subname === this.topicsEvents[te].subInfo[s].subname) {
                     subFound = true;
@@ -407,13 +407,13 @@ class Kafkav2 extends EventEmitterCl {
     try {
       // Create a kafka client
       const combinedProps = this.props.client || {};
-      if (this.props.client.ssl.ca) {
+      if (this.props.client.ssl && this.props.client.ssl.ca) {
         combinedProps.ssl.ca = [fs.readFileSync(this.props.client.ssl.ca, 'utf-8')];
       }
-      if (this.props.client.ssl.cert) {
+      if (this.props.client.ssl && this.props.client.ssl.cert) {
         combinedProps.ssl.cert = fs.readFileSync(this.props.client.ssl.cert, 'utf-8');
       }
-      if (this.props.client.ssl.key) {
+      if (this.props.client.ssl && this.props.client.ssl.key) {
         combinedProps.ssl.key = fs.readFileSync(this.props.client.ssl.key, 'utf-8');
       }
 
@@ -423,22 +423,18 @@ class Kafkav2 extends EventEmitterCl {
 
       this.consumer = this.KafkaClient.consumer(this.props.consumer || {});
 
-      const isPassingFiltering = (topic, message) => {
+      const isPassingFiltering = (topic, message, filters) => {
         const topicConfig = this.topicsEvents.find((e) => e.topic === topic);
 
         if (!topicConfig) return true;
 
         if (!topicConfig.subInfo || !Array.isArray(topicConfig.subInfo)) return true;
-
-        const allFilters = topicConfig.subInfo.filter((e) => Array.isArray(e.filters) && e.filters.length > 0);
-        if (allFilters.length === 0) return true;
-        const matched = allFilters.find((s) => {
-          for (let f = 0; f < s.filters.length; f += 1) {
-            const re = new RegExp(s.filters[f]);
-            if (message.value.toString().match(re)) {
-              log.debug('Matched filter: ', re);
-              return true;
-            }
+        if (filters.length === 0) return true;
+        const matched = filters.find((filter) => {
+          const re = new RegExp(filter);
+          if (message.value.toString().match(re)) {
+            log.debug('Matched filter: ', re);
+            return true;
           }
           return false;
         });
@@ -463,27 +459,8 @@ class Kafkav2 extends EventEmitterCl {
             log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
             const newMsg = message;
             let avroUsed = false;
-            let desiredTopic = 'kafka';
-
-            // find the topic here to change the offset
-            for (let t = 0; t < this.topicsEvents.length; t += 1) {
-              if (this.topicsEvents[t].topic === topic && this.topicsEvents[t].partition === partition) {
-                if (this.topicsEvents[t].offset < message.offset) {
-                  this.topicsEvents[t].offset = message.offset;
-                }
-                // determine if we are using AVRO and set flag
-                if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
-                  avroUsed = true;
-                }
-                // set desired topic
-                if (this.topicsEvents[t].subInfo && this.topicsEvents[t].subInfo.rabbit) {
-                  desiredTopic = this.topicsEvents[t].subInfo.rabbit;
-                } else {
-                  desiredTopic = this.topicsEvents[t].topic;
-                }
-                break;
-              }
-            }
+            let desiredTopic = [];
+            let allowedPartition = false;
 
             // if using avro
             if (this.registry && avroUsed) {
@@ -514,18 +491,45 @@ class Kafkav2 extends EventEmitterCl {
                 }
               }, fastInt);
             }
+            // find the topic here to change the offset
+            for (let t = 0; t < this.topicsEvents.length; t += 1) {
+              if (this.topicsEvents[t].topic === topic && this.topicsEvents[t].partition === partition) {
+                allowedPartition = true;
+                if (this.topicsEvents[t].offset < message.offset) {
+                  this.topicsEvents[t].offset = message.offset;
+                }
+                // determine if we are using AVRO and set flag
+                if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
+                  avroUsed = true;
+                }
+                // set desired topic for messages passing filtering
+                if (this.topicsEvents[t].subInfo) {
+                  for (let k = 0; k < this.topicsEvents[t].subInfo.length; k += 1) {
+                    let passesFiltering = true;
+                    if (this.topicsEvents[t].subInfo[k].filters) {
+                      passesFiltering = isPassingFiltering(topic, newMsg, this.topicsEvents[t].subInfo[k].filters);
+                    }
+                    if (this.topicsEvents[t].subInfo[k].rabbit && passesFiltering) {
+                      desiredTopic.push(this.topicsEvents[t].subInfo[k].rabbit);
+                    } else if (passesFiltering) {
+                      desiredTopic.push(this.topicsEvents[t].topic);
+                    }
+                  }
+                }
+                break;
+              }
+            }
 
-            const goodMsg = isPassingFiltering(topic, newMsg);
-            if (goodMsg) {
+            if (desiredTopic.length !== 0 && allowedPartition) {
               log.info(`Message: '${message.value.toString()}' being CONSUMED as it does meet the filter condition`);
-              if (mytopics.includes(desiredTopic)) {
+              if (desiredTopic.every((val) => mytopics.includes(val))) {
                 log.info(`Emitting: ${message.value.toString()} TO ${desiredTopic}`);
               } else {
                 log.info(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
                 log.info(`Emitting: ${newMsg.value.toString()} TO kafka`);
                 desiredTopic = 'kafka';
               }
-              if (this.props.parseMessage) {
+              if (this.props.parseMessage || this.props.parseMessage === undefined) {
                 let messageObj;
                 const wrapper = this.props.wrapMessage || 'payload';
                 try {
@@ -533,9 +537,13 @@ class Kafkav2 extends EventEmitterCl {
                 } catch (ex) {
                   messageObj = { [wrapper]: newMsg.value.toString() };
                 }
-                eventSystem.publish(desiredTopic, messageObj);
+                for (let j = 0; j < desiredTopic.length; j += 1) {
+                  eventSystem.publish(desiredTopic[j], messageObj);
+                }
               } else {
-                eventSystem.publish(desiredTopic, newMsg.value.toString());
+                for (let j = 0; j < desiredTopic.length; j += 1) {
+                  eventSystem.publish(desiredTopic[j], newMsg);
+                }
               }
             } else {
               log.info(`Message: '${newMsg.value.toString()}' being DROPPED as it fails to meet any filter condition`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-kafkav2",
-  "version": "0.3.13",
+  "version": "0.3.14",
   "description": "Itential adapter to connect to kafka",
   "main": "adapter.js",
   "wizardVersion": "2.35.0",

package/pronghorn.json

@@ -9,9 +9,7 @@
     "admin"
   ],
   "topics": {
-    "kafka": {},
-    "first_kafka_topic": {},
-    "test-topic": {}
+    "kafka": {}
   },
   "methods": [
     {