@itentialopensource/adapter-kafkav2 0.3.13 → 0.3.15

package/CHANGELOG.md

@@ -1,4 +1,20 @@
 
+## 0.3.15 [02-17-2023]
+
+* Updated readme
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!2
+
+---
+
+## 0.3.14 [02-16-2023]
+
+* updated to allow multiple subscribers per topic entry and fixed partition logic
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!1
+
+---
+
 ## 0.3.13 [02-07-2023]
 
 * Bug fixes and performance improvements

package/README.md

@@ -148,9 +148,66 @@ Example of adapter's configuration with filtering applied:
   }
 ],
 ```
-
 When operator skips to provide `filters` property or the property contains no actual filters set, then all messages are passed.
 
+Example of adapter's configuration with multiple paritions and subscribers per topic:
+```json
+      "topics": [
+        {
+          "name": "topic1",
+          "always": true,
+          "partition": 0,
+          "subscriberInfo": [
+            {
+              "subname": "default",
+              "filters": [
+                "PIXI",
+                "DRED",
+                "F: (\\w+), L: (\\w+)",
+                "\\d{3,4}"
+              ],
+              "rabbit": "topic1",
+              "throttle": {}
+            },
+            {
+              "subname": "sub2",
+              "filters": [
+                "[abc]",
+                "F: (\\w+), L: (\\w+)",
+                "\\d{3,4}"
+              ],
+              "rabbit": "topic1-s2",
+              "throttle": {}
+            }
+          ]
+        },
+        {
+          "name": "topic1",
+          "always": true,
+          "partition": 1,
+          "subscriberInfo": [
+            {
+              "subname": "default",
+              "filters": [
+                "PIXI",
+                "DRED",
+                "[abc]",
+                "F: (\\w+), L: (\\w+)",
+                "\\d{3,4}"
+              ],
+              "rabbit": "topic1-p1",
+              "throttle": {}
+            }
+          ]
+        },
+        {
+          "name": "test-6",
+          "always": true
+        }
+      ],
+```
+Note that if no parition is supplied only messages on partition 0 will be consumed. Additionally, if no rabbit topic is supplied, events will be published to a topic with the same name as the Kafka topic. For example, topic `test-6` above will be published to the `test-6` rabbit topic. 
+
 ### Connection Properties
 
 These base properties are used to connect to Kafka upon the adapter initially coming up. It is important to set these properties appropriately.
@@ -211,6 +268,10 @@ The following properties are used to define the Kafka Consumer. These properties
 | encoding | If set to 'buffer', values will be returned as raw buffer objects..|
 | keyEncoding | |
 
+## Parsing Properties
+
+The `parseMessage` property allows the user to define how they want the Kafka message to be published to IAP's event system. If `parseMessage` is set to true or omitted, the value of the Kafka message will be parsed as either an object or string and wrapped in an outer object. The wrapper object's key can be defined with the property `wrapMessage`, or the default value `payload` can be used if omitted. If `parseMessage` is set to false, the entire kafka payload, including metadata, would be returned and the message itself would need to be transformed at a later point.
+
 ## Testing an Itential Product Adapter
 
 Mocha is generally used to test all Itential Product Adapters. There are unit tests as well as integration tests performed. Integration tests can generally be run as standalone using mock data and running the adapter in stub mode, or as integrated. When running integrated, every effort is made to prevent environmental failures, however there is still a possibility.

package/adapter.js

@@ -217,7 +217,7 @@ class Kafkav2 extends EventEmitterCl {
             {
               subname: 'default',
               filters: [],
-              rabbit: 'kafka',
+              rabbit: this.topicsEvents[t].topic,
               throttle: {}
             }
           ];
@@ -260,7 +260,7 @@ class Kafkav2 extends EventEmitterCl {
             {
               subname: 'default',
               filters: [],
-              rabbit: 'kafka',
+              rabbit: this.props.topics[t].name,
               throttle: {}
             }
           ];
@@ -293,7 +293,7 @@ class Kafkav2 extends EventEmitterCl {
               // check to see if we need to add a subscriber
               for (let sub = 0; sub < subInfo.length; sub += 1) {
                 let subFound = false;
-                for (let s = 0; s <= this.topicsEvents[te].subInfo.length; s += 1) {
+                for (let s = 0; s < this.topicsEvents[te].subInfo.length; s += 1) {
                   // if the subscriber is found, no need to add anything
                   if (subInfo[sub].subname === this.topicsEvents[te].subInfo[s].subname) {
                     subFound = true;
@@ -407,13 +407,13 @@ class Kafkav2 extends EventEmitterCl {
     try {
       // Create a kafka client
       const combinedProps = this.props.client || {};
-      if (this.props.client.ssl.ca) {
+      if (this.props.client.ssl && this.props.client.ssl.ca) {
         combinedProps.ssl.ca = [fs.readFileSync(this.props.client.ssl.ca, 'utf-8')];
       }
-      if (this.props.client.ssl.cert) {
+      if (this.props.client.ssl && this.props.client.ssl.cert) {
         combinedProps.ssl.cert = fs.readFileSync(this.props.client.ssl.cert, 'utf-8');
       }
-      if (this.props.client.ssl.key) {
+      if (this.props.client.ssl && this.props.client.ssl.key) {
         combinedProps.ssl.key = fs.readFileSync(this.props.client.ssl.key, 'utf-8');
       }
 
@@ -423,22 +423,18 @@ class Kafkav2 extends EventEmitterCl {
 
       this.consumer = this.KafkaClient.consumer(this.props.consumer || {});
 
-      const isPassingFiltering = (topic, message) => {
+      const isPassingFiltering = (topic, message, filters) => {
         const topicConfig = this.topicsEvents.find((e) => e.topic === topic);
 
         if (!topicConfig) return true;
 
         if (!topicConfig.subInfo || !Array.isArray(topicConfig.subInfo)) return true;
-
-        const allFilters = topicConfig.subInfo.filter((e) => Array.isArray(e.filters) && e.filters.length > 0);
-        if (allFilters.length === 0) return true;
-        const matched = allFilters.find((s) => {
-          for (let f = 0; f < s.filters.length; f += 1) {
-            const re = new RegExp(s.filters[f]);
-            if (message.value.toString().match(re)) {
-              log.debug('Matched filter: ', re);
-              return true;
-            }
+        if (filters.length === 0) return true;
+        const matched = filters.find((filter) => {
+          const re = new RegExp(filter);
+          if (message.value.toString().match(re)) {
+            log.debug('Matched filter: ', re);
+            return true;
           }
           return false;
         });
@@ -463,27 +459,8 @@ class Kafkav2 extends EventEmitterCl {
             log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
             const newMsg = message;
             let avroUsed = false;
-            let desiredTopic = 'kafka';
-
-            // find the topic here to change the offset
-            for (let t = 0; t < this.topicsEvents.length; t += 1) {
-              if (this.topicsEvents[t].topic === topic && this.topicsEvents[t].partition === partition) {
-                if (this.topicsEvents[t].offset < message.offset) {
-                  this.topicsEvents[t].offset = message.offset;
-                }
-                // determine if we are using AVRO and set flag
-                if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
-                  avroUsed = true;
-                }
-                // set desired topic
-                if (this.topicsEvents[t].subInfo && this.topicsEvents[t].subInfo.rabbit) {
-                  desiredTopic = this.topicsEvents[t].subInfo.rabbit;
-                } else {
-                  desiredTopic = this.topicsEvents[t].topic;
-                }
-                break;
-              }
-            }
+            let desiredTopic = [];
+            let allowedPartition = false;
 
             // if using avro
             if (this.registry && avroUsed) {
@@ -514,18 +491,45 @@ class Kafkav2 extends EventEmitterCl {
                 }
               }, fastInt);
             }
+            // find the topic here to change the offset
+            for (let t = 0; t < this.topicsEvents.length; t += 1) {
+              if (this.topicsEvents[t].topic === topic && this.topicsEvents[t].partition === partition) {
+                allowedPartition = true;
+                if (this.topicsEvents[t].offset < message.offset) {
+                  this.topicsEvents[t].offset = message.offset;
+                }
+                // determine if we are using AVRO and set flag
+                if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
+                  avroUsed = true;
+                }
+                // set desired topic for messages passing filtering
+                if (this.topicsEvents[t].subInfo) {
+                  for (let k = 0; k < this.topicsEvents[t].subInfo.length; k += 1) {
+                    let passesFiltering = true;
+                    if (this.topicsEvents[t].subInfo[k].filters) {
+                      passesFiltering = isPassingFiltering(topic, newMsg, this.topicsEvents[t].subInfo[k].filters);
+                    }
+                    if (this.topicsEvents[t].subInfo[k].rabbit && passesFiltering) {
+                      desiredTopic.push(this.topicsEvents[t].subInfo[k].rabbit);
+                    } else if (passesFiltering) {
+                      desiredTopic.push(this.topicsEvents[t].topic);
+                    }
+                  }
+                }
+                break;
+              }
+            }
 
-            const goodMsg = isPassingFiltering(topic, newMsg);
-            if (goodMsg) {
+            if (desiredTopic.length !== 0 && allowedPartition) {
               log.info(`Message: '${message.value.toString()}' being CONSUMED as it does meet the filter condition`);
-              if (mytopics.includes(desiredTopic)) {
+              if (desiredTopic.every((val) => mytopics.includes(val))) {
                 log.info(`Emitting: ${message.value.toString()} TO ${desiredTopic}`);
               } else {
                 log.info(`The desired topic ${desiredTopic} does not exist. If it should, make sure it is in pronghorn.json`);
                 log.info(`Emitting: ${newMsg.value.toString()} TO kafka`);
                 desiredTopic = 'kafka';
               }
-              if (this.props.parseMessage) {
+              if (this.props.parseMessage || this.props.parseMessage === undefined) {
                 let messageObj;
                 const wrapper = this.props.wrapMessage || 'payload';
                 try {
@@ -533,9 +537,13 @@ class Kafkav2 extends EventEmitterCl {
                 } catch (ex) {
                   messageObj = { [wrapper]: newMsg.value.toString() };
                 }
-                eventSystem.publish(desiredTopic, messageObj);
+                for (let j = 0; j < desiredTopic.length; j += 1) {
+                  eventSystem.publish(desiredTopic[j], messageObj);
+                }
               } else {
-                eventSystem.publish(desiredTopic, newMsg.value.toString());
+                for (let j = 0; j < desiredTopic.length; j += 1) {
+                  eventSystem.publish(desiredTopic[j], newMsg);
+                }
               }
             } else {
               log.info(`Message: '${newMsg.value.toString()}' being DROPPED as it fails to meet any filter condition`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-kafkav2",
-  "version": "0.3.13",
+  "version": "0.3.15",
   "description": "Itential adapter to connect to kafka",
   "main": "adapter.js",
   "wizardVersion": "2.35.0",

package/pronghorn.json

@@ -9,9 +9,7 @@
     "admin"
   ],
   "topics": {
-    "kafka": {},
-    "first_kafka_topic": {},
-    "test-topic": {}
+    "kafka": {}
   },
   "methods": [
     {