@itentialopensource/adapter-kafkav2 0.19.0 → 0.21.0

package/CHANGELOG.md

@@ -1,4 +1,20 @@
 
+## 0.21.0 [01-22-2024]
+
+* Minor/topic db dedupe
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!16
+
+---
+
+## 0.20.0 [01-22-2024]
+
+* Minor/topic db dedupe
+
+See merge request itentialopensource/adapters/notification-messaging/adapter-kafkav2!16
+
+---
+
 ## 0.19.0 [01-08-2024]
 
 * Migration changes

package/adapter.js

@@ -20,6 +20,8 @@ const util = require('util');
 /* Fetch in the other needed components for the this Adaptor */
 // const EventEmitterCl = require('events').EventEmitter;
 const AdapterBaseCl = require(path.join(__dirname, 'adapterBase.js'));
+const DBUtil = require(path.join(__dirname, 'utils', 'dbUtil.js'));
+
 const request = require('request');
 
 let needRestart = false;
@@ -195,6 +197,12 @@ class Kafkav2 extends AdapterBaseCl {
     this.id = prongid;
     myid = prongid;
 
+    this.dbUtil = new DBUtil(prongid, properties, __dirname);
+    this.dbUtil.dboptions = {};
+    // const databaseName = this.props.mongo ? this.props.mongo.database : undefined;
+    // const dbInfo = { dburl: this.dbUtil.dburl, dboptions: this.dbUtil.dboptions, database: databaseName };
+    this.allow_connect = false;
+
     // put topics from file into memory
     this.topicsEvents = [];
     let topicsFile = path.join(__dirname, `/.topics-${this.id}.json`);
@@ -465,10 +473,61 @@ class Kafkav2 extends AdapterBaseCl {
     ({ errors } = errorData);
 
     // rewrite the topics file for persistence
+    let databaseIsAlive = false;
+    const dedupeCollectionName = `dedupe-${myid}`;
     if (this.props && this.props.stub === false) {
       const intTime = this.props.interval_time || 30000;
-      setInterval(() => {
+      setInterval(async () => {
         try {
+          let documents;
+
+          // Write to Database if connected
+          if (databaseIsAlive) {
+            const topicNames = this.topicsEvents.map((topicEvent) => topicEvent.topic);
+            const promises = [];
+            this.topicsEvents.forEach((t, ind) => {
+              const eachTopic = this.topicsEvents[ind];
+              eachTopic.offset = parseInt(eachTopic.offset, 10);
+              const filter = { topic: eachTopic.topic, offset: { $lt: eachTopic.offset } };
+              const update = { $set: { offset: eachTopic.offset } };
+              const atomicDBupdate = this.dbUtil.findAndModify(dedupeCollectionName, filter, [], update, false, undefined, false)
+                .then((findRes) => {
+                  if (findRes && findRes.lastErrorObject && findRes.lastErrorObject.updatedExisting) {
+                    log.debug(`Updated topic: ${eachTopic.topic} to offset: ${eachTopic.offset}`);
+                  }
+                }).catch((reason) => {
+                  log.error(`Couldn't update topic: ${eachTopic.topic}: ${reason}`);
+                });
+              promises.push(atomicDBupdate);
+            });
+
+            try {
+              await Promise.all(promises);
+              // Then read from it (important if only some topics are updated)
+              const options = { filter: { topic: { $in: topicNames } }, limit: 0 };
+              documents = await this.dbUtil.find(dedupeCollectionName, options, undefined, false);
+            } catch (err) {
+              log.error(err);
+            }
+          }
+
+          // Then update local .topics-id.json
+          if (documents) {
+            documents.forEach((document) => {
+              if (!document.topic) {
+                log.warn(`document ${document} does not contain property 'topic' when it should`);
+                return;
+              }
+              const topicEvent = this.topicsEvents.find((t) => t.topic === document.topic);
+              if (!topicEvent) {
+                log.warn(`topicEvents does not have topic: ${document.topic} when it should`);
+                return;
+              }
+
+              topicEvent.offset = Math.max(topicEvent.offset, document.offset);
+            });
+          }
+
           const split = topicsFile.split('/');
           log.debug(`Update ${split[split.length - 1]} file.`);
           fs.writeFileSync(topicsFile, JSON.stringify(this.topicsEvents, null, 2));
@@ -477,6 +536,47 @@ class Kafkav2 extends AdapterBaseCl {
         }
       }, intTime);
     }
+
+    const setupDatabase = async (dbUtil, collectionName, setupDBInfo, topicEvents) => {
+      await dbUtil.createCollection(collectionName, setupDBInfo, false);
+      const dbInserts = [];
+      topicEvents.forEach((topicEvent) => {
+        const insert = dbUtil.findAndModify(collectionName, { topic: topicEvent.topic }, [], { $setOnInsert: topicEvent }, true, setupDBInfo, false);
+        dbInserts.push(insert);
+      });
+      await Promise.all(dbInserts);
+
+      const topicNames = topicEvents.map((t) => t.topic);
+      const options = { filter: { topic: { $in: topicNames } }, limit: 0 };
+      const documents = await this.dbUtil.find(collectionName, options, undefined, false);
+      topicEvents.forEach((t, ind) => {
+        const topicEvent = this.topicsEvents[ind];
+        const doc = documents.find((d) => d.topic === topicEvent.topic);
+        if (!doc) throw new Error(`There is no document with topic ${topicEvent.topic} when there should be`);
+        topicEvent.offset = Math.max(topicEvent.offset, doc.offset);
+      });
+      return topicEvents;
+    };
+    // Connect and set up database + topics
+    this.dbUtil.connect()
+      .then((connectionInfo) => {
+        if (connectionInfo.isAlive) {
+          log.info('Connected to adapter database');
+        } else {
+          throw new Error('Could not connect to adapter database');
+        }
+      })
+      .then(async () => {
+        const databaseTopics = await setupDatabase(this.dbUtil, dedupeCollectionName, undefined, this.topicsEvents);
+        this.topicsEvents = databaseTopics;
+        databaseIsAlive = true;
+        this.allow_connect = true;
+      })
+      .catch((err) => {
+        databaseIsAlive = false;
+        log.error(`Database Connection error: ${err}`);
+        this.allow_connect = true;
+      });
   }
 
   /**
@@ -485,6 +585,15 @@ class Kafkav2 extends AdapterBaseCl {
    * @function connect
    */
   connect() {
+    // Wait to read from database before connecting
+    if (!this.allow_connect) {
+      const retryIn = 1000;
+      const sleep = new Promise((r) => setTimeout(r, retryIn));
+      sleep.then(() => this.connect());
+      log.warn('Waiting to connect...');
+      return;
+    }
+    log.info('Connecting...');
     const meth = 'adapter-connect';
     const origin = `${this.id}-${meth}`;
     log.trace(origin);
@@ -518,7 +627,6 @@ class Kafkav2 extends AdapterBaseCl {
         if (this.props.client.ssl && this.props.client.ssl.key) {
           combinedProps.ssl.key = fs.readFileSync(this.props.client.ssl.key, 'utf-8');
         }
-
         this.KafkaClient = new Kafka(combinedProps);
 
         if (this.props.client && this.props.client.producersasl) {
@@ -604,7 +712,16 @@ class Kafkav2 extends AdapterBaseCl {
               }
             }, intTime);
           }
-          await this.consumer.run({
+
+          const topicOffsetsToSeek = this.topicsEvents.map((t) => (
+            {
+              topic: t.topic,
+              partition: t.partition,
+              offset: Number.parseInt(t.offset, 10) + 1 // Seek to the next offset you're going to read
+            }
+          ));
+
+          const runPromise = this.consumer.run({
             eachMessage: async ({ topic, partition, message }) => {
               log.info(`PROCESSING NEW MESSAGE ON TOPIC: ${topic} PARTITION: ${partition} WITH OFFSET: ${message.offset}`);
               const newMsg = message;
@@ -645,8 +762,12 @@ class Kafkav2 extends AdapterBaseCl {
               for (let t = 0; t < this.topicsEvents.length; t += 1) {
                 if (this.topicsEvents[t].topic === topic && (this.topicsEvents[t].partition === partition || (Array.isArray(this.topicsEvents[t].partition) && this.topicsEvents[t].partition.includes(partition)))) {
                   allowedPartition = true;
-                  if (this.topicsEvents[t].offset < message.offset) {
-                    this.topicsEvents[t].offset = message.offset;
+                  const topicOffset = parseInt(this.topicsEvents[t].offset, 10);
+                  const messageOffset = parseInt(message.offset, 10);
+                  if (Number.isNaN(topicOffset) || Number.isNaN(messageOffset)) throw new Error('Topic offset or message offset is NaN');
+                  // if (this.topicsEvents[t].offset < message.offset) {
+                  if (topicOffset < messageOffset) {
+                    this.topicsEvents[t].offset = messageOffset;
                   }
                   // determine if we are using AVRO and set flag
                   if (this.topicsEvents[t].avro && this.topicsEvents[t].avro.toUpperCase() === 'YES') {
@@ -702,6 +823,13 @@ class Kafkav2 extends AdapterBaseCl {
               }
             }
           });
+
+          // Set consumer offsets to match database
+          topicOffsetsToSeek.forEach((topicPartitionOffset) => {
+            this.consumer.seek(topicPartitionOffset);
+          });
+
+          await runPromise;
         };
         run().catch((err) => console.error(`[consumer] ${err.message}`, err));
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-kafkav2",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "description": "Itential adapter to connect to kafka",
   "main": "adapter.js",
   "wizardVersion": "2.35.0",
@@ -14,7 +14,6 @@
     "test:baseunit": "mocha test/unit/adapterBaseTestUnit.js --LOG=error",
     "test:unit": "mocha test/unit/adapterTestUnit.js --LOG=error",
     "test:integration": "mocha test/integration/adapterTestIntegration.js --LOG=error",
-    "test:cover": "nyc --reporter html --reporter text mocha --reporter dot test/*",
     "test": "npm run test:unit && npm run test:integration",
     "deploy": "npm publish --registry=https://registry.npmjs.org --access=public",
     "build": "npm run deploy"
@@ -49,6 +48,7 @@
     "fs-extra": "^11.1.1",
     "json-query": "^2.2.2",
     "kafkajs": "2.2.3",
+    "mongodb": "^6.2.0",
     "patch-package": "^6.4.7",
     "readline-sync": "^1.4.10",
     "request": "^2.88.2",
@@ -61,7 +61,6 @@
     "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-json": "^3.1.0",
     "mocha": "^9.0.1",
-    "nyc": "^15.1.0",
     "testdouble": "^3.18.0",
     "winston": "^3.3.3"
   },

package/pronghorn.json

@@ -9,9 +9,7 @@
     "admin"
   ],
   "topics": {
-    "kafka": {},
-    "test-topic": {},
-    "test_topic": {}
+    "kafka": {}
   },
   "methods": [
     {