@jetit/publisher 1.1.3 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "0.0.11",

package/src/lib/redis/streams.js

@@ -43,6 +43,7 @@ class Streams {
         var _a;
         this.eventsListened = [];
         this.instanceId = `${serviceName}:${(0, id_1.generateID)('HEX', 'FE')}`;
+        console.log(this.instanceId);
         this.consumerGroupName = `cg-${serviceName}`;
         const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
         setTimeout(() => this.runClear(cleanUpInterval), 3000);
@@ -52,7 +53,7 @@ class Streams {
     }
     runClear(cleanUpInterval) {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            console.log('Running Publisher Clearance');
+            console.log('Running Publisher Clearance', this.eventsListened);
             this.clearDuplicationCheckKeys();
             for (const eventName of this.eventsListened) {
                 process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
@@ -277,16 +278,17 @@ class Streams {
                 const [, streamMessages] = result[0];
                 if (!streamMessages)
                     return;
-                console.log(`Unprocessed events: ${streamMessages.length}`);
+                console.log(`Unprocessed events:  ${streamMessages.length}`);
                 for (const [id, data] of streamMessages) {
+                    const transaction = this.redisGroups.multi({ pipeline: true });
                     const eventData = JSON.parse(data[1]);
-                    const transaction = this.redisGroups.multi();
                     // Republishing the events
                     transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
                     transaction.publish(eventName, '');
                     transaction.xack(streamName, this.consumerGroupName, id);
                     yield transaction.exec().catch(publisherErrorHandler);
                     console.log(`Event ${eventName} with ID: ${id} published`);
+                    yield transaction.exec();
                 }
             }
         });
@@ -315,11 +317,13 @@ class Streams {
             if (!consumers || consumers.length === 0)
                 return;
             for (const [consumer, pendingCount] of consumers) {
+                const transaction = this.redisGroups.multi({ pipeline: true });
                 if (parseInt(pendingCount) > 0) {
                     const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
                     for (const [messageId] of pending) {
                         const claimedMessage = (yield this.redisGroups.xclaim(streamName, this.consumerGroupName, this.instanceId, idleTimeout, messageId));
                         if (claimedMessage) {
+                            console.log({ claimedMessage: JSON.stringify(claimedMessage) });
                             const [, data] = claimedMessage[0];
                             const eventData = JSON.parse(data[1]);
                             const transaction = this.redisGroups.multi();
@@ -330,6 +334,7 @@ class Streams {
                         }
                     }
                 }
+                yield transaction.exec();
             }
         });
     }
@@ -355,7 +360,7 @@ class Streams {
      */
     close() {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            this.clearSubscribedEvents();
+            yield this.clearSubscribedEvents();
             if (this.redisPublisher) {
                 yield this.redisPublisher.quit();
             }
@@ -373,11 +378,17 @@ class Streams {
     clearSubscribedEvents() {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
             console.log(`${this.eventsListened.length} events to be cleared`);
+            let x = this.eventsListened.length;
             for (const eventName of this.eventsListened) {
+                console.log(`${eventName} is being cleared in publisher`);
                 yield this.redisGroups.srem(`${eventName}`, this.consumerGroupName);
+                console.log(`${eventName} is removed from ${this.consumerGroupName}`);
                 // Releasing all claims based on info from: https://redis.io/commands/xgroup-delconsumer/
                 yield this.releaseAllClaims(eventName);
+                console.log(`${eventName} removes all claims`);
                 yield this.redisGroups.xgroup('DELCONSUMER', eventName, this.consumerGroupName, this.instanceId);
+                console.log(`${eventName} is deleted as a consumer from ${this.consumerGroupName}, ${this.instanceId}`);
+                console.log(x--);
             }
         });
     }
@@ -388,14 +399,18 @@ class Streams {
     }
     releaseAllClaims(eventName) {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            // Retrieve the pending messages for the consumer
-            const pendingMessages = (yield this.redisGroups.xpending(eventName, this.consumerGroupName, '-', '+', 1000, this.instanceId));
+            /**
+             * Retrieve the pending messages for the consumer. Note this only fetches the last
+             * 10000 events assigned to this consumer. This function has been modified to make sure
+             * that there is a temp instance that claims all this messages
+             */
+            const pendingMessages = (yield this.redisGroups.xpending(eventName, this.consumerGroupName, '-', '+', 10000, this.instanceId));
             if (pendingMessages && pendingMessages.length > 0) {
+                console.log(`${pendingMessages.length} messages to clean up.`);
                 const transaction = this.redisGroups.multi({ pipeline: true });
+                const tempConsumerId = `${this.instanceId}-temp`;
                 for (const [messageId] of pendingMessages) {
-                    // Setting the idle time to 10000 ms to make sure this consumer shuts down completely
-                    const idleTime = 10000;
-                    transaction.xclaim(eventName, this.consumerGroupName, this.instanceId, idleTime);
+                    transaction.xclaim(eventName, this.consumerGroupName, tempConsumerId, 10, messageId);
                 }
                 yield transaction.exec();
             }