@jetit/publisher 1.1.0 → 1.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "0.0.11",

package/src/lib/redis/scheduler.js

@@ -51,7 +51,7 @@ class ScheduledProcessor {
                  * Instead of using the publish method directly, the entire logic is
                  * copy pasted to reduce the case of failure.
                  */
-                const transaction = this.redisPublisher.multi();
+                const transaction = this.redisPublisher.multi({ pipeline: true });
                 eventData.eventId = (0, id_1.generateID)('HEX', 'FF');
                 transaction.zrem('se', eventString);
                 const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);

package/src/lib/redis/streams.d.ts

@@ -27,6 +27,7 @@ export declare class Streams {
      * const streams = new Streams('POS');
      */
     constructor(serviceName: string);
+    private runClear;
     private createConsumerGroup;
     private isDuplicateMessage;
     private clearDuplicationCheckKeys;
@@ -124,7 +125,7 @@ export declare class Streams {
      * // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
      * await streams.recoverCrashedConsumerMessages('order.created', 10000);
      */
-    recoverCrashedConsumerMessages(eventName: string, idleTimeout: number): Promise<void>;
+    recoverCrashedConsumerMessages(eventName: string, idleTimeout?: number): Promise<void>;
     /**
      * This method allows the possibility of a graceful shutdown by cleaning up the
      * redis connections.

package/src/lib/redis/streams.js

@@ -6,6 +6,9 @@ const registry_1 = require("./registry");
 const rxjs_1 = require("rxjs");
 const id_1 = require("@jetit/id");
 const groups_1 = require("./groups");
+function publisherErrorHandler(error) {
+    console.error('Publisher Error: ', error);
+}
 class Streams {
     get redisPublisher() {
         if (!this._redisPublisher)
@@ -41,14 +44,24 @@ class Streams {
         this.eventsListened = [];
         this.instanceId = `${serviceName}:${(0, id_1.generateID)('HEX', 'FE')}`;
         this.consumerGroupName = `cg-${serviceName}`;
-        const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || '1000 * 60 * 60', 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
+        const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
+        setTimeout(() => this.runClear(cleanUpInterval), 3000);
         this.cleanUpTimer = setInterval(() => {
+            this.runClear(cleanUpInterval);
+        }, cleanUpInterval);
+    }
+    runClear(cleanUpInterval) {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
+            console.log('Running Publisher Clearance');
             this.clearDuplicationCheckKeys();
             for (const eventName of this.eventsListened) {
-                this.cleanupAcknowledgedMessages(eventName, cleanUpInterval);
-                this.republishUnprocessedEvents(eventName);
+                process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
+                    yield this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
+                    yield this.republishUnprocessedEvents(eventName).catch(publisherErrorHandler);
+                    yield this.recoverCrashedConsumerMessages(eventName).catch(publisherErrorHandler);
+                }));
             }
-        }, cleanUpInterval);
+        });
     }
     createConsumerGroup(eventName) {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
@@ -78,14 +91,16 @@ class Streams {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const processedMessagesKeyPattern = `pm:${this.consumerGroupName}:*`;
             let cursor = '0';
+            const transaction = this.redisGroups.pipeline();
             do {
                 const [nextCursor, keys] = yield this.redisGroups.scan(cursor, 'MATCH', processedMessagesKeyPattern);
                 cursor = nextCursor;
                 for (const key of keys) {
                     const oneHourAgo = Date.now() - 60 * 60 * 1000;
-                    yield this.redisGroups.zremrangebyscore(key, '-inf', oneHourAgo);
+                    transaction.zremrangebyscore(key, '-inf', oneHourAgo);
                 }
             } while (cursor !== '0');
+            yield transaction.exec();
         });
     }
     publish(data) {
@@ -95,7 +110,7 @@ class Streams {
             data.eventId = (0, id_1.generateID)('HEX', 'FF');
             if (!data.createdAt)
                 data.createdAt = Date.now();
-            const transaction = this.redisPublisher.multi();
+            const transaction = this.redisPublisher.multi({ pipeline: true });
             const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
             if (consumerGroups.length > 0) {
                 console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
@@ -203,35 +218,37 @@ class Streams {
             const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
             const streamName = `${eventName}:${this.consumerGroupName}`;
             this.redisSubscriber.subscribe(eventName);
-            const processMessage = () => tslib_1.__awaiter(this, void 0, void 0, function* () {
+            const processMessage = () => {
                 try {
-                    const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
-                    if (result) {
-                        const [, streamMessages] = result[0];
-                        for (const [id, data] of streamMessages) {
-                            const eventData = JSON.parse(data[1]);
-                            const messageId = eventData.eventId;
-                            const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
-                            if (isDuplicate) {
-                                console.warn(`Duplicate message detected: ${messageId}`);
-                                yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
-                                continue;
+                    process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
+                        const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
+                        if (result) {
+                            const [, streamMessages] = result[0];
+                            for (const [id, data] of streamMessages) {
+                                const eventData = JSON.parse(data[1]);
+                                const messageId = eventData.eventId;
+                                const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
+                                if (isDuplicate) {
+                                    console.warn(`Duplicate message detected: ${messageId}`);
+                                    yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
+                                    continue;
+                                }
+                                bs.next(eventData);
+                                const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
+                                const currentTime = Date.now();
+                                const transaction = this.redisGroups.multi({ pipeline: true });
+                                transaction.zadd(pmKey, currentTime, messageId);
+                                transaction.xack(streamName, this.consumerGroupName, id);
+                                transaction.zadd(`ack:${streamName}`, Date.now(), id);
+                                yield transaction.exec();
                             }
-                            bs.next(eventData);
-                            const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
-                            const currentTime = Date.now();
-                            const transaction = this.redisGroups.multi();
-                            transaction.zadd(pmKey, currentTime, messageId);
-                            transaction.xack(streamName, this.consumerGroupName, id);
-                            transaction.zadd(`ack:${streamName}`, Date.now(), id);
-                            yield transaction.exec();
                         }
-                    }
+                    }));
                 }
                 catch (e) {
                     console.error(JSON.stringify(e));
                 }
-            });
+            };
             this.redisSubscriber.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
                 yield processMessage();
             }));
@@ -258,17 +275,19 @@ class Streams {
             const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'STREAMS', streamName, '>');
             if (result) {
                 const [, streamMessages] = result[0];
+                if (!streamMessages)
+                    return;
                 console.log(`Unprocessed events: ${streamMessages.length}`);
+                const transaction = this.redisGroups.multi({ pipeline: true });
                 for (const [id, data] of streamMessages) {
                     const eventData = JSON.parse(data[1]);
-                    const transaction = this.redisGroups.multi();
                     // Republishing the events
                     transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
                     transaction.publish(eventName, '');
                     transaction.xack(streamName, this.consumerGroupName, id);
-                    yield transaction.exec();
                     console.log(`Event ${eventName} with ID: ${id} published`);
                 }
+                yield transaction.exec();
             }
         });
     }
@@ -286,13 +305,16 @@ class Streams {
      * // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
      * await streams.recoverCrashedConsumerMessages('order.created', 10000);
      */
-    recoverCrashedConsumerMessages(eventName, idleTimeout) {
+    recoverCrashedConsumerMessages(eventName, idleTimeout = 30000) {
         return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const streamName = `${eventName}:${this.consumerGroupName}`;
             const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName));
             if (!pendingMessages)
                 return;
             const [, minId, maxId, consumers] = pendingMessages;
+            if (!consumers || consumers.length === 0)
+                return;
+            const transaction = this.redisGroups.multi({ pipeline: true });
             for (const [consumer, pendingCount] of consumers) {
                 if (parseInt(pendingCount) > 0) {
                     const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
@@ -301,15 +323,14 @@ class Streams {
                         if (claimedMessage) {
                             const [, data] = claimedMessage[0];
                             const eventData = JSON.parse(data[1]);
-                            const transaction = this.redisGroups.multi();
                             transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
                             transaction.publish(eventName, '');
                             transaction.xack(streamName, this.consumerGroupName, messageId);
-                            yield transaction.exec();
                         }
                     }
                 }
             }
+            yield transaction.exec();
         });
     }
     /**
@@ -368,7 +389,7 @@ class Streams {
             const cleanupThreshold = Date.now() - interval;
             const acknowledgedMessages = yield this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
             if (acknowledgedMessages && acknowledgedMessages.length > 0) {
-                const transaction = this.redisGroups.multi();
+                const transaction = this.redisGroups.multi({ pipeline: true });
                 // Remove acknowledged messages from the stream
                 for (const messageId of acknowledgedMessages) {
                     transaction.xdel(streamName, messageId);