@jetit/publisher 2.0.2 → 3.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "2.0.2",
+  "version": "3.0.0",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "0.0.11",
@@ -8,7 +8,7 @@
     "rxjs": "^7.8.0"
   },
   "peerDependencies": {
-    "tslib": "2.5.0"
+    "tslib": "2.5.3"
   },
   "main": "./src/index.js",
   "types": "./src/index.d.ts"

package/src/lib/publisher.d.ts

@@ -1,3 +1,4 @@
 export { Streams as Publisher } from './redis/streams';
 export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
 export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';
+export { UTILS as StreamUtilityFunctions } from './redis/utils';

package/src/lib/publisher.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.__SCHEDULER_INTERNALS__ = exports.setRedisConfig = exports.Publisher = void 0;
+exports.StreamUtilityFunctions = exports.__SCHEDULER_INTERNALS__ = exports.setRedisConfig = exports.Publisher = void 0;
 var streams_1 = require("./redis/streams");
 Object.defineProperty(exports, "Publisher", { enumerable: true, get: function () { return streams_1.Streams; } });
 var registry_1 = require("./redis/registry");
 Object.defineProperty(exports, "setRedisConfig", { enumerable: true, get: function () { return registry_1.setRedisConnectionSettings; } });
 var scheduler_1 = require("./redis/scheduler");
 Object.defineProperty(exports, "__SCHEDULER_INTERNALS__", { enumerable: true, get: function () { return scheduler_1.ScheduledProcessor; } });
+var utils_1 = require("./redis/utils");
+Object.defineProperty(exports, "StreamUtilityFunctions", { enumerable: true, get: function () { return utils_1.UTILS; } });

package/src/lib/redis/registry.js

@@ -1,17 +1,24 @@
 "use strict";
-var _a, _b;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
-const tslib_1 = require("tslib");
 const ioredis_1 = require("ioredis");
 class RedisRegistry {
     static attemptConnection(connectionKey, storeRef = 0) {
         let ref;
         if (RedisRegistry.options.cluster) {
-            ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: Object.assign(Object.assign({}, RedisRegistry.options.cluster.options.redisOptions), { db: storeRef }) }));
+            ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, {
+                ...RedisRegistry.options.cluster.options,
+                redisOptions: {
+                    ...RedisRegistry.options.cluster.options.redisOptions,
+                    db: storeRef,
+                },
+            });
         }
         else
-            ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+            ref = new ioredis_1.default({
+                ...RedisRegistry.options.redis,
+                db: storeRef,
+            });
         RedisRegistry.registry.set(connectionKey, ref);
         RedisRegistry.handleDisconnects(ref, connectionKey, storeRef);
         return ref;
@@ -24,8 +31,8 @@ class RedisRegistry {
         });
     }
     static handlePing(connection) {
-        setInterval(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const res = yield Promise.race([
+        setInterval(async () => {
+            const res = await Promise.race([
                 connection.ping(),
                 new Promise((res) => {
                     setTimeout(() => {
@@ -38,17 +45,26 @@ class RedisRegistry {
                 console.error('PUBLISHER: failed to ping redis, disconnecting and restarting service.');
                 process.exit(0);
             }
-        }), 2000);
+        }, 2000);
     }
     static getConnection(connectionType = 'primary', storeRef = 0) {
         const connectionKey = `${connectionType}${storeRef}`;
         let ref = this.registry.get(connectionKey);
         if (!ref) {
             if (RedisRegistry.options.cluster) {
-                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: Object.assign(Object.assign({}, RedisRegistry.options.cluster.options.redisOptions), { db: storeRef }) }));
+                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, {
+                    ...RedisRegistry.options.cluster.options,
+                    redisOptions: {
+                        ...RedisRegistry.options.cluster.options.redisOptions,
+                        db: storeRef,
+                    },
+                });
             }
             else
-                ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+                ref = new ioredis_1.default({
+                    ...RedisRegistry.options.redis,
+                    db: storeRef,
+                });
         }
         return ref;
     }
@@ -59,14 +75,14 @@ class RedisRegistry {
         return RedisRegistry.options;
     }
 }
+exports.RedisRegistry = RedisRegistry;
 RedisRegistry.registry = new Map();
 RedisRegistry.options = {
     redis: {
-        port: parseInt((_a = process.env['REDIS_PORT']) !== null && _a !== void 0 ? _a : '6379'),
-        host: (_b = process.env['REDIS_HOST']) !== null && _b !== void 0 ? _b : 'localhost',
+        port: parseInt(process.env['REDIS_PORT'] ?? '6379'),
+        host: process.env['REDIS_HOST'] ?? 'localhost',
     },
 };
-exports.RedisRegistry = RedisRegistry;
 /**
  * This function is used to set Redis Connection options per instance. If no
  * options are provided, then the service connects as to a single instance

package/src/lib/redis/scheduler.js

@@ -1,11 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ScheduledProcessor = void 0;
-const tslib_1 = require("tslib");
 const id_1 = require("@jetit/id");
 const rxjs_1 = require("rxjs");
 const registry_1 = require("./registry");
-const groups_1 = require("./groups");
+const utils_1 = require("./utils");
 /**
  * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
  * meant to be used internally by the scheduler application
@@ -36,48 +35,49 @@ class ScheduledProcessor {
             }
         });
     }
-    processScheduledEvents() {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const currentTime = new Date().getTime();
-            const events = yield this.redisPublisher.zrangebyscore('se', 0, currentTime);
-            console.log('Events to process:', events.length);
-            for (const eventString of events) {
-                const eventData = JSON.parse(eventString);
-                /**
-                 * Remove the event from the Redis Sorted Set first. Please note that
-                 * there is a chance of failure here if the process crashes before
-                 * the event is published. In that case, the event will be lost.
-                 *
-                 * Instead of using the publish method directly, the entire logic is
-                 * copy pasted to reduce the case of failure.
-                 */
-                eventData.eventId = (0, id_1.generateID)('HEX', 'FF');
-                yield this.redisPublisher.zrem('se', eventString);
-                const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);
-                console.log('Scheduled Publishing to consumer groups: ', consumerGroups, 'with id ', eventData.eventId, '...');
-                for (const consumerGroup of consumerGroups) {
-                    // Publish the event to each consumer group's stream
-                    const streamName = `${eventData.eventName}:${consumerGroup}`;
-                    yield this.redisPublisher.xadd(streamName, '*', 'data', JSON.stringify(eventData));
-                }
-                if (eventData.repeatInterval) {
-                    const nextEventTime = currentTime + eventData.repeatInterval;
-                    const nextEventString = JSON.stringify(Object.assign({}, eventData));
-                    yield this.redisPublisher.zadd('se', nextEventTime, nextEventString);
-                }
-                yield this.redisPublisher.publish(eventData.eventName, '');
+    async processScheduledEvents() {
+        const currentTime = new Date().getTime();
+        const events = await this.redisPublisher.zrangebyscore('se', 0, currentTime);
+        console.log('Events to process:', events.length);
+        for (const eventString of events) {
+            const eventData = JSON.parse(eventString);
+            /**
+             * Remove the event from the Redis Sorted Set first. Please note that
+             * there is a chance of failure here if the process crashes before
+             * the event is published. In that case, the event will be lost.
+             *
+             * Instead of using the publish method directly, the entire logic is
+             * copy pasted to reduce the case of failure.
+             */
+            eventData.eventId = (0, id_1.generateID)('HEX', 'FF');
+            await this.redisPublisher.zrem('se', eventString);
+            const consumerGroups = await (0, utils_1.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);
+            console.log('Scheduled Publishing to consumer groups: ', consumerGroups, 'with id ', eventData.eventId, '...');
+            let key = '*';
+            for (const consumerGroup of consumerGroups) {
+                // Publish the event to each consumer group's stream
+                const streamName = `${eventData.eventName}:${consumerGroup}`;
+                const generatedKey = await this.redisPublisher
+                    .xadd(streamName, '*', 'data', JSON.stringify(eventData))
+                    .catch((e) => console.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
+                if (key === '*')
+                    key = generatedKey ?? key;
             }
-        });
+            if (eventData.repeatInterval) {
+                const nextEventTime = currentTime + eventData.repeatInterval;
+                const nextEventString = JSON.stringify({ ...eventData });
+                await this.redisPublisher.zadd('se', nextEventTime, nextEventString);
+            }
+            await (0, utils_1.notifySubscribers)(this.redisPublisher, eventData.eventName, key);
+        }
     }
     getAllScheduledEvents() {
         return this.redisPublisher.zrange('se', 0, -1);
     }
-    close() {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            if (this.scheduledMessagesTimer) {
-                this.scheduledMessagesTimer.unsubscribe();
-            }
-        });
+    async close() {
+        if (this.scheduledMessagesTimer) {
+            this.scheduledMessagesTimer.unsubscribe();
+        }
     }
 }
 exports.ScheduledProcessor = ScheduledProcessor;

package/src/lib/redis/streams.d.ts

@@ -26,8 +26,6 @@ export declare class Streams {
      */
     constructor(serviceName: string);
     private runClear;
-    private isDuplicateMessage;
-    private clearDuplicationCheckKeys;
     /**
      * Publishes an event with the given data to the Redis event stream.
      *
@@ -121,6 +119,5 @@ export declare class Streams {
      * }
      */
     close(): Promise<void>;
-    private scanAndClaimAUnclaimedMessage;
     private cleanupAcknowledgedMessages;
 }

package/src/lib/redis/streams.js

@@ -1,13 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Streams = void 0;
-const tslib_1 = require("tslib");
 const registry_1 = require("./registry");
 const rxjs_1 = require("rxjs");
 const id_1 = require("@jetit/id");
-const groups_1 = require("./groups");
+const utils_1 = require("./utils");
 function publisherErrorHandler(error) {
-    console.error('PUBLISHER UNHANDLED ERROR: ', error);
+    console.error('PUBLISHER UNHANDLED ERROR: ', JSON.stringify(error));
 }
 class Streams {
     get redisPublisher() {
@@ -35,114 +34,78 @@ class Streams {
      * const streams = new Streams('POS');
      */
     constructor(serviceName) {
-        var _a, _b;
         this.eventsListened = [];
-        this.instanceUniqueId = (_a = process.env['INSTANCE_ID']) !== null && _a !== void 0 ? _a : (0, id_1.generateID)('HEX', 'FE');
+        this.instanceUniqueId = process.env['INSTANCE_ID'] ?? (0, id_1.generateID)('HEX', 'FE');
         this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
         this.consumerGroupName = `cg-${serviceName}`;
         console.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
-        const cleanUpInterval = (_b = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10)) !== null && _b !== void 0 ? _b : 1000 * 60 * 60;
-        setTimeout(() => this.runClear(cleanUpInterval), 60000);
+        const cleanUpInterval = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10) ?? 1000 * 60 * 60;
         this.cleanUpTimer = setInterval(() => {
             this.runClear(cleanUpInterval);
         }, cleanUpInterval);
     }
-    runClear(cleanUpInterval) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            console.log('PUBLISHER: Running Clearance', this.eventsListened);
-            this.clearDuplicationCheckKeys();
-            for (const eventName of this.eventsListened) {
-                process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
-                    /** This removes the messages from the stream after they have been processed according to cleanup interval */
-                    yield this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
-                    console.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
-                    /**
-                     * This process scans and claims any unclaimed message according to the cleanup interval.
-                     * This triggers a cascaded reaction down the chain as message by message is claimed and processed
-                     */
-                    const streamName = `${eventName}:${this.consumerGroupName}`;
-                    yield this.scanAndClaimAUnclaimedMessage(streamName).catch(publisherErrorHandler);
-                    console.log(`Unclaimed messages for ${streamName}`);
-                }));
-            }
-        });
-    }
-    isDuplicateMessage(streamName, messageId) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
-            const temp = yield Promise.race([
-                this.redisGroups.zscore(processedMessagesKey, messageId),
-                /** ioRedis doesnt seem to return the nil event. So waiting for 100ms before moving on */
-                new Promise((res) => setTimeout(() => res(null), 100)),
-            ]);
-            return temp !== null;
-        });
-    }
-    clearDuplicationCheckKeys() {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const processedMessagesKeyPattern = `pm:${this.consumerGroupName}:*`;
-            let cursor = '0';
-            do {
-                const [nextCursor, keys] = yield this.redisGroups.scan(cursor, 'MATCH', processedMessagesKeyPattern);
-                cursor = nextCursor;
-                for (const key of keys) {
-                    const oneHourAgo = Date.now() - 60 * 60 * 1000;
-                    yield this.redisGroups.zremrangebyscore(key, '-inf', oneHourAgo);
-                }
-            } while (cursor !== '0');
-        });
+    async runClear(cleanUpInterval) {
+        console.log('PUBLISHER: Running Clearance', this.eventsListened);
+        for (const eventName of this.eventsListened) {
+            process.nextTick(async () => {
+                /** This removes the messages from the stream after they have been processed according to cleanup interval */
+                await this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
+                console.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
+            });
+        }
     }
-    publish(data) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            if (data.eventId)
-                data.republishEvent = data.eventId;
-            data.eventId = (0, id_1.generateID)('HEX', 'FF');
-            if (!data.createdAt)
-                data.createdAt = Date.now();
-            const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
-            if (consumerGroups.length > 0) {
-                console.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
-                try {
-                    for (const consumerGroup of consumerGroups) {
-                        // Publish the event to each consumer group's stream
-                        const streamName = `${data.eventName}:${consumerGroup}`;
-                        yield this.redisPublisher.xadd(streamName, '*', 'data', JSON.stringify(data));
-                    }
-                    yield this.redisPublisher.publish(data.eventName, '');
-                }
-                catch (error) {
-                    console.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
-                    throw new Error('Publisher Error');
+    async publish(data) {
+        if (data.eventId)
+            data.republishEvent = data.eventId;
+        data.eventId = (0, id_1.generateID)('HEX', 'FF');
+        if (!data.createdAt)
+            data.createdAt = Date.now();
+        const consumerGroups = await (0, utils_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
+        let key = '*';
+        if (consumerGroups.length > 0) {
+            console.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
+            try {
+                for (const consumerGroup of consumerGroups) {
+                    // Publish the event to each consumer group's stream
+                    const streamName = `${data.eventName}:${consumerGroup}`;
+                    const generatedKey = await this.redisPublisher
+                        .xadd(streamName, key, 'data', JSON.stringify(data))
+                        .catch((e) => console.error(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(data)}, ${e} `));
+                    if (key === '*')
+                        key = generatedKey ?? key;
                 }
+                await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key);
             }
-            else
-                console.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
-        });
-    }
-    scheduledPublish(scheduledTime, eventData, uniquePerInstance = false, repeatInterval = 0) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const currentTime = new Date();
-            delete eventData.repeatInterval;
-            if (repeatInterval > 0) {
-                eventData.repeatInterval = repeatInterval;
-            }
-            if (scheduledTime < currentTime) {
-                throw new Error('PUBLISHER: Cannot schedule an event in the past');
+            catch (error) {
+                console.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
+                throw new Error('Publisher Error');
             }
-            else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
-                yield this.publish(eventData);
-            }
-            else {
-                if (uniquePerInstance === true) {
-                    const existingJob = yield this.redisPublisher.zscore('se', JSON.stringify(eventData));
-                    if (existingJob) {
-                        console.log(`PUBLISHER: Job with data '${eventData}' already exists. Skipping.`);
-                        return;
-                    }
+        }
+        else
+            console.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
+    }
+    async scheduledPublish(scheduledTime, eventData, uniquePerInstance = false, repeatInterval = 0) {
+        const currentTime = new Date();
+        delete eventData.repeatInterval;
+        if (repeatInterval > 0) {
+            eventData.repeatInterval = repeatInterval;
+        }
+        if (scheduledTime < currentTime) {
+            throw new Error('PUBLISHER: Cannot schedule an event in the past');
+        }
+        else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
+            await this.publish(eventData);
+        }
+        else {
+            if (uniquePerInstance === true) {
+                const existingJob = await this.redisPublisher.zscore('se', JSON.stringify(eventData));
+                if (existingJob) {
+                    console.log(`PUBLISHER: Job with data '${eventData}' already exists. Skipping.`);
+                    return;
                 }
-                yield this.redisPublisher.zadd('se', scheduledTime.getTime(), JSON.stringify(eventData));
             }
-        });
+            await this.redisPublisher.zadd('se', scheduledTime.getTime(), JSON.stringify(eventData));
+        }
     }
     /**
      * Listens for events with the given name and returns an Observable that emits an EventData<T> object
@@ -184,98 +147,75 @@ class Streams {
             return (0, rxjs_1.throwError)(() => new Error(error.message));
         }));
     }
-    createConsumerAndRegister(eventName) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const streamName = `${eventName}:${this.consumerGroupName}`;
-            const key = `instance:${this.instanceId}:subscribedEvents`;
-            const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
-            this.eventsListened.push(eventName);
-            yield this.redisGroups
-                .xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM')
-                .then((v) => {
-                console.log(`Group created created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
-            })
-                .catch((e) => {
-                console.log(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
-            });
-            const createConsumerStatus = (yield this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId));
-            yield this.redisGroups.sadd(key, eventName);
-            const addToCGSet = yield this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
-            const addToFlushSet = yield this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName);
-            console.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
-            return createConsumerStatus === 0 || createConsumerStatus === 1;
+    async createConsumerAndRegister(eventName) {
+        const streamName = `${eventName}:${this.consumerGroupName}`;
+        const key = `instance:${this.instanceId}:subscribedEvents`;
+        const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
+        this.eventsListened.push(eventName);
+        await this.redisGroups
+            .xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM')
+            .then(() => {
+            console.log(`Group created created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+        })
+            .catch((e) => {
+            console.error(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
         });
+        const createConsumerStatus = (await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId));
+        await this.redisGroups.sadd(key, eventName);
+        const addToCGSet = await this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
+        const addToFlushSet = await this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName);
+        console.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
+        return createConsumerStatus === 0 || createConsumerStatus === 1;
     }
     listenInternals(eventName) {
-        /** Create the return observable */
         const bs = new rxjs_1.BehaviorSubject(null);
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
-        /** This gets called the first time the stream is registered to pickup any messages from the previous subscription */
         const streamName = `${eventName}:${this.consumerGroupName}`;
-        const processMessage = (redisClient) => tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const racePr = new Promise((resolve, _) => {
-                setTimeout(resolve, 1300, 'RACE');
-            });
-            console.log(`PUBLISHER: processMessage called for ${streamName} cgn: ${this.consumerGroupName} inst: ${this.instanceId}`);
+        const processMessage = async (redisClient, messageId, processPending = false) => {
+            console.log(`PUBLISHER: Processing message ${messageId} for ${streamName}`);
             try {
-                const prResult = yield Promise.race([
-                    redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 1000, 'STREAMS', streamName, '>'),
-                    racePr,
-                ]);
-                console.log(`PUBLISHER: Promise race resolved with ${JSON.stringify(prResult)}`);
-                if (prResult && prResult === 'RACE')
-                    return;
-                const result = prResult;
-                console.log(`PUBLISHER: XREADGROUP returned with ${JSON.stringify(result[0])}`);
-                if (result) {
-                    const [, streamMessages] = result[0];
-                    for (const [id, data] of streamMessages) {
-                        const eventData = JSON.parse(data[1]);
-                        const messageId = eventData.eventId;
-                        const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
-                        if (isDuplicate) {
-                            console.warn(`Duplicate message detected: ${messageId}`);
-                            yield redisClient.xack(streamName, this.consumerGroupName, id);
-                            continue;
-                        }
-                        bs.next(eventData);
-                        const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
-                        const currentTime = Date.now();
-                        yield redisClient.zadd(pmKey, currentTime, messageId);
-                        yield redisClient.xack(streamName, this.consumerGroupName, id);
-                        yield redisClient.zadd(`ack:${streamName}`, Date.now(), id);
+                const messages = await redisClient.xrange(streamName, messageId, messageId);
+                if (messages && messages.length) {
+                    const eventData = JSON.parse(messages[0][1][1]);
+                    bs.next(eventData);
+                    await redisClient.xack(streamName, this.consumerGroupName, messageId);
+                    await redisClient.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
+                }
+                else {
+                    console.warn(`PUBLISHER: Message ${messageId} not found for ${streamName}`);
+                }
+                /** Process Unprocessed Message if this is a main tree, otherwise limit to processing 100 messages that are unacknowledged */
+                if (!processPending) {
+                    const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, 25);
+                    if (unprocessedMessageIds.count > 25) {
+                        console.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
+                    }
+                    for (const id of unprocessedMessageIds.messageIds) {
+                        console.log(`PUBLISHER: Reporcessing unprocessed message with id: ${id}`);
+                        await processMessage(redisClient, id, true);
                     }
                 }
-                this.scanAndClaimAUnclaimedMessage(streamName)
-                    .then()
-                    .catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
             }
             catch (e) {
-                console.error(`PUBLISHER: ${JSON.stringify(e)}`);
+                console.error(`PUBLISHER: Error processing message ${messageId} for ${streamName}`, e);
             }
-        });
+        };
+        /** Register the consumer and setup the Observable */
         this.createConsumerAndRegister(eventName)
             .then((consumerRegistered) => {
             if (!consumerRegistered)
                 throw new Error('PUBLISHER: Cannot setup consumer');
-            /** Create new REDIS connection and subscribe */
             const eventStreamClient = registry_1.RedisRegistry.getConnection(`sub-${eventName}`);
             eventStreamClient.subscribe(eventName).then(() => {
-                console.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName} with ${JSON.stringify({
-                    cluster: eventStreamClient.isCluster,
-                })}`);
+                console.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName}`);
+            });
+            eventStreamClient.on('message', async (channel, messageId) => {
+                console.log(`PUBLISHER: Stream Notification Received for event ${eventName} with message ID ${messageId}`);
+                await processMessage(this.redisGroups, messageId);
             });
-            eventStreamClient.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
-                console.log(`PUBLISHER: Stream Notification Received for event ${eventName}`);
-                yield processMessage(this.redisGroups);
-            }));
-            this.scanAndClaimAUnclaimedMessage(streamName)
-                .then()
-                .catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
         })
             .catch((e) => {
-            console.error(`PUBLISHER: ${JSON.stringify(e)}`);
-            throw e;
+            console.error(`PUBLISHER: Error during consumer registration for ${eventName}`, e);
         });
         return observable;
     }
@@ -299,50 +239,32 @@ class Streams {
      *   process.exit(0);
      * }
      */
-    close() {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            if (this.cleanUpTimer) {
-                clearInterval(this.cleanUpTimer);
-            }
-            if (this.redisPublisher) {
-                yield this.redisPublisher.quit();
-            }
-            for (const eventName of this.eventsListened) {
-                registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
-            }
-            if (this.redisGroups) {
-                yield this.redisGroups.quit();
-            }
-        });
-    }
-    scanAndClaimAUnclaimedMessage(streamName) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const rows = yield this.redisGroups.xautoclaim(streamName, this.consumerGroupName, this.instanceId, 500, '0-0', 'COUNT', 1);
-            if (rows && rows[0] !== '0-0') {
-                console.log(`PUBLISHER: Handling pending unclaimed Message from ${streamName} for ${this.instanceId}`);
-                yield this.redisPublisher.publish(streamName.split(':')[0], '');
-                return this.scanAndClaimAUnclaimedMessage(streamName);
-            }
-            else {
-                console.log(`PUBLISHER: No previous messages found for ${streamName}`);
-            }
-            return;
-        });
+    async close() {
+        if (this.cleanUpTimer) {
+            clearInterval(this.cleanUpTimer);
+        }
+        if (this.redisPublisher) {
+            await this.redisPublisher.quit();
+        }
+        for (const eventName of this.eventsListened) {
+            registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
+        }
+        if (this.redisGroups) {
+            await this.redisGroups.quit();
+        }
     }
-    cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const streamName = `${eventName}:${this.consumerGroupName}`;
-            const cleanupThreshold = Date.now() - interval;
-            const acknowledgedMessages = yield this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
-            if (acknowledgedMessages && acknowledgedMessages.length > 0) {
-                // Remove acknowledged messages from the stream
-                for (const messageId of acknowledgedMessages) {
-                    yield this.redisGroups.xdel(streamName, messageId);
-                }
-                // Remove acknowledged messages from the Sorted Set
-                yield this.redisGroups.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+    async cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
+        const streamName = `${eventName}:${this.consumerGroupName}`;
+        const cleanupThreshold = Date.now() - interval;
+        const acknowledgedMessages = await this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+        if (acknowledgedMessages && acknowledgedMessages.length > 0) {
+            // Remove acknowledged messages from the stream
+            for (const messageId of acknowledgedMessages) {
+                await this.redisGroups.xdel(streamName, messageId);
             }
-        });
+            // Remove acknowledged messages from the Sorted Set
+            await this.redisGroups.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+        }
     }
 }
 exports.Streams = Streams;

package/src/lib/redis/utils.d.ts

@@ -0,0 +1,18 @@
+import { RedisType } from './registry';
+export declare function getAllConsumerGroups(eventName: string, redisConnection: RedisType): Promise<string[]>;
+export declare function getUnacknowledgedMessages(redisClient: RedisType, consumerGroupName: string, streamName: string, count?: number): Promise<{
+    count: number;
+    messageIds: string[];
+    messages?: unknown[];
+}>;
+export declare function getMessageStatesCount(redisClient: RedisType, streamName: string, consumerGroup: string): Promise<{
+    acknowledged: number;
+    unacknowledged: number;
+}>;
+export declare function notifySubscribers(redisClient: RedisType, eventName: string, messageId: string): Promise<void>;
+export declare const UTILS: {
+    getMessageStatesCount: typeof getMessageStatesCount;
+    getUnacknowledgedMessages: typeof getUnacknowledgedMessages;
+    getAllConsumerGroupsForEvent: typeof getAllConsumerGroups;
+    notifySubscribers: typeof notifySubscribers;
+};

package/src/lib/redis/utils.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UTILS = exports.notifySubscribers = exports.getMessageStatesCount = exports.getUnacknowledgedMessages = exports.getAllConsumerGroups = void 0;
+async function getAllConsumerGroups(eventName, redisConnection) {
+    const consumerGroups = await redisConnection.smembers(`${eventName}`);
+    return consumerGroups;
+}
+exports.getAllConsumerGroups = getAllConsumerGroups;
+async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamName, count = 500) {
+    try {
+        // Get pending messages summary
+        const summary = await redisClient.xpending(streamName, consumerGroupName);
+        if (!summary || summary[1] === 0) {
+            // If count is zero
+            return { count: 0, messageIds: [] };
+        }
+        // Use the smallest and largest IDs to get a detailed range
+        const pendingMessageCount = summary[1];
+        // Get detailed information in the range
+        const pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, '-', '+', count));
+        return {
+            count: pendingMessageCount,
+            messageIds: pendingMessages.map((message) => message[0]),
+            messages: pendingMessages,
+        };
+    }
+    catch (error) {
+        console.error(`PUBLISHER: Error fetching unacknowledged messages for ${streamName}`, error);
+        return { count: 0, messageIds: [] };
+    }
+}
+exports.getUnacknowledgedMessages = getUnacknowledgedMessages;
+async function getMessageStatesCount(redisClient, streamName, consumerGroup) {
+    try {
+        const pendingInfo = (await redisClient.xpending(streamName, consumerGroup));
+        const totalCount = await redisClient.xlen(streamName);
+        return {
+            acknowledged: totalCount - pendingInfo[1],
+            unacknowledged: pendingInfo[1],
+        };
+    }
+    catch (error) {
+        console.error(`PUBLISHER: Error fetching message states count for ${streamName}`, error);
+        return { acknowledged: 0, unacknowledged: 0 };
+    }
+}
+exports.getMessageStatesCount = getMessageStatesCount;
+async function notifySubscribers(redisClient, eventName, messageId) {
+    await redisClient.publish(eventName, messageId);
+}
+exports.notifySubscribers = notifySubscribers;
+exports.UTILS = {
+    getMessageStatesCount,
+    getUnacknowledgedMessages,
+    getAllConsumerGroupsForEvent: getAllConsumerGroups,
+    notifySubscribers,
+};

package/src/lib/redis/groups.d.ts

@@ -1,2 +0,0 @@
-import { RedisType } from './registry';
-export declare function getAllConsumerGroups(eventName: string, redisConnection: RedisType): Promise<string[]>;

package/src/lib/redis/groups.js

@@ -1,11 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAllConsumerGroups = void 0;
-const tslib_1 = require("tslib");
-function getAllConsumerGroups(eventName, redisConnection) {
-    return tslib_1.__awaiter(this, void 0, void 0, function* () {
-        const consumerGroups = yield redisConnection.smembers(`${eventName}`);
-        return consumerGroups;
-    });
-}
-exports.getAllConsumerGroups = getAllConsumerGroups;