@jetit/publisher 5.2.2 → 5.4.0

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.2.2",
+  "version": "5.4.0",
   "type": "commonjs",
   "dependencies": {
-    "@jetit/id": "^0.0.12",
-    "events": "3.3.0",
+    "@jetit/id": "^0.0.13",
     "ioredis": "^5.3.0",
     "rxjs": "^7.8.0",
-    "tslib": "2.5.0"
+    "tslib": "2.7.0"
   },
+  "types": "./src/index.d.ts",
   "main": "./src/index.js"
-}
+}

package/src/lib/monitoring/collector.js

@@ -111,33 +111,26 @@ class MetricsCollector {
         const individualDepths = {};
         let cursor = '0';
         do {
-            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', `ack:*:cg-*`, 'COUNT', 100);
+            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', '*:cg-*', 'COUNT', 100);
             cursor = nextCursor;
             if (keys.length > 0) {
-                const pipeline = this.redisClient.pipeline();
-                keys.forEach((key) => {
-                    pipeline.xlen(key.slice(4));
-                    pipeline.zcard(key);
-                });
-                const results = await pipeline.exec();
-                if (!results) {
-                    continue;
-                }
-                for (let i = 0; i < results.length; i += 2) {
-                    const key = keys[i / 2];
-                    const [streamLengthErr, streamLength] = results[i];
-                    const [ackCountErr, ackCount] = results[i + 1];
-                    if (streamLengthErr) {
-                        console.error(`Error getting length for key: ${streamLengthErr}`);
-                        continue;
+                for (const streamKey of keys) {
+                    try {
+                        // Get stream length and pending info
+                        const streamLength = await this.redisClient.xlen(streamKey);
+                        // Extract consumer group name from stream key (format: eventName:cg-serviceName)
+                        const consumerGroup = streamKey.split(':')[1];
+                        // XPENDING returns [count, min-id, max-id, consumer-list]
+                        const pendingInfo = await this.redisClient.xpending(streamKey, consumerGroup);
+                        const totalPending = pendingInfo ? Number(pendingInfo[0]) : 0;
+                        const queueDepth = Math.max(0, streamLength - totalPending);
+                        totalDepth += queueDepth;
+                        individualDepths[streamKey] = queueDepth;
                     }
-                    if (ackCountErr) {
-                        console.error(`Error getting ack count for key: ${ackCountErr}`);
+                    catch (error) {
+                        console.error(`Error processing key ${streamKey}:`, error);
                         continue;
                     }
-                    const queueDepth = Math.max(0, streamLength - ackCount);
-                    totalDepth += queueDepth;
-                    individualDepths[key] = queueDepth;
                 }
             }
         } while (cursor !== '0');

package/src/lib/performance/circuit_breaker.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { EventEmitter } from 'events';
 import { RedisType } from '../redis/registry';
 import { EventData, IStreamsConfig } from '../redis/types';

package/src/lib/redis/batch.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.publishScheduledBatch = exports.publishBatch = void 0;
+exports.publishBatch = publishBatch;
+exports.publishScheduledBatch = publishScheduledBatch;
 const id_1 = require("@jetit/id");
 const tracker_1 = require("../monitoring/tracker");
 const logger_1 = require("./logger");
@@ -72,8 +73,6 @@ async function publishBatchWithRetry(streams, events, options) {
 function publishBatch(streams, events, options = {}) {
     return publishBatchWithRetry(streams, events, options);
 }
-exports.publishBatch = publishBatch;
 function publishScheduledBatch(streams, events, options) {
     return publishBatchWithRetry(streams, events, options);
 }
-exports.publishScheduledBatch = publishScheduledBatch;

package/src/lib/redis/registry.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
+exports.RedisRegistry = void 0;
+exports.setRedisConnectionSettings = setRedisConnectionSettings;
 const ioredis_1 = require("ioredis");
 const logger_1 = require("./logger");
 class RedisRegistry {
@@ -103,4 +104,3 @@ RedisRegistry.options = {
 function setRedisConnectionSettings(options) {
     RedisRegistry.setOptions(options);
 }
-exports.setRedisConnectionSettings = setRedisConnectionSettings;

package/src/lib/redis/streams.d.ts

@@ -200,6 +200,10 @@ export declare class Streams {
      * circuit is OPEN
      */
     processStoredEvents(): Promise<void>;
+    /**
+     * Acknowledges a message and updates the last acknowledged message ID.
+     * This is used to track cleanup progress and ensure we don't delete unprocessed messages.
+     */
     acknowledgeMessage(ackKey: string): Promise<void>;
     private frameMessageKey;
     private demergeMessageKey;

package/src/lib/redis/streams.js

@@ -257,131 +257,151 @@ class Streams {
         const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
         this.eventsListened.push(eventName);
         try {
-            // Check if the consumer group already exists
-            let groupInfo = [];
+            // Try to create the consumer group and consumer in one go
+            // If group doesn't exist, this will create it with MKSTREAM
+            // If group exists but consumer doesn't, this will create just the consumer
+            // If both exist, this will be a no-op
             try {
-                groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
-            }
-            catch (e) {
-                // Do nothing
-            }
-            let groupExists = false;
-            for (const group of groupInfo) {
-                if (group[1] === this.consumerGroupName) {
-                    groupExists = true;
-                    break;
-                }
-            }
-            if (!groupExists) {
                 await this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
                 logger_1.PUBLISHER_LOGGER.log(`Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
-            else {
+            catch (e) {
+                // BUSYGROUP error means group already exists, which is fine
+                if (!e.message.includes('BUSYGROUP')) {
+                    throw e;
+                }
                 logger_1.PUBLISHER_LOGGER.log(`Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
+            // Create consumer (idempotent operation)
+            const createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
+            // Register event and consumer group in parallel
+            const [addToCGSet, addToFlushSet] = await Promise.all([
+                this.redisGroups.sadd(`${eventName}`, this.consumerGroupName),
+                this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName),
+                this.redisGroups.sadd(key, eventName),
+            ]);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered with ${this.instanceId} under ${this.consumerGroupName} with status ${JSON.stringify({
+                createConsumerStatus,
+                addToCGSet,
+                addToFlushSet,
+            })}`);
+            return true;
         }
-        catch (e) {
-            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
-        }
-        // Check if the consumer already exists in the group
-        let consumers = [];
-        try {
-            consumers = (await this.redisGroups.xinfo('CONSUMERS', streamName, this.consumerGroupName));
-        }
-        catch (e) {
-            // Do nothing
-        }
-        let consumerExists = false;
-        for (const consumer of consumers) {
-            if (consumer[1] === this.instanceId) {
-                consumerExists = true;
-                break;
-            }
-        }
-        let createConsumerStatus;
-        if (!consumerExists) {
-            createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
-        }
-        else {
-            createConsumerStatus = 0; // Consumer already exists
-            logger_1.PUBLISHER_LOGGER.log(`Consumer already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName, instanceId: this.instanceId })}`);
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Consumer registration failed for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}:`, error);
+            return false;
         }
-        await this.redisGroups.sadd(key, eventName);
-        const addToCGSet = await this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
-        const addToFlushSet = await this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName);
-        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
-        return createConsumerStatus === 0 || createConsumerStatus === 1;
     }
     listenInternals(eventName, subscriptionId, eventFilter, filterKeepAlive = 24 * 60 * 60 * 1000, publishOnceGuarantee = false, externalAcknowledgement = false) {
-        if (!this.subscriptions.has(eventName)) {
-            this.subscriptions.set(eventName, new Map());
+        // Get or create subscription map for this event
+        const eventSubscriptions = this.subscriptions.get(eventName) || new Map();
+        const isNewSubscription = !this.subscriptions.has(eventName);
+        if (isNewSubscription) {
+            this.subscriptions.set(eventName, eventSubscriptions);
         }
         const bs = new rxjs_1.BehaviorSubject(null);
-        const subscription = {
+        // Making the subscription Immutable
+        const subscription = Object.freeze({
             subject: bs,
             filter: eventFilter,
             lastMatchTime: Date.now(),
             keepAlive: filterKeepAlive,
-        };
-        this.subscriptions.get(eventName).set(subscriptionId, subscription);
-        const timer = (0, rxjs_1.interval)(10000).subscribe(async () => {
-            /** Clear earlier unprocessed messages. Runs every 10 seconds */
-            await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
         });
+        eventSubscriptions.set(subscriptionId, subscription);
+        // Return early if not first subscription
+        if (!isNewSubscription) {
+            return bs.asObservable().pipe((0, rxjs_1.skip)(1));
+        }
+        const cleanupInterval = 10000; // 10 seconds
+        const timer = (0, rxjs_1.interval)(cleanupInterval).subscribe({
+            next: async () => {
+                try {
+                    await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
+                }
+                catch (error) {
+                    logger_1.PUBLISHER_LOGGER.error('Error in running recurring cleanup task:', error);
+                }
+            },
+            error: (error) => {
+                logger_1.PUBLISHER_LOGGER.error('Fatal error in cleanup timer:', error);
+            },
+        });
+        // Create observable with proper cleanup
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
-            /** Cleanup timer */
             timer.unsubscribe();
+            // Clean up subscription on completion
+            this.removeSubscription(eventName, subscriptionId);
         }));
-        const streamName = `${eventName}:${this.consumerGroupName}`;
         const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
+            // Skip processing if subscription was removed. This is needed because the processing is independent of the subscription
+            if (!this.subscriptions.has(eventName)) {
+                return;
+            }
+            const streamName = `${eventName}:${this.consumerGroupName}`;
             logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Processing message ${messageId} for ${streamName}`);
             try {
-                try {
-                    /**
-                     * Check if the message is already acquired by another client and is pending.
-                     * This check should only happen if the message is not a multicast message
-                     */
-                    const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
-                    if (pendingDetails[2] === 0 && !multicast) {
-                        logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: MACK ${messageId} for ${streamName}`);
-                        return;
+                // Skip XPENDING check for:
+                // 1. Multicast messages (meant for all consumers)
+                // 2. Initial message processing (messageId = '0')
+                // 3. Pending message reprocessing
+                if (!multicast && messageId !== '0' && !processPending) {
+                    try {
+                        const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
+                        if (pendingDetails[2] === 0) {
+                            logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Message ${messageId} already processed for ${streamName}`);
+                            return;
+                        }
+                    }
+                    catch (e) {
+                        // If XPENDING fails, continue with processing
+                        logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: XPENDING check failed for ${messageId}, continuing with processing`);
                     }
-                }
-                catch (e) {
-                    // Ignore the xpending error and continue
-                    logger_1.PUBLISHER_LOGGER.error('XPENDING ERROR: To be handled');
-                    logger_1.PUBLISHER_LOGGER.warn(JSON.stringify(e));
                 }
                 let eventData;
-                /**
-                 * Both multicast messages and pending messages cannot be read by xreadgroup
-                 * Multicast messages should not be claimed by a single consumer. And pending messages
-                 * are usually behind in the stream so XREADGROUP will not read them and hence
-                 * they need to be read using XRANGE.
-                 */
                 tracker.startRedisOperation();
-                if (multicast || processPending) {
-                    const messages = await redisClient.xrange(streamName, messageId, messageId);
-                    if (messages && messages.length) {
-                        try {
-                            eventData = JSON.parse(messages[0][1][1]);
-                        }
-                        catch (e) {
-                            console.error(`JSON parsing failed for the following message ${messages[0][1][1]} in the publisher.`);
+                try {
+                    /**
+                     * Both multicast messages and pending messages cannot be read by xreadgroup
+                     * Multicast messages should not be claimed by a single consumer. And pending messages
+                     * are usually behind in the stream so XREADGROUP will not read them and hence
+                     * they need to be read using XRANGE.
+                     */
+                    if (multicast || processPending) {
+                        const messages = await redisClient.xrange(streamName, messageId, messageId);
+                        if (messages?.length) {
+                            try {
+                                eventData = JSON.parse(messages[0][1][1]);
+                            }
+                            catch (error) {
+                                logger_1.PUBLISHER_LOGGER.error(`JSON parsing failed for message: ${messages[0][1][1]}`);
+                                return;
+                            }
                         }
                     }
-                }
-                else {
-                    const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
-                    if (messages && messages.length) {
-                        if (messageId === '0') {
-                            messageId = messages[0][1][0][0];
-                            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                    else {
+                        const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
+                        if (messages?.length) {
+                            if (messageId === '0') {
+                                messageId = messages[0][1][0][0];
+                                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                            }
+                            try {
+                                eventData = JSON.parse(messages[0][1][0][1][1]);
+                            }
+                            catch (error) {
+                                logger_1.PUBLISHER_LOGGER.error(`JSON parsing failed for message: ${messages[0][1][0][1][1]}`);
+                                return;
+                            }
                         }
-                        eventData = JSON.parse(messages[0][1][0][1][1]);
                     }
                 }
-                tracker.endRedisOperation();
+                catch (error) {
+                    logger_1.PUBLISHER_LOGGER.error('Error retrieving or parsing event data:', error);
+                    return;
+                }
+                finally {
+                    tracker.endRedisOperation();
+                }
                 tracker.startProcessing();
                 if (eventData) {
                     if (publishOnceGuarantee) {
@@ -396,61 +416,68 @@ class Streams {
                         const ackKey = this.frameMessageKey(streamName, messageId);
                         const subscriptions = this.subscriptions.get(eventName);
                         if (subscriptions) {
+                            const currentTime = Date.now();
                             const subscriptionEntries = Array.from(subscriptions.entries());
-                            for (let i = 0; i < subscriptionEntries.length; i++) {
-                                const [subId, sub] = subscriptionEntries[i];
-                                if (!sub.filter || sub.filter(eventData)) {
-                                    sub.subject.next({ ...eventData, ackKey });
-                                    sub.lastMatchTime = Date.now();
+                            // Process subscriptions in parallel for better performance
+                            await Promise.all(subscriptionEntries.map(async ([subId, sub]) => {
+                                try {
+                                    if (!sub.filter || sub.filter(eventData)) {
+                                        sub.subject.next({ ...eventData, ackKey });
+                                        sub.lastMatchTime = currentTime;
+                                    }
+                                    else if (currentTime - sub.lastMatchTime > sub.keepAlive) {
+                                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: No matching events for ${eventName} (Subscription ${subId}) in the last ${sub.keepAlive / 1000 / 60 / 60} hours`);
+                                        sub.lastMatchTime = currentTime;
+                                    }
                                 }
-                                else if (Date.now() - sub.lastMatchTime > sub.keepAlive) {
-                                    /**
-                                     * Reset the lastMatch time every day by default. For now only
-                                     * log the data. Should add functionality to remove the filter
-                                     * if its not used at all to gain minor improvements in
-                                     * performace
-                                     */
-                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: No matching events for ${eventName} (Subscription ${subId}) in the last ${sub.keepAlive / 1000 / 60 / 60} hours`);
-                                    sub.lastMatchTime = Date.now();
+                                catch (error) {
+                                    // Log error but don't fail entire processing
+                                    logger_1.PUBLISHER_LOGGER.error(`Error processing subscription ${subId}:`, error);
                                 }
-                            }
+                            }));
+                        }
+                        // Acknowledge message if needed
+                        if (!externalAcknowledgement) {
+                            await this.acknowledgeMessage(ackKey);
                         }
-                        if (!externalAcknowledgement)
-                            this.acknowledgeMessage(ackKey);
+                        // Update metrics
+                        const currentTime = Date.now();
+                        tracker.incrementMessageRate('subscribe', eventData.eventName);
+                        const processingTime = currentTime - eventData.createdAt;
+                        tracker.addProcessingTime(processingTime);
+                        tracker.setConsumerLag(this.consumerGroupName, currentTime - eventData.createdAt);
                     }
-                    catch (processingError) {
-                        logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, processingError);
+                    catch (error) {
+                        logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, error);
                         const dlqEvent = {
                             ...eventData,
-                            failureReason: processingError.message,
+                            failureReason: error.message,
                             retryCount: (eventData.retryCount || 0) + 1,
                             originalStream: streamName,
                             consumerGroupName: this.consumerGroupName,
                             timestamp: Date.now(),
                         };
                         await this.dlq.addToDLQ(dlqEvent);
+                        // Don't rethrow to prevent message loss
                     }
-                    tracker.incrementMessageRate('subscribe', eventData.eventName);
-                    const processingTime = Date.now() - eventData.createdAt;
-                    tracker.addProcessingTime(processingTime);
-                    const lag = Date.now() - eventData.createdAt;
-                    tracker.setConsumerLag(this.consumerGroupName, lag);
                 }
                 else {
                     logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Message ${messageId} not found for ${streamName}`);
                 }
                 tracker.endProcessing();
-                /** Process Unprocessed Message if this is a main tree, otherwise limit to processing 100 messages that are unacknowledged */
+                /** Process Unprocessed Messages with rate limiting */
                 if (!processPending) {
                     const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, this.instanceId);
                     if (unprocessedMessageIds.countOnThisConsumer &&
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
-                    for (const id of unprocessedMessageIds.messageIds) {
-                        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${id}`);
+                    // Process messages with rate limiting
+                    const processWithDelay = async (id, index) => {
+                        await new Promise((resolve) => setTimeout(resolve, index * 20));
                         await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
-                    }
+                    };
+                    unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));
                 }
             }
             catch (e) {
@@ -550,17 +577,30 @@ class Streams {
     }
     async cleanupAcknowledgedMessages(eventName, interval = this.config.acknowledgedMessageCleanupInterval) {
         const streamName = `${eventName}:${this.consumerGroupName}`;
-        const cleanupThreshold = Date.now() - interval;
-        const acknowledgedMessages = await this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
-        if (acknowledgedMessages && acknowledgedMessages.length > 0) {
-            // Remove acknowledged messages from the stream in batches
-            const batchSize = 100;
-            for (let i = 0; i < acknowledgedMessages.length; i += batchSize) {
-                const batch = acknowledgedMessages.slice(i, i + batchSize);
-                await this.redisGroups.xdel(streamName, ...batch);
+        const lastAckKey = `last_ack:${streamName}`;
+        const oneHourAgo = Date.now() - interval;
+        try {
+            // Get consumer group info to check if consumers are active
+            const groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
+            // If no active consumers, leave stream as is
+            if (!groupInfo || !groupInfo.some((group) => group.consumers > 0)) {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No active consumers for ${streamName}, leaving stream as is`);
+                return;
+            }
+            // Get last acknowledged message ID
+            const lastAckId = await this.redisGroups.get(lastAckKey);
+            if (!lastAckId) {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No acknowledged messages for ${streamName}`);
+                return;
             }
-            // Remove acknowledged messages from the Sorted Set
-            await this.redisGroups.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+            // Extract timestamp from message ID
+            const [timestamp] = lastAckId.split('-').map(Number);
+            const cleanupThreshold = Math.min(timestamp, oneHourAgo);
+            await this.redisGroups.xtrim(streamName, 'MINID', cleanupThreshold);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Cleaned up messages before last acknowledged message ${timestamp} from ${streamName}`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${streamName}:`, error);
         }
     }
     async getDiagnosticData(events) {
@@ -673,10 +713,21 @@ class Streams {
             await this.circuitBreaker.clearStoredEvents();
         }
     }
+    /**
+     * Acknowledges a message and updates the last acknowledged message ID.
+     * This is used to track cleanup progress and ensure we don't delete unprocessed messages.
+     */
     async acknowledgeMessage(ackKey) {
         const { streamName, messageId } = this.demergeMessageKey(ackKey);
-        await this.redisGroups.xack(streamName, this.consumerGroupName, messageId);
-        await this.redisGroups.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
+        const lastAckKey = `last_ack:${streamName}`;
+        try {
+            // Update last acknowledged ID and acknowledge message atomically
+            await Promise.all([this.redisGroups.xack(streamName, this.consumerGroupName, messageId), this.redisGroups.set(lastAckKey, messageId)]);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`Error acknowledging message ${messageId} for ${streamName}:`, error);
+            throw error;
+        }
     }
     frameMessageKey(streamName, messageId) {
         return `${streamName}##${messageId}`;

package/src/lib/redis/utils.js

@@ -1,12 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UTILS = exports.decodeScheduledMessage = exports.encodeScheduledMessage = exports.removedScheduledJob = exports.notifySubscribers = exports.getMessageStatesCount = exports.getUnacknowledgedMessages = exports.getSummaryOnStreamConsumerGroup = exports.getAllConsumerGroups = void 0;
+exports.UTILS = void 0;
+exports.getAllConsumerGroups = getAllConsumerGroups;
+exports.getSummaryOnStreamConsumerGroup = getSummaryOnStreamConsumerGroup;
+exports.getUnacknowledgedMessages = getUnacknowledgedMessages;
+exports.getMessageStatesCount = getMessageStatesCount;
+exports.notifySubscribers = notifySubscribers;
+exports.removedScheduledJob = removedScheduledJob;
+exports.encodeScheduledMessage = encodeScheduledMessage;
+exports.decodeScheduledMessage = decodeScheduledMessage;
 const logger_1 = require("./logger");
 async function getAllConsumerGroups(eventName, redisConnection) {
     const consumerGroups = await redisConnection.smembers(`${eventName}`);
     return consumerGroups;
 }
-exports.getAllConsumerGroups = getAllConsumerGroups;
 function* getSummaryOnStreamConsumerGroup(redisClient, consumerGroupName, streamName) {
     const [count, , , consumers] = (yield redisClient.xpending(streamName, consumerGroupName));
     yield {
@@ -21,7 +28,6 @@ function* getSummaryOnStreamConsumerGroup(redisClient, consumerGroupName, stream
             : [],
     };
 }
-exports.getSummaryOnStreamConsumerGroup = getSummaryOnStreamConsumerGroup;
 async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamName, consumerName, count = 500) {
     try {
         // Get pending messages summary
@@ -32,12 +38,16 @@ async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamN
         }
         // Use the smallest and largest IDs to get a detailed range
         const pendingMessageCount = summary[0];
-        // Get detailed information in the range
-        let pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, '-', '+', count, consumerName));
+        // Create a message ID for 2 seconds ago to exclude recent messages
+        const oneSecondAgo = Date.now() - 2000;
+        const minId = '0-0'; // Start from beginning
+        const maxId = `${oneSecondAgo}-0`; // Only include messages older than 1 second
+        // Get detailed information in the range, excluding recent messages
+        let pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, minId, maxId, count, consumerName));
         /** If no pending messages on consumer, fetch messages from other consumers that haven't been claimed for more than 10s */
         if (count > pendingMessages.length && pendingMessages.length === 0) {
             await redisClient.xautoclaim(streamName, consumerGroupName, consumerName, 10000, '0-0', 'COUNT', 100);
-            pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, '-', '+', count, consumerName));
+            pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, minId, maxId, count, consumerName));
         }
         return {
             count: pendingMessageCount,
@@ -51,7 +61,6 @@ async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamN
         return { count: 0, messageIds: [] };
     }
 }
-exports.getUnacknowledgedMessages = getUnacknowledgedMessages;
 async function getMessageStatesCount(redisClient, streamName, consumerGroup) {
     try {
         const pendingInfo = (await redisClient.xpending(streamName, consumerGroup));
@@ -66,11 +75,9 @@ async function getMessageStatesCount(redisClient, streamName, consumerGroup) {
         return { acknowledged: 0, unacknowledged: 0 };
     }
 }
-exports.getMessageStatesCount = getMessageStatesCount;
 async function notifySubscribers(redisClient, eventName, messageId, multicast = false) {
     await redisClient.publish(eventName, JSON.stringify({ messageId, multicast }));
 }
-exports.notifySubscribers = notifySubscribers;
 async function removedScheduledJob(redisClient, eventString) {
     const currentTime = new Date().getTime();
     const events = await redisClient.zrangebyscore('se', 0, currentTime);
@@ -79,7 +86,6 @@ async function removedScheduledJob(redisClient, eventString) {
     const eventsLater = await redisClient.zrangebyscore('se', 0, currentTime);
     logger_1.PUBLISHER_LOGGER.log(`Total Events in scheduled queue: ${eventsLater.length}`);
 }
-exports.removedScheduledJob = removedScheduledJob;
 function encodeScheduledMessage(data) {
     const eventName = data.eventName;
     const eventData = JSON.stringify(data.data);
@@ -87,7 +93,6 @@ function encodeScheduledMessage(data) {
     const eventDataBuffer = Buffer.from(eventData, 'utf8').toString('base64');
     return `${eventName}%%${eventDataBuffer}%%${repeatInterval}`;
 }
-exports.encodeScheduledMessage = encodeScheduledMessage;
 function decodeScheduledMessage(data) {
     const parts = data.split('%%');
     const eventName = parts[0];
@@ -97,7 +102,6 @@ function decodeScheduledMessage(data) {
         repeatInterval: parseInt(parts[2]),
     };
 }
-exports.decodeScheduledMessage = decodeScheduledMessage;
 exports.UTILS = {
     getMessageStatesCount,
     getUnacknowledgedMessages,