@jetit/publisher 4.1.1 → 5.1.1

package/src/lib/redis/streams.js

@@ -3,12 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Streams = void 0;
 const id_1 = require("@jetit/id");
 const rxjs_1 = require("rxjs");
+const collector_1 = require("../monitoring/collector");
+const tracker_1 = require("../monitoring/tracker");
+const circuit_breaker_1 = require("../performance/circuit_breaker");
+const dlq_1 = require("./dlq");
+const duplication_1 = require("./duplication");
 const logger_1 = require("./logger");
 const registry_1 = require("./registry");
 const utils_1 = require("./utils");
-function publisherErrorHandler(error) {
-    logger_1.PUBLISHER_LOGGER.error('PUBLISHER UNHANDLED ERROR: ', JSON.stringify(error));
-}
 class Streams {
     get redisPublisher() {
         if (!this._redisPublisher)
@@ -34,61 +36,145 @@ class Streams {
      * // Create a new Streams instance for the "POS" service
      * const streams = new Streams('POS');
      */
-    constructor(serviceName) {
+    constructor(serviceName, config = {}) {
         this.eventsListened = [];
+        this.subscriptions = new Map();
+        this.DEFAULT_STREAMS_CONFIG = {
+            immediatePublishThreshold: 500,
+            unprocessedMessageThreshold: 25,
+            acknowledgedMessageCleanupInterval: 60 * 60 * 1000, // 1 hour
+            cleanUpInterval: 1000 * 60 * 60,
+            dlqEventThreshold: 2000,
+            filterKeepAlive: 24 * 60 * 60 * 1000,
+            duplicationCheckWindow: 84600,
+            circuitBreaker: {
+                enabled: true,
+                errorThreshold: 50,
+                errorThresholdPercentage: 50,
+                openStateDuration: 30000,
+                halfOpenStateMaxAttempts: 10,
+                maxStoredEvents: 10000,
+            },
+        };
+        /** Initialise Config properties */
+        this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
         this.instanceUniqueId = process.env['INSTANCE_ID'] ?? (0, id_1.generateID)('HEX', 'FE');
         this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
         this.consumerGroupName = `cg-${serviceName}`;
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
-        const cleanUpInterval = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10) ?? 1000 * 60 * 60;
+        const cleanUpInterval = this.config.cleanUpInterval ?? parseInt(process.env['CLEANUP_INTERVAL'] ?? `${this.config.cleanUpInterval}`, 10);
         this.cleanUpTimer = setInterval(() => {
-            this.runClear(cleanUpInterval);
+            this.runClear(cleanUpInterval).catch((error) => {
+                logger_1.PUBLISHER_LOGGER.error('Error during cleanup:', error);
+            });
         }, cleanUpInterval);
+        this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
+        this.metricsCollector = new collector_1.MetricsCollector({
+            redisClient: this.redisPublisher,
+            collectionInterval: 60000,
+            retentionPeriod: 6 * 60 * 60 * 1000,
+        }, this.dlq);
+        this.duplicateChecker = new duplication_1.ContentBasedDeduplication(this.redisPublisher, this.config.duplicationCheckWindow);
+        this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
+        if (this.config.circuitBreaker.enabled)
+            this.setupCircuitBreakerListeners();
+    }
+    setupCircuitBreakerListeners() {
+        this.circuitBreaker.on('stateChange', async (newState) => {
+            logger_1.PUBLISHER_LOGGER.log(`Circuit breaker state changed to: ${circuit_breaker_1.CircuitState[newState]}`);
+            if (newState === circuit_breaker_1.CircuitState.CLOSED) {
+                await this.processStoredEvents();
+            }
+        });
     }
     async runClear(cleanUpInterval) {
         logger_1.PUBLISHER_LOGGER.log('PUBLISHER: Running Clearance', this.eventsListened);
-        for (const eventName of this.eventsListened) {
-            process.nextTick(async () => {
-                /** This removes the messages from the stream after they have been processed according to cleanup interval */
-                await this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
-                logger_1.PUBLISHER_LOGGER.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
-            });
-        }
+        const cleanupPromises = this.eventsListened.map((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval)
+            .then(() => {
+            logger_1.PUBLISHER_LOGGER.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
+        })
+            .catch((error) => {
+            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${eventName}:`, error);
+        }));
+        await Promise.all(cleanupPromises);
     }
     async publish(data, multicast = false) {
-        const publishStartTime = process.hrtime();
+        const tracker = new tracker_1.MetricsTracker();
         if (data.eventId)
             data.republishEvent = data.eventId;
         data.eventId = (0, id_1.generateID)('HEX', 'FF');
         if (!data.createdAt)
             data.createdAt = Date.now();
-        const consumerGroups = await (0, utils_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
-        let key = '*';
-        if (consumerGroups.length > 0) {
-            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
-            try {
+        /**
+         * This is a simplified description of the circuit breaker code
+         * 1. If the circuit breaker is enabled, then we check if the circuit is closed and messages are allowed to pass
+         * 2. If the circuit is open, then the event is stored and an error is logged notifying that the system has recorded
+         * a large number of publisher failures (controlled by errorThreshold and errorThresholdPercentage). This emits an
+         * event and the time of change of state is recorded. The circuit remains open for 30 seconds, allowing the system to
+         * correct itself as much as possible. All events during this time end up in the buffer
+         * 3. If the 30s is expired, the system resets to a half open state where new events are allowed to go through and a
+         * success state is recorded.
+         * 4. In the half open state if there are 10 attempts that succeed, then the circuit goes to closed and all pending
+         * messages are published.
+         *
+         * Here is a pictorial representation
+         *
+         *    [Failure threshold met]
+         *  CLOSED --------------------> OPEN
+         *     ^                         |
+         *     |                         |
+         *     |  [Max success           | [Open duration elapsed]
+         *     |   in half-open]         |
+         *     |                         v
+         *     -------------------- HALF-OPEN
+         *
+         * These properties are configurable
+         */
+        if (this.config.circuitBreaker.enabled && !this.circuitBreaker.isAllowed()) {
+            await this.circuitBreaker.storeEvent(data);
+            logger_1.PUBLISHER_LOGGER.error('Circuit is open, event stored for later processing');
+            return 'CIRCUIT_BREAKER_FLOW';
+        }
+        try {
+            tracker.startRedisOperation();
+            const consumerGroups = await (0, utils_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
+            tracker.endRedisOperation();
+            let key = '*';
+            if (consumerGroups.length > 0) {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
                 for (const consumerGroup of consumerGroups) {
                     // Publish the event to each consumer group's stream
                     const streamName = `${data.eventName}:${consumerGroup}`;
-                    const generatedKey = await this.redisPublisher
-                        .xadd(streamName, key, 'data', JSON.stringify(data))
-                        .catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(data)}, ${e} `));
+                    tracker.startRedisOperation();
+                    const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
+                    tracker.endRedisOperation();
+                    tracker.incrementEventCount();
+                    tracker.incrementMessageRate('publish', data.eventName);
+                    if (this.metricsCollector) {
+                        this.metricsCollector.addMetrics(tracker.getMetrics());
+                    }
                     if (key === '*')
                         key = generatedKey ?? key;
                 }
                 await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
+                await this.circuitBreaker.recordSuccess();
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
             }
-            catch (error) {
-                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
-                throw new Error('Publisher Error');
+            const metrics = tracker.getMetrics();
+            this.logPerformance(`PTIME;${key};${data.eventName};${Date.now()};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime}`);
+            return key;
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
+            tracker.incrementErrorCount('publish');
+            if (this.metricsCollector) {
+                this.metricsCollector.addMetrics(tracker.getMetrics());
             }
+            await this.circuitBreaker.recordFailure();
+            throw new Error('Publisher Error');
         }
-        else
-            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
-        const publishEndTime = process.hrtime(publishStartTime);
-        const elapsedTime = publishEndTime[0] * 1000 + publishEndTime[1] / 1000000;
-        logger_1.PERFORMANCE_LOGGER.log(`PTIME;${key};${data.eventName};${Date.now()};${elapsedTime}`);
-        return key;
     }
     async scheduledPublish(scheduledTime, eventData, uniquePerInstance = false, repeatInterval = 0, multicast = false) {
         const currentTime = new Date();
@@ -99,7 +185,7 @@ class Streams {
         if (scheduledTime < currentTime) {
             throw new Error('PUBLISHER: Cannot schedule an event in the past');
         }
-        else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
+        else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= this.config.immediatePublishThreshold) {
             await this.publish(eventData, multicast);
         }
         else {
@@ -107,7 +193,7 @@ class Streams {
             if (uniquePerInstance === true) {
                 const existingJob = await this.redisPublisher.zscore('se', key);
                 if (existingJob) {
-                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Job with data '${eventData}' already exists. Skipping.`);
+                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Job with data '${JSON.stringify(eventData)}' already exists. Skipping.`);
                     return;
                 }
             }
@@ -141,17 +227,27 @@ class Streams {
      *   PUBLISHER_LOGGER.log('New order created:', event.data);
      * });
      */
-    listen(eventName, maxRetries = 5, initialDelay = 1000) {
-        return this.listenInternals(eventName).pipe((0, rxjs_1.retry)({
-            count: maxRetries,
+    listen(eventName, listenerOptions) {
+        const options = {
+            maxRetries: this.config.maxRetries,
+            initialDelay: this.config.initialRetryDelay,
+            filterKeepAlive: this.config.filterKeepAlive,
+            publishOnceGuarantee: false,
+            ...listenerOptions,
+        };
+        const subscriptionId = (0, id_1.generateID)('HEX');
+        return this.listenInternals(eventName, subscriptionId, options.eventFilter, options.filterKeepAlive, options.publishOnceGuarantee).pipe((0, rxjs_1.retry)({
+            count: options.maxRetries,
             delay: (error, retryAttempt) => {
-                const delay = initialDelay * Math.pow(2, retryAttempt);
+                const delay = options.initialDelay * Math.pow(2, retryAttempt);
                 logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
                 return (0, rxjs_1.timer)(delay);
             },
         }), (0, rxjs_1.catchError)((error) => {
-            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen after ${maxRetries} retries: ${error.message}`);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen after ${options.maxRetries} retries: ${error.message}`);
             return (0, rxjs_1.throwError)(() => new Error(error.message));
+        }), (0, rxjs_1.finalize)(() => {
+            this.removeSubscription(eventName, subscriptionId);
         }));
     }
     async createConsumerAndRegister(eventName) {
@@ -159,41 +255,94 @@ class Streams {
         const key = `instance:${this.instanceId}:subscribedEvents`;
         const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
         this.eventsListened.push(eventName);
-        await this.redisGroups
-            .xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM')
-            .then(() => {
-            logger_1.PUBLISHER_LOGGER.log(`Group created created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
-        })
-            .catch((e) => {
+        try {
+            // Check if the consumer group already exists
+            let groupInfo = [];
+            try {
+                groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
+            }
+            catch (e) {
+                // Do nothing
+            }
+            let groupExists = false;
+            for (const group of groupInfo) {
+                if (group[1] === this.consumerGroupName) {
+                    groupExists = true;
+                    break;
+                }
+            }
+            if (!groupExists) {
+                await this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
+                logger_1.PUBLISHER_LOGGER.log(`Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.log(`Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+            }
+        }
+        catch (e) {
             logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
-        });
-        const createConsumerStatus = (await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId));
+        }
+        // Check if the consumer already exists in the group
+        let consumers = [];
+        try {
+            consumers = (await this.redisGroups.xinfo('CONSUMERS', streamName, this.consumerGroupName));
+        }
+        catch (e) {
+            // Do nothing
+        }
+        let consumerExists = false;
+        for (const consumer of consumers) {
+            if (consumer[1] === this.instanceId) {
+                consumerExists = true;
+                break;
+            }
+        }
+        let createConsumerStatus;
+        if (!consumerExists) {
+            createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
+        }
+        else {
+            createConsumerStatus = 0; // Consumer already exists
+            logger_1.PUBLISHER_LOGGER.log(`Consumer already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName, instanceId: this.instanceId })}`);
+        }
         await this.redisGroups.sadd(key, eventName);
         const addToCGSet = await this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
         const addToFlushSet = await this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName);
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
         return createConsumerStatus === 0 || createConsumerStatus === 1;
     }
-    listenInternals(eventName) {
+    listenInternals(eventName, subscriptionId, eventFilter, filterKeepAlive = 24 * 60 * 60 * 1000, publishOnceGuarantee = false) {
+        if (!this.subscriptions.has(eventName)) {
+            this.subscriptions.set(eventName, new Map());
+        }
         const bs = new rxjs_1.BehaviorSubject(null);
+        const subscription = {
+            subject: bs,
+            filter: eventFilter,
+            lastMatchTime: Date.now(),
+            keepAlive: filterKeepAlive,
+        };
+        this.subscriptions.get(eventName).set(subscriptionId, subscription);
         const timer = (0, rxjs_1.interval)(10000).subscribe(async () => {
             /** Clear earlier unprocessed messages. Runs every 10 seconds */
-            await processMessage(this.redisGroups, '0', false);
+            await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
         });
+        let lastMatchTime = Date.now();
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
             /** Cleanup timer */
             timer.unsubscribe();
         }));
         const streamName = `${eventName}:${this.consumerGroupName}`;
-        const processMessage = async (redisClient, messageId, multicast = false, processPending = false) => {
+        const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
             logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Processing message ${messageId} for ${streamName}`);
             try {
                 try {
                     /**
                      * Check if the message is already acquired by another client and is pending.
+                     * This check should only happen if the message is not a multicast message
                      */
                     const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
-                    if (pendingDetails[2] === 0 && multicast === false) {
+                    if (pendingDetails[2] === 0 && !multicast) {
                         logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: MACK ${messageId} for ${streamName}`);
                         return;
                     }
@@ -210,40 +359,118 @@ class Streams {
                  * are usually behind in the stream so XREADGROUP will not read them and hence
                  * they need to be read using XRANGE.
                  */
-                if (multicast === true || processPending) {
+                tracker.startRedisOperation();
+                if (multicast || processPending) {
                     const messages = await redisClient.xrange(streamName, messageId, messageId);
                     if (messages && messages.length) {
-                        eventData = JSON.parse(messages[0][1][1]);
+                        try {
+                            eventData = JSON.parse(messages[0][1][1]);
+                        }
+                        catch (e) {
+                            console.error(`JSON parsing failed for the following message ${messages[0][1][1]} in the publisher.`);
+                        }
                     }
                 }
                 else {
                     const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
                     if (messages && messages.length) {
+                        if (messageId === '0') {
+                            messageId = messages[0][1][0][0];
+                            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                        }
                         eventData = JSON.parse(messages[0][1][0][1][1]);
                     }
                 }
+                tracker.endRedisOperation();
+                tracker.startProcessing();
                 if (eventData) {
-                    bs.next(eventData);
-                    await redisClient.xack(streamName, this.consumerGroupName, messageId);
-                    await redisClient.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
+                    if (publishOnceGuarantee) {
+                        if (await this.duplicateChecker.isDuplicate(eventData, this.consumerGroupName)) {
+                            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Duplicate event detected, skipping processing for consumer group ${this.consumerGroupName}`);
+                            tracker.incrementDuplicateEvent();
+                            await redisClient.xack(streamName, this.consumerGroupName, messageId);
+                            return;
+                        }
+                    }
+                    try {
+                        const subscriptions = this.subscriptions.get(eventName);
+                        if (subscriptions) {
+                            const subscriptionEntries = Array.from(subscriptions.entries());
+                            for (let i = 0; i < subscriptionEntries.length; i++) {
+                                const [subId, sub] = subscriptionEntries[i];
+                                if (!sub.filter || sub.filter(eventData)) {
+                                    sub.subject.next(eventData);
+                                    sub.lastMatchTime = Date.now();
+                                }
+                                else if (Date.now() - sub.lastMatchTime > sub.keepAlive) {
+                                    /**
+                                     * Reset the lastMatch time every day by default. For now only
+                                     * log the data. Should add functionality to remove the filter
+                                     * if its not used at all to gain minor improvements in
+                                     * performace
+                                     */
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: No matching events for ${eventName} (Subscription ${subId}) in the last ${sub.keepAlive / 1000 / 60 / 60} hours`);
+                                    sub.lastMatchTime = Date.now();
+                                }
+                            }
+                        }
+                        await redisClient.xack(streamName, this.consumerGroupName, messageId);
+                        await redisClient.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
+                    }
+                    catch (processingError) {
+                        logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, processingError);
+                        const dlqEvent = {
+                            ...eventData,
+                            failureReason: processingError.message,
+                            retryCount: (eventData.retryCount || 0) + 1,
+                            originalStream: streamName,
+                            consumerGroupName: this.consumerGroupName,
+                            timestamp: Date.now(),
+                        };
+                        await this.dlq.addToDLQ(dlqEvent);
+                    }
+                    tracker.incrementMessageRate('subscribe', eventData.eventName);
+                    const processingTime = Date.now() - eventData.createdAt;
+                    tracker.addProcessingTime(processingTime);
+                    const lag = Date.now() - eventData.createdAt;
+                    tracker.setConsumerLag(this.consumerGroupName, lag);
                 }
                 else {
                     logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Message ${messageId} not found for ${streamName}`);
                 }
+                tracker.endProcessing();
                 /** Process Unprocessed Message if this is a main tree, otherwise limit to processing 100 messages that are unacknowledged */
                 if (!processPending) {
                     const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, this.instanceId);
-                    if (unprocessedMessageIds.countOnThisConsumer && unprocessedMessageIds.countOnThisConsumer > 25) {
+                    if (unprocessedMessageIds.countOnThisConsumer &&
+                        unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
                     for (const id of unprocessedMessageIds.messageIds) {
-                        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reporcessing unprocessed message with id: ${id}`);
-                        await processMessage(redisClient, id, multicast, true);
+                        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${id}`);
+                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
                     }
                 }
             }
             catch (e) {
                 logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error processing message ${messageId} for ${streamName}`, e);
+                if (!multicast) {
+                    const dlqEvent = {
+                        eventId: messageId,
+                        eventName: streamName.split(':')[0],
+                        data: {},
+                        failureReason: e.message,
+                        retryCount: 1,
+                        originalStream: streamName,
+                        timestamp: Date.now(),
+                        consumerGroupName: this.consumerGroupName,
+                    };
+                    await this.dlq.addToDLQ(dlqEvent);
+                    tracker.incrementErrorCount('subscribe');
+                    if (this.metricsCollector) {
+                        this.metricsCollector.addMetrics(tracker.getMetrics());
+                    }
+                }
             }
         };
         /** Register the consumer and setup the Observable */
@@ -256,7 +483,7 @@ class Streams {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName}`);
             });
             eventStreamClient.on('message', async (channel, data) => {
-                const subscribeStartTime = process.hrtime();
+                const tracker = new tracker_1.MetricsTracker();
                 let messageIdRead, multicastRead;
                 try {
                     const { messageId, multicast } = JSON.parse(data);
@@ -268,10 +495,12 @@ class Streams {
                     multicastRead = false;
                 }
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Stream Notification Received for event ${eventName} with message ID ${messageIdRead}`);
-                await processMessage(this.redisGroups, messageIdRead, multicastRead);
-                const subscribendTime = process.hrtime(subscribeStartTime);
-                const elapsedTime = subscribendTime[0] * 1000 + subscribendTime[1] / 1000000;
-                logger_1.PERFORMANCE_LOGGER.log(`STIME;${messageIdRead};${data.eventName};${Date.now()};${elapsedTime}`);
+                await processMessage(this.redisGroups, messageIdRead, tracker, multicastRead);
+                const metrics = tracker.getMetrics();
+                logger_1.PERFORMANCE_LOGGER.log(`STIME;${messageIdRead};${data.eventName};${Date.now()};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime}`);
+                if (this.metricsCollector) {
+                    this.metricsCollector.addMetrics(tracker.getMetrics());
+                }
             });
         })
             .catch((e) => {
@@ -300,27 +529,34 @@ class Streams {
      * }
      */
     async close() {
-        if (this.cleanUpTimer) {
-            clearInterval(this.cleanUpTimer);
-        }
-        if (this.redisPublisher) {
-            await this.redisPublisher.quit();
-        }
-        for (const eventName of this.eventsListened) {
-            registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
+        try {
+            if (this.cleanUpTimer) {
+                clearInterval(this.cleanUpTimer);
+            }
+            if (this._redisPublisher) {
+                await this._redisPublisher.quit();
+            }
+            for (const eventName of this.eventsListened) {
+                await registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
+            }
+            if (this._redisGroups) {
+                await this._redisGroups.quit();
+            }
         }
-        if (this.redisGroups) {
-            await this.redisGroups.quit();
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('Error during cleanup:', error);
         }
     }
-    async cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
+    async cleanupAcknowledgedMessages(eventName, interval = this.config.acknowledgedMessageCleanupInterval) {
         const streamName = `${eventName}:${this.consumerGroupName}`;
         const cleanupThreshold = Date.now() - interval;
         const acknowledgedMessages = await this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
         if (acknowledgedMessages && acknowledgedMessages.length > 0) {
-            // Remove acknowledged messages from the stream
-            for (const messageId of acknowledgedMessages) {
-                await this.redisGroups.xdel(streamName, messageId);
+            // Remove acknowledged messages from the stream in batches
+            const batchSize = 100;
+            for (let i = 0; i < acknowledgedMessages.length; i += batchSize) {
+                const batch = acknowledgedMessages.slice(i, i + batchSize);
+                await this.redisGroups.xdel(streamName, ...batch);
             }
             // Remove acknowledged messages from the Sorted Set
             await this.redisGroups.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
@@ -342,5 +578,99 @@ class Streams {
         const returnData = await await Promise.all(tempPromises);
         return { status: 'SUCCESS', data: returnData, message: 'We recommend not running this in times of heavy load' };
     }
+    logPerformance(message) {
+        if (this.config.performanceLogger) {
+            this.config.performanceLogger(message);
+        }
+        else {
+            logger_1.PERFORMANCE_LOGGER.log(message);
+        }
+    }
+    /**
+     * @description
+     * This method is use to retry an event that has ended in the dead letter queue,
+     * which happens after the first retry.
+     */
+    async retryFromDLQ(eventId) {
+        return this.dlq.retryFromDLQ(eventId);
+    }
+    /**
+     * @description
+     * This returns the number of items and the rate at which events are added
+     * to the queue. The queue is global and hence remains as is
+     */
+    async getDLQStats() {
+        return this.dlq.getDLQStats();
+    }
+    removeSubscription(eventName, subscriptionId) {
+        const eventSubscriptions = this.subscriptions.get(eventName);
+        if (eventSubscriptions) {
+            eventSubscriptions.delete(subscriptionId);
+            if (eventSubscriptions.size === 0) {
+                this.subscriptions.delete(eventName);
+            }
+        }
+    }
+    /**
+     * @description
+     * This is a simple helper utility that can be used externally to create alerts based
+     * on thresholds that can be provided into the function. It returns true/false for each
+     * key that is provided. Not all keys are required
+     */
+    async checkThresholds(thresholds) {
+        if (!this.metricsCollector) {
+            throw new Error('Metrics collection is not enabled');
+        }
+        const latestMetrics = await this.metricsCollector.getLatestMetrics();
+        if (!latestMetrics) {
+            throw new Error('No metrics available');
+        }
+        const alerts = {};
+        for (const [key, threshold] of Object.entries(thresholds)) {
+            if (key in latestMetrics) {
+                alerts[key] = latestMetrics[key] > threshold;
+            }
+        }
+        return alerts;
+    }
+    /**
+     * @description
+     * This will return you the stats of the publisher for the last 6 hours after cleaning
+     */
+    async getMetrics(startTime, endTime) {
+        return this.metricsCollector.getMetrics(startTime, endTime);
+    }
+    /**
+     * @description
+     * This will return you the latest stats of the publisher
+     */
+    async getLatestMetrics() {
+        return this.metricsCollector.getLatestMetrics();
+    }
+    /**
+     * @description
+     * This returns the status of the performance control setup. This includes
+     * the circuit breaker
+     */
+    async getPerformanceControlStatus() {
+        const circuitBreakerState = this.circuitBreaker.getState();
+        return {
+            circuitBreakerState,
+        };
+    }
+    /**
+     * @description
+     * This is a manual control to process stored events in case the
+     * circuit is OPEN
+     */
+    async processStoredEvents() {
+        if (this.circuitBreaker.getState() === circuit_breaker_1.CircuitState.CLOSED) {
+            const storedEvents = await this.circuitBreaker.getStoredEvents();
+            for (const event of storedEvents) {
+                await this.publish(event);
+            }
+            await this.circuitBreaker.clearStoredEvents();
+        }
+    }
 }
 exports.Streams = Streams;