@jetit/publisher 5.6.1 → 6.0.0

package/src/lib/redis/streams-lite.js

@@ -0,0 +1,734 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamsLite = void 0;
+const id_1 = require("@jetit/id");
+const rxjs_1 = require("rxjs");
+const collector_1 = require("../monitoring/collector");
+const tracker_1 = require("../monitoring/tracker");
+const circuit_breaker_1 = require("../performance/circuit_breaker");
+const dlq_1 = require("./dlq");
+const duplication_1 = require("./duplication");
+const logger_1 = require("./logger");
+const registry_1 = require("./registry");
+const utils_1 = require("./utils");
+class StreamsLite {
+    // Removed Lua script and related properties (findHighestStreamIdScript, findHighestStreamIdScriptSHA, optimizationActiveUntil)
+    get redisPublisher() {
+        if (!this._redisPublisher)
+            this._redisPublisher = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'publish');
+        return this._redisPublisher;
+    }
+    get redisGroups() {
+        if (!this._redisGroups)
+            this._redisGroups = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'groups');
+        return this._redisGroups;
+    }
+    /**
+     * Creates a new Streams instance for a given service.
+     *
+     * The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
+     * It also sets up an interval timer for clearing expired messages from Redis and another interval timer
+     * for processing scheduled events at regular intervals.
+     *
+     * @param serviceName - A unique name for the service that will be using this Streams instance.
+     *
+     * @example
+     *
+     * // Create a new Streams instance for the "POS" service
+     * const streams = new Streams('POS');
+     */
+    constructor(serviceName, config = {}, redisConnectionId = 'default') {
+        this.redisConnectionId = redisConnectionId;
+        this.eventsListened = [];
+        this.subscriptions = new Map();
+        this.DEFAULT_STREAMS_CONFIG = {
+            immediatePublishThreshold: 500,
+            unprocessedMessageThreshold: 25,
+            acknowledgedMessageCleanupInterval: 60 * 60 * 1000, // 1 hour
+            cleanUpInterval: 1000 * 60 * 60,
+            dlqEventThreshold: 2000,
+            filterKeepAlive: 24 * 60 * 60 * 1000,
+            duplicationCheckWindow: 84600,
+            circuitBreaker: {
+                enabled: true,
+                errorThreshold: 50,
+                errorThresholdPercentage: 50,
+                openStateDuration: 30000,
+                halfOpenStateMaxAttempts: 10,
+                maxStoredEvents: 10000,
+            },
+            // New defaults
+            optimizationDurationMs: 2 * 60 * 1000, // 2 minutes
+            optimizationThreshold: 20, // Enable optimization for >20 consumer groups
+        };
+        /** Initialise Config properties */
+        this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
+        this.instanceUniqueId = process.env['INSTANCE_ID'] ?? (0, id_1.generateID)('HEX', 'FE');
+        this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
+        this.consumerGroupName = `cg-${serviceName}`;
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId} and with config: ${JSON.stringify(this.config)}`);
+        const cleanUpInterval = this.config.cleanUpInterval ?? parseInt(process.env['CLEANUP_INTERVAL'] ?? `${this.config.cleanUpInterval}`, 10);
+        this.cleanUpTimer = setInterval(() => {
+            this.runClear(cleanUpInterval).catch((error) => {
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error during cleanup:', error);
+            });
+        }, cleanUpInterval);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
+        this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
+        this.metricsCollector = new collector_1.MetricsCollector({
+            redisClient: this.redisPublisher,
+            collectionInterval: 60000,
+            retentionPeriod: 6 * 60 * 60 * 1000,
+        }, this.dlq);
+        this.duplicateChecker = new duplication_1.ContentBasedDeduplication(this.redisPublisher, this.config.duplicationCheckWindow);
+        this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
+        if (this.config.circuitBreaker.enabled)
+            this.setupCircuitBreakerListeners();
+    }
+    setupCircuitBreakerListeners() {
+        this.circuitBreaker.on('stateChange', async (newState) => {
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Circuit breaker state changed to: ${circuit_breaker_1.CircuitState[newState]}`);
+            if (newState === circuit_breaker_1.CircuitState.CLOSED) {
+                await this.processStoredEvents();
+            }
+        });
+    }
+    async runClear(cleanUpInterval) {
+        logger_1.PUBLISHER_LOGGER.log('PUBLISHER: Running Clearance', this.eventsListened);
+        const cleanupPromises = this.eventsListened.map((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval)
+            .then(() => {
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Cleanup process for Acknowledged messages completed for ${eventName}`);
+        })
+            .catch((error) => {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during cleanup for ${eventName}:`, error);
+        }));
+        await Promise.all(cleanupPromises);
+    }
+    async publish(data, multicast = false) {
+        const tracker = new tracker_1.MetricsTracker();
+        if (data.eventId)
+            data.republishEvent = data.eventId;
+        data.eventId = (0, id_1.generateID)('HEX', 'FF');
+        if (!data.createdAt)
+            data.createdAt = Date.now();
+        /**
+         * This is a simplified description of the circuit breaker code
+         * 1. If the circuit breaker is enabled, then we check if the circuit is closed and messages are allowed to pass
+         * 2. If the circuit is open, then the event is stored and an error is logged notifying that the system has recorded
+         * a large number of publisher failures (controlled by errorThreshold and errorThresholdPercentage). This emits an
+         * event and the time of change of state is recorded. The circuit remains open for 30 seconds, allowing the system to
+         * correct itself as much as possible. All events during this time end up in the buffer
+         * 3. If the 30s is expired, the system resets to a half open state where new events are allowed to go through and a
+         * success state is recorded.
+         * 4. In the half open state if there are 10 attempts that succeed, then the circuit goes to closed and all pending
+         * messages are published.
+         *
+         * Here is a pictorial representation
+         *
+         *    [Failure threshold met]
+         *  CLOSED --------------------> OPEN
+         *     ^                         |
+         *     |                         |
+         *     |  [Max success           | [Open duration elapsed]
+         *     |   in half-open]         |
+         *     |                         v
+         *     -------------------- HALF-OPEN
+         *
+         * These properties are configurable
+         */
+        if (this.config.circuitBreaker.enabled && !this.circuitBreaker.isAllowed()) {
+            await this.circuitBreaker.storeEvent(data);
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Circuit is open, event stored for later processing');
+            return 'CIRCUIT_BREAKER_FLOW';
+        }
+        try {
+            const streamName = `sl:${data.eventName}`; // Use sl: prefix
+            let key = '*'; // Default to auto-generated ID
+            tracker.startRedisOperation();
+            try {
+                // Publish directly to the single stream
+                const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
+                tracker.endRedisOperation();
+                if (generatedKey === null) {
+                    // Handle the case where xadd failed unexpectedly
+                    throw new Error(`XADD command failed to return a message ID for stream ${streamName}`);
+                }
+                key = generatedKey; // Assign the non-null key
+                // Increment metrics
+                tracker.incrementEventCount();
+                tracker.incrementMessageRate('publish', data.eventName);
+                if (this.metricsCollector) {
+                    this.metricsCollector.addMetrics(tracker.getMetrics());
+                }
+                // Notify subscribers
+                await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
+                await this.circuitBreaker.recordSuccess();
+            }
+            catch (error) {
+                tracker.endRedisOperation(); // Ensure tracker ends even on error
+                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error publishing to stream ${streamName}:`, error);
+                throw error; // Rethrow the error
+            }
+            const metrics = tracker.getMetrics();
+            this.logPerformance(`PTIME;${key};${data.eventName};${Date.now()};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime}`);
+            return key;
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
+            tracker.incrementErrorCount('publish');
+            if (this.metricsCollector) {
+                this.metricsCollector.addMetrics(tracker.getMetrics());
+            }
+            await this.circuitBreaker.recordFailure();
+            throw new Error('Publisher Error');
+        }
+    }
+    /**
+     * Listens for events with the given name and returns an Observable that emits an EventData<T> object
+     * each time a new event is received.
+     *
+     * The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
+     * that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
+     *
+     * If an error occurs while subscribing, the method logs the error to the PUBLISHER_LOGGER and throws
+     * an error. This is done to prevent the service from continuing without a proper event subscription.
+     *
+     * There is retry logic with exponential backoff to handle error cases. These are also controllable by the
+     * calling service
+     *
+     * @param eventName - The name of the event to listen for.
+     *
+     * @returns An Observable that emits an EventData<T> object each time a new event is received.
+     *
+     * @example
+     *
+     * // Listen for "order.created" events
+     * const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
+     *
+     * // Subscribe to the Observable and log each new event
+     * orderCreated.subscribe((event) => {
+     *   PUBLISHER_LOGGER.log('New order created:', event.data);
+     * });
+     */
+    listen(eventName, listenerOptions) {
+        const options = {
+            maxRetries: this.config.maxRetries,
+            initialDelay: this.config.initialRetryDelay,
+            filterKeepAlive: this.config.filterKeepAlive,
+            publishOnceGuarantee: false,
+            externalAcknowledgement: false,
+            ...listenerOptions,
+        };
+        const subscriptionId = (0, id_1.generateID)('HEX');
+        return this.listenInternals(eventName, subscriptionId, options.eventFilter, options.filterKeepAlive, options.publishOnceGuarantee, options.externalAcknowledgement).pipe((0, rxjs_1.retry)({
+            count: options.maxRetries,
+            delay: (error, retryAttempt) => {
+                const delay = options.initialDelay * Math.pow(2, retryAttempt);
+                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
+                return (0, rxjs_1.timer)(delay);
+            },
+        }), (0, rxjs_1.catchError)((error) => {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen after ${options.maxRetries} retries: ${error.message}`);
+            return (0, rxjs_1.throwError)(() => new Error(error.message));
+        }), (0, rxjs_1.finalize)(() => {
+            this.removeSubscription(eventName, subscriptionId);
+        }));
+    }
+    async createConsumerAndRegister(eventName) {
+        const streamName = `sl:${eventName}`; // Use sl: prefix
+        const key = `instance:${this.instanceId}:subscribedEvents`;
+        const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
+        if (!this.eventsListened.includes(eventName)) {
+            this.eventsListened.push(eventName);
+        }
+        try {
+            // Try to create the consumer group on the single stream
+            try {
+                await this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Group ${this.consumerGroupName} created for stream ${streamName}`);
+            }
+            catch (e) {
+                // BUSYGROUP error means group already exists, which is fine
+                if (!e.message.includes('BUSYGROUP') || !e.includes('BUSYGROUP')) {
+                    throw e;
+                }
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Group ${this.consumerGroupName} already exists for stream ${streamName}`);
+            }
+            // Create consumer (idempotent operation) - Note: CREATECONSUMER is not needed if group exists
+            // const createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
+            // Register event and consumer group in parallel
+            const [, addToFlushSet] = await Promise.all([
+                this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName),
+                this.redisGroups.sadd(key, eventName), // Still track subscribed events per instance
+            ]);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer ${this.instanceId} registered for group ${this.consumerGroupName} on stream ${streamName} with status ${JSON.stringify({
+                addToFlushSet,
+            })}`);
+            return true;
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Consumer registration failed for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}:`, error);
+            return false;
+        }
+    }
+    listenInternals(eventName, subscriptionId, eventFilter, filterKeepAlive = 24 * 60 * 60 * 1000, publishOnceGuarantee = false, externalAcknowledgement = false) {
+        const eventSubscriptions = this.subscriptions.get(eventName) || new Map();
+        const isNewSubscription = !this.subscriptions.has(eventName);
+        if (isNewSubscription) {
+            this.subscriptions.set(eventName, eventSubscriptions);
+        }
+        const bs = new rxjs_1.BehaviorSubject(null);
+        const subscription = {
+            subject: bs,
+            filter: eventFilter,
+            lastMatchTime: Date.now(),
+            keepAlive: filterKeepAlive,
+        };
+        eventSubscriptions.set(subscriptionId, subscription);
+        if (!isNewSubscription) {
+            return bs.asObservable().pipe((0, rxjs_1.skip)(1));
+        }
+        const cleanupInterval = 10000;
+        const timer = (0, rxjs_1.interval)(cleanupInterval).subscribe({
+            next: async () => {
+                try {
+                    await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
+                }
+                catch (error) {
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
+                }
+            },
+            error: (error) => {
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
+            },
+        });
+        const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
+            timer.unsubscribe();
+            this.removeSubscription(eventName, subscriptionId);
+        }));
+        const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
+            // Skip processing if subscription was removed. This is needed because the processing is independent of the subscription
+            if (!this.subscriptions.has(eventName)) {
+                return;
+            }
+            const streamName = `sl:${eventName}`; // Use sl: prefix
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Processing message ${messageId} for ${streamName} / ${this.consumerGroupName}`);
+            try {
+                // Removed redundant XPENDING check block
+                let eventData;
+                tracker.startRedisOperation();
+                try {
+                    /**
+                     * Both multicast messages and pending messages cannot be read by xreadgroup
+                     * Multicast messages should not be claimed by a single consumer. And pending messages
+                     * are usually behind in the stream so XREADGROUP will not read them and hence
+                     * they need to be read using XRANGE.
+                     */
+                    // Simplified condition: XRANGE is only needed for multicast messages now
+                    if (multicast) {
+                        // XRANGE is still needed for multicast as XREADGROUP won't deliver the same message to all consumers
+                        const messages = await redisClient.xrange(streamName, messageId, messageId);
+                        if (messages?.length) {
+                            try {
+                                eventData = JSON.parse(messages[0][1][1]);
+                            }
+                            catch (error) {
+                                logger_1.PUBLISHER_LOGGER.error(`JSON parsing failed for message: ${messages[0][1][1]}`);
+                                return;
+                            }
+                        }
+                    }
+                    else {
+                        // Use XREADGROUP for regular consumer group processing
+                        const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
+                        // Handle potential null response from BLOCK timeout
+                        if (!messages || messages.length === 0) {
+                            // PUBLISHER_LOGGER.log(`PUBLISHER: No new messages for ${streamName}/${this.consumerGroupName}`);
+                            return; // No new messages
+                        }
+                        if (messages[0]?.[1]?.[0]) {
+                            // Check structure carefully
+                            const messageIdRead = messages[0][1][0][0];
+                            // Log if the message ID read is different from expected (can happen with retries/claims)
+                            if (messageId !== '0' && messageIdRead !== messageId) {
+                                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reading message ${messageIdRead} instead of expected ${messageId} for ${streamName}`);
+                            }
+                            messageId = messageIdRead; // Update messageId to the one actually read
+                            try {
+                                eventData = JSON.parse(messages[0][1][0][1][1]);
+                            }
+                            catch (error) {
+                                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: JSON parsing failed for message: ${messages[0][1][0][1][1]}`);
+                                return;
+                            }
+                        }
+                    }
+                }
+                catch (error) {
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error retrieving or parsing event data:', error);
+                    return;
+                }
+                finally {
+                    tracker.endRedisOperation();
+                }
+                tracker.startProcessing();
+                if (eventData) {
+                    if (publishOnceGuarantee) {
+                        if (await this.duplicateChecker.isDuplicate(eventData, this.consumerGroupName)) {
+                            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Duplicate event detected, skipping processing for consumer group ${this.consumerGroupName}`);
+                            tracker.incrementDuplicateEvent();
+                            await redisClient.xack(streamName, this.consumerGroupName, messageId);
+                            return;
+                        }
+                    }
+                    try {
+                        const ackKey = this.frameMessageKey(streamName, messageId);
+                        const subscriptions = this.subscriptions.get(eventName);
+                        if (subscriptions) {
+                            const currentTime = Date.now();
+                            const subscriptionEntries = Array.from(subscriptions.entries());
+                            // Process subscriptions in parallel for better performance
+                            await Promise.all(subscriptionEntries.map(async ([subId, sub]) => {
+                                try {
+                                    if (!sub.filter || sub.filter(eventData)) {
+                                        sub.subject.next({ ...eventData, ackKey });
+                                        sub.lastMatchTime = currentTime;
+                                    }
+                                    else if (currentTime - sub.lastMatchTime > sub.keepAlive) {
+                                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: No matching events for ${eventName} (Subscription ${subId}) in the last ${sub.keepAlive / 1000 / 60 / 60} hours`);
+                                        sub.lastMatchTime = currentTime;
+                                    }
+                                }
+                                catch (error) {
+                                    // Log error but don't fail entire processing
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error processing subscription ${subId}:`, error);
+                                }
+                            }));
+                        }
+                        // Acknowledge message if needed
+                        if (!externalAcknowledgement) {
+                            await this.acknowledgeMessage(ackKey);
+                        }
+                        // Update metrics
+                        const currentTime = Date.now();
+                        tracker.incrementMessageRate('subscribe', eventData.eventName);
+                        const processingTime = currentTime - eventData.createdAt;
+                        tracker.addProcessingTime(processingTime);
+                        tracker.setConsumerLag(this.consumerGroupName, currentTime - eventData.createdAt);
+                    }
+                    catch (error) {
+                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Processing error for message ${messageId}:`, error);
+                        const dlqEvent = {
+                            ...eventData,
+                            failureReason: error.message,
+                            retryCount: (eventData.retryCount || 0) + 1,
+                            originalStream: streamName,
+                            consumerGroupName: this.consumerGroupName,
+                            timestamp: Date.now(),
+                        };
+                        await this.dlq.addToDLQ(dlqEvent);
+                        // Don't rethrow to prevent message loss
+                    }
+                }
+                else {
+                    // This case might happen if XRANGE was used for a multicast message that was deleted before reading
+                    // Or if XREADGROUP returned an empty message array unexpectedly
+                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No event data found for message ${messageId} in ${streamName}`);
+                }
+                tracker.endProcessing();
+                // Ensure custom pending message processing logic is fully removed
+            }
+            catch (e) {
+                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error processing message ${messageId} for ${streamName}/${this.consumerGroupName}`, e);
+                if (!multicast) {
+                    const dlqEvent = {
+                        eventId: messageId,
+                        eventName: eventName, // Use base event name
+                        data: {}, // Consider adding actual data if available
+                        failureReason: e.message,
+                        retryCount: 1,
+                        originalStream: streamName, // Keep full stream name
+                        timestamp: Date.now(),
+                        consumerGroupName: this.consumerGroupName,
+                    };
+                    await this.dlq.addToDLQ(dlqEvent);
+                    tracker.incrementErrorCount('subscribe');
+                    if (this.metricsCollector) {
+                        this.metricsCollector.addMetrics(tracker.getMetrics());
+                    }
+                }
+            }
+        };
+        /** Register the consumer and setup the Observable */
+        this.createConsumerAndRegister(eventName)
+            .then((consumerRegistered) => {
+            if (!consumerRegistered)
+                throw new Error('PUBLISHER: Cannot setup consumer');
+            const eventStreamClient = registry_1.RedisRegistry.getConnection(this.redisConnectionId, `sub-${eventName}`);
+            eventStreamClient.subscribe(eventName).then(() => {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName}`);
+            });
+            eventStreamClient.on('message', async (channel, data) => {
+                const tracker = new tracker_1.MetricsTracker();
+                let messageIdRead, multicastRead;
+                try {
+                    const { messageId, multicast } = JSON.parse(data);
+                    messageIdRead = messageId;
+                    multicastRead = multicast;
+                }
+                catch (e) {
+                    messageIdRead = data;
+                    multicastRead = false;
+                }
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Stream Notification Received for event ${eventName} with message ID ${messageIdRead}`);
+                await processMessage(this.redisGroups, messageIdRead, tracker, multicastRead);
+                const metrics = tracker.getMetrics();
+                logger_1.PERFORMANCE_LOGGER.log(`STIME;${messageIdRead};${eventName};${Date.now()};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime}`);
+                if (this.metricsCollector) {
+                    this.metricsCollector.addMetrics(tracker.getMetrics());
+                }
+            });
+        })
+            .catch((e) => {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during consumer registration for ${eventName}`, e);
+        });
+        return observable;
+    }
+    /**
+     * This method allows the possibility of a graceful shutdown by cleaning up the
+     * redis connections.
+     *
+     * In all services where the library is used, its better to implement this method
+     *
+     * process.on('SIGTERM', shutdown);
+     * process.on('SIGINT', shutdown);
+     *
+     * async function shutdown(): Promise<void> {
+     *   PUBLISHER_LOGGER.log('Graceful shutdown initiated.');
+     *   try {
+     *     await streams.close();
+     *     PUBLISHER_LOGGER.log('Resources and connections successfully closed.');
+     *   } catch (error) {
+     *     PUBLISHER_LOGGER.error('Error during graceful shutdown:', error);
+     *   }
+     *   process.exit(0);
+     * }
+     */
+    async close() {
+        try {
+            // Removed reset of optimization state and script SHA
+            if (this.cleanUpTimer) {
+                clearInterval(this.cleanUpTimer);
+            }
+            if (this._redisPublisher) {
+                await this._redisPublisher.quit();
+            }
+            for (const eventName of this.eventsListened) {
+                await registry_1.RedisRegistry.getConnection(this.redisConnectionId, `sub-${eventName}`).quit();
+            }
+            if (this._redisGroups) {
+                await this._redisGroups.quit();
+            }
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error during cleanup:', error);
+        }
+    }
+    // Updated cleanup logic
+    async cleanupAcknowledgedMessages(eventName, interval = this.config.acknowledgedMessageCleanupInterval) {
+        const streamName = `sl:${eventName}`; // Use sl: prefix
+        const twentyFourHoursAgoTimestamp = Date.now() - 24 * 60 * 60 * 1000;
+        let minLastAckTimestamp = Date.now(); // Default to now if no groups/acks found
+        try {
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Started cleanup for stream ${streamName}`);
+            // 1. Get all consumer groups for the base event name
+            const consumerGroups = await (0, utils_1.getAllConsumerGroups)(eventName, this.redisPublisher); // Use base event name to find groups
+            if (!consumerGroups || consumerGroups.length === 0) {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No consumer groups found for event ${eventName}, skipping cleanup for ${streamName}`);
+                // Optionally, still trim based on 24 hours if desired, even without consumers
+                // await this.redisGroups.xtrim(streamName, 'MINID', twentyFourHoursAgoTimestamp);
+                return;
+            }
+            // 2. Get the last acknowledged ID for each consumer group on this stream
+            const lastAckPromises = consumerGroups.map(async (cg) => {
+                const lastAckKey = `last_ack:${streamName}:${cg}`; // Key includes stream and group
+                const lastAckId = await this.redisGroups.get(lastAckKey);
+                if (lastAckId) {
+                    const [timestamp] = lastAckId.split('-').map(Number);
+                    return timestamp;
+                }
+                return null; // Return null if no ack ID found for this group
+            });
+            const lastAckTimestamps = (await Promise.all(lastAckPromises)).filter((ts) => ts !== null);
+            // 3. Find the minimum last acknowledged timestamp across all groups
+            if (lastAckTimestamps.length > 0) {
+                minLastAckTimestamp = Math.min(...lastAckTimestamps);
+            }
+            else {
+                // If no groups have acknowledged anything, maybe default to 24 hours ago?
+                // Or keep minLastAckTimestamp as Date.now() to effectively only use the 24h rule.
+                // Let's default to 24h ago if no acks found, to allow trimming old messages.
+                minLastAckTimestamp = twentyFourHoursAgoTimestamp;
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No acknowledged messages found for any group on ${streamName}, using 24h threshold.`);
+            }
+            // 4. Determine the final cleanup threshold
+            const cleanupThreshold = Math.max(twentyFourHoursAgoTimestamp, minLastAckTimestamp);
+            const cleanupThresholdDate = new Date(cleanupThreshold).toISOString();
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Determined cleanup threshold for ${streamName}: ${cleanupThreshold} (${cleanupThresholdDate}) (minAck: ${new Date(minLastAckTimestamp).toISOString()}, 24hAgo: ${new Date(twentyFourHoursAgoTimestamp).toISOString()})`);
+            // 5. Perform XTRIM
+            // Check if stream exists before trimming (optional, XTRIM handles non-existent streams gracefully)
+            const streamExists = await this.redisGroups.exists(streamName);
+            if (streamExists) {
+                await this.redisGroups.xtrim(streamName, 'MINID', cleanupThreshold);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Performed XTRIM on ${streamName} with MINID ${cleanupThreshold}`);
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Stream ${streamName} does not exist, skipping XTRIM.`);
+            }
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during cleanup for ${streamName}:`, error);
+        }
+    }
+    async getDiagnosticData(events) {
+        if (events.length > 100) {
+            return { status: 'ERROR', message: 'Please pass in a maximum of 100 elements to fetch diagnostics' };
+        }
+        const tempPromises = events.map(async (baseEventName) => {
+            const streamName = `sl:${baseEventName}`; // Use sl: prefix
+            const consumerGroups = await (0, utils_1.getAllConsumerGroups)(baseEventName, this.redisPublisher); // Get groups based on base name
+            const consumerGroupMap = await Promise.all(consumerGroups.map(async (consumerGroup) => {
+                // Query the single stream for this specific consumer group
+                const diagnostics = await (0, utils_1.getSummaryOnStreamConsumerGroup)(this.redisGroups, consumerGroup, streamName);
+                return { consumerGroup, diagnostics };
+            }));
+            return { eventName: baseEventName, streamName, consumerGroupMap }; // Include stream name in output
+        });
+        const returnData = await Promise.all(tempPromises);
+        return { status: 'SUCCESS', data: returnData, message: 'We recommend not running this in times of heavy load' };
+    }
+    logPerformance(message) {
+        if (this.config.performanceLogger) {
+            this.config.performanceLogger(message);
+        }
+        else {
+            logger_1.PERFORMANCE_LOGGER.log(message);
+        }
+    }
+    /**
+     * @description
+     * This method is use to retry an event that has ended in the dead letter queue,
+     * which happens after the first retry.
+     */
+    async retryFromDLQ(eventId) {
+        return this.dlq.retryFromDLQ(eventId);
+    }
+    /**
+     * @description
+     * This returns the number of items and the rate at which events are added
+     * to the queue. The queue is global and hence remains as is
+     */
+    async getDLQStats() {
+        return this.dlq.getDLQStats();
+    }
+    removeSubscription(eventName, subscriptionId) {
+        const eventSubscriptions = this.subscriptions.get(eventName);
+        if (eventSubscriptions) {
+            eventSubscriptions.delete(subscriptionId);
+            if (eventSubscriptions.size === 0) {
+                this.subscriptions.delete(eventName);
+            }
+        }
+    }
+    /**
+     * @description
+     * This is a simple helper utility that can be used externally to create alerts based
+     * on thresholds that can be provided into the function. It returns true/false for each
+     * key that is provided. Not all keys are required
+     */
+    async checkThresholds(thresholds) {
+        if (!this.metricsCollector) {
+            throw new Error('Metrics collection is not enabled');
+        }
+        const latestMetrics = await this.metricsCollector.getLatestMetrics();
+        if (!latestMetrics) {
+            throw new Error('No metrics available');
+        }
+        const alerts = {};
+        for (const [key, threshold] of Object.entries(thresholds)) {
+            if (key in latestMetrics) {
+                alerts[key] = latestMetrics[key] > threshold;
+            }
+        }
+        return alerts;
+    }
+    /**
+     * @description
+     * This will return you the stats of the publisher for the last 6 hours after cleaning
+     */
+    async getMetrics(startTime, endTime) {
+        return this.metricsCollector.getMetrics(startTime, endTime);
+    }
+    /**
+     * @description
+     * This will return you the latest stats of the publisher
+     */
+    async getLatestMetrics() {
+        return this.metricsCollector.getLatestMetrics();
+    }
+    /**
+     * @description
+     * This returns the status of the performance control setup. This includes
+     * the circuit breaker
+     */
+    async getPerformanceControlStatus() {
+        const circuitBreakerState = this.circuitBreaker.getState();
+        return {
+            circuitBreakerState,
+        };
+    }
+    /**
+     * @description
+     * This is a manual control to process stored events in case the
+     * circuit is OPEN
+     */
+    async processStoredEvents() {
+        if (this.circuitBreaker.getState() === circuit_breaker_1.CircuitState.CLOSED) {
+            const storedEvents = await this.circuitBreaker.getStoredEvents();
+            for (const event of storedEvents) {
+                await this.publish(event);
+            }
+            await this.circuitBreaker.clearStoredEvents();
+        }
+    }
+    /**
+     * Acknowledges a message and updates the last acknowledged message ID for the specific consumer group.
+     * This is used to track cleanup progress and ensure we don't delete unprocessed messages.
+     */
+    async acknowledgeMessage(ackKey) {
+        const { streamName, messageId } = this.demergeMessageKey(ackKey);
+        // Construct the specific last_ack key for this stream and consumer group
+        const lastAckKey = `last_ack:${streamName}:${this.consumerGroupName}`;
+        const validMessageId = messageId === '0' ? '0-0' : messageId; // Ensure valid format if '0'
+        try {
+            // Update last acknowledged ID and acknowledge message atomically
+            await Promise.all([
+                this.redisGroups.xack(streamName, this.consumerGroupName, validMessageId),
+                this.redisGroups.set(lastAckKey, validMessageId), // Use the specific key
+            ]);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error acknowledging message ${validMessageId} for ${streamName}/${this.consumerGroupName}:`, error);
+            throw error;
+        }
+    }
+    // Keep framing/demerging simple for now, assuming streamName includes sl: prefix
+    frameMessageKey(streamName, messageId) {
+        return `${streamName}##${messageId}`;
+    }
+    demergeMessageKey(messageKey) {
+        const [streamName, messageId] = messageKey.split('##');
+        return { streamName, messageId };
+    }
+}
+exports.StreamsLite = StreamsLite;