@jetit/publisher 1.0.0

package/README.md

@@ -0,0 +1,43 @@
+# publisher
+
+publisher is a library for implementing an event-driven architecture using Redis PUB/SUB and Redis Streams. It provides a simple and scalable mechanism for publishing and consuming events in real-time, and supports features such as message deduplication, consumer group management, and scheduled event publishing.
+
+## Simple Example
+
+```typescript
+import { Publisher, EventData } from '@jetit/streams';
+
+// Create an instance of the publisher
+const streams = new Streams('Websockets');
+
+// Publish an event
+const eventData: EventData<{ message: string }> = {
+    eventName: 'my-event',
+    data: { message: 'Hello, world!' }
+};
+
+await streams.publish(eventData);
+
+// Subscribe to an event
+streams.listen('my-event').subscribe(event => {
+    console.log(`Received event: ${event.eventName}`, event.data);
+});
+```
+
+## Possible use cases
+
+1. Microservices communication: If your system is composed of multiple microservices, the publisher can be used to facilitate communication between them by publishing and listening to events.
+
+2. Event sourcing and CQRS: In an event-sourced system, the publisher can be used to store and process events that represent the state changes of the system, enabling Command Query Responsibility Segregation (CQRS) by separating the read and write models.
+
+3. Task queues: The publisher can be used to create task queues for distributing workloads among different worker instances, ensuring that tasks are processed in the order they were created.
+
+4. Data streaming and processing: The publisher can be used to ingest and process large volumes of data in real-time, such as log files, clickstream data, or other event-based data.
+
+5. Distributed system coordination: In a distributed system, the publisher can be used for coordination between different components, such as managing leader election or maintaining configuration information.
+
+6. Real-time analytics and monitoring: The publisher can be used to collect and process real-time analytics data, such as user behavior, application performance metrics, or system monitoring information.
+
+7. Event-driven workflows: You can use the publisher to create event-driven workflows, where each step in the workflow is triggered by the completion of a previous step. This can be useful for orchestrating complex, multi-step processes.
+
+8. Message broadcasting: The publisher can be used to broadcast messages to multiple consumers or subscribers, allowing for efficient and scalable communication in applications with many components or services.

package/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@jetit/publisher",
+  "version": "1.0.0",
+  "type": "module",
+  "dependencies": {
+    "@jetit/id": "0.0.6",
+    "ioredis": "5.3.1",
+    "rxjs": "7.8.0"
+  },
+  "peerDependencies": {
+    "@types/ioredis-mock": "8.2.1",
+    "tslib": "2.5.0"
+  },
+  "main": "./src/index.js",
+  "types": "./src/index.d.ts"
+}

package/src/index.d.ts

@@ -0,0 +1 @@
+export * from './lib/publisher';

package/src/index.js

@@ -0,0 +1 @@
+export * from './lib/publisher';

package/src/lib/publisher.d.ts

@@ -0,0 +1,3 @@
+export { Streams as Publisher } from './redis/streams';
+export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
+export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';

package/src/lib/publisher.js

@@ -0,0 +1,3 @@
+export { Streams as Publisher } from './redis/streams';
+export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
+export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';

package/src/lib/redis/registry.d.ts

@@ -0,0 +1,20 @@
+import Redis, { Cluster } from 'ioredis';
+import { IOptions } from './types';
+export type RedisType = Redis | Cluster;
+export declare class RedisRegistry {
+    private static registry;
+    private static options;
+    static attemptConnection(connectionKey: string, storeRef?: number): Redis;
+    static getConnection(connectionType?: string, storeRef?: number): RedisType;
+    static setOptions(options: IOptions): void;
+    static _getOptions(): IOptions;
+}
+/**
+ * This function is used to set Redis Connection options per instance. If no
+ * options are provided, then the service connects as to a single instance
+ * with the environment values from REDIS_PORT and REDIS_HOST. if those
+ * environment values are not provided, it attempts to connect to localhost:6379
+ *
+ * @param options
+ */
+export declare function setRedisConnectionSettings(options: IOptions): void;

package/src/lib/redis/registry.js

@@ -0,0 +1,52 @@
+var _a, _b;
+import Redis, { Cluster } from 'ioredis';
+export class RedisRegistry {
+    static attemptConnection(connectionKey, storeRef = 0) {
+        let ref;
+        if (RedisRegistry.options.cluster) {
+            ref = new Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
+                    db: storeRef,
+                } }));
+        }
+        ref = new Redis(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+        RedisRegistry.registry.set(connectionKey, ref);
+        return ref;
+    }
+    static getConnection(connectionType = 'primary', storeRef = 0) {
+        const connectionKey = `${connectionType}${storeRef}`;
+        let ref = this.registry.get(connectionKey);
+        if (!ref) {
+            if (RedisRegistry.options.cluster) {
+                ref = new Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
+                        db: storeRef,
+                    } }));
+            }
+            ref = new Redis(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+        }
+        return ref;
+    }
+    static setOptions(options) {
+        RedisRegistry.options = options;
+    }
+    static _getOptions() {
+        return RedisRegistry.options;
+    }
+}
+RedisRegistry.registry = new Map();
+RedisRegistry.options = {
+    redis: {
+        port: parseInt((_a = process.env['REDIS_PORT']) !== null && _a !== void 0 ? _a : '6379'),
+        host: (_b = process.env['REDIS_HOST']) !== null && _b !== void 0 ? _b : 'localhost',
+    },
+};
+/**
+ * This function is used to set Redis Connection options per instance. If no
+ * options are provided, then the service connects as to a single instance
+ * with the environment values from REDIS_PORT and REDIS_HOST. if those
+ * environment values are not provided, it attempts to connect to localhost:6379
+ *
+ * @param options
+ */
+export function setRedisConnectionSettings(options) {
+    RedisRegistry.setOptions(options);
+}

package/src/lib/redis/scheduler.d.ts

@@ -0,0 +1,14 @@
+import { RedisType } from './registry';
+/**
+ * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
+ * meant to be used internally by the scheduler application
+ */
+export declare class ScheduledProcessor {
+    private scheduledMessagesTimer;
+    private _redisPublisher?;
+    get redisPublisher(): RedisType;
+    constructor(duration?: number);
+    private processScheduledEvents;
+    getAllScheduledEvents(): Promise<Array<string>>;
+    close(): Promise<void>;
+}

package/src/lib/redis/scheduler.js

@@ -0,0 +1,55 @@
+import { __awaiter } from "tslib";
+import { generateID } from '@jetit/id';
+import { interval } from 'rxjs';
+import { RedisRegistry } from './registry';
+/**
+ * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
+ * meant to be used internally by the scheduler application
+ */
+export class ScheduledProcessor {
+    get redisPublisher() {
+        if (!this._redisPublisher)
+            this._redisPublisher = RedisRegistry.getConnection('publish');
+        return this._redisPublisher;
+    }
+    constructor(duration = 1000) {
+        this.scheduledMessagesTimer = interval(duration).subscribe(() => {
+            this.processScheduledEvents().catch((error) => {
+                console.error('Error while processing scheduled events:', error);
+            });
+        });
+    }
+    processScheduledEvents() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const currentTime = new Date().getTime();
+            const events = yield this.redisPublisher.zrangebyscore('se', 0, currentTime);
+            for (const eventString of events) {
+                const eventData = JSON.parse(eventString);
+                /**
+                 * Remove the event from the Redis Sorted Set first. Please note that
+                 * there is a chance of failure here if the process crashes before
+                 * the event is published. In that case, the event will be lost.
+                 *
+                 * Instead of using the publish method directly, the entire logic is
+                 * copy pasted to reduce the case of failure.
+                 */
+                const transaction = this.redisPublisher.multi();
+                eventData.eventId = generateID('HEX', 'FF');
+                transaction.zrem('se', eventString);
+                transaction.xadd(eventData.eventName, '*', 'data', JSON.stringify(eventData));
+                transaction.publish(eventData.eventName, '');
+                yield transaction.exec();
+            }
+        });
+    }
+    getAllScheduledEvents() {
+        return this.redisPublisher.zrange('se', 0, -1);
+    }
+    close() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.scheduledMessagesTimer) {
+                this.scheduledMessagesTimer.unsubscribe();
+            }
+        });
+    }
+}

package/src/lib/redis/streams.d.ts

@@ -0,0 +1,153 @@
+import { RedisType } from './registry';
+import { EventData } from './types';
+import { Observable } from 'rxjs';
+export declare class Streams {
+    private _redisPublisher?;
+    private _redisSubscriber?;
+    private _redisGroups?;
+    private consumerGroupName;
+    private instanceId;
+    private cleanUpTimer;
+    private eventsListened;
+    get redisPublisher(): RedisType;
+    get redisSubscriber(): RedisType;
+    get redisGroups(): RedisType;
+    /**
+     * Creates a new Streams instance for a given service.
+     *
+     * The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
+     * It also sets up an interval timer for clearing expired messages from Redis and another interval timer
+     * for processing scheduled events at regular intervals.
+     *
+     * @param serviceName - A unique name for the service that will be using this Streams instance.
+     *
+     * @example
+     *
+     * // Create a new Streams instance for the "POS" service
+     * const streams = new Streams('POS');
+     */
+    constructor(serviceName: string);
+    private createStream;
+    private isDuplicateMessage;
+    private clearDuplicationCheckKeys;
+    /**
+     * Publishes an event with the given data to the Redis event stream.
+     *
+     * The method generates a unique event ID for each event, and adds it to the event data to handle message
+     * deduplication.
+     *
+     * @param data - An EventData<T> object containing the data for the event.
+     *
+     * @returns A Promise that resolves when the event has been published to the Redis stream.
+     *
+     * @example
+     *
+     * // Publish an "order.created" event with the given order data
+     * const orderData = { id: '123', customerName: 'John Doe', amount: 100 };
+     * const eventData = { eventName: 'order.created', data: orderData };
+     * await streams.publish(eventData);
+     */
+    publish<T>(data: EventData<T>): Promise<void>;
+    /**
+     * Schedules an event to be published at a specified future time. Thee event gets published if the
+     * differnece between the current time and the scheduled time is less than 500ms.
+     *
+     * @param scheduledTime - The Date object representing the future time when the event should be published.
+     * @param eventData - The event data object, containing the event name and its associated data.
+     *
+     * @throws Error - Throws an error if the scheduled time is in the past.
+     *
+     * @example
+     *
+     * const streams = new Streams('app-service');
+     *
+     * const futureTime = new Date(Date.now() + 10000); // 10 seconds from now
+     * const eventData: EventData<string> = {
+     *   eventName: 'order.created',
+     *   data: 'Order data'
+     * };
+     *
+     * await streams.scheduledPublish(futureTime, eventData);
+     */
+    scheduledPublish<T>(scheduledTime: Date, eventData: EventData<T>, uniquePerInstance?: boolean): Promise<void>;
+    /**
+     * Listens for events with the given name and returns an Observable that emits an EventData<T> object
+     * each time a new event is received.
+     *
+     * The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
+     * that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
+     *
+     * If an error occurs while subscribing, the method logs the error to the console and throws
+     * an error. This is done to prevent the service from continuing without a proper event subscription.
+     *
+     * There is retry logic with exponential backoff to handle error cases. These are also controllable by the
+     * calling service
+     *
+     * @param eventName - The name of the event to listen for.
+     *
+     * @returns An Observable that emits an EventData<T> object each time a new event is received.
+     *
+     * @example
+     *
+     * // Listen for "order.created" events
+     * const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
+     *
+     * // Subscribe to the Observable and log each new event
+     * orderCreated.subscribe((event) => {
+     *   console.log('New order created:', event.data);
+     * });
+     */
+    listen<T>(eventName: string, maxRetries?: number, initialDelay?: number): Observable<EventData<T>>;
+    private listenInternals;
+    /**
+     * This method takes all messages allocated to this instance and republishes them so
+     * that other instances of this service can receive and process them.
+     *
+     * This needs to be handled every 1-2 minutes if the queue becomes too long and messages
+     * are not being processed.
+     *
+     * Ideal implementation would be to wrap this inside a setInterval
+     * @param streamName
+     */
+    republishUnprocessedEvents(eventName: string): Promise<void>;
+    /**
+     * This method is used to claim messages in the event of a service crash. This library currently
+     * does not detect a service crash. This needs to be built as an extension of Kubernetes and
+     * a standalone service that notifies this service to process the events that are marked as
+     * pending
+     *
+     * @param streamName
+     * @param idleTimeout
+     *
+     *  * @example
+     *
+     * // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
+     * await streams.recoverCrashedConsumerMessages('order.created', 10000);
+     */
+    recoverCrashedConsumerMessages(eventName: string, idleTimeout: number): Promise<void>;
+    /**
+     * This method allows the possibility of a graceful shutdown by cleaning up the
+     * redis connections.
+     *
+     * In all services where the library is used, its better to implement this method
+     *
+     * process.on('SIGTERM', shutdown);
+     * process.on('SIGINT', shutdown);
+     *
+     * async function shutdown(): Promise<void> {
+     *   console.log('Graceful shutdown initiated.');
+     *   try {
+     *     await streams.close();
+     *     console.log('Resources and connections successfully closed.');
+     *   } catch (error) {
+     *     console.error('Error during graceful shutdown:', error);
+     *   }
+     *   process.exit(0);
+     * }
+     */
+    close(): Promise<void>;
+    private clearSubscribedEvents;
+    private registerConsumerGroup;
+    private getAllConsumerGroups;
+    private cleanupAcknowledgedMessages;
+}

package/src/lib/redis/streams.js

@@ -0,0 +1,393 @@
+import { __awaiter } from "tslib";
+import { RedisRegistry } from './registry';
+import { BehaviorSubject, skip, timer, interval, catchError, retry, throwError } from 'rxjs';
+import { generateID } from '@jetit/id';
+export class Streams {
+    get redisPublisher() {
+        if (!this._redisPublisher)
+            this._redisPublisher = RedisRegistry.getConnection('publish');
+        return this._redisPublisher;
+    }
+    get redisSubscriber() {
+        if (!this._redisSubscriber)
+            this._redisSubscriber = RedisRegistry.getConnection('subscriber');
+        return this._redisSubscriber;
+    }
+    get redisGroups() {
+        if (!this._redisGroups)
+            this._redisGroups = RedisRegistry.getConnection('groups');
+        return this._redisGroups;
+    }
+    /**
+     * Creates a new Streams instance for a given service.
+     *
+     * The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
+     * It also sets up an interval timer for clearing expired messages from Redis and another interval timer
+     * for processing scheduled events at regular intervals.
+     *
+     * @param serviceName - A unique name for the service that will be using this Streams instance.
+     *
+     * @example
+     *
+     * // Create a new Streams instance for the "POS" service
+     * const streams = new Streams('POS');
+     */
+    constructor(serviceName) {
+        var _a;
+        this.eventsListened = [];
+        this.instanceId = `${serviceName}:${generateID('HEX', 'FE')}`;
+        this.consumerGroupName = `cg-${serviceName}`;
+        const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || '1000 * 60 * 60', 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
+        this.cleanUpTimer = interval(cleanUpInterval).subscribe(() => {
+            this.clearDuplicationCheckKeys();
+            this.eventsListened.forEach((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval));
+        });
+    }
+    createStream(eventName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const streamName = `${eventName}:${this.consumerGroupName}`;
+                yield this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
+            }
+            catch (error) {
+                if (error.message !== 'BUSYGROUP Consumer Group name already exists') {
+                    throw error;
+                }
+            }
+        });
+    }
+    isDuplicateMessage(streamName, messageId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
+            const temp = yield Promise.race([
+                this.redisGroups.zscore(processedMessagesKey, messageId),
+                /** ioRedis doesnt seem to return the nil event. So waiting for 100ms before moving on */
+                new Promise((res) => setTimeout(() => res(null), 100)),
+            ]);
+            return temp !== null;
+        });
+    }
+    clearDuplicationCheckKeys() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const processedMessagesKeyPattern = `pm:${this.consumerGroupName}:*`;
+            let cursor = '0';
+            do {
+                const [nextCursor, keys] = yield this.redisGroups.scan(cursor, 'MATCH', processedMessagesKeyPattern);
+                cursor = nextCursor;
+                for (const key of keys) {
+                    const oneHourAgo = Date.now() - 60 * 60 * 1000;
+                    yield this.redisGroups.zremrangebyscore(key, '-inf', oneHourAgo);
+                }
+            } while (cursor !== '0');
+        });
+    }
+    /**
+     * Publishes an event with the given data to the Redis event stream.
+     *
+     * The method generates a unique event ID for each event, and adds it to the event data to handle message
+     * deduplication.
+     *
+     * @param data - An EventData<T> object containing the data for the event.
+     *
+     * @returns A Promise that resolves when the event has been published to the Redis stream.
+     *
+     * @example
+     *
+     * // Publish an "order.created" event with the given order data
+     * const orderData = { id: '123', customerName: 'John Doe', amount: 100 };
+     * const eventData = { eventName: 'order.created', data: orderData };
+     * await streams.publish(eventData);
+     */
+    publish(data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            data.eventId = generateID('HEX', 'FF'); // Added a unique Id to handle Message deduplication
+            const transaction = this.redisPublisher.multi();
+            const consumerGroups = yield this.getAllConsumerGroups(data.eventName);
+            if (consumerGroups.length > 0) {
+                console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
+                for (const consumerGroup of consumerGroups) {
+                    // Publish the event to each consumer group's stream
+                    const streamName = `${data.eventName}:${consumerGroup}`;
+                    transaction.xadd(streamName, '*', 'data', JSON.stringify(data));
+                }
+                transaction.publish(data.eventName, '');
+                yield transaction.exec().catch((error) => {
+                    console.error(`Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
+                    throw new Error('Publisher Error');
+                });
+            }
+        });
+    }
+    /**
+     * Schedules an event to be published at a specified future time. Thee event gets published if the
+     * differnece between the current time and the scheduled time is less than 500ms.
+     *
+     * @param scheduledTime - The Date object representing the future time when the event should be published.
+     * @param eventData - The event data object, containing the event name and its associated data.
+     *
+     * @throws Error - Throws an error if the scheduled time is in the past.
+     *
+     * @example
+     *
+     * const streams = new Streams('app-service');
+     *
+     * const futureTime = new Date(Date.now() + 10000); // 10 seconds from now
+     * const eventData: EventData<string> = {
+     *   eventName: 'order.created',
+     *   data: 'Order data'
+     * };
+     *
+     * await streams.scheduledPublish(futureTime, eventData);
+     */
+    scheduledPublish(scheduledTime, eventData, uniquePerInstance = false) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const currentTime = new Date();
+            if (scheduledTime < currentTime) {
+                throw new Error('Cannot schedule an event in the past');
+            }
+            else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
+                yield this.publish(eventData);
+            }
+            else {
+                if (uniquePerInstance === true) {
+                    const existingJob = yield this.redisPublisher.zscore('se', JSON.stringify(eventData));
+                    if (existingJob) {
+                        console.log(`Job with data '${eventData}' already exists. Skipping.`);
+                        return;
+                    }
+                }
+                yield this.redisPublisher.zadd('se', scheduledTime.getTime(), JSON.stringify(eventData));
+            }
+        });
+    }
+    /**
+     * Listens for events with the given name and returns an Observable that emits an EventData<T> object
+     * each time a new event is received.
+     *
+     * The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
+     * that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
+     *
+     * If an error occurs while subscribing, the method logs the error to the console and throws
+     * an error. This is done to prevent the service from continuing without a proper event subscription.
+     *
+     * There is retry logic with exponential backoff to handle error cases. These are also controllable by the
+     * calling service
+     *
+     * @param eventName - The name of the event to listen for.
+     *
+     * @returns An Observable that emits an EventData<T> object each time a new event is received.
+     *
+     * @example
+     *
+     * // Listen for "order.created" events
+     * const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
+     *
+     * // Subscribe to the Observable and log each new event
+     * orderCreated.subscribe((event) => {
+     *   console.log('New order created:', event.data);
+     * });
+     */
+    listen(eventName, maxRetries = 5, initialDelay = 1000) {
+        this.registerConsumerGroup(eventName); // Registers the consumer group for listening to the message
+        this.eventsListened.push(eventName);
+        return this.listenInternals(eventName).pipe(retry({
+            count: maxRetries,
+            delay: (error, retryAttempt) => {
+                const delay = initialDelay * Math.pow(2, retryAttempt);
+                console.error(`Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
+                return timer(delay);
+            },
+        }), catchError((error) => {
+            console.error(`Error in listen after ${maxRetries} retries: ${error.message}`);
+            return throwError(() => new Error(error.message));
+        }));
+    }
+    listenInternals(eventName) {
+        try {
+            this.createStream(eventName);
+            const bs = new BehaviorSubject(null);
+            const observable = bs.asObservable().pipe(skip(1));
+            const streamName = `${eventName}:${this.consumerGroupName}`;
+            this.redisSubscriber.subscribe(eventName);
+            const processMessage = () => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
+                    if (result) {
+                        const [, streamMessages] = result[0];
+                        for (const [id, data] of streamMessages) {
+                            const eventData = JSON.parse(data[1]);
+                            const messageId = eventData.eventId;
+                            const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
+                            if (isDuplicate) {
+                                console.warn(`Duplicate message detected: ${messageId}`);
+                                yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
+                                continue;
+                            }
+                            bs.next(eventData);
+                            const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
+                            const currentTime = Date.now();
+                            const transaction = this.redisGroups.multi();
+                            transaction.zadd(pmKey, currentTime, messageId);
+                            transaction.xack(streamName, this.consumerGroupName, id);
+                            transaction.zadd(`ack:${streamName}`, Date.now(), id);
+                            yield transaction.exec();
+                        }
+                    }
+                }
+                catch (e) {
+                    console.error(JSON.stringify(e));
+                }
+            });
+            this.redisSubscriber.on('message', () => __awaiter(this, void 0, void 0, function* () {
+                processMessage();
+            }));
+            return observable;
+        }
+        catch (e) {
+            console.error(JSON.stringify(e));
+            throw e;
+        }
+    }
+    /**
+     * This method takes all messages allocated to this instance and republishes them so
+     * that other instances of this service can receive and process them.
+     *
+     * This needs to be handled every 1-2 minutes if the queue becomes too long and messages
+     * are not being processed.
+     *
+     * Ideal implementation would be to wrap this inside a setInterval
+     * @param streamName
+     */
+    republishUnprocessedEvents(eventName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const streamName = `${eventName}:${this.consumerGroupName}`;
+            const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'STREAMS', streamName, '>');
+            if (result) {
+                const [, streamMessages] = result[0];
+                for (const [id, data] of streamMessages) {
+                    const eventData = JSON.parse(data[1]);
+                    console.log(`Unprocessed event: ${id}, data:`, eventData);
+                    const transaction = this.redisGroups.multi();
+                    // Republishing the events
+                    transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
+                    transaction.publish(eventName, '');
+                    transaction.xack(streamName, this.consumerGroupName, id);
+                    yield transaction.exec();
+                }
+            }
+        });
+    }
+    /**
+     * This method is used to claim messages in the event of a service crash. This library currently
+     * does not detect a service crash. This needs to be built as an extension of Kubernetes and
+     * a standalone service that notifies this service to process the events that are marked as
+     * pending
+     *
+     * @param streamName
+     * @param idleTimeout
+     *
+     *  * @example
+     *
+     * // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
+     * await streams.recoverCrashedConsumerMessages('order.created', 10000);
+     */
+    recoverCrashedConsumerMessages(eventName, idleTimeout) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const streamName = `${eventName}:${this.consumerGroupName}`;
+            const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName));
+            if (!pendingMessages)
+                return;
+            const [, minId, maxId, consumers] = pendingMessages;
+            for (const [consumer, pendingCount] of consumers) {
+                if (parseInt(pendingCount) > 0) {
+                    const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
+                    for (const [messageId] of pending) {
+                        const claimedMessage = (yield this.redisGroups.xclaim(streamName, this.consumerGroupName, this.instanceId, idleTimeout, messageId));
+                        if (claimedMessage) {
+                            const [, data] = claimedMessage[0];
+                            const eventData = JSON.parse(data[1]);
+                            const transaction = this.redisGroups.multi();
+                            transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
+                            transaction.publish(eventName, '');
+                            transaction.xack(streamName, this.consumerGroupName, messageId);
+                            yield transaction.exec();
+                        }
+                    }
+                }
+            }
+        });
+    }
+    /**
+     * This method allows the possibility of a graceful shutdown by cleaning up the
+     * redis connections.
+     *
+     * In all services where the library is used, its better to implement this method
+     *
+     * process.on('SIGTERM', shutdown);
+     * process.on('SIGINT', shutdown);
+     *
+     * async function shutdown(): Promise<void> {
+     *   console.log('Graceful shutdown initiated.');
+     *   try {
+     *     await streams.close();
+     *     console.log('Resources and connections successfully closed.');
+     *   } catch (error) {
+     *     console.error('Error during graceful shutdown:', error);
+     *   }
+     *   process.exit(0);
+     * }
+     */
+    close() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.clearSubscribedEvents();
+            if (this.redisPublisher) {
+                yield this.redisPublisher.quit();
+            }
+            if (this.redisSubscriber) {
+                yield this.redisSubscriber.quit();
+            }
+            if (this.redisGroups) {
+                yield this.redisGroups.quit();
+            }
+            if (this.cleanUpTimer) {
+                this.cleanUpTimer.unsubscribe();
+            }
+        });
+    }
+    clearSubscribedEvents() {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log(`${this.eventsListened.length} events to be cleared`);
+            for (const eventName of this.eventsListened) {
+                yield this.redisGroups.srem(`consumerGroups:${eventName}`, this.consumerGroupName);
+            }
+        });
+    }
+    registerConsumerGroup(eventName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.redisGroups.sadd(`consumerGroups:${eventName}`, this.consumerGroupName);
+        });
+    }
+    getAllConsumerGroups(eventName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const consumerGroups = yield this.redisGroups.smembers(`consumerGroups:${eventName}`);
+            return consumerGroups;
+        });
+    }
+    cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const streamName = `${eventName}:${this.consumerGroupName}`;
+            const cleanupThreshold = Date.now() - interval;
+            const acknowledgedMessages = yield this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+            if (acknowledgedMessages && acknowledgedMessages.length > 0) {
+                const transaction = this.redisGroups.multi();
+                // Remove acknowledged messages from the stream
+                for (const messageId of acknowledgedMessages) {
+                    transaction.xdel(streamName, messageId);
+                }
+                // Remove acknowledged messages from the Sorted Set
+                transaction.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+                yield transaction.exec();
+            }
+        });
+    }
+}

package/src/lib/redis/types.d.ts

@@ -0,0 +1,15 @@
+export type EventData<T> = {
+    data: T;
+    eventName: string;
+    eventId?: string;
+};
+export type PendingMessages = [never, never, string, string, Array<[string, number]>];
+export type ClaimedMessages = Array<[never, string]>;
+import { ClusterNode, ClusterOptions, RedisOptions } from 'ioredis';
+export interface IOptions {
+    redis?: RedisOptions;
+    cluster?: {
+        options: ClusterOptions;
+        nodes: Array<ClusterNode>;
+    };
+}

package/src/lib/redis/types.js

@@ -0,0 +1 @@
+export {};