@jetit/publisher 1.0.3 → 1.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "0.0.6",
@@ -12,5 +12,6 @@
     "tslib": "2.5.0"
   },
   "main": "./src/index.js",
-  "types": "./src/index.d.ts"
+  "types": "./src/index.d.ts",
+  "readme":"README.md"
 }

package/src/README.md

@@ -0,0 +1,43 @@
+# publisher
+
+publisher is a library for implementing an event-driven architecture using Redis PUB/SUB and Redis Streams. It provides a simple and scalable mechanism for publishing and consuming events in real-time, and supports features such as message deduplication, consumer group management, and scheduled event publishing.
+
+## Simple Example
+
+```typescript
+import { Publisher, EventData } from '@jetit/streams';
+
+// Create an instance of the publisher
+const streams = new Streams('Websockets');
+
+// Publish an event
+const eventData: EventData<{ message: string }> = {
+    eventName: 'my-event',
+    data: { message: 'Hello, world!' }
+};
+
+await streams.publish(eventData);
+
+// Subscribe to an event
+streams.listen('my-event').subscribe(event => {
+    console.log(`Received event: ${event.eventName}`, event.data);
+});
+```
+
+## Possible use cases
+
+1. Microservices communication: If your system is composed of multiple microservices, the publisher can be used to facilitate communication between them by publishing and listening to events.
+
+2. Event sourcing and CQRS: In an event-sourced system, the publisher can be used to store and process events that represent the state changes of the system, enabling Command Query Responsibility Segregation (CQRS) by separating the read and write models.
+
+3. Task queues: The publisher can be used to create task queues for distributing workloads among different worker instances, ensuring that tasks are processed in the order they were created.
+
+4. Data streaming and processing: The publisher can be used to ingest and process large volumes of data in real-time, such as log files, clickstream data, or other event-based data.
+
+5. Distributed system coordination: In a distributed system, the publisher can be used for coordination between different components, such as managing leader election or maintaining configuration information.
+
+6. Real-time analytics and monitoring: The publisher can be used to collect and process real-time analytics data, such as user behavior, application performance metrics, or system monitoring information.
+
+7. Event-driven workflows: You can use the publisher to create event-driven workflows, where each step in the workflow is triggered by the completion of a previous step. This can be useful for orchestrating complex, multi-step processes.
+
+8. Message broadcasting: The publisher can be used to broadcast messages to multiple consumers or subscribers, allowing for efficient and scalable communication in applications with many components or services.

package/src/index.d.ts

@@ -0,0 +1 @@
+export * from './lib/publisher';

package/src/index.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./lib/publisher"), exports);

package/src/lib/publisher.d.ts

@@ -0,0 +1,3 @@
+export { Streams as Publisher } from './redis/streams';
+export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
+export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';

package/src/lib/publisher.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.__SCHEDULER_INTERNALS__ = exports.setRedisConfig = exports.Publisher = void 0;
+var streams_1 = require("./redis/streams");
+Object.defineProperty(exports, "Publisher", { enumerable: true, get: function () { return streams_1.Streams; } });
+var registry_1 = require("./redis/registry");
+Object.defineProperty(exports, "setRedisConfig", { enumerable: true, get: function () { return registry_1.setRedisConnectionSettings; } });
+var scheduler_1 = require("./redis/scheduler");
+Object.defineProperty(exports, "__SCHEDULER_INTERNALS__", { enumerable: true, get: function () { return scheduler_1.ScheduledProcessor; } });

package/src/lib/redis/groups.d.ts

@@ -0,0 +1,2 @@
+import { RedisType } from './registry';
+export declare function getAllConsumerGroups(eventName: string, redisConnection: RedisType): Promise<string[]>;

package/src/lib/redis/groups.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAllConsumerGroups = void 0;
+const tslib_1 = require("tslib");
+function getAllConsumerGroups(eventName, redisConnection) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        const consumerGroups = yield redisConnection.smembers(`consumerGroups:${eventName}`);
+        return consumerGroups;
+    });
+}
+exports.getAllConsumerGroups = getAllConsumerGroups;

package/src/lib/redis/registry.d.ts

@@ -0,0 +1,20 @@
+import Redis, { Cluster } from 'ioredis';
+import { IOptions } from './types';
+export type RedisType = Redis | Cluster;
+export declare class RedisRegistry {
+    private static registry;
+    private static options;
+    static attemptConnection(connectionKey: string, storeRef?: number): Redis;
+    static getConnection(connectionType?: string, storeRef?: number): RedisType;
+    static setOptions(options: IOptions): void;
+    static _getOptions(): IOptions;
+}
+/**
+ * This function is used to set Redis Connection options per instance. If no
+ * options are provided, then the service connects as to a single instance
+ * with the environment values from REDIS_PORT and REDIS_HOST. if those
+ * environment values are not provided, it attempts to connect to localhost:6379
+ *
+ * @param options
+ */
+export declare function setRedisConnectionSettings(options: IOptions): void;

package/src/lib/redis/registry.js

@@ -0,0 +1,57 @@
+"use strict";
+var _a, _b;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
+const ioredis_1 = require("ioredis");
+class RedisRegistry {
+    static attemptConnection(connectionKey, storeRef = 0) {
+        let ref;
+        if (RedisRegistry.options.cluster) {
+            ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
+                    db: storeRef,
+                } }));
+        }
+        ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+        RedisRegistry.registry.set(connectionKey, ref);
+        return ref;
+    }
+    static getConnection(connectionType = 'primary', storeRef = 0) {
+        const connectionKey = `${connectionType}${storeRef}`;
+        let ref = this.registry.get(connectionKey);
+        if (!ref) {
+            if (RedisRegistry.options.cluster) {
+                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
+                        db: storeRef,
+                    } }));
+            }
+            ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+        }
+        return ref;
+    }
+    static setOptions(options) {
+        RedisRegistry.options = options;
+    }
+    static _getOptions() {
+        return RedisRegistry.options;
+    }
+}
+RedisRegistry.registry = new Map();
+RedisRegistry.options = {
+    redis: {
+        port: parseInt((_a = process.env['REDIS_PORT']) !== null && _a !== void 0 ? _a : '6379'),
+        host: (_b = process.env['REDIS_HOST']) !== null && _b !== void 0 ? _b : 'localhost',
+    },
+};
+exports.RedisRegistry = RedisRegistry;
+/**
+ * This function is used to set Redis Connection options per instance. If no
+ * options are provided, then the service connects as to a single instance
+ * with the environment values from REDIS_PORT and REDIS_HOST. if those
+ * environment values are not provided, it attempts to connect to localhost:6379
+ *
+ * @param options
+ */
+function setRedisConnectionSettings(options) {
+    RedisRegistry.setOptions(options);
+}
+exports.setRedisConnectionSettings = setRedisConnectionSettings;

package/src/lib/redis/scheduler.d.ts

@@ -0,0 +1,15 @@
+import { RedisType } from './registry';
+/**
+ * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
+ * meant to be used internally by the scheduler application
+ */
+export declare class ScheduledProcessor {
+    private scheduledMessagesTimer;
+    private _redisPublisher?;
+    private previousTaskCompleted;
+    get redisPublisher(): RedisType;
+    constructor(duration?: number);
+    private processScheduledEvents;
+    getAllScheduledEvents(): Promise<Array<string>>;
+    close(): Promise<void>;
+}

package/src/lib/redis/scheduler.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScheduledProcessor = void 0;
+const tslib_1 = require("tslib");
+const id_1 = require("@jetit/id");
+const rxjs_1 = require("rxjs");
+const registry_1 = require("./registry");
+const groups_1 = require("./groups");
+/**
+ * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
+ * meant to be used internally by the scheduler application
+ */
+class ScheduledProcessor {
+    get redisPublisher() {
+        if (!this._redisPublisher)
+            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
+        return this._redisPublisher;
+    }
+    constructor(duration = 1000) {
+        this.previousTaskCompleted = true;
+        this.scheduledMessagesTimer = (0, rxjs_1.interval)(duration).subscribe(() => {
+            console.log('Checking Streams messages at ', new Date().toISOString(), '...');
+            /** Do not run scheduler if the previous run is not completed */
+            if (this.previousTaskCompleted) {
+                this.previousTaskCompleted = false;
+                this.processScheduledEvents()
+                    .catch((error) => {
+                    console.error('Error while processing scheduled events:', error);
+                })
+                    .then(() => {
+                    this.previousTaskCompleted = true;
+                });
+            }
+            else {
+                console.log('Skipping current scheduler run because previous run is in progress');
+            }
+        });
+    }
+    processScheduledEvents() {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
+            const currentTime = new Date().getTime();
+            const events = yield this.redisPublisher.zrangebyscore('se', 0, currentTime);
+            console.log('Events to process:', events.length);
+            for (const eventString of events) {
+                const eventData = JSON.parse(eventString);
+                /**
+                 * Remove the event from the Redis Sorted Set first. Please note that
+                 * there is a chance of failure here if the process crashes before
+                 * the event is published. In that case, the event will be lost.
+                 *
+                 * Instead of using the publish method directly, the entire logic is
+                 * copy pasted to reduce the case of failure.
+                 */
+                const transaction = this.redisPublisher.multi();
+                eventData.eventId = (0, id_1.generateID)('HEX', 'FF');
+                transaction.zrem('se', eventString);
+                const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);
+                console.log('Scheduled Publishing to consumer groups: ', consumerGroups, 'with id ', eventData.eventId, '...');
+                for (const consumerGroup of consumerGroups) {
+                    // Publish the event to each consumer group's stream
+                    const streamName = `${eventData.eventName}:${consumerGroup}`;
+                    transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
+                }
+                transaction.publish(eventData.eventName, '');
+                yield transaction.exec();
+            }
+        });
+    }
+    getAllScheduledEvents() {
+        return this.redisPublisher.zrange('se', 0, -1);
+    }
+    close() {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
+            if (this.scheduledMessagesTimer) {
+                this.scheduledMessagesTimer.unsubscribe();
+            }
+        });
+    }
+}
+exports.ScheduledProcessor = ScheduledProcessor;

package/src/lib/redis/streams.d.ts

@@ -0,0 +1,152 @@
+import { RedisType } from './registry';
+import { EventData } from './types';
+import { Observable } from 'rxjs';
+export declare class Streams {
+    private _redisPublisher?;
+    private _redisSubscriber?;
+    private _redisGroups?;
+    private consumerGroupName;
+    private instanceId;
+    private cleanUpTimer;
+    private eventsListened;
+    get redisPublisher(): RedisType;
+    get redisSubscriber(): RedisType;
+    get redisGroups(): RedisType;
+    /**
+     * Creates a new Streams instance for a given service.
+     *
+     * The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
+     * It also sets up an interval timer for clearing expired messages from Redis and another interval timer
+     * for processing scheduled events at regular intervals.
+     *
+     * @param serviceName - A unique name for the service that will be using this Streams instance.
+     *
+     * @example
+     *
+     * // Create a new Streams instance for the "POS" service
+     * const streams = new Streams('POS');
+     */
+    constructor(serviceName: string);
+    private createConsumerGroup;
+    private isDuplicateMessage;
+    private clearDuplicationCheckKeys;
+    /**
+     * Publishes an event with the given data to the Redis event stream.
+     *
+     * The method generates a unique event ID for each event, and adds it to the event data to handle message
+     * deduplication.
+     *
+     * @param data - An EventData<T> object containing the data for the event.
+     *
+     * @returns A Promise that resolves when the event has been published to the Redis stream.
+     *
+     * @example
+     *
+     * // Publish an "order.created" event with the given order data
+     * const orderData = { id: '123', customerName: 'John Doe', amount: 100 };
+     * const eventData = { eventName: 'order.created', data: orderData };
+     * await streams.publish(eventData);
+     */
+    publish<T>(data: EventData<T>): Promise<void>;
+    /**
+     * Schedules an event to be published at a specified future time. Thee event gets published if the
+     * differnece between the current time and the scheduled time is less than 500ms.
+     *
+     * @param scheduledTime - The Date object representing the future time when the event should be published.
+     * @param eventData - The event data object, containing the event name and its associated data.
+     *
+     * @throws Error - Throws an error if the scheduled time is in the past.
+     *
+     * @example
+     *
+     * const streams = new Streams('app-service');
+     *
+     * const futureTime = new Date(Date.now() + 10000); // 10 seconds from now
+     * const eventData: EventData<string> = {
+     *   eventName: 'order.created',
+     *   data: 'Order data'
+     * };
+     *
+     * await streams.scheduledPublish(futureTime, eventData);
+     */
+    scheduledPublish<T>(scheduledTime: Date, eventData: EventData<T>, uniquePerInstance?: boolean): Promise<void>;
+    /**
+     * Listens for events with the given name and returns an Observable that emits an EventData<T> object
+     * each time a new event is received.
+     *
+     * The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
+     * that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
+     *
+     * If an error occurs while subscribing, the method logs the error to the console and throws
+     * an error. This is done to prevent the service from continuing without a proper event subscription.
+     *
+     * There is retry logic with exponential backoff to handle error cases. These are also controllable by the
+     * calling service
+     *
+     * @param eventName - The name of the event to listen for.
+     *
+     * @returns An Observable that emits an EventData<T> object each time a new event is received.
+     *
+     * @example
+     *
+     * // Listen for "order.created" events
+     * const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
+     *
+     * // Subscribe to the Observable and log each new event
+     * orderCreated.subscribe((event) => {
+     *   console.log('New order created:', event.data);
+     * });
+     */
+    listen<T>(eventName: string, maxRetries?: number, initialDelay?: number): Observable<EventData<T>>;
+    private listenInternals;
+    /**
+     * This method takes all messages allocated to this instance and republishes them so
+     * that other instances of this service can receive and process them.
+     *
+     * This needs to be handled every 1-2 minutes if the queue becomes too long and messages
+     * are not being processed.
+     *
+     * Ideal implementation would be to wrap this inside a setInterval
+     * @param streamName
+     */
+    republishUnprocessedEvents(eventName: string): Promise<void>;
+    /**
+     * This method is used to claim messages in the event of a service crash. This library currently
+     * does not detect a service crash. This needs to be built as an extension of Kubernetes and
+     * a standalone service that notifies this service to process the events that are marked as
+     * pending
+     *
+     * @param streamName
+     * @param idleTimeout
+     *
+     *  * @example
+     *
+     * // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
+     * await streams.recoverCrashedConsumerMessages('order.created', 10000);
+     */
+    recoverCrashedConsumerMessages(eventName: string, idleTimeout: number): Promise<void>;
+    /**
+     * This method allows the possibility of a graceful shutdown by cleaning up the
+     * redis connections.
+     *
+     * In all services where the library is used, its better to implement this method
+     *
+     * process.on('SIGTERM', shutdown);
+     * process.on('SIGINT', shutdown);
+     *
+     * async function shutdown(): Promise<void> {
+     *   console.log('Graceful shutdown initiated.');
+     *   try {
+     *     await streams.close();
+     *     console.log('Resources and connections successfully closed.');
+     *   } catch (error) {
+     *     console.error('Error during graceful shutdown:', error);
+     *   }
+     *   process.exit(0);
+     * }
+     */
+    close(): Promise<void>;
+    private clearSubscribedEvents;
+    private registerConsumerGroup;
+    private cleanupAcknowledgedMessages;
+}