@jetit/publisher 1.0.3 → 1.0.5

package/index.js

@@ -1,553 +0,0 @@
-define("src/lib/redis/types", ["require", "exports"], function (require, exports) {
-    "use strict";
-    Object.defineProperty(exports, "__esModule", { value: true });
-});
-define("src/lib/redis/registry", ["require", "exports", "ioredis"], function (require, exports, ioredis_1) {
-    "use strict";
-    var _a, _b;
-    Object.defineProperty(exports, "__esModule", { value: true });
-    exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
-    class RedisRegistry {
-        static attemptConnection(connectionKey, storeRef = 0) {
-            let ref;
-            if (RedisRegistry.options.cluster) {
-                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
-                        db: storeRef,
-                    } }));
-            }
-            ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
-            RedisRegistry.registry.set(connectionKey, ref);
-            return ref;
-        }
-        static getConnection(connectionType = 'primary', storeRef = 0) {
-            const connectionKey = `${connectionType}${storeRef}`;
-            let ref = this.registry.get(connectionKey);
-            if (!ref) {
-                if (RedisRegistry.options.cluster) {
-                    ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
-                            db: storeRef,
-                        } }));
-                }
-                ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
-            }
-            return ref;
-        }
-        static setOptions(options) {
-            RedisRegistry.options = options;
-        }
-        static _getOptions() {
-            return RedisRegistry.options;
-        }
-    }
-    RedisRegistry.registry = new Map();
-    RedisRegistry.options = {
-        redis: {
-            port: parseInt((_a = process.env['REDIS_PORT']) !== null && _a !== void 0 ? _a : '6379'),
-            host: (_b = process.env['REDIS_HOST']) !== null && _b !== void 0 ? _b : 'localhost',
-        },
-    };
-    exports.RedisRegistry = RedisRegistry;
-    /**
-     * This function is used to set Redis Connection options per instance. If no
-     * options are provided, then the service connects as to a single instance
-     * with the environment values from REDIS_PORT and REDIS_HOST. if those
-     * environment values are not provided, it attempts to connect to localhost:6379
-     *
-     * @param options
-     */
-    function setRedisConnectionSettings(options) {
-        RedisRegistry.setOptions(options);
-    }
-    exports.setRedisConnectionSettings = setRedisConnectionSettings;
-});
-define("src/lib/redis/groups", ["require", "exports", "tslib"], function (require, exports, tslib_1) {
-    "use strict";
-    Object.defineProperty(exports, "__esModule", { value: true });
-    exports.getAllConsumerGroups = void 0;
-    function getAllConsumerGroups(eventName, redisConnection) {
-        return tslib_1.__awaiter(this, void 0, void 0, function* () {
-            const consumerGroups = yield redisConnection.smembers(`consumerGroups:${eventName}`);
-            return consumerGroups;
-        });
-    }
-    exports.getAllConsumerGroups = getAllConsumerGroups;
-});
-define("src/lib/redis/streams", ["require", "exports", "tslib", "src/lib/redis/registry", "rxjs", "@jetit/id", "src/lib/redis/groups"], function (require, exports, tslib_2, registry_1, rxjs_1, id_1, groups_1) {
-    "use strict";
-    Object.defineProperty(exports, "__esModule", { value: true });
-    exports.Streams = void 0;
-    class Streams {
-        get redisPublisher() {
-            if (!this._redisPublisher)
-                this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
-            return this._redisPublisher;
-        }
-        get redisSubscriber() {
-            if (!this._redisSubscriber)
-                this._redisSubscriber = registry_1.RedisRegistry.getConnection('subscriber');
-            return this._redisSubscriber;
-        }
-        get redisGroups() {
-            if (!this._redisGroups)
-                this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
-            return this._redisGroups;
-        }
-        /**
-         * Creates a new Streams instance for a given service.
-         *
-         * The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
-         * It also sets up an interval timer for clearing expired messages from Redis and another interval timer
-         * for processing scheduled events at regular intervals.
-         *
-         * @param serviceName - A unique name for the service that will be using this Streams instance.
-         *
-         * @example
-         *
-         * // Create a new Streams instance for the "POS" service
-         * const streams = new Streams('POS');
-         */
-        constructor(serviceName) {
-            var _a;
-            this.eventsListened = [];
-            this.instanceId = `${serviceName}:${(0, id_1.generateID)('HEX', 'FE')}`;
-            this.consumerGroupName = `cg-${serviceName}`;
-            const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || '1000 * 60 * 60', 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
-            this.cleanUpTimer = (0, rxjs_1.interval)(cleanUpInterval).subscribe(() => {
-                this.clearDuplicationCheckKeys();
-                this.eventsListened.forEach((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval));
-            });
-        }
-        createConsumerGroup(eventName) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                try {
-                    const streamName = `${eventName}:${this.consumerGroupName}`;
-                    yield this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
-                }
-                catch (error) {
-                    if (error.message !== 'BUSYGROUP Consumer Group name already exists') {
-                        throw error;
-                    }
-                }
-            });
-        }
-        isDuplicateMessage(streamName, messageId) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
-                const temp = yield Promise.race([
-                    this.redisGroups.zscore(processedMessagesKey, messageId),
-                    /** ioRedis doesnt seem to return the nil event. So waiting for 100ms before moving on */
-                    new Promise((res) => setTimeout(() => res(null), 100)),
-                ]);
-                return temp !== null;
-            });
-        }
-        clearDuplicationCheckKeys() {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                const processedMessagesKeyPattern = `pm:${this.consumerGroupName}:*`;
-                let cursor = '0';
-                do {
-                    const [nextCursor, keys] = yield this.redisGroups.scan(cursor, 'MATCH', processedMessagesKeyPattern);
-                    cursor = nextCursor;
-                    for (const key of keys) {
-                        const oneHourAgo = Date.now() - 60 * 60 * 1000;
-                        yield this.redisGroups.zremrangebyscore(key, '-inf', oneHourAgo);
-                    }
-                } while (cursor !== '0');
-            });
-        }
-        /**
-         * Publishes an event with the given data to the Redis event stream.
-         *
-         * The method generates a unique event ID for each event, and adds it to the event data to handle message
-         * deduplication.
-         *
-         * @param data - An EventData<T> object containing the data for the event.
-         *
-         * @returns A Promise that resolves when the event has been published to the Redis stream.
-         *
-         * @example
-         *
-         * // Publish an "order.created" event with the given order data
-         * const orderData = { id: '123', customerName: 'John Doe', amount: 100 };
-         * const eventData = { eventName: 'order.created', data: orderData };
-         * await streams.publish(eventData);
-         */
-        publish(data) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                data.eventId = (0, id_1.generateID)('HEX', 'FF'); // Added a unique Id to handle Message deduplication
-                const transaction = this.redisPublisher.multi();
-                const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisGroups);
-                if (consumerGroups.length > 0) {
-                    console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
-                    for (const consumerGroup of consumerGroups) {
-                        // Publish the event to each consumer group's stream
-                        const streamName = `${data.eventName}:${consumerGroup}`;
-                        transaction.xadd(streamName, '*', 'data', JSON.stringify(data));
-                    }
-                    transaction.publish(data.eventName, '');
-                    yield transaction.exec().catch((error) => {
-                        console.error(`Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
-                        throw new Error('Publisher Error');
-                    });
-                }
-            });
-        }
-        /**
-         * Schedules an event to be published at a specified future time. Thee event gets published if the
-         * differnece between the current time and the scheduled time is less than 500ms.
-         *
-         * @param scheduledTime - The Date object representing the future time when the event should be published.
-         * @param eventData - The event data object, containing the event name and its associated data.
-         *
-         * @throws Error - Throws an error if the scheduled time is in the past.
-         *
-         * @example
-         *
-         * const streams = new Streams('app-service');
-         *
-         * const futureTime = new Date(Date.now() + 10000); // 10 seconds from now
-         * const eventData: EventData<string> = {
-         *   eventName: 'order.created',
-         *   data: 'Order data'
-         * };
-         *
-         * await streams.scheduledPublish(futureTime, eventData);
-         */
-        scheduledPublish(scheduledTime, eventData, uniquePerInstance = false) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                const currentTime = new Date();
-                if (scheduledTime < currentTime) {
-                    throw new Error('Cannot schedule an event in the past');
-                }
-                else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
-                    yield this.publish(eventData);
-                }
-                else {
-                    if (uniquePerInstance === true) {
-                        const existingJob = yield this.redisPublisher.zscore('se', JSON.stringify(eventData));
-                        if (existingJob) {
-                            console.log(`Job with data '${eventData}' already exists. Skipping.`);
-                            return;
-                        }
-                    }
-                    yield this.redisPublisher.zadd('se', scheduledTime.getTime(), JSON.stringify(eventData));
-                }
-            });
-        }
-        /**
-         * Listens for events with the given name and returns an Observable that emits an EventData<T> object
-         * each time a new event is received.
-         *
-         * The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
-         * that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
-         *
-         * If an error occurs while subscribing, the method logs the error to the console and throws
-         * an error. This is done to prevent the service from continuing without a proper event subscription.
-         *
-         * There is retry logic with exponential backoff to handle error cases. These are also controllable by the
-         * calling service
-         *
-         * @param eventName - The name of the event to listen for.
-         *
-         * @returns An Observable that emits an EventData<T> object each time a new event is received.
-         *
-         * @example
-         *
-         * // Listen for "order.created" events
-         * const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
-         *
-         * // Subscribe to the Observable and log each new event
-         * orderCreated.subscribe((event) => {
-         *   console.log('New order created:', event.data);
-         * });
-         */
-        listen(eventName, maxRetries = 5, initialDelay = 1000) {
-            this.registerConsumerGroup(eventName); // Registers the consumer group for listening to the message
-            this.eventsListened.push(eventName);
-            return this.listenInternals(eventName).pipe((0, rxjs_1.retry)({
-                count: maxRetries,
-                delay: (error, retryAttempt) => {
-                    const delay = initialDelay * Math.pow(2, retryAttempt);
-                    console.error(`Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
-                    return (0, rxjs_1.timer)(delay);
-                },
-            }), (0, rxjs_1.catchError)((error) => {
-                console.error(`Error in listen after ${maxRetries} retries: ${error.message}`);
-                return (0, rxjs_1.throwError)(() => new Error(error.message));
-            }));
-        }
-        listenInternals(eventName) {
-            try {
-                this.createConsumerGroup(eventName);
-                const bs = new rxjs_1.BehaviorSubject(null);
-                const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
-                const streamName = `${eventName}:${this.consumerGroupName}`;
-                this.redisSubscriber.subscribe(eventName);
-                const processMessage = () => tslib_2.__awaiter(this, void 0, void 0, function* () {
-                    try {
-                        const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
-                        if (result) {
-                            const [, streamMessages] = result[0];
-                            for (const [id, data] of streamMessages) {
-                                const eventData = JSON.parse(data[1]);
-                                const messageId = eventData.eventId;
-                                const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
-                                if (isDuplicate) {
-                                    console.warn(`Duplicate message detected: ${messageId}`);
-                                    yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
-                                    continue;
-                                }
-                                bs.next(eventData);
-                                const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
-                                const currentTime = Date.now();
-                                const transaction = this.redisGroups.multi();
-                                transaction.zadd(pmKey, currentTime, messageId);
-                                transaction.xack(streamName, this.consumerGroupName, id);
-                                transaction.zadd(`ack:${streamName}`, Date.now(), id);
-                                yield transaction.exec();
-                            }
-                        }
-                    }
-                    catch (e) {
-                        console.error(JSON.stringify(e));
-                    }
-                });
-                this.redisSubscriber.on('message', () => tslib_2.__awaiter(this, void 0, void 0, function* () {
-                    yield processMessage();
-                }));
-                return observable;
-            }
-            catch (e) {
-                console.error(JSON.stringify(e));
-                throw e;
-            }
-        }
-        /**
-         * This method takes all messages allocated to this instance and republishes them so
-         * that other instances of this service can receive and process them.
-         *
-         * This needs to be handled every 1-2 minutes if the queue becomes too long and messages
-         * are not being processed.
-         *
-         * Ideal implementation would be to wrap this inside a setInterval
-         * @param streamName
-         */
-        republishUnprocessedEvents(eventName) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                const streamName = `${eventName}:${this.consumerGroupName}`;
-                const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'STREAMS', streamName, '>');
-                if (result) {
-                    const [, streamMessages] = result[0];
-                    for (const [id, data] of streamMessages) {
-                        const eventData = JSON.parse(data[1]);
-                        console.log(`Unprocessed event: ${id}, data:`, eventData);
-                        const transaction = this.redisGroups.multi();
-                        // Republishing the events
-                        transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
-                        transaction.publish(eventName, '');
-                        transaction.xack(streamName, this.consumerGroupName, id);
-                        yield transaction.exec();
-                    }
-                }
-            });
-        }
-        /**
-         * This method is used to claim messages in the event of a service crash. This library currently
-         * does not detect a service crash. This needs to be built as an extension of Kubernetes and
-         * a standalone service that notifies this service to process the events that are marked as
-         * pending
-         *
-         * @param streamName
-         * @param idleTimeout
-         *
-         *  * @example
-         *
-         * // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
-         * await streams.recoverCrashedConsumerMessages('order.created', 10000);
-         */
-        recoverCrashedConsumerMessages(eventName, idleTimeout) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                const streamName = `${eventName}:${this.consumerGroupName}`;
-                const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName));
-                if (!pendingMessages)
-                    return;
-                const [, minId, maxId, consumers] = pendingMessages;
-                for (const [consumer, pendingCount] of consumers) {
-                    if (parseInt(pendingCount) > 0) {
-                        const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
-                        for (const [messageId] of pending) {
-                            const claimedMessage = (yield this.redisGroups.xclaim(streamName, this.consumerGroupName, this.instanceId, idleTimeout, messageId));
-                            if (claimedMessage) {
-                                const [, data] = claimedMessage[0];
-                                const eventData = JSON.parse(data[1]);
-                                const transaction = this.redisGroups.multi();
-                                transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
-                                transaction.publish(eventName, '');
-                                transaction.xack(streamName, this.consumerGroupName, messageId);
-                                yield transaction.exec();
-                            }
-                        }
-                    }
-                }
-            });
-        }
-        /**
-         * This method allows the possibility of a graceful shutdown by cleaning up the
-         * redis connections.
-         *
-         * In all services where the library is used, its better to implement this method
-         *
-         * process.on('SIGTERM', shutdown);
-         * process.on('SIGINT', shutdown);
-         *
-         * async function shutdown(): Promise<void> {
-         *   console.log('Graceful shutdown initiated.');
-         *   try {
-         *     await streams.close();
-         *     console.log('Resources and connections successfully closed.');
-         *   } catch (error) {
-         *     console.error('Error during graceful shutdown:', error);
-         *   }
-         *   process.exit(0);
-         * }
-         */
-        close() {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                this.clearSubscribedEvents();
-                if (this.redisPublisher) {
-                    yield this.redisPublisher.quit();
-                }
-                if (this.redisSubscriber) {
-                    yield this.redisSubscriber.quit();
-                }
-                if (this.redisGroups) {
-                    yield this.redisGroups.quit();
-                }
-                if (this.cleanUpTimer) {
-                    this.cleanUpTimer.unsubscribe();
-                }
-            });
-        }
-        clearSubscribedEvents() {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                console.log(`${this.eventsListened.length} events to be cleared`);
-                for (const eventName of this.eventsListened) {
-                    yield this.redisGroups.srem(`consumerGroups:${eventName}`, this.consumerGroupName);
-                }
-            });
-        }
-        registerConsumerGroup(eventName) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                yield this.redisGroups.sadd(`consumerGroups:${eventName}`, this.consumerGroupName);
-            });
-        }
-        cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
-            return tslib_2.__awaiter(this, void 0, void 0, function* () {
-                const streamName = `${eventName}:${this.consumerGroupName}`;
-                const cleanupThreshold = Date.now() - interval;
-                const acknowledgedMessages = yield this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
-                if (acknowledgedMessages && acknowledgedMessages.length > 0) {
-                    const transaction = this.redisGroups.multi();
-                    // Remove acknowledged messages from the stream
-                    for (const messageId of acknowledgedMessages) {
-                        transaction.xdel(streamName, messageId);
-                    }
-                    // Remove acknowledged messages from the Sorted Set
-                    transaction.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
-                    yield transaction.exec();
-                }
-            });
-        }
-    }
-    exports.Streams = Streams;
-});
-define("src/lib/redis/scheduler", ["require", "exports", "tslib", "@jetit/id", "rxjs", "src/lib/redis/registry", "src/lib/redis/groups"], function (require, exports, tslib_3, id_2, rxjs_2, registry_2, groups_2) {
-    "use strict";
-    Object.defineProperty(exports, "__esModule", { value: true });
-    exports.ScheduledProcessor = void 0;
-    /**
-     * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
-     * meant to be used internally by the scheduler application
-     */
-    class ScheduledProcessor {
-        get redisPublisher() {
-            if (!this._redisPublisher)
-                this._redisPublisher = registry_2.RedisRegistry.getConnection('publish');
-            return this._redisPublisher;
-        }
-        constructor(duration = 1000) {
-            this.previousTaskCompleted = true;
-            this.scheduledMessagesTimer = (0, rxjs_2.interval)(duration).subscribe(() => {
-                console.log('Checking Streams messages at ', new Date().toISOString(), '...');
-                /** Do not run scheduler if the previous run is not completed */
-                if (this.previousTaskCompleted) {
-                    this.previousTaskCompleted = false;
-                    this.processScheduledEvents()
-                        .catch((error) => {
-                        console.error('Error while processing scheduled events:', error);
-                    })
-                        .then(() => {
-                        this.previousTaskCompleted = true;
-                    });
-                }
-                else {
-                    console.log('Skipping current scheduler run because previous run is in progress');
-                }
-            });
-        }
-        processScheduledEvents() {
-            return tslib_3.__awaiter(this, void 0, void 0, function* () {
-                const currentTime = new Date().getTime();
-                const events = yield this.redisPublisher.zrangebyscore('se', 0, currentTime);
-                console.log('Events to process:', events.length);
-                for (const eventString of events) {
-                    const eventData = JSON.parse(eventString);
-                    /**
-                     * Remove the event from the Redis Sorted Set first. Please note that
-                     * there is a chance of failure here if the process crashes before
-                     * the event is published. In that case, the event will be lost.
-                     *
-                     * Instead of using the publish method directly, the entire logic is
-                     * copy pasted to reduce the case of failure.
-                     */
-                    const transaction = this.redisPublisher.multi();
-                    eventData.eventId = (0, id_2.generateID)('HEX', 'FF');
-                    transaction.zrem('se', eventString);
-                    const consumerGroups = yield (0, groups_2.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);
-                    console.log('Scheduled Publishing to consumer groups: ', consumerGroups, 'with id ', eventData.eventId, '...');
-                    for (const consumerGroup of consumerGroups) {
-                        // Publish the event to each consumer group's stream
-                        const streamName = `${eventData.eventName}:${consumerGroup}`;
-                        transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
-                    }
-                    transaction.publish(eventData.eventName, '');
-                    yield transaction.exec();
-                }
-            });
-        }
-        getAllScheduledEvents() {
-            return this.redisPublisher.zrange('se', 0, -1);
-        }
-        close() {
-            return tslib_3.__awaiter(this, void 0, void 0, function* () {
-                if (this.scheduledMessagesTimer) {
-                    this.scheduledMessagesTimer.unsubscribe();
-                }
-            });
-        }
-    }
-    exports.ScheduledProcessor = ScheduledProcessor;
-});
-define("src/lib/publisher", ["require", "exports", "src/lib/redis/streams", "src/lib/redis/registry", "src/lib/redis/scheduler"], function (require, exports, streams_1, registry_3, scheduler_1) {
-    "use strict";
-    Object.defineProperty(exports, "__esModule", { value: true });
-    exports.__SCHEDULER_INTERNALS__ = exports.setRedisConfig = exports.Publisher = void 0;
-    Object.defineProperty(exports, "Publisher", { enumerable: true, get: function () { return streams_1.Streams; } });
-    Object.defineProperty(exports, "setRedisConfig", { enumerable: true, get: function () { return registry_3.setRedisConnectionSettings; } });
-    Object.defineProperty(exports, "__SCHEDULER_INTERNALS__", { enumerable: true, get: function () { return scheduler_1.ScheduledProcessor; } });
-});
-define("src/index", ["require", "exports", "tslib", "src/lib/publisher"], function (require, exports, tslib_4, publisher_1) {
-    "use strict";
-    Object.defineProperty(exports, "__esModule", { value: true });
-    tslib_4.__exportStar(publisher_1, exports);
-});