@jetit/publisher 1.0.0 → 1.0.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@jetit/publisher",
-  "version": "1.0.0",
-  "type": "module",
+  "version": "1.0.2",
+  "type": "commonjs",
   "dependencies": {
     "@jetit/id": "0.0.6",
     "ioredis": "5.3.1",

package/src/index.js

@@ -1 +1,4 @@
-export * from './lib/publisher';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./lib/publisher"), exports);

package/src/lib/publisher.js

@@ -1,3 +1,9 @@
-export { Streams as Publisher } from './redis/streams';
-export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
-export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.__SCHEDULER_INTERNALS__ = exports.setRedisConfig = exports.Publisher = void 0;
+var streams_1 = require("./redis/streams");
+Object.defineProperty(exports, "Publisher", { enumerable: true, get: function () { return streams_1.Streams; } });
+var registry_1 = require("./redis/registry");
+Object.defineProperty(exports, "setRedisConfig", { enumerable: true, get: function () { return registry_1.setRedisConnectionSettings; } });
+var scheduler_1 = require("./redis/scheduler");
+Object.defineProperty(exports, "__SCHEDULER_INTERNALS__", { enumerable: true, get: function () { return scheduler_1.ScheduledProcessor; } });

package/src/lib/redis/groups.d.ts

@@ -0,0 +1,2 @@
+import { RedisType } from './registry';
+export declare function getAllConsumerGroups(eventName: string, redisConnection: RedisType): Promise<string[]>;

package/src/lib/redis/groups.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAllConsumerGroups = void 0;
+const tslib_1 = require("tslib");
+function getAllConsumerGroups(eventName, redisConnection) {
+    return tslib_1.__awaiter(this, void 0, void 0, function* () {
+        const consumerGroups = yield redisConnection.smembers(`consumerGroups:${eventName}`);
+        return consumerGroups;
+    });
+}
+exports.getAllConsumerGroups = getAllConsumerGroups;

package/src/lib/redis/registry.js

@@ -1,14 +1,17 @@
+"use strict";
 var _a, _b;
-import Redis, { Cluster } from 'ioredis';
-export class RedisRegistry {
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
+const ioredis_1 = require("ioredis");
+class RedisRegistry {
     static attemptConnection(connectionKey, storeRef = 0) {
         let ref;
         if (RedisRegistry.options.cluster) {
-            ref = new Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
+            ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
                     db: storeRef,
                 } }));
         }
-        ref = new Redis(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+        ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
         RedisRegistry.registry.set(connectionKey, ref);
         return ref;
     }
@@ -17,11 +20,11 @@ export class RedisRegistry {
         let ref = this.registry.get(connectionKey);
         if (!ref) {
             if (RedisRegistry.options.cluster) {
-                ref = new Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
+                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, Object.assign(Object.assign({}, RedisRegistry.options.cluster.options), { redisOptions: {
                         db: storeRef,
                     } }));
             }
-            ref = new Redis(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
+            ref = new ioredis_1.default(Object.assign(Object.assign({}, RedisRegistry.options.redis), { db: storeRef }));
         }
         return ref;
     }
@@ -39,6 +42,7 @@ RedisRegistry.options = {
         host: (_b = process.env['REDIS_HOST']) !== null && _b !== void 0 ? _b : 'localhost',
     },
 };
+exports.RedisRegistry = RedisRegistry;
 /**
  * This function is used to set Redis Connection options per instance. If no
  * options are provided, then the service connects as to a single instance
@@ -47,6 +51,7 @@ RedisRegistry.options = {
  *
  * @param options
  */
-export function setRedisConnectionSettings(options) {
+function setRedisConnectionSettings(options) {
     RedisRegistry.setOptions(options);
 }
+exports.setRedisConnectionSettings = setRedisConnectionSettings;

package/src/lib/redis/scheduler.d.ts

@@ -6,6 +6,7 @@ import { RedisType } from './registry';
 export declare class ScheduledProcessor {
     private scheduledMessagesTimer;
     private _redisPublisher?;
+    private previousTaskCompleted;
     get redisPublisher(): RedisType;
     constructor(duration?: number);
     private processScheduledEvents;

package/src/lib/redis/scheduler.js

@@ -1,28 +1,46 @@
-import { __awaiter } from "tslib";
-import { generateID } from '@jetit/id';
-import { interval } from 'rxjs';
-import { RedisRegistry } from './registry';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScheduledProcessor = void 0;
+const tslib_1 = require("tslib");
+const id_1 = require("@jetit/id");
+const rxjs_1 = require("rxjs");
+const registry_1 = require("./registry");
+const groups_1 = require("./groups");
 /**
  * DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
  * meant to be used internally by the scheduler application
  */
-export class ScheduledProcessor {
+class ScheduledProcessor {
     get redisPublisher() {
         if (!this._redisPublisher)
-            this._redisPublisher = RedisRegistry.getConnection('publish');
+            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
         return this._redisPublisher;
     }
     constructor(duration = 1000) {
-        this.scheduledMessagesTimer = interval(duration).subscribe(() => {
-            this.processScheduledEvents().catch((error) => {
-                console.error('Error while processing scheduled events:', error);
-            });
+        this.previousTaskCompleted = true;
+        this.scheduledMessagesTimer = (0, rxjs_1.interval)(duration).subscribe(() => {
+            console.log('Checking Streams messages at ', new Date().toISOString(), '...');
+            /** Do not run scheduler if the previous run is not completed */
+            if (this.previousTaskCompleted) {
+                this.previousTaskCompleted = false;
+                this.processScheduledEvents()
+                    .catch((error) => {
+                    console.error('Error while processing scheduled events:', error);
+                })
+                    .then(() => {
+                    this.previousTaskCompleted = true;
+                });
+            }
+            else {
+                console.log('Skipping current scheduler run because previous run is in progress');
+            }
         });
     }
     processScheduledEvents() {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const currentTime = new Date().getTime();
             const events = yield this.redisPublisher.zrangebyscore('se', 0, currentTime);
+            console.log('Events to process:', events.length);
             for (const eventString of events) {
                 const eventData = JSON.parse(eventString);
                 /**
@@ -34,9 +52,15 @@ export class ScheduledProcessor {
                  * copy pasted to reduce the case of failure.
                  */
                 const transaction = this.redisPublisher.multi();
-                eventData.eventId = generateID('HEX', 'FF');
+                eventData.eventId = (0, id_1.generateID)('HEX', 'FF');
                 transaction.zrem('se', eventString);
-                transaction.xadd(eventData.eventName, '*', 'data', JSON.stringify(eventData));
+                const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);
+                console.log('Scheduled Publishing to consumer groups: ', consumerGroups, 'with id ', eventData.eventId, '...');
+                for (const consumerGroup of consumerGroups) {
+                    // Publish the event to each consumer group's stream
+                    const streamName = `${eventData.eventName}:${consumerGroup}`;
+                    transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
+                }
                 transaction.publish(eventData.eventName, '');
                 yield transaction.exec();
             }
@@ -46,10 +70,11 @@ export class ScheduledProcessor {
         return this.redisPublisher.zrange('se', 0, -1);
     }
     close() {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             if (this.scheduledMessagesTimer) {
                 this.scheduledMessagesTimer.unsubscribe();
             }
         });
     }
 }
+exports.ScheduledProcessor = ScheduledProcessor;

package/src/lib/redis/streams.d.ts

@@ -27,7 +27,7 @@ export declare class Streams {
      * const streams = new Streams('POS');
      */
     constructor(serviceName: string);
-    private createStream;
+    private createConsumerGroup;
     private isDuplicateMessage;
     private clearDuplicationCheckKeys;
     /**
@@ -148,6 +148,5 @@ export declare class Streams {
     close(): Promise<void>;
     private clearSubscribedEvents;
     private registerConsumerGroup;
-    private getAllConsumerGroups;
     private cleanupAcknowledgedMessages;
 }

package/src/lib/redis/streams.js

@@ -1,21 +1,25 @@
-import { __awaiter } from "tslib";
-import { RedisRegistry } from './registry';
-import { BehaviorSubject, skip, timer, interval, catchError, retry, throwError } from 'rxjs';
-import { generateID } from '@jetit/id';
-export class Streams {
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Streams = void 0;
+const tslib_1 = require("tslib");
+const registry_1 = require("./registry");
+const rxjs_1 = require("rxjs");
+const id_1 = require("@jetit/id");
+const groups_1 = require("./groups");
+class Streams {
     get redisPublisher() {
         if (!this._redisPublisher)
-            this._redisPublisher = RedisRegistry.getConnection('publish');
+            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
         return this._redisPublisher;
     }
     get redisSubscriber() {
         if (!this._redisSubscriber)
-            this._redisSubscriber = RedisRegistry.getConnection('subscriber');
+            this._redisSubscriber = registry_1.RedisRegistry.getConnection('subscriber');
         return this._redisSubscriber;
     }
     get redisGroups() {
         if (!this._redisGroups)
-            this._redisGroups = RedisRegistry.getConnection('groups');
+            this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
         return this._redisGroups;
     }
     /**
@@ -35,16 +39,16 @@ export class Streams {
     constructor(serviceName) {
         var _a;
         this.eventsListened = [];
-        this.instanceId = `${serviceName}:${generateID('HEX', 'FE')}`;
+        this.instanceId = `${serviceName}:${(0, id_1.generateID)('HEX', 'FE')}`;
         this.consumerGroupName = `cg-${serviceName}`;
         const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || '1000 * 60 * 60', 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
-        this.cleanUpTimer = interval(cleanUpInterval).subscribe(() => {
+        this.cleanUpTimer = (0, rxjs_1.interval)(cleanUpInterval).subscribe(() => {
             this.clearDuplicationCheckKeys();
             this.eventsListened.forEach((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval));
         });
     }
-    createStream(eventName) {
-        return __awaiter(this, void 0, void 0, function* () {
+    createConsumerGroup(eventName) {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             try {
                 const streamName = `${eventName}:${this.consumerGroupName}`;
                 yield this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
@@ -57,7 +61,7 @@ export class Streams {
         });
     }
     isDuplicateMessage(streamName, messageId) {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
             const temp = yield Promise.race([
                 this.redisGroups.zscore(processedMessagesKey, messageId),
@@ -68,7 +72,7 @@ export class Streams {
         });
     }
     clearDuplicationCheckKeys() {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const processedMessagesKeyPattern = `pm:${this.consumerGroupName}:*`;
             let cursor = '0';
             do {
@@ -99,10 +103,10 @@ export class Streams {
      * await streams.publish(eventData);
      */
     publish(data) {
-        return __awaiter(this, void 0, void 0, function* () {
-            data.eventId = generateID('HEX', 'FF'); // Added a unique Id to handle Message deduplication
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
+            data.eventId = (0, id_1.generateID)('HEX', 'FF'); // Added a unique Id to handle Message deduplication
             const transaction = this.redisPublisher.multi();
-            const consumerGroups = yield this.getAllConsumerGroups(data.eventName);
+            const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisGroups);
             if (consumerGroups.length > 0) {
                 console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
                 for (const consumerGroup of consumerGroups) {
@@ -140,7 +144,7 @@ export class Streams {
      * await streams.scheduledPublish(futureTime, eventData);
      */
     scheduledPublish(scheduledTime, eventData, uniquePerInstance = false) {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const currentTime = new Date();
             if (scheduledTime < currentTime) {
                 throw new Error('Cannot schedule an event in the past');
@@ -190,26 +194,26 @@ export class Streams {
     listen(eventName, maxRetries = 5, initialDelay = 1000) {
         this.registerConsumerGroup(eventName); // Registers the consumer group for listening to the message
         this.eventsListened.push(eventName);
-        return this.listenInternals(eventName).pipe(retry({
+        return this.listenInternals(eventName).pipe((0, rxjs_1.retry)({
             count: maxRetries,
             delay: (error, retryAttempt) => {
                 const delay = initialDelay * Math.pow(2, retryAttempt);
                 console.error(`Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
-                return timer(delay);
+                return (0, rxjs_1.timer)(delay);
             },
-        }), catchError((error) => {
+        }), (0, rxjs_1.catchError)((error) => {
             console.error(`Error in listen after ${maxRetries} retries: ${error.message}`);
-            return throwError(() => new Error(error.message));
+            return (0, rxjs_1.throwError)(() => new Error(error.message));
         }));
     }
     listenInternals(eventName) {
         try {
-            this.createStream(eventName);
-            const bs = new BehaviorSubject(null);
-            const observable = bs.asObservable().pipe(skip(1));
+            this.createConsumerGroup(eventName);
+            const bs = new rxjs_1.BehaviorSubject(null);
+            const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
             const streamName = `${eventName}:${this.consumerGroupName}`;
             this.redisSubscriber.subscribe(eventName);
-            const processMessage = () => __awaiter(this, void 0, void 0, function* () {
+            const processMessage = () => tslib_1.__awaiter(this, void 0, void 0, function* () {
                 try {
                     const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
                     if (result) {
@@ -238,8 +242,8 @@ export class Streams {
                     console.error(JSON.stringify(e));
                 }
             });
-            this.redisSubscriber.on('message', () => __awaiter(this, void 0, void 0, function* () {
-                processMessage();
+            this.redisSubscriber.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
+                yield processMessage();
             }));
             return observable;
         }
@@ -259,7 +263,7 @@ export class Streams {
      * @param streamName
      */
     republishUnprocessedEvents(eventName) {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const streamName = `${eventName}:${this.consumerGroupName}`;
             const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'STREAMS', streamName, '>');
             if (result) {
@@ -292,7 +296,7 @@ export class Streams {
      * await streams.recoverCrashedConsumerMessages('order.created', 10000);
      */
     recoverCrashedConsumerMessages(eventName, idleTimeout) {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const streamName = `${eventName}:${this.consumerGroupName}`;
             const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName));
             if (!pendingMessages)
@@ -338,7 +342,7 @@ export class Streams {
      * }
      */
     close() {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             this.clearSubscribedEvents();
             if (this.redisPublisher) {
                 yield this.redisPublisher.quit();
@@ -355,7 +359,7 @@ export class Streams {
         });
     }
     clearSubscribedEvents() {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             console.log(`${this.eventsListened.length} events to be cleared`);
             for (const eventName of this.eventsListened) {
                 yield this.redisGroups.srem(`consumerGroups:${eventName}`, this.consumerGroupName);
@@ -363,18 +367,12 @@ export class Streams {
         });
     }
     registerConsumerGroup(eventName) {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             yield this.redisGroups.sadd(`consumerGroups:${eventName}`, this.consumerGroupName);
         });
     }
-    getAllConsumerGroups(eventName) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const consumerGroups = yield this.redisGroups.smembers(`consumerGroups:${eventName}`);
-            return consumerGroups;
-        });
-    }
     cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
-        return __awaiter(this, void 0, void 0, function* () {
+        return tslib_1.__awaiter(this, void 0, void 0, function* () {
             const streamName = `${eventName}:${this.consumerGroupName}`;
             const cleanupThreshold = Date.now() - interval;
             const acknowledgedMessages = yield this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
@@ -391,3 +389,4 @@ export class Streams {
         });
     }
 }
+exports.Streams = Streams;

package/src/lib/redis/types.js

@@ -1 +1,2 @@
-export {};
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });