@jetit/publisher 1.6.0 → 1.6.2

package/README.md

@@ -2,6 +2,164 @@
 
 publisher is a library for implementing an event-driven architecture using Redis PUB/SUB and Redis Streams. It provides a simple and scalable mechanism for publishing and consuming events in real-time, and supports features such as message deduplication, consumer group management, and scheduled event publishing.
 
+## IMPORTANT NOTE
+This project currently does not have a means to clean up inactive consumers. This means that if you have a consumer that is no longer active, it will continue to receive events until it is removed from the consumer group. This is a known issue and will be addressed in a future release. A workaround is to use the following code to remove inactive consumers from the consumer group as part of your process cleanup:
+
+```javascript
+const ioredis = require(`ioredis`);
+
+const env = process.env;
+
+async function bootstrap() {
+    const connection = new ioredis.Redis({
+        host: env.REDIS_HOST,
+        port: parseInt(env.REDIS_PORT),
+    });
+    console.log(`Redis Connection Status ${connection.status}`);
+    await waitForSeconds(0.3);
+    console.log(`Redis Connection Status ${connection.status}`);
+    const instanceUniqueId = env.INSTANCE_ID;
+    if (!instanceUniqueId) {
+        console.log(`Unique instance ID is not available`);
+        return;
+    }
+    console.log(`Instance Unique ID : ${instanceUniqueId}`);
+    const consumerGroupName = await getConsumerGroupName(connection, instanceUniqueId);
+    if (!consumerGroupName) {
+        console.log(`Consumer is not available, so graceful shutdown is not necessary.`);
+        return;
+    }
+    console.log(`Consumer Group Name : ${consumerGroupName}`);
+    const instanceId = getInstanceId(consumerGroupName.slice(3), instanceUniqueId);
+    console.log(`Instance ID : ${instanceId}`);
+    const subscribedEvents = await getAllEventsForInstance(connection, instanceId);
+    console.log(`Subscribed Events : ${JSON.stringify(subscribedEvents)}`);
+    await clearSubscribedEvents(connection, consumerGroupName, instanceId, subscribedEvents);
+    await deleteConsumerGroupNameForInstance(connection, instanceId);
+    await deleteAllEventsFroInstance(connection, instanceId);
+}
+
+/**
+ *
+ * @param {ioredis.Redis|ioredis.Cluster} connection
+ * @param {string} consumerGroupName
+ * @param {string} instanceId
+ * @param {Array<string>} events
+ */
+async function clearSubscribedEvents(connection, consumerGroupName, instanceId, events) {
+    return Promise.all(
+        events.map(async (eventName) => {
+            console.log(`${eventName} is being cleared in publisher`);
+            const streamName = `${eventName}:${consumerGroupName}`;
+            console.log(`${streamName} is being removed.`);
+            await connection.srem(`${eventName}`, consumerGroupName);
+            console.log(`${eventName} is removed from ${consumerGroupName}`);
+            // Releasing all claims based on info from: https://redis.io/commands/xgroup-delconsumer/
+            await releaseAllClaims(connection, streamName, consumerGroupName, instanceId);
+            console.log(`${eventName} removes all claims`);
+            await connection.xgroup(`DELCONSUMER`, streamName, consumerGroupName, instanceId);
+            console.log(`${eventName} is deleted as a consumer from ${consumerGroupName}, ${instanceId}`);
+        })
+    );
+}
+
+/**
+ *
+ * @param {ioredis.Redis|ioredis.Cluster} connection
+ * @param {string} streamName
+ * @param {string} consumerGroupName
+ * @param {string} instanceId
+ */
+async function releaseAllClaims(connection, streamName, consumerGroupName, instanceId) {
+    /**
+     * Retrieve the pending messages for the consumer. Note this only fetches the last
+     * 10000 events assigned to this consumer. This function has been modified to make sure
+     * that there is a temp instance that claims all this messages
+     */
+    const pendingMessages = await connection.xpending(streamName, consumerGroupName, `-`, `+`, 10000, instanceId);
+
+    if (pendingMessages && pendingMessages.length > 0) {
+        console.log(`${pendingMessages.length} messages to clean up.`);
+        const transaction = connection.multi({ pipeline: true });
+        const tempConsumerId = `${consumerGroupName}-temp`;
+        for (const [messageId] of pendingMessages) {
+            transaction.xclaim(streamName, consumerGroupName, tempConsumerId, 10, messageId);
+        }
+        await transaction.exec();
+    }
+}
+
+/**
+ *
+ * @param {string} serviceName
+ * @param {string} instanceUniqueId
+ */
+function getInstanceId(serviceName, instanceUniqueId) {
+    const instanceId = `${serviceName}:${instanceUniqueId}`;
+    console.log(`Generated Instance ID : ${instanceId}`);
+    return instanceId;
+}
+
+/**
+ *
+ * @param {ioredis.Redis|ioredis.Cluster} serviceName
+ * @param {string} instanceId
+ * @returns {Promise<string>} consumer group name
+ */
+async function getConsumerGroupName(connection, instanceId) {
+    const key = `instance:${instanceId}:consumerGroupName`;
+    console.log(`Get consumer group name called for key : ${key}`);
+    return await connection.get(key);
+}
+
+/**
+ *
+ * @param {ioredis.Redis|ioredis.Cluster} connection
+ * @param {string} instanceId
+ * @returns
+ */
+async function deleteConsumerGroupNameForInstance(connection, instanceId) {
+    return await connection.del(`instance:${instanceId}:consumerGroupName`);
+}
+
+/**
+ *
+ * @param {ioredis.Redis|ioredis.Cluster} connection
+ * @param {string} instanceId
+ * @returns {Promise<Array<events>>} subscribed events for this instance
+ */
+async function getAllEventsForInstance(connection, instanceId) {
+    const key = `instance:${instanceId}:subscribedEvents`;
+    console.log(`Get consumer group events : ${key}`);
+    return (await connection.sscan(key, 0))[1];
+}
+
+/**
+ *
+ * @param {ioredis.Redis|ioredis.Cluster} connection
+ * @param {string} instanceId
+ */
+async function deleteAllEventsFroInstance(connection, instanceId) {
+    return await connection.del(`instance:${instanceId}:subscribedEvents`);
+}
+
+async function waitForSeconds(seconds = 10) {
+    return new Promise((res, _) => setTimeout(() => res(), seconds * 1000));
+}
+
+/**
+ * Start
+ */
+
+bootstrap()
+    .then(() => process.exit(0))
+    .catch((e) => {
+        console.error(e);
+        process.exit(1);
+    });
+
+```
+
 ## Simple Example
 
 ```typescript

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@jetit/publisher",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "0.0.11",
     "ioredis": "^5.3.0",
-    "rxjs": "^7.0.0"
+    "rxjs": "^7.8.0"
   },
   "peerDependencies": {
     "tslib": "2.5.0"

package/src/lib/redis/streams.js

@@ -223,10 +223,20 @@ class Streams {
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
         /** This gets called the first time the stream is registered to pickup any messages from the previous subscription */
         const streamName = `${eventName}:${this.consumerGroupName}`;
-        const processMessage = () => tslib_1.__awaiter(this, void 0, void 0, function* () {
+        const processMessage = (redisClient) => tslib_1.__awaiter(this, void 0, void 0, function* () {
+            const racePr = new Promise((resolve, _) => {
+                setTimeout(resolve, 1100, 'RACE');
+            });
             console.log(`PUBLISHER: processMessage called for ${streamName} cgn: ${this.consumerGroupName} inst: ${this.instanceId}`);
             try {
-                const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
+                const prResult = yield Promise.race([
+                    redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 1000, 'STREAMS', streamName, '>'),
+                    racePr,
+                ]);
+                console.log(`PUBLISHER: Promise race resolved with ${JSON.stringify(prResult)}`);
+                if (prResult && prResult === 'RACE')
+                    return;
+                const result = prResult;
                 console.log(`PUBLISHER: XREADGROUP returned with ${JSON.stringify(result[0])}`);
                 if (result) {
                     const [, streamMessages] = result[0];
@@ -236,13 +246,13 @@ class Streams {
                         const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
                         if (isDuplicate) {
                             console.warn(`Duplicate message detected: ${messageId}`);
-                            yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
+                            yield redisClient.xack(streamName, this.consumerGroupName, id);
                             continue;
                         }
                         bs.next(eventData);
                         const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
                         const currentTime = Date.now();
-                        const transaction = this.redisGroups.multi({ pipeline: true });
+                        const transaction = redisClient.multi({ pipeline: true });
                         transaction.zadd(pmKey, currentTime, messageId);
                         transaction.xack(streamName, this.consumerGroupName, id);
                         transaction.zadd(`ack:${streamName}`, Date.now(), id);
@@ -270,7 +280,7 @@ class Streams {
             });
             eventStreamClient.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
                 console.log(`PUBLISHER: Stream Notification Recieved for event ${eventName}`);
-                yield processMessage();
+                yield processMessage(this.redisGroups);
             }));
             this.scanAndClaimAUnclaimedMessage(streamName)
                 .then()