@cap-js-community/event-queue 1.8.1 → 1.8.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.8.1",
+  "version": "1.8.3",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -75,13 +75,19 @@
         "disableRedis": false
       },
       "[test]": {
-        "isEventQueueActive": true,
+        "isEventQueueActive": false,
         "registerAsEventProcessor": false,
         "updatePeriodicEvents": false,
         "insertEventsBeforeCommit": false
       }
     },
     "requires": {
+      "redis-eventQueue": {
+        "options": {},
+        "vcap": {
+          "label": "redis-cache"
+        }
+      },
       "event-queue": {
         "model": "@cap-js-community/event-queue"
       }

package/src/config.js

@@ -113,7 +113,7 @@ class Config {
   }
 
   _checkRedisIsBound() {
-    return !!this.#env.redisCredentialsFromEnv;
+    return !!this.#env.redisRequires?.credentials;
   }
 
   shouldBeProcessedInThisApplication(type, subType) {

package/src/initialize.js

@@ -10,7 +10,7 @@ const VError = require("verror");
 const runner = require("./runner/runner");
 const dbHandler = require("./dbHandler");
 const config = require("./config");
-const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redis/redisSub");
+const redisSub = require("./redis/redisSub");
 const redis = require("./shared/redis");
 const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 const { getAllTenantIds } = require("./shared/cdsHelper");
@@ -22,6 +22,7 @@ const readFileAsync = promisify(fs.readFile);
 
 const VERROR_CLUSTER_NAME = "EventQueueInitialization";
 const COMPONENT = "eventQueue/initialize";
+const TIMEOUT_SHUTDOWN = 2500;
 
 const CONFIG_VARS = [
   ["configFilePath", null],
@@ -148,7 +149,7 @@ const registerEventProcessors = () => {
   const errorHandler = (err) => cds.log(COMPONENT).error("error during init runner", err);
 
   if (config.redisEnabled) {
-    initEventQueueRedisSubscribe();
+    redisSub.initEventQueueRedisSubscribe();
     config.attachConfigChangeHandler();
     if (config.isMultiTenancy) {
       runner.multiTenancyRedis().catch(errorHandler);
@@ -186,9 +187,24 @@ const mixConfigVarsWithEnv = (options) => {
 };
 
 const registerCdsShutdown = () => {
+  const isTestProfile = cds.env.profiles.find((profile) => profile.includes("test"));
+  if (isTestProfile) {
+    return;
+  }
   cds.on("shutdown", async () => {
-    await distributedLock.shutdownHandler();
-    await Promise.allSettled([redis.closeMainClient(), closeSubscribeClient()]);
+    return await new Promise((resolve) => {
+      const timeoutRef = setTimeout(() => {
+        clearTimeout(timeoutRef);
+        cds.log(COMPONENT).info("shutdown timeout reached - some locks might not have been released!");
+        resolve();
+      }, TIMEOUT_SHUTDOWN);
+      distributedLock.shutdownHandler().then(() =>
+        Promise.allSettled([redis.closeMainClient(), redis.closeSubscribeClient()]).then((result) => {
+          clearTimeout(timeoutRef);
+          resolve(result);
+        })
+      );
+    });
   });
 };
 

package/src/redis/redisSub.js

@@ -9,12 +9,12 @@ const common = require("../shared/common");
 
 const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
 const COMPONENT_NAME = "/eventQueue/redisSub";
-let subscriberClientPromise;
 
 const initEventQueueRedisSubscribe = () => {
-  if (subscriberClientPromise || !config.redisEnabled) {
+  if (initEventQueueRedisSubscribe._initDone || !config.redisEnabled) {
     return;
   }
+  initEventQueueRedisSubscribe._initDone = true;
   redis.subscribeRedisChannel(config.redisOptions, EVENT_MESSAGE_CHANNEL, _messageHandlerProcessEvents);
 };
 
@@ -83,20 +83,8 @@ const _messageHandlerProcessEvents = async (messageData) => {
   }
 };
 
-const closeSubscribeClient = async () => {
-  try {
-    const client = await subscriberClientPromise;
-    if (client?.quit) {
-      await client.quit();
-    }
-  } catch (err) {
-    // ignore errors during shutdown
-  }
-};
-
 module.exports = {
   initEventQueueRedisSubscribe,
-  closeSubscribeClient,
   __: {
     _messageHandlerProcessEvents,
   },

package/src/shared/env.js

@@ -1,25 +1,24 @@
 "use strict";
 
+const cds = require("@sap/cds");
+
 let instance;
 
 class Env {
-  #vcapServices;
   #vcapApplication;
   #vcapApplicationInstance;
 
   constructor() {
     try {
-      this.#vcapServices = JSON.parse(process.env.VCAP_SERVICES);
       this.#vcapApplication = JSON.parse(process.env.VCAP_APPLICATION);
     } catch {
-      this.#vcapServices = {};
       this.#vcapApplication = {};
     }
     this.#vcapApplicationInstance = Number(process.env.CF_INSTANCE_INDEX);
   }
 
-  get redisCredentialsFromEnv() {
-    return this.#vcapServices["redis-cache"]?.[0]?.credentials;
+  get redisRequires() {
+    return cds.requires["redis-eventQueue"] || cds.requires["redis"];
   }
 
   get applicationName() {
@@ -30,14 +29,6 @@ class Env {
     return this.#vcapApplicationInstance;
   }
 
-  set vcapServices(value) {
-    this.#vcapServices = value;
-  }
-
-  get vcapServices() {
-    return this.#vcapServices;
-  }
-
   set applicationInstance(value) {
     this.#vcapApplicationInstance = value;
   }

package/src/shared/redis.js

@@ -9,7 +9,8 @@ const COMPONENT_NAME = "/eventQueue/shared/redis";
 const LOG_AFTER_SEC = 5;
 
 let mainClientPromise;
-const subscriberChannelClientPromise = {};
+let subscriberClientPromise;
+const subscribedChannels = {};
 let lastErrorLog = Date.now();
 
 const createMainClientAndConnect = (options) => {
@@ -18,7 +19,8 @@ const createMainClientAndConnect = (options) => {
   }
 
   const errorHandlerCreateClient = (err) => {
-    cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
+    mainClientPromise?.then?.(_resilientClientClose);
+    cds.log(COMPONENT_NAME).error("error from redis main client:", err);
     mainClientPromise = null;
     setTimeout(() => createMainClientAndConnect(options), LOG_AFTER_SEC * 1000).unref();
   };
@@ -27,24 +29,30 @@ const createMainClientAndConnect = (options) => {
   return mainClientPromise;
 };
 
-const _createClientBase = (redisOptions) => {
+const _createClientBase = (redisOptions = {}) => {
   const env = getEnvInstance();
   try {
-    const credentials = env.redisCredentialsFromEnv;
-    const redisIsCluster = credentials.cluster_mode;
-    const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
-    if (redisIsCluster) {
+    const { credentials, options } = env.redisRequires;
+    const socket = Object.assign(
+      {
+        host: credentials.hostname,
+        tls: !!credentials.tls,
+        port: credentials.port,
+      },
+      options?.socket,
+      redisOptions.socket
+    );
+    const socketOptions = Object.assign({}, options, redisOptions, {
+      password: redisOptions.password ?? options.password ?? credentials.password,
+      socket,
+    });
+    if (credentials.cluster_mode) {
       return redis.createCluster({
-        rootNodes: [{ url }],
-        // https://github.com/redis/node-redis/issues/1782
-        defaults: {
-          password: credentials.password,
-          socket: { tls: credentials.tls },
-          ...redisOptions,
-        },
+        rootNodes: [socketOptions],
+        defaults: socketOptions,
       });
     }
-    return redis.createClient({ url, ...redisOptions });
+    return redis.createClient(socketOptions);
   } catch (err) {
     throw EventQueueError.redisConnectionFailure(err);
   }
@@ -57,7 +65,7 @@ const createClientAndConnect = async (options, errorHandlerCreateClient, isConne
       client.on("error", (err) => {
         const dateNow = Date.now();
         if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
-          cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
+          cds.log(COMPONENT_NAME).error("error redis client:", err);
           lastErrorLog = dateNow;
         }
       });
@@ -78,21 +86,48 @@ const createClientAndConnect = async (options, errorHandlerCreateClient, isConne
 };
 
 const subscribeRedisChannel = (options, channel, subscribeHandler) => {
+  subscribedChannels[channel] = subscribeHandler;
   const errorHandlerCreateClient = (err) => {
     cds.log(COMPONENT_NAME).error(`error from redis client for pub/sub failed for channel ${channel}`, err);
-    subscriberChannelClientPromise[channel] = null;
-    setTimeout(() => subscribeRedisChannel(options, channel, subscribeHandler), LOG_AFTER_SEC * 1000).unref();
+    subscriberClientPromise?.then?.(_resilientClientClose);
+    subscriberClientPromise = null;
+    setTimeout(() => _subscribeChannels(options, subscribedChannels, subscribeHandler), LOG_AFTER_SEC * 1000).unref();
   };
 
-  subscriberChannelClientPromise[channel] = createClientAndConnect(options, errorHandlerCreateClient)
+  _subscribeChannels(options, { [channel]: subscribeHandler }, errorHandlerCreateClient);
+};
+
+const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClient) => {
+  subscriberClientPromise = createClientAndConnect(options, errorHandlerCreateClient)
     .then((client) => {
-      cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
-      client.subscribe(channel, subscribeHandler).catch(errorHandlerCreateClient);
+      for (const channel in subscribedChannels) {
+        const fn = subscribedChannels[channel];
+        client._subscribedChannels ??= {};
+        if (client._subscribedChannels[channel]) {
+          continue;
+        }
+        cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
+        client
+          .subscribe(channel, fn)
+          .then(() => {
+            client._subscribedChannels ??= {};
+            client._subscribedChannels[channel] = 1;
+          })
+          .catch(() => {
+            cds.log(COMPONENT_NAME).error("error subscribe to channel - retrying...");
+            setTimeout(() => _subscribeChannels(options, [channel], fn), LOG_AFTER_SEC * 1000).unref();
+          });
+      }
     })
     .catch((err) => {
       cds
         .log(COMPONENT_NAME)
-        .error(`error from redis client for pub/sub failed during startup - trying to reconnect - ${channel}`, err);
+        .error(
+          `error from redis client for pub/sub failed during startup - trying to reconnect - ${Object.keys(
+            subscribedChannels
+          ).join(", ")}`,
+          err
+        );
     });
 };
 
@@ -102,11 +137,13 @@ const publishMessage = async (options, channel, message) => {
 };
 
 const closeMainClient = async () => {
-  try {
-    await _resilientClientClose(await mainClientPromise);
-  } catch (err) {
-    // ignore errors during shutdown
-  }
+  await _resilientClientClose(await mainClientPromise);
+  cds.log(COMPONENT_NAME).info("main redis client closed!");
+};
+
+const closeSubscribeClient = async () => {
+  await _resilientClientClose(await subscriberClientPromise);
+  cds.log(COMPONENT_NAME).info("subscribe redis client closed!");
 };
 
 const _resilientClientClose = async (client) => {
@@ -115,7 +152,7 @@ const _resilientClientClose = async (client) => {
       await client.quit();
     }
   } catch (err) {
-    // ignore errors during shutdown
+    cds.log(COMPONENT_NAME).info("error during redis close - continuing...", err);
   }
 };
 
@@ -145,5 +182,6 @@ module.exports = {
   subscribeRedisChannel,
   publishMessage,
   closeMainClient,
+  closeSubscribeClient,
   connectionCheck,
 };