@cap-js-community/event-queue 1.3.3 → 1.3.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [
@@ -44,7 +44,7 @@
   "dependencies": {
     "redis": "4.6.13",
     "verror": "1.10.1",
-    "yaml": "2.4.0"
+    "yaml": "2.4.1"
   },
   "devDependencies": {
     "@cap-js/hana": "^0.0.6",
@@ -67,6 +67,16 @@
     "url": "https://github.com/cap-js-community/event-queue.git"
   },
   "cds": {
+    "eventQueue": {
+      "[production]": {
+        "disableRedis": false
+      },
+      "[test]": {
+        "registerAsEventProcessor": false,
+        "isRunnerDeactivated": true,
+        "updatePeriodicEvents": false
+      }
+    },
     "requires": {
       "event-queue": {
         "model": "@cap-js-community/event-queue"

package/src/EventQueueProcessorBase.js

@@ -10,6 +10,7 @@ const { arrayToFlatMap } = require("./shared/common");
 const eventScheduler = require("./shared/eventScheduler");
 const eventConfig = require("./config");
 const PerformanceTracer = require("./shared/PerformanceTracer");
+const { broadcastEvent } = require("./redisPubSub");
 
 const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
 const COMPONENT_NAME = "/eventQueue/EventQueueProcessorBase";
@@ -1002,11 +1003,11 @@ class EventQueueProcessorBase {
   }
 
   async scheduleNextPeriodEvent(queueEntry) {
-    const intervalInSec = this.#eventConfig.interval * 1000;
+    const intervalInMs = this.#eventConfig.interval * 1000;
     const newEvent = {
       type: this.#eventType,
       subType: this.#eventSubType,
-      startAfter: new Date(new Date(queueEntry.startAfter).getTime() + intervalInSec),
+      startAfter: new Date(new Date(queueEntry.startAfter).getTime() + intervalInMs),
     };
     const { relative } = this.#eventSchedulerInstance.calculateOffset(
       this.#eventType,
@@ -1015,11 +1016,13 @@ class EventQueueProcessorBase {
     );
 
     // more than one interval behind - shift tick to keep up
-    if (relative < 0 && Math.abs(relative) >= intervalInSec) {
+    if (relative < 0 && Math.abs(relative) >= intervalInMs) {
+      const plannedStartAfter = newEvent.startAfter;
       newEvent.startAfter = new Date(Date.now() + 5 * 1000);
       this.logger.info("interval adjusted because shifted more than one interval", {
         eventType: this.#eventType,
         eventSubType: this.#eventSubType,
+        plannedStartAfter,
         newStartAfter: newEvent.startAfter,
       });
     }
@@ -1027,7 +1030,7 @@ class EventQueueProcessorBase {
     this.tx._skipEventQueueBroadcase = true;
     await this.tx.run(INSERT.into(this.#config.tableNameEventQueue).entries({ ...newEvent }));
     this.tx._skipEventQueueBroadcase = false;
-    if (intervalInSec < this.#config.runInterval * 1.5) {
+    if (intervalInMs < this.#config.runInterval * 1.5) {
       this.#handleDelayedEvents([newEvent]);
       const { relative: relativeAfterSchedule } = this.#eventSchedulerInstance.calculateOffset(
         this.#eventType,
@@ -1108,6 +1111,18 @@ class EventQueueProcessorBase {
     this.__processTx = null;
   }
 
+  broadCastEvent() {
+    setTimeout(() => {
+      broadcastEvent(this.__baseContext.tenant, this.#eventType, this.#eventSubType).catch((err) => {
+        this.logger.error("could not execute scheduled event", err, {
+          tenantId: this.__baseContext.tenant,
+          type: this.#eventType,
+          subType: this.#eventSubType,
+        });
+      });
+    }, 1000).unref();
+  }
+
   get logger() {
     return this.__logger ?? this.__baseLogger;
   }

package/src/config.js

@@ -99,7 +99,8 @@ class Config {
   }
 
   checkRedisEnabled() {
-    this.#redisEnabled = !this.#disableRedis && this._checkRedisIsBound() && this.#env.isOnCF;
+    this.#redisEnabled = !this.#disableRedis && this._checkRedisIsBound();
+    return this.#redisEnabled;
   }
 
   attachConfigChangeHandler() {

package/src/initialize.js

@@ -12,7 +12,7 @@ const runner = require("./runner");
 const dbHandler = require("./dbHandler");
 const config = require("./config");
 const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
-const { closeMainClient } = require("./shared/redis");
+const redis = require("./shared/redis");
 const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 const { getAllTenantIds } = require("./shared/cdsHelper");
 const { EventProcessingStatus } = require("./constants");
@@ -33,7 +33,7 @@ const CONFIG_VARS = [
   ["runInterval", 25 * 60 * 1000],
   ["tableNameEventQueue", BASE_TABLES.EVENT],
   ["tableNameEventLock", BASE_TABLES.LOCK],
-  ["disableRedis", false],
+  ["disableRedis", true],
   ["skipCsnCheck", false],
   ["updatePeriodicEvents", true],
   ["thresholdLoggingEventProcessing", 50],
@@ -84,14 +84,19 @@ const initialize = async ({
   );
 
   const logger = cds.log(COMPONENT);
-  config.checkRedisEnabled();
+  const redisEnabled = config.checkRedisEnabled();
+  let resolveFn;
+  let initFinished = new Promise((resolve) => (resolveFn = resolve));
   cds.on("connect", (service) => {
     if (service.name === "db") {
       config.processEventsAfterPublish && dbHandler.registerEventQueueDbHandler(service);
       config.cleanupLocksAndEventsForDev && registerCleanupForDevDb().catch(() => {});
-      registerEventProcessors();
+      initFinished.then(registerEventProcessors);
     }
   });
+  if (redisEnabled) {
+    config.redisEnabled = await redis.connectionCheck();
+  }
   config.fileContent = await readConfigFromFile(config.configFilePath);
 
   !config.skipCsnCheck && (await csnCheck());
@@ -105,6 +110,7 @@ const initialize = async ({
     redisEnabled: config.redisEnabled,
     runInterval: config.runInterval,
   });
+  resolveFn();
 };
 
 const readConfigFromFile = async (configFilepath) => {
@@ -219,7 +225,7 @@ const mixConfigVarsWithEnv = (...args) => {
 
 const registerCdsShutdown = () => {
   cds.on("shutdown", async () => {
-    await Promise.allSettled([closeMainClient(), closeSubscribeClient()]);
+    await Promise.allSettled([redis.closeMainClient(), closeSubscribeClient()]);
   });
 };
 

package/src/processEventQueue.js

@@ -117,6 +117,7 @@ const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime
   if (new Date(startTime.getTime() + config.runInterval) > new Date()) {
     return true;
   }
+
   eventTypeInstance.logTimeExceeded(iterationCounter);
   return false;
 };

package/src/redisPubSub.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const { promisify } = require("util");
+
 const cds = require("@sap/cds");
 
 const redis = require("./shared/redis");
@@ -10,7 +12,10 @@ const { getSubdomainForTenantId } = require("./shared/cdsHelper");
 
 const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
 const COMPONENT_NAME = "/eventQueue/redisPubSub";
+const TRIES_FOR_PUBLISH_PERIODIC_EVENT = 10;
+const SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT = 30 * 1000;
 
+const wait = promisify(setTimeout);
 let subscriberClientPromise;
 
 const initEventQueueRedisSubscribe = () => {
@@ -107,23 +112,34 @@ const broadcastEvent = async (tenantId, type, subType) => {
       }
       return;
     }
-    const result = await checkLockExistsAndReturnValue(
-      new cds.EventContext({ tenant: tenantId }),
-      [type, subType].join("##")
-    );
-    if (result) {
-      logger.debug("skip publish redis event as no lock is available", {
+    const eventConfig = config.getEventConfig(type, subType);
+    for (let i = 0; i < TRIES_FOR_PUBLISH_PERIODIC_EVENT; i++) {
+      const result = await checkLockExistsAndReturnValue(
+        new cds.EventContext({ tenant: tenantId }),
+        [type, subType].join("##")
+      );
+      if (result) {
+        logger.debug("skip publish redis event as no lock is available", {
+          type,
+          subType,
+          index: i,
+          isPeriodic: eventConfig.isPeriodic,
+          waitInterval: SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT,
+        });
+        if (!eventConfig.isPeriodic) {
+          break;
+        }
+        await wait(SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT);
+        continue;
+      }
+      logger.debug("publishing redis event", {
+        tenantId,
         type,
         subType,
       });
-      return;
+      await redis.publishMessage(EVENT_MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
+      break;
     }
-    logger.debug("publishing redis event", {
-      tenantId,
-      type,
-      subType,
-    });
-    await redis.publishMessage(EVENT_MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
   } catch (err) {
     logger.error("publish event failed!", err, {
       tenantId,

package/src/shared/env.js

@@ -4,12 +4,9 @@ let instance;
 
 class Env {
   #isLocal;
-  #isOnCF;
   #vcapServices;
 
   constructor() {
-    this.#isLocal = process.env.USER !== "vcap";
-    this.#isOnCF = !this.#isLocal;
     try {
       this.#vcapServices = JSON.parse(process.env.VCAP_SERVICES);
     } catch {
@@ -28,13 +25,6 @@ class Env {
     return this.#isLocal;
   }
 
-  set isOnCF(value) {
-    this.#isOnCF = value;
-  }
-  get isOnCF() {
-    return this.#isOnCF;
-  }
-
   set vcapServices(value) {
     this.#vcapServices = value;
   }

package/src/shared/redis.js

@@ -26,29 +26,23 @@ const createMainClientAndConnect = () => {
 
 const _createClientBase = () => {
   const env = getEnvInstance();
-  if (env.isOnCF) {
-    try {
-      const credentials = env.getRedisCredentialsFromEnv();
-      const redisIsCluster = credentials.cluster_mode;
-      const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
-      if (redisIsCluster) {
-        return redis.createCluster({
-          rootNodes: [{ url }],
-          // https://github.com/redis/node-redis/issues/1782
-          defaults: {
-            password: credentials.password,
-            socket: { tls: credentials.tls },
-          },
-        });
-      }
-      return redis.createClient({ url });
-    } catch (err) {
-      throw EventQueueError.redisConnectionFailure(err);
+  try {
+    const credentials = env.getRedisCredentialsFromEnv();
+    const redisIsCluster = credentials.cluster_mode;
+    const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
+    if (redisIsCluster) {
+      return redis.createCluster({
+        rootNodes: [{ url }],
+        // https://github.com/redis/node-redis/issues/1782
+        defaults: {
+          password: credentials.password,
+          socket: { tls: credentials.tls },
+        },
+      });
     }
-  } else {
-    return redis.createClient({
-      socket: { reconnectStrategy: _localReconnectStrategy },
-    });
+    return redis.createClient({ url });
+  } catch (err) {
+    throw EventQueueError.redisConnectionFailure(err);
   }
 };
 
@@ -93,11 +87,16 @@ const publishMessage = async (channel, message) => {
   return await client.publish(channel, message);
 };
 
-const _localReconnectStrategy = () => EventQueueError.redisNoReconnect();
-
 const closeMainClient = async () => {
   try {
-    const client = await mainClientPromise;
+    await _resilientClientClose(await mainClientPromise);
+  } catch (err) {
+    // ignore errors during shutdown
+  }
+};
+
+const _resilientClientClose = async (client) => {
+  try {
     if (client?.quit) {
       await client.quit();
     }
@@ -106,10 +105,28 @@ const closeMainClient = async () => {
   }
 };
 
+const connectionCheck = async () => {
+  return new Promise((resolve, reject) => {
+    createClientAndConnect(reject)
+      .then((client) => {
+        if (client) {
+          _resilientClientClose(client);
+          resolve();
+        } else {
+          reject(new Error());
+        }
+      })
+      .catch(reject);
+  })
+    .then(() => true)
+    .catch(() => false);
+};
+
 module.exports = {
   createClientAndConnect,
   createMainClientAndConnect,
   subscribeRedisChannel,
   publishMessage,
   closeMainClient,
+  connectionCheck,
 };