@cap-js-community/event-queue 1.3.5 → 1.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.3.5",
+  "version": "1.4.0",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [

package/src/EventQueueProcessorBase.js

@@ -10,7 +10,7 @@ const { arrayToFlatMap } = require("./shared/common");
 const eventScheduler = require("./shared/eventScheduler");
 const eventConfig = require("./config");
 const PerformanceTracer = require("./shared/PerformanceTracer");
-const { broadcastEvent } = require("./redisPubSub");
+const { broadcastEvent } = require("./redis/redisPub");
 
 const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
 const COMPONENT_NAME = "/eventQueue/EventQueueProcessorBase";
@@ -1113,7 +1113,7 @@ class EventQueueProcessorBase {
 
   broadCastEvent() {
     setTimeout(() => {
-      broadcastEvent(this.__baseContext.tenant, this.#eventType, this.#eventSubType).catch((err) => {
+      broadcastEvent(this.__baseContext.tenant, { type: this.#eventType, subType: this.#eventSubType }).catch((err) => {
         this.logger.error("could not execute scheduled event", err, {
           tenantId: this.__baseContext.tenant,
           type: this.#eventType,

package/src/config.js

@@ -34,6 +34,11 @@ const BASE_PERIODIC_EVENTS = [
   },
 ];
 
+const BASE_TABLES = {
+  EVENT: "sap.eventqueue.Event",
+  LOCK: "sap.eventqueue.Lock",
+};
+
 class Config {
   #logger;
   #config;
@@ -61,6 +66,7 @@ class Config {
   #userId;
   #enableTxConsistencyCheck;
   #cleanupLocksAndEventsForDev;
+  #redisOptions;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -105,7 +111,7 @@ class Config {
 
   attachConfigChangeHandler() {
     this.#attachBlockListChangeHandler();
-    redis.subscribeRedisChannel(REDIS_CONFIG_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.#redisOptions, REDIS_CONFIG_CHANNEL, (messageData) => {
       try {
         const { key, value } = JSON.parse(messageData);
         if (this[key] !== value) {
@@ -125,13 +131,13 @@ class Config {
       this.#logger.info("redis not connected, config change won't be published", { key, value });
       return;
     }
-    redis.publishMessage(REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
+    redis.publishMessage(this.#redisOptions, REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
       this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
     });
   }
 
   #attachBlockListChangeHandler() {
-    redis.subscribeRedisChannel(REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.#redisOptions, REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
       try {
         const { command, key, tenant } = JSON.parse(messageData);
         if (command === COMMAND_BLOCK) {
@@ -160,7 +166,11 @@ class Config {
     }
 
     redis
-      .publishMessage(REDIS_CONFIG_BLOCKLIST_CHANNEL, JSON.stringify({ command: COMMAND_BLOCK, key, tenant }))
+      .publishMessage(
+        this.#redisOptions,
+        REDIS_CONFIG_BLOCKLIST_CHANNEL,
+        JSON.stringify({ command: COMMAND_BLOCK, key, tenant })
+      )
       .catch((error) => {
         this.#logger.error(`publishing config block failed key: ${key}`, error);
       });
@@ -189,7 +199,11 @@ class Config {
     }
 
     redis
-      .publishMessage(REDIS_CONFIG_BLOCKLIST_CHANNEL, JSON.stringify({ command: COMMAND_UNBLOCK, key, tenant }))
+      .publishMessage(
+        this.#redisOptions,
+        REDIS_CONFIG_BLOCKLIST_CHANNEL,
+        JSON.stringify({ command: COMMAND_UNBLOCK, key, tenant })
+      )
       .catch((error) => {
         this.#logger.error(`publishing config block failed key: ${key}`, error);
       });
@@ -386,19 +400,11 @@ class Config {
   }
 
   get tableNameEventQueue() {
-    return this.#tableNameEventQueue;
-  }
-
-  set tableNameEventQueue(value) {
-    this.#tableNameEventQueue = value;
+    return BASE_TABLES.EVENT;
   }
 
   get tableNameEventLock() {
-    return this.#tableNameEventLock;
-  }
-
-  set tableNameEventLock(value) {
-    this.#tableNameEventLock = value;
+    return BASE_TABLES.LOCK;
   }
 
   set configFilePath(value) {
@@ -489,6 +495,14 @@ class Config {
     return this.#cleanupLocksAndEventsForDev;
   }
 
+  set redisOptions(value) {
+    this.#redisOptions = value;
+  }
+
+  get redisOptions() {
+    return this.#redisOptions;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }

package/src/dbHandler.js

@@ -2,7 +2,7 @@
 
 const cds = require("@sap/cds");
 
-const { broadcastEvent } = require("./redisPubSub");
+const { broadcastEvent } = require("./redis/redisPub");
 const config = require("./config");
 
 const COMPONENT_NAME = "/eventQueue/dbHandler";
@@ -31,14 +31,17 @@ const registerEventQueueDbHandler = (dbService) => {
 
     eventCombinations.length &&
       req.on("succeeded", () => {
-        for (const eventCombination of eventCombinations) {
-          broadcastEvent(req.tenant, ...eventCombination.split("##")).catch((err) => {
-            cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
-              tenant: req.tenant,
-              eventCombination,
-            });
+        const events = eventCombinations.map((eventCombination) => {
+          const [type, subType] = eventCombination.split("##");
+          return { type, subType };
+        });
+
+        broadcastEvent(req.tenant, events).catch((err) => {
+          cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
+            tenant: req.tenant,
+            events,
           });
-        }
+        });
       });
   });
 };

package/src/initialize.js

@@ -7,11 +7,10 @@ const cds = require("@sap/cds");
 const yaml = require("yaml");
 const VError = require("verror");
 
-const EventQueueError = require("./EventQueueError");
-const runner = require("./runner");
+const runner = require("./runner/runner");
 const dbHandler = require("./dbHandler");
 const config = require("./config");
-const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
+const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redis/redisSub");
 const redis = require("./shared/redis");
 const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 const { getAllTenantIds } = require("./shared/cdsHelper");
@@ -21,26 +20,21 @@ const readFileAsync = promisify(fs.readFile);
 
 const VERROR_CLUSTER_NAME = "EventQueueInitialization";
 const COMPONENT = "eventQueue/initialize";
-const BASE_TABLES = {
-  EVENT: "sap.eventqueue.Event",
-  LOCK: "sap.eventqueue.Lock",
-};
+
 const CONFIG_VARS = [
   ["configFilePath", null],
   ["registerAsEventProcessor", true],
   ["processEventsAfterPublish", true],
   ["isEventQueueActive", true],
   ["runInterval", 25 * 60 * 1000],
-  ["tableNameEventQueue", BASE_TABLES.EVENT],
-  ["tableNameEventLock", BASE_TABLES.LOCK],
   ["disableRedis", true],
-  ["skipCsnCheck", false],
   ["updatePeriodicEvents", true],
   ["thresholdLoggingEventProcessing", 50],
   ["useAsCAPOutbox", false],
   ["userId", null],
   ["enableTxConsistencyCheck", false],
   ["cleanupLocksAndEventsForDev", false],
+  ["redisOptions", {}],
 ];
 
 const initialize = async ({
@@ -49,16 +43,14 @@ const initialize = async ({
   processEventsAfterPublish,
   isEventQueueActive,
   runInterval,
-  tableNameEventQueue,
-  tableNameEventLock,
   disableRedis,
-  skipCsnCheck,
   updatePeriodicEvents,
   thresholdLoggingEventProcessing,
   useAsCAPOutbox,
   userId,
   enableTxConsistencyCheck,
   cleanupLocksAndEventsForDev,
+  redisOptions,
 } = {}) => {
   if (config.initialized) {
     return;
@@ -71,16 +63,14 @@ const initialize = async ({
     processEventsAfterPublish,
     isEventQueueActive,
     runInterval,
-    tableNameEventQueue,
-    tableNameEventLock,
     disableRedis,
-    skipCsnCheck,
     updatePeriodicEvents,
     thresholdLoggingEventProcessing,
     useAsCAPOutbox,
     userId,
     enableTxConsistencyCheck,
-    cleanupLocksAndEventsForDev
+    cleanupLocksAndEventsForDev,
+    redisOptions
   );
 
   const logger = cds.log(COMPONENT);
@@ -95,12 +85,10 @@ const initialize = async ({
     }
   });
   if (redisEnabled) {
-    config.redisEnabled = await redis.connectionCheck();
+    config.redisEnabled = await redis.connectionCheck(config.redisOptions);
   }
   config.fileContent = await readConfigFromFile(config.configFilePath);
 
-  !config.skipCsnCheck && (await csnCheck());
-
   monkeyPatchCAPOutbox();
   registerCdsShutdown();
   logger.info("event queue initialized", {
@@ -172,50 +160,6 @@ const monkeyPatchCAPOutbox = () => {
   }
 };
 
-const csnCheck = async () => {
-  cds.on("loaded", async (csn) => {
-    if (csn.namespace === "cds.xt") {
-      return;
-    }
-    const eventCsn = csn.definitions[config.tableNameEventQueue];
-    if (!eventCsn) {
-      throw EventQueueError.missingTableInCsn(config.tableNameEventQueue);
-    }
-
-    const lockCsn = csn.definitions[config.tableNameEventLock];
-    if (!lockCsn) {
-      throw EventQueueError.missingTableInCsn(config.tableNameEventLock);
-    }
-
-    if (config.tableNameEventQueue === BASE_TABLES.EVENT && config.tableNameEventLock === BASE_TABLES.LOCK) {
-      return; // no need to check base tables
-    }
-
-    const baseEvent = csn.definitions["sap.eventqueue.Event"];
-    const baseLock = csn.definitions["sap.eventqueue.Lock"];
-
-    checkCustomTable(baseEvent, eventCsn);
-    checkCustomTable(baseLock, lockCsn);
-  });
-};
-
-const checkCustomTable = (baseCsn, customCsn) => {
-  for (const columnName in baseCsn.elements) {
-    if (!customCsn.elements[columnName]) {
-      throw EventQueueError.missingElementInTable(customCsn.name, columnName);
-    }
-
-    if (
-      customCsn.elements[columnName].type !== "cds.Association" &&
-      customCsn.elements[columnName].type !== baseCsn.elements[columnName].type &&
-      columnName === "status" &&
-      customCsn.elements[columnName].type !== "cds.Integer"
-    ) {
-      throw EventQueueError.typeMismatchInTable(customCsn.name, columnName);
-    }
-  }
-};
-
 const mixConfigVarsWithEnv = (...args) => {
   CONFIG_VARS.forEach(([configName, defaultValue], index) => {
     const configValue = args[index];

package/src/redis/redisPub.js

@@ -0,0 +1,93 @@
+"use strict";
+
+const { promisify } = require("util");
+
+const cds = require("@sap/cds");
+
+const redis = require("../shared/redis");
+const { checkLockExistsAndReturnValue } = require("../shared/distributedLock");
+const config = require("../config");
+const { getSubdomainForTenantId } = require("../shared/cdsHelper");
+const { runEventCombinationForTenant } = require("../runner/runnerHelper");
+
+const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
+const COMPONENT_NAME = "/eventQueue/redisPub";
+const TRIES_FOR_PUBLISH_PERIODIC_EVENT = 10;
+const SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT = 30 * 1000;
+
+const wait = promisify(setTimeout);
+
+const broadcastEvent = async (tenantId, events) => {
+  const logger = cds.log(COMPONENT_NAME);
+  events = Array.isArray(events) ? events : [events];
+  try {
+    if (!config.isEventQueueActive) {
+      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {});
+      return;
+    }
+    if (!config.redisEnabled) {
+      if (config.registerAsEventProcessor) {
+        let context = {};
+        if (tenantId) {
+          const subdomain = await getSubdomainForTenantId(tenantId);
+          const user = new cds.User.Privileged(config.userId);
+          context = {
+            // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+            tenant: tenantId,
+            user,
+            http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+          };
+        }
+
+        return await cds.tx(context, async ({ context }) => {
+          for (const { type, subType } of events) {
+            await runEventCombinationForTenant(context, type, subType);
+          }
+        });
+      }
+      return;
+    }
+    for (const { type, subType } of events) {
+      const eventConfig = config.getEventConfig(type, subType);
+      for (let i = 0; i < TRIES_FOR_PUBLISH_PERIODIC_EVENT; i++) {
+        const result = await checkLockExistsAndReturnValue(
+          new cds.EventContext({ tenant: tenantId }),
+          [type, subType].join("##")
+        );
+        if (result) {
+          logger.debug("skip publish redis event as no lock is available", {
+            type,
+            subType,
+            index: i,
+            isPeriodic: eventConfig.isPeriodic,
+            waitInterval: SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT,
+          });
+          if (!eventConfig.isPeriodic) {
+            break;
+          }
+          await wait(SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT);
+          continue;
+        }
+        logger.debug("publishing redis event", {
+          tenantId,
+          type,
+          subType,
+        });
+        await redis.publishMessage(
+          config.redisOptions,
+          EVENT_MESSAGE_CHANNEL,
+          JSON.stringify({ tenantId, type, subType })
+        );
+        break;
+      }
+    }
+  } catch (err) {
+    logger.error("publish events failed!", err, {
+      tenantId,
+    });
+  }
+};
+
+module.exports = {
+  broadcastEvent,
+};

package/src/redis/redisSub.js

@@ -0,0 +1,95 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const redis = require("../shared/redis");
+const config = require("../config");
+const { getSubdomainForTenantId } = require("../shared/cdsHelper");
+const runnerHelper = require("../runner/runnerHelper");
+
+const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
+const COMPONENT_NAME = "/eventQueue/redisSub";
+let subscriberClientPromise;
+
+const initEventQueueRedisSubscribe = () => {
+  if (subscriberClientPromise || !config.redisEnabled) {
+    return;
+  }
+  redis.subscribeRedisChannel(config.redisOptions, EVENT_MESSAGE_CHANNEL, _messageHandlerProcessEvents);
+};
+
+const _messageHandlerProcessEvents = async (messageData) => {
+  const logger = cds.log(COMPONENT_NAME);
+  try {
+    const { tenantId, type, subType } = JSON.parse(messageData);
+    logger.debug("received redis event", {
+      tenantId,
+      type,
+      subType,
+    });
+    if (!config.isEventQueueActive) {
+      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {
+        type,
+        subType,
+      });
+      return;
+    }
+
+    const subdomain = await getSubdomainForTenantId(tenantId);
+    const user = new cds.User.Privileged(config.userId);
+    const tenantContext = {
+      tenant: tenantId,
+      user,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    };
+
+    if (!config.getEventConfig(type, subType)) {
+      if (config.isCapOutboxEvent(type)) {
+        try {
+          const service = await cds.connect.to(subType);
+          cds.outboxed(service);
+        } catch (err) {
+          logger.error("could not connect to outboxed service", err, {
+            type,
+            subType,
+          });
+          return;
+        }
+      } else {
+        logger.error("cannot find configuration for published event. Event won't be processed", {
+          type,
+          subType,
+        });
+        return;
+      }
+    }
+
+    return await cds.tx(tenantContext, async ({ context }) => {
+      return await runnerHelper.runEventCombinationForTenant(context, type, subType);
+    });
+  } catch (err) {
+    logger.error("could not parse event information", {
+      messageData,
+    });
+  }
+};
+
+const closeSubscribeClient = async () => {
+  try {
+    const client = await subscriberClientPromise;
+    if (client?.quit) {
+      await client.quit();
+    }
+  } catch (err) {
+    // ignore errors during shutdown
+  }
+};
+
+module.exports = {
+  initEventQueueRedisSubscribe,
+  closeSubscribeClient,
+  __: {
+    _messageHandlerProcessEvents,
+  },
+};

package/src/runner/openEvents.js

@@ -0,0 +1,52 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const eventConfig = require("../config");
+const { EventProcessingStatus } = require("../constants");
+
+const getOpenQueueEntries = async (tx) => {
+  const startTime = new Date();
+  const refDateStartAfter = new Date(startTime.getTime() + eventConfig.runInterval * 1.2);
+  const entries = await tx.run(
+    SELECT.from(eventConfig.tableNameEventQueue)
+      .where(
+        "( startAfter IS NULL OR startAfter <=",
+        refDateStartAfter.toISOString(),
+        " ) AND ( status =",
+        EventProcessingStatus.Open,
+        "OR ( status =",
+        EventProcessingStatus.Error,
+        ") OR ( status =",
+        EventProcessingStatus.InProgress,
+        "AND lastAttemptTimestamp <=",
+        new Date(startTime.getTime() - eventConfig.globalTxTimeout).toISOString(),
+        ") )"
+      )
+      .columns("type", "subType")
+      .groupBy("type", "subType")
+  );
+
+  const result = [];
+  for (const { type, subType } of entries) {
+    if (type.startsWith("CAP_OUTBOX")) {
+      if (cds.requires[subType]) {
+        result.push({ type, subType });
+      } else {
+        const service = await cds.connect.to(subType).catch(() => {});
+        if (service) {
+          result.push({ type, subType });
+        }
+      }
+    } else {
+      if (eventConfig.getEventConfig(type, subType)) {
+        result.push({ type, subType });
+      }
+    }
+  }
+  return result;
+};
+
+module.exports = {
+  getOpenQueueEntries,
+};

package/src/{runner.js → runner/runner.js}

@@ -1,32 +1,33 @@
 "use strict";
 
 const { randomUUID } = require("crypto");
-const { AsyncResource } = require("async_hooks");
 
 const cds = require("@sap/cds");
 
-const eventQueueConfig = require("./config");
-const { processEventQueue } = require("./processEventQueue");
-const WorkerQueue = require("./shared/WorkerQueue");
-const cdsHelper = require("./shared/cdsHelper");
-const distributedLock = require("./shared/distributedLock");
-const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
-const { getSubdomainForTenantId } = require("./shared/cdsHelper");
-const periodicEvents = require("./periodicEvents");
-const { hashStringTo32Bit } = require("./shared/common");
-const config = require("./config");
-const { Priorities } = require("./constants");
+const eventQueueConfig = require("../config");
+const WorkerQueue = require("../shared/WorkerQueue");
+const cdsHelper = require("../shared/cdsHelper");
+const distributedLock = require("../shared/distributedLock");
+const SetIntervalDriftSafe = require("../shared/SetIntervalDriftSafe");
+const { getSubdomainForTenantId } = require("../shared/cdsHelper");
+const periodicEvents = require("../periodicEvents");
+const { hashStringTo32Bit } = require("../shared/common");
+const config = require("../config");
+const redisPub = require("../redis/redisPub");
+const openEvents = require("./openEvents");
+const { runEventCombinationForTenant } = require("./runnerHelper");
 
 const COMPONENT_NAME = "/eventQueue/runner";
 const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
 const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
-const EVENT_QUEUE_RUN_PERIODIC_EVENT = "EVENT_QUEUE_RUN_PERIODIC_EVENT";
+const EVENT_QUEUE_RUN_REDIS_CHECK = "EVENT_QUEUE_RUN_REDIS_CHECK";
+const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "EVENT_QUEUE_UPDATE_PERIODIC_EVENTS";
 const OFFSET_FIRST_RUN = 10 * 1000;
 
 let tenantIdHash;
 let singleRunDone;
 
-const singleTenant = () => _scheduleFunction(_checkPeriodicEventsSingleTenant, _singleTenantDb);
+const singleTenant = () => _scheduleFunction(_checkPeriodicEventsSingleTenantOneTime, _singleTenantDb);
 
 const multiTenancyDb = () => _scheduleFunction(async () => {}, _multiTenancyDb);
 
@@ -68,19 +69,14 @@ const _scheduleFunction = async (singleRunFn, periodicFn) => {
 
 const _multiTenancyRedis = async () => {
   const logger = cds.log(COMPONENT_NAME);
-  const emptyContext = new cds.EventContext({});
-  logger.info("executing event queue run for multi instance and tenant");
-  const tenantIds = await cdsHelper.getAllTenantIds();
-  await _checkPeriodicEventUpdate(tenantIds);
-
-  const runId = await _acquireRunId(emptyContext);
-
-  if (!runId) {
-    logger.error("could not acquire runId, skip processing events!");
-    return;
+  try {
+    logger.info("executing event queue run for multi instance and tenant");
+    const tenantIds = await cdsHelper.getAllTenantIds();
+    await _checkPeriodicEventUpdate(tenantIds);
+    return await _executeEventsAllTenantsRedis(tenantIds);
+  } catch (err) {
+    logger.info("executing event queue run for multi instance and tenant failed", err);
   }
-
-  return await _executeEventsAllTenants(tenantIds, runId);
 };
 
 const _checkPeriodicEventUpdate = async (tenantIds) => {
@@ -100,80 +96,119 @@ const _checkPeriodicEventUpdate = async (tenantIds) => {
   }
 };
 
-const _executeEventsAllTenants = (tenantIds, runId) => {
-  const events = eventQueueConfig.allEvents;
-  const product = tenantIds.reduce((result, tenantId) => {
-    events.forEach((event) => {
-      result.push([tenantId, event]);
+const _executeEventsAllTenantsRedis = async (tenantIds) => {
+  const logger = cds.log(COMPONENT_NAME);
+  try {
+    // NOTE: do checks for all tenants on the same app instance --> acquire lock tenant independent
+    //       distribute from this instance to all others
+    const dummyContext = new cds.EventContext({});
+    const couldAcquireLock = await distributedLock.acquireLock(dummyContext, EVENT_QUEUE_RUN_REDIS_CHECK, {
+      expiryTime: eventQueueConfig.runInterval * 0.95,
+      tenantScoped: false,
     });
-    return result;
-  }, []);
+    if (!couldAcquireLock) {
+      return;
+    }
 
-  return Promise.allSettled(
-    product.map(async ([tenantId, eventConfig]) => {
-      const subdomain = await getSubdomainForTenantId(tenantId);
-      const user = new cds.User.Privileged(config.userId);
-      const tenantContext = {
-        tenant: tenantId,
-        user,
-        // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-        http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-      };
-      const label = `${eventConfig.type}_${eventConfig.subType}`;
-      return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
-        return await cds.tx(tenantContext, async ({ context }) => {
-          try {
-            const lockId = `${runId}_${label}`;
-            const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-              expiryTime: eventQueueConfig.runInterval * 0.95,
-            });
-            if (!couldAcquireLock) {
-              return;
-            }
-            await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, true);
-          } catch (err) {
-            cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-              tenantId,
-            });
-          }
+    for (const tenantId of tenantIds) {
+      await cds.tx({ tenant: tenantId }, async (tx) => {
+        const entries = await openEvents.getOpenQueueEntries(tx);
+        logger.info("broadcasting events for run", {
+          tenantId,
+          entries: entries.length,
+        });
+        if (!entries.length) {
+          return;
+        }
+        await redisPub.broadcastEvent(tenantId, entries).catch((err) => {
+          logger.error("broadcasting event failed", err, {
+            tenantId,
+            entries: entries.length,
+          });
         });
       });
-    })
-  );
+    }
+  } catch (err) {
+    logger.info("executing event queue run for multi instance and tenant failed", err);
+  }
 };
 
-const _executePeriodicEventsAllTenants = async (tenantIds, runId) => {
-  return await Promise.allSettled(
-    tenantIds.map(async (tenantId) => {
-      const label = `UPDATE_PERIODIC_EVENTS_${tenantId}`;
-      return await WorkerQueue.instance.addToQueue(1, label, Priorities.Low, async () => {
-        try {
-          const subdomain = await getSubdomainForTenantId(tenantId);
-          const user = new cds.User.Privileged(config.userId);
-          const tenantContext = {
-            tenant: tenantId,
-            user,
-            // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-            http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-          };
+const _executeEventsAllTenants = async (tenantIds, runId) => {
+  const promises = [];
+
+  for (const tenantId of tenantIds) {
+    const subdomain = await getSubdomainForTenantId(tenantId);
+    const user = new cds.User.Privileged(config.userId);
+    const tenantContext = {
+      tenant: tenantId,
+      user,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    };
+    const events = await cds.tx(tenantContext, async (tx) => {
+      return await openEvents.getOpenQueueEntries(tx);
+    });
+
+    if (!events.length) {
+      continue;
+    }
 
+    promises.concat(
+      events.map(async (openEvent) => {
+        const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
+        const label = `${eventConfig.type}_${eventConfig.subType}`;
+        return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
           return await cds.tx(tenantContext, async ({ context }) => {
-            const couldAcquireLock = await distributedLock.acquireLock(context, runId, {
-              expiryTime: eventQueueConfig.runInterval * 0.95,
-            });
-            if (!couldAcquireLock) {
-              return;
+            try {
+              const lockId = `${runId}_${label}`;
+              const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+                expiryTime: eventQueueConfig.runInterval * 0.95,
+              });
+              if (!couldAcquireLock) {
+                return;
+              }
+              await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, true);
+            } catch (err) {
+              cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
+                tenantId,
+              });
             }
-            await _checkPeriodicEventsSingleTenant(context);
           });
-        } catch (err) {
-          cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-            tenantId,
+        });
+      })
+    );
+  }
+  return Promise.allSettled(promises);
+};
+
+const _executePeriodicEventsAllTenants = async (tenantIds) => {
+  for (const tenantId of tenantIds) {
+    try {
+      const subdomain = await getSubdomainForTenantId(tenantId);
+      const user = new cds.User.Privileged(config.userId);
+      const tenantContext = {
+        tenant: tenantId,
+        user,
+        // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+        http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+      };
+      await cds.tx(tenantContext, async ({ context }) => {
+        if (!config.redisEnabled) {
+          const couldAcquireLock = await distributedLock.acquireLock(context, EVENT_QUEUE_UPDATE_PERIODIC_EVENTS, {
+            expiryTime: eventQueueConfig.runInterval * 0.95,
           });
+          if (!couldAcquireLock) {
+            return;
+          }
         }
+        await _checkPeriodicEventsSingleTenant(context);
       });
-    })
-  );
+    } catch (err) {
+      cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
+        tenantId,
+      });
+    }
+  }
 };
 
 const _singleTenantDb = async (tenantId) => {
@@ -265,30 +300,6 @@ const _calculateOffsetForFirstRun = async () => {
   return offsetDependingOnLastRun;
 };
 
-const runEventCombinationForTenant = async (context, type, subType, skipWorkerPool) => {
-  try {
-    if (skipWorkerPool) {
-      return await processEventQueue(context, type, subType);
-    } else {
-      const eventConfig = eventQueueConfig.getEventConfig(type, subType);
-      const label = `${type}_${subType}`;
-      return await WorkerQueue.instance.addToQueue(
-        eventConfig.load,
-        label,
-        eventConfig.priority,
-        AsyncResource.bind(async () => await processEventQueue(context, type, subType))
-      );
-    }
-  } catch (err) {
-    const logger = cds.log(COMPONENT_NAME);
-    logger.error("error executing event combination for tenant", err, {
-      tenantId: context.tenant,
-      type,
-      subType,
-    });
-  }
-};
-
 const _multiTenancyDb = async () => {
   const logger = cds.log(COMPONENT_NAME);
   try {
@@ -305,14 +316,29 @@ const _multiTenancyPeriodicEvents = async (tenantIds) => {
   const logger = cds.log(COMPONENT_NAME);
   try {
     logger.info("executing event queue update periodic events");
+
+    if (config.redisEnabled) {
+      const dummyContext = new cds.EventContext({});
+      const couldAcquireLock = await distributedLock.acquireLock(dummyContext, EVENT_QUEUE_UPDATE_PERIODIC_EVENTS, {
+        expiryTime: 60 * 1000, // short living lock --> assume we do not have 2 onboards within 1 minute
+        tenantScoped: false,
+      });
+      if (!couldAcquireLock) {
+        return;
+      }
+    }
+
     tenantIds = tenantIds ?? (await cdsHelper.getAllTenantIds());
-    return await _executePeriodicEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_PERIODIC_EVENT);
+    return await _executePeriodicEventsAllTenants(tenantIds);
   } catch (err) {
     logger.error("Couldn't fetch tenant ids for updating periodic event processing!", err);
   }
 };
 
-const _checkPeriodicEventsSingleTenant = async (context = {}) => {
+const _checkPeriodicEventsSingleTenantOneTime = () =>
+  cds.tx({}, async (tx) => await periodicEvents.checkAndInsertPeriodicEvents(tx.context));
+
+const _checkPeriodicEventsSingleTenant = async (context) => {
   const logger = cds.log(COMPONENT_NAME);
   if (!eventQueueConfig.updatePeriodicEvents || !eventQueueConfig.periodicEvents.length) {
     logger.info("updating of periodic events is disabled or no periodic events configured", {
@@ -326,9 +352,7 @@ const _checkPeriodicEventsSingleTenant = async (context = {}) => {
       tenantId: context.tenant,
       subdomain: context.http?.req.authInfo.getSubdomain(),
     });
-    await cdsHelper.executeInNewTransaction(context, "update-periodic-events", async (tx) => {
-      await periodicEvents.checkAndInsertPeriodicEvents(tx.context);
-    });
+    await periodicEvents.checkAndInsertPeriodicEvents(context);
   } catch (err) {
     logger.error("Couldn't update periodic events for tenant! Next try after defined interval.", err, {
       tenantId: context.tenant,
@@ -341,7 +365,6 @@ module.exports = {
   singleTenant,
   multiTenancyDb,
   multiTenancyRedis,
-  runEventCombinationForTenant,
   __: {
     _singleTenantDb,
     _multiTenancyRedis,

package/src/runner/runnerHelper.js

@@ -0,0 +1,37 @@
+"use strict";
+
+const { AsyncResource } = require("async_hooks");
+
+const cds = require("@sap/cds");
+
+const { processEventQueue } = require("../processEventQueue");
+const eventQueueConfig = require("../config");
+const WorkerQueue = require("../shared/WorkerQueue");
+
+const COMPONENT_NAME = "/eventQueue/runnerHelper";
+
+const runEventCombinationForTenant = async (context, type, subType, skipWorkerPool) => {
+  try {
+    if (skipWorkerPool) {
+      return await processEventQueue(context, type, subType);
+    } else {
+      const eventConfig = eventQueueConfig.getEventConfig(type, subType);
+      const label = `${type}_${subType}`;
+      return await WorkerQueue.instance.addToQueue(
+        eventConfig.load,
+        label,
+        eventConfig.priority,
+        AsyncResource.bind(async () => await processEventQueue(context, type, subType))
+      );
+    }
+  } catch (err) {
+    const logger = cds.log(COMPONENT_NAME);
+    logger.error("error executing event combination for tenant", err, {
+      tenantId: context.tenant,
+      type,
+      subType,
+    });
+  }
+};
+
+module.exports = { runEventCombinationForTenant };

package/src/shared/cdsHelper.js

@@ -129,6 +129,15 @@ const getAllTenantIds = async () => {
   if (!config.isMultiTenancy) {
     return null;
   }
+
+  // NOTE: tmp workaround until cds-mtxs fixes the connect.to service
+  for (let i = 0; i < 10; i++) {
+    if (cds.services["saas-registry"]) {
+      break;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+
   const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");
   const response = await ssp.get("/tenant");
   return response

package/src/shared/distributedLock.js

@@ -54,7 +54,7 @@ const checkLockExistsAndReturnValue = async (context, key, { tenantScoped = true
 };
 
 const _acquireLockRedis = async (context, fullKey, expiryTime, { value = "true", overrideValue = false } = {}) => {
-  const client = await redis.createMainClientAndConnect();
+  const client = await redis.createMainClientAndConnect(config.redisOptions);
   const result = await client.set(fullKey, value, {
     PX: expiryTime,
     ...(overrideValue ? null : { NX: true }),
@@ -63,7 +63,7 @@ const _acquireLockRedis = async (context, fullKey, expiryTime, { value = "true",
 };
 
 const _checkLockExistsRedis = async (context, fullKey) => {
-  const client = await redis.createMainClientAndConnect();
+  const client = await redis.createMainClientAndConnect(config.redisOptions);
   return await client.get(fullKey);
 };
 
@@ -76,7 +76,7 @@ const _checkLockExistsDb = async (context, fullKey) => {
 };
 
 const _releaseLockRedis = async (context, fullKey) => {
-  const client = await redis.createMainClientAndConnect();
+  const client = await redis.createMainClientAndConnect(config.redisOptions);
   await client.del(fullKey);
 };
 

package/src/shared/eventScheduler.js

@@ -2,7 +2,7 @@
 
 const cds = require("@sap/cds");
 
-const { broadcastEvent } = require("../redisPubSub");
+const redisPub = require("../redis/redisPub");
 const config = require("./../config");
 
 const COMPONENT_NAME = "/eventQueue/shared/eventScheduler";
@@ -26,7 +26,7 @@ class EventScheduler {
     });
     setTimeout(() => {
       delete this.#scheduledEvents[key];
-      broadcastEvent(tenantId, type, subType).catch((err) => {
+      redisPub.broadcastEvent(tenantId, { type, subType }).catch((err) => {
         cds.log(COMPONENT_NAME).error("could not execute scheduled event", err, {
           tenantId,
           type,

package/src/shared/redis.js

@@ -6,11 +6,13 @@ const { getEnvInstance } = require("./env");
 const EventQueueError = require("../EventQueueError");
 
 const COMPONENT_NAME = "/eventQueue/shared/redis";
+const LOG_AFTER_SEC = 5;
 
 let mainClientPromise;
 const subscriberChannelClientPromise = {};
+let lastErrorLog = Date.now();
 
-const createMainClientAndConnect = () => {
+const createMainClientAndConnect = (options) => {
   if (mainClientPromise) {
     return mainClientPromise;
   }
@@ -18,13 +20,14 @@ const createMainClientAndConnect = () => {
   const errorHandlerCreateClient = (err) => {
     cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
     mainClientPromise = null;
-    setTimeout(createMainClientAndConnect, 5 * 1000).unref();
+    setTimeout(() => createMainClientAndConnect(options), LOG_AFTER_SEC * 1000).unref();
   };
-  mainClientPromise = createClientAndConnect(errorHandlerCreateClient);
+
+  mainClientPromise = createClientAndConnect(options, errorHandlerCreateClient);
   return mainClientPromise;
 };
 
-const _createClientBase = () => {
+const _createClientBase = (redisOptions) => {
   const env = getEnvInstance();
   try {
     const credentials = env.getRedisCredentialsFromEnv();
@@ -37,40 +40,49 @@ const _createClientBase = () => {
         defaults: {
           password: credentials.password,
           socket: { tls: credentials.tls },
+          ...redisOptions,
         },
       });
     }
-    return redis.createClient({ url });
+    return redis.createClient({ url, ...redisOptions });
   } catch (err) {
     throw EventQueueError.redisConnectionFailure(err);
   }
 };
 
-const createClientAndConnect = async (errorHandlerCreateClient) => {
-  let client = null;
-  try {
-    client = _createClientBase();
-  } catch (err) {
-    throw EventQueueError.redisConnectionFailure(err);
-  }
-
-  client.on("error", errorHandlerCreateClient);
-
+const createClientAndConnect = async (options, errorHandlerCreateClient) => {
   try {
+    const client = _createClientBase(options);
     await client.connect();
+    client.on("error", (err) => {
+      const dateNow = Date.now();
+      if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
+        cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
+        lastErrorLog = dateNow;
+      }
+    });
+
+    client.on("reconnecting", () => {
+      const dateNow = Date.now();
+      if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
+        cds.log(COMPONENT_NAME).info("redis client trying reconnect...");
+        lastErrorLog = dateNow;
+      }
+    });
+    return client;
   } catch (err) {
     errorHandlerCreateClient(err);
   }
-  return client;
 };
 
-const subscribeRedisChannel = (channel, subscribeHandler) => {
+const subscribeRedisChannel = (options, channel, subscribeHandler) => {
   const errorHandlerCreateClient = (err) => {
     cds.log(COMPONENT_NAME).error(`error from redis client for pub/sub failed for channel ${channel}`, err);
     subscriberChannelClientPromise[channel] = null;
-    setTimeout(() => subscribeRedisChannel(channel, subscribeHandler), 5 * 1000).unref();
+    setTimeout(() => subscribeRedisChannel(options, channel, subscribeHandler), LOG_AFTER_SEC * 1000).unref();
   };
-  subscriberChannelClientPromise[channel] = createClientAndConnect(errorHandlerCreateClient)
+
+  subscriberChannelClientPromise[channel] = createClientAndConnect(options, errorHandlerCreateClient)
     .then((client) => {
       cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
       client.subscribe(channel, subscribeHandler).catch(errorHandlerCreateClient);
@@ -82,8 +94,8 @@ const subscribeRedisChannel = (channel, subscribeHandler) => {
     });
 };
 
-const publishMessage = async (channel, message) => {
-  const client = await createMainClientAndConnect();
+const publishMessage = async (options, channel, message) => {
+  const client = await createMainClientAndConnect(options);
   return await client.publish(channel, message);
 };
 
@@ -105,9 +117,9 @@ const _resilientClientClose = async (client) => {
   }
 };
 
-const connectionCheck = async () => {
+const connectionCheck = async (options) => {
   return new Promise((resolve, reject) => {
-    createClientAndConnect(reject)
+    createClientAndConnect(options, reject)
       .then((client) => {
         if (client) {
           _resilientClientClose(client);

package/src/redisPubSub.js

@@ -1,170 +0,0 @@
-"use strict";
-
-const { promisify } = require("util");
-
-const cds = require("@sap/cds");
-
-const redis = require("./shared/redis");
-const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
-const config = require("./config");
-const runner = require("./runner");
-const { getSubdomainForTenantId } = require("./shared/cdsHelper");
-
-const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
-const COMPONENT_NAME = "/eventQueue/redisPubSub";
-const TRIES_FOR_PUBLISH_PERIODIC_EVENT = 10;
-const SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT = 30 * 1000;
-
-const wait = promisify(setTimeout);
-let subscriberClientPromise;
-
-const initEventQueueRedisSubscribe = () => {
-  if (subscriberClientPromise || !config.redisEnabled) {
-    return;
-  }
-  redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL, _messageHandlerProcessEvents);
-};
-
-const _messageHandlerProcessEvents = async (messageData) => {
-  const logger = cds.log(COMPONENT_NAME);
-  try {
-    const { tenantId, type, subType } = JSON.parse(messageData);
-    logger.debug("received redis event", {
-      tenantId,
-      type,
-      subType,
-    });
-    if (!config.isEventQueueActive) {
-      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {
-        type,
-        subType,
-      });
-      return;
-    }
-
-    const subdomain = await getSubdomainForTenantId(tenantId);
-    const user = new cds.User.Privileged(config.userId);
-    const tenantContext = {
-      tenant: tenantId,
-      user,
-      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-    };
-
-    if (!config.getEventConfig(type, subType)) {
-      if (config.isCapOutboxEvent(type)) {
-        try {
-          const service = await cds.connect.to(subType);
-          cds.outboxed(service);
-        } catch (err) {
-          logger.error("could not connect to outboxed service", err, {
-            type,
-            subType,
-          });
-          return;
-        }
-      } else {
-        logger.error("cannot find configuration for published event. Event won't be processed", {
-          type,
-          subType,
-        });
-        return;
-      }
-    }
-
-    return await cds.tx(tenantContext, async ({ context }) => {
-      return await runner.runEventCombinationForTenant(context, type, subType);
-    });
-  } catch (err) {
-    logger.error("could not parse event information", {
-      messageData,
-    });
-  }
-};
-
-const broadcastEvent = async (tenantId, type, subType) => {
-  const logger = cds.log(COMPONENT_NAME);
-  try {
-    if (!config.isEventQueueActive) {
-      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {
-        type,
-        subType,
-      });
-      return;
-    }
-    if (!config.redisEnabled) {
-      if (config.registerAsEventProcessor) {
-        let context = {};
-        if (tenantId) {
-          const subdomain = await getSubdomainForTenantId(tenantId);
-          const user = new cds.User.Privileged(config.userId);
-          context = {
-            // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-            tenant: tenantId,
-            user,
-            http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-          };
-        }
-
-        return await cds.tx(context, async ({ context }) => {
-          return await runner.runEventCombinationForTenant(context, type, subType);
-        });
-      }
-      return;
-    }
-    const eventConfig = config.getEventConfig(type, subType);
-    for (let i = 0; i < TRIES_FOR_PUBLISH_PERIODIC_EVENT; i++) {
-      const result = await checkLockExistsAndReturnValue(
-        new cds.EventContext({ tenant: tenantId }),
-        [type, subType].join("##")
-      );
-      if (result) {
-        logger.debug("skip publish redis event as no lock is available", {
-          type,
-          subType,
-          index: i,
-          isPeriodic: eventConfig.isPeriodic,
-          waitInterval: SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT,
-        });
-        if (!eventConfig.isPeriodic) {
-          break;
-        }
-        await wait(SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT);
-        continue;
-      }
-      logger.debug("publishing redis event", {
-        tenantId,
-        type,
-        subType,
-      });
-      await redis.publishMessage(EVENT_MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
-      break;
-    }
-  } catch (err) {
-    logger.error("publish event failed!", err, {
-      tenantId,
-      type,
-      subType,
-    });
-  }
-};
-
-const closeSubscribeClient = async () => {
-  try {
-    const client = await subscriberClientPromise;
-    if (client?.quit) {
-      await client.quit();
-    }
-  } catch (err) {
-    // ignore errors during shutdown
-  }
-};
-
-module.exports = {
-  initEventQueueRedisSubscribe,
-  broadcastEvent,
-  closeSubscribeClient,
-  __: {
-    _messageHandlerProcessEvents,
-  },
-};