@cap-js-community/event-queue 1.11.0-beta.2 → 1.11.0-beta.4

package/db/Lock.cds

@@ -1,6 +1,5 @@
 namespace sap.eventqueue;
 
-using cuid from '@sap/cds/common';
 using managed from '@sap/cds/common';
 
 @assert.unique.semanticKey: [code]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.11.0-beta.2",
+  "version": "1.11.0-beta.4",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -48,23 +48,23 @@
   },
   "dependencies": {
     "@sap/xssec": "^4.6.0",
-    "cron-parser": "^5.2.0",
+    "cron-parser": "^5.3.1",
     "redis": "^4.7.0",
     "verror": "^1.10.1",
     "yaml": "^2.7.1"
   },
   "devDependencies": {
-    "@cap-js/cds-test": "^0.3.0",
-    "@cap-js/hana": "^1.8.0",
-    "@cap-js/sqlite": "^1.10.0",
-    "@sap/cds": "^8.9.0",
-    "@sap/cds-dk": "^8.8.0",
+    "@cap-js/cds-test": "^0.4.0",
+    "@cap-js/hana": "^2.2.0",
+    "@cap-js/sqlite": "^2.0.1",
+    "@sap/cds": "^9.3.1",
+    "@sap/cds-dk": "^9.3.1",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jest": "^28.6.0",
     "eslint-plugin-node": "^11.1.0",
     "express": "^4.21.2",
-    "hdb": "^0.19.10",
+    "hdb": "^2.25.1",
     "jest": "^29.7.0",
     "prettier": "^2.8.8",
     "sqlite3": "^5.1.7",

package/src/EventQueueProcessorBase.js

@@ -1016,7 +1016,7 @@ class EventQueueProcessorBase {
 
     // NOTE: do not pass current date as we always want to calc. a future date
     const cronExpression = CronExpressionParser.parse(this.#eventConfig.cron, {
-      tz: eventConfig.tz,
+      tz: this.#eventConfig.tz,
     });
     return cronExpression.next();
   }

package/src/config.js

@@ -28,6 +28,7 @@ const DEFAULT_CHECK_FOR_NEXT_CHUNK = true;
 const SUFFIX_PERIODIC = "_PERIODIC";
 const CAP_EVENT_TYPE = "CAP_OUTBOX";
 const CAP_PARALLEL_DEFAULT = 5;
+const CAP_MAX_ATTEMPTS_DEFAULT = 5;
 const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
 const PRIORITIES = Object.values(Priorities);
 const UTC_DEFAULT = false;
@@ -113,11 +114,12 @@ class Config {
   #redisNamespace;
   #publishEventBlockList;
   #crashOnRedisUnavailable;
-  #tenantIdFilterTokenInfoCb;
+  #tenantIdFilterAuthContextCb;
   #tenantIdFilterEventProcessingCb;
   #configEvents;
   #configPeriodicEvents;
   #enableAdminService;
+  #disableProcessingOfSuspendedTenants;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -195,9 +197,7 @@ class Config {
             result = config.value.test(this.#env.applicationName);
           } else {
             const shouldBeProcessedBasedOnAppName = appNameConfig[this.#env.applicationName];
-            if (!shouldBeProcessedBasedOnAppName) {
-              result = config.value === this.#env.applicationName;
-            }
+            result = !!shouldBeProcessedBasedOnAppName;
           }
           if (result) {
             break;
@@ -387,7 +387,7 @@ class Config {
       kind: config.kind ?? "persistent-outbox",
       selectMaxChunkSize: config.selectMaxChunkSize ?? config.chunkSize,
       parallelEventProcessing: config.parallelEventProcessing ?? (config.parallel && CAP_PARALLEL_DEFAULT),
-      retryAttempts: config.retryAttempts ?? config.maxAttempts,
+      retryAttempts: config.retryAttempts ?? config.maxAttempts ?? CAP_MAX_ATTEMPTS_DEFAULT,
       ...config,
     });
     eventConfig.internalEvent = true;
@@ -483,10 +483,11 @@ class Config {
               delete base.interval;
             }
 
-            result[fnName] = Object.assign(
+            const subType = `${name}.${fnName}`;
+            result[subType] = Object.assign(
               {
                 type: CAP_EVENT_TYPE,
-                subType: `${name}.${fnName}`,
+                subType,
                 impl: "./outbox/EventQueueGenericOutboxHandler",
                 internalEvent: true,
               },
@@ -770,12 +771,12 @@ class Config {
     this.#crashOnRedisUnavailable = value;
   }
 
-  get tenantIdFilterTokenInfo() {
-    return this.#tenantIdFilterTokenInfoCb;
+  get tenantIdFilterAuthContext() {
+    return this.#tenantIdFilterAuthContextCb;
   }
 
-  set tenantIdFilterTokenInfo(value) {
-    this.#tenantIdFilterTokenInfoCb = value;
+  set tenantIdFilterAuthContext(value) {
+    this.#tenantIdFilterAuthContextCb = value;
   }
 
   get tenantIdFilterEventProcessing() {
@@ -984,6 +985,14 @@ class Config {
     this.#enableAdminService = value;
   }
 
+  get disableProcessingOfSuspendedTenants() {
+    return this.#disableProcessingOfSuspendedTenants;
+  }
+
+  set disableProcessingOfSuspendedTenants(value) {
+    this.#disableProcessingOfSuspendedTenants = value;
+  }
+
   /**
     @return { Config }
   **/

package/src/constants.js

@@ -22,6 +22,6 @@ module.exports = {
   },
   TenantIdCheckTypes: {
     eventProcessing: "eventProcessing",
-    getTokenInfo: "getTokenInfo",
+    getAuthContext: "getAuthContext",
   },
 };

package/src/index.d.ts

@@ -14,7 +14,7 @@ export declare type EventProcessingStatusType = (typeof EventProcessingStatus)[E
 
 export declare const TenantIdCheckTypes: {
   eventProcessing: "eventProcessing";
-  getTokenInfo: "getTokenInfo";
+  getAuthContext: "getAuthContext";
 };
 
 export declare const TransactionMode: {
@@ -215,8 +215,8 @@ declare class Config {
   get publishEventBlockList(): any;
   set crashOnRedisUnavailable(value: any);
   get crashOnRedisUnavailable(): any;
-  set tenantIdFilterTokenInfo(value: any);
-  get tenantIdFilterTokenInfo(): any;
+  set tenantIdFilterAuthContext(value: any);
+  get tenantIdFilterAuthContext(): any;
   set tenantIdFilterEventProcessing(value: any);
   get tenantIdFilterEventProcessing(): any;
   set runInterval(value: any);

package/src/initialize.js

@@ -47,6 +47,7 @@ const CONFIG_VARS = [
   ["publishEventBlockList", true],
   ["crashOnRedisUnavailable", false],
   ["enableAdminService", false],
+  ["disableProcessingOfSuspendedTenants", true],
 ];
 
 /**

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -61,7 +61,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
         }
       } else {
         for (const actionName in genericClusterEvents) {
-          const msg = new cds.Request({
+          const reg = new cds.Request({
             event: EVENT_QUEUE_ACTIONS.CLUSTER,
             user: this.context.user,
             eventQueue: {
@@ -74,14 +74,14 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
                 this.#clusterByDataProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
             },
           });
-          const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
+          const clusterResult = await this.__srvUnboxed.tx(this.context).send(reg);
           if (this.#validateCluster(clusterResult)) {
             Object.assign(clusterMap, clusterResult);
           } else {
             this.logger.error(
               "cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
               {
-                handler: msg.event,
+                handler: reg.event,
                 clusterResult: JSON.stringify(clusterResult),
               }
             );
@@ -92,7 +92,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     }
 
     for (const actionName in specificClusterEvents) {
-      const msg = new cds.Request({
+      const reg = new cds.Request({
         event: `${EVENT_QUEUE_ACTIONS.CLUSTER}.${actionName}`,
         user: this.context.user,
         eventQueue: {
@@ -105,14 +105,14 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
             this.#clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
         },
       });
-      const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
+      const clusterResult = await this.__srvUnboxed.tx(this.context).send(reg);
       if (this.#validateCluster(clusterResult)) {
         Object.assign(clusterMap, clusterResult);
       } else {
         this.logger.error(
           "cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
           {
-            handler: msg.event,
+            handler: reg.event,
             clusterResult: JSON.stringify(clusterResult),
           }
         );
@@ -264,12 +264,12 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       return payload;
     }
 
-    const { msg, userId } = this.#buildDispatchData(this.context, payload, {
+    const { reg, userId } = this.#buildDispatchData(this.context, payload, {
       queueEntries: [queueEntry],
     });
-    msg.event = handlerName;
-    await this.#setContextUser(this.context, userId, msg);
-    const data = await this.__srvUnboxed.tx(this.context).send(msg);
+    reg.event = handlerName;
+    await this.#setContextUser(this.context, userId, reg);
+    const data = await this.__srvUnboxed.tx(this.context).send(reg);
     if (data) {
       payload.data = data;
       return payload;
@@ -285,12 +285,12 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       return await super.hookForExceededEvents(exceededEvent);
     }
 
-    const { msg, userId } = this.#buildDispatchData(this.context, exceededEvent.payload, {
+    const { reg, userId } = this.#buildDispatchData(this.context, exceededEvent.payload, {
       queueEntries: [exceededEvent],
     });
-    await this.#setContextUser(this.context, userId, msg);
-    msg.event = handlerName;
-    await this.__srvUnboxed.tx(this.context).send(msg);
+    await this.#setContextUser(this.context, userId, reg);
+    reg.event = handlerName;
+    await this.__srvUnboxed.tx(this.context).send(reg);
   }
 
   // NOTE: Currently not exposed to CAP service; we wait for a valid use case
@@ -310,37 +310,39 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
 
   async processPeriodicEvent(processContext, key, queueEntry) {
     const [, action] = this.eventSubType.split(".");
-    const msg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
-    await this.#setContextUser(processContext, config.userId, msg);
-    await this.__srvUnboxed.tx(processContext).emit(msg);
+    const reg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
+    await this.#setContextUser(processContext, config.userId, reg);
+    await this.__srvUnboxed.tx(processContext).emit(reg);
   }
 
   #buildDispatchData(context, payload, { key, queueEntries } = {}) {
     const { useEventQueueUser } = this.eventConfig;
     const userId = useEventQueueUser ? config.userId : payload.contextUser;
-    const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
+    const reg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
     const invocationFn = payload._fromSend ? "send" : "emit";
-    delete msg._fromSend; // TODO: this changes the source object --> check after multiple invocations
-    delete msg.contextUser;
-    msg.eventQueue = { processor: this, key, queueEntries, payload };
-    return { msg, userId, invocationFn };
+    delete reg._fromSend;
+    delete reg.contextUser;
+    reg.eventQueue = { processor: this, key, queueEntries, payload };
+    return { reg, userId, invocationFn };
   }
 
-  async #setContextUser(context, userId, data) {
+  async #setContextUser(context, userId, reg) {
+    const authInfo = await common.getAuthContext(context.tenant);
     context.user = new cds.User.Privileged({
       id: userId,
-      tokenInfo: await common.getTokenInfo(this.baseContext.tenant),
+      authInfo,
+      tokenInfo: authInfo?.token,
     });
-    if (data) {
-      data.user = context.user;
+    if (reg) {
+      reg.user = context.user;
     }
   }
 
   async processEvent(processContext, key, queueEntries, payload) {
     try {
-      const { userId, invocationFn, msg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
-      await this.#setContextUser(processContext, userId, msg);
-      const result = await this.__srvUnboxed.tx(processContext)[invocationFn](msg);
+      const { userId, invocationFn, reg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
+      await this.#setContextUser(processContext, userId, reg);
+      const result = await this.__srvUnboxed.tx(processContext)[invocationFn](reg);
       return this.#determineResultStatus(result, queueEntries);
     } catch (err) {
       this.logger.error("error processing outboxed service call", err, {

package/src/outbox/eventQueueAsOutbox.js

@@ -52,7 +52,7 @@ function outboxed(srv, customOpts) {
       outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
     }
 
-    if (outboxOpts.kind === "persistent-outbox") {
+    if (["persistent-outbox", "persistent-queue"].includes(outboxOpts.kind)) {
       await _mapToEventAndPublish(context, srv.name, req, !!specificSettings);
       return;
     }

package/src/periodicEvents.js

@@ -7,11 +7,17 @@ const { EventProcessingStatus } = require("./constants");
 const { processChunkedSync } = require("./shared/common");
 const eventConfig = require("./config");
 
-const COMPONENT_NAME = "/eventQueue/periodicEvents";
 const CHUNK_SIZE_INSERT_PERIODIC_EVENTS = 4;
 
+const ALLOWED_PERIODIC_SEC_DIFF = 30;
+
+const COMPONENT_NAME = "/eventQueue/periodicEvents";
+
 const checkAndInsertPeriodicEvents = async (context) => {
   const now = new Date();
+  cds.log(COMPONENT_NAME).info("updating periodic events", {
+    tenant: context.tenant,
+  });
   const tx = cds.tx(context);
   const baseCqn = SELECT.from(eventConfig.tableNameEventQueue)
     .where([
@@ -117,11 +123,16 @@ const _determineChangedCron = (existingEventsCron) => {
     const config = eventConfig.getEventConfig(event.type, event.subType);
     const eventStartAfter = new Date(event.startAfter);
     const eventCreatedAt = new Date(event.createdAt);
+    const randomOffset = config.randomOffset ?? eventConfig.randomOffsetPeriodicEvents ?? 0;
     const cronExpression = CronExpressionParser.parse(config.cron, {
       currentDate: eventCreatedAt,
       tz: config.tz,
     });
-    return Math.abs(cronExpression.next().getTime() - eventStartAfter.getTime()) > 30 * 1000; // report as changed if diff created than 30 seconds
+    // report as changed if diff created than ALLOWED_PERIODIC_SEC_DIFF + the random event offset seconds
+    return (
+      Math.abs(cronExpression.next().getTime() - eventStartAfter.getTime()) >
+      (ALLOWED_PERIODIC_SEC_DIFF + randomOffset) * 1000
+    );
   });
 };
 

package/src/processEventQueue.js

@@ -318,8 +318,11 @@ const _checkEventIsBlocked = async (baseInstance) => {
 
 const _processEvent = async (eventTypeInstance, processContext, key, queueEntries, payload) => {
   let traceContext;
-  if (queueEntries.length === 1 && eventTypeInstance.inheritTraceContext) {
-    traceContext = queueEntries[0].context?.traceContext;
+  if (eventTypeInstance.inheritTraceContext) {
+    const uniqueTraceContext = [...new Set(queueEntries.map((entry) => entry.context?.traceContext).filter((a) => a))];
+    if (uniqueTraceContext.length === 1) {
+      traceContext = uniqueTraceContext[0];
+    }
   }
 
   return await trace(

package/src/redis/redisPub.js

@@ -119,7 +119,8 @@ const _processLocalWithoutRedis = async (tenantId, events) => {
     let context = {};
     if (tenantId) {
       const user = await cds.tx({ tenant: tenantId }, async () => {
-        return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
+        const authInfo = await common.getAuthContext(tenantId);
+        return new cds.User.Privileged({ id: config.userId, authInfo, tokenInfo: authInfo.token });
       });
       context = {
         tenant: tenantId,

package/src/redis/redisSub.js

@@ -78,7 +78,8 @@ const _messageHandlerProcessEvents = async (messageData) => {
     }
 
     const user = await cds.tx({ tenant: tenantId }, async () => {
-      return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
+      const authInfo = await common.getAuthContext(tenantId);
+      return new cds.User.Privileged({ id: config.userId, authInfo, tokenInfo: authInfo?.token });
     });
     const tenantContext = {
       tenant: tenantId,

package/src/runner/openEvents.js

@@ -38,34 +38,27 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
   for (const { type, subType } of entries) {
     if (eventConfig.isCapOutboxEvent(type)) {
       const [srvName, actionName] = subType.split(".");
-      cds.connect
-        .to(srvName)
-        .then((service) => {
-          if (!filterAppSpecificEvents) {
-            return; // will be done in finally
-          }
-
+      try {
+        const service = await cds.connect.to(srvName);
+        if (filterAppSpecificEvents) {
           if (!service) {
-            return;
+            continue;
           }
           cds.outboxed(service);
           if (actionName) {
             config.addCAPOutboxEventSpecificAction(srvName, actionName);
           }
-          if (filterAppSpecificEvents) {
-            if (eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
-              result.push({ type, subType });
-            }
-          } else {
-            result.push({ type, subType });
-          }
-        })
-        .catch(() => {})
-        .finally(() => {
-          if (!filterAppSpecificEvents) {
+          if (eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
             result.push({ type, subType });
           }
-        });
+        }
+      } catch {
+        /* ignore catch */
+      } finally {
+        if (!filterAppSpecificEvents) {
+          result.push({ type, subType });
+        }
+      }
     } else {
       if (filterAppSpecificEvents) {
         if (

package/src/runner/runner.js

@@ -138,16 +138,22 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
     if (!couldAcquireLock) {
       return;
     }
+  } catch (err) {
+    logger.error("executing event queue run for multi instance and tenant failed", err);
+  }
 
-    for (const tenantId of tenantIds) {
+  for (const tenantId of tenantIds) {
+    try {
       await cds.tx({ tenant: tenantId }, async (tx) => {
         await trace(
           tx.context,
           "get-openEvents-and-publish",
           async () => {
+            const authInfo = await common.getAuthContext(tenantId);
             tx.context.user = new cds.User.Privileged({
               id: config.userId,
-              tokenInfo: await common.getTokenInfo(tenantId),
+              authInfo,
+              tokenInfo: authInfo?.token,
             });
             const entries = await openEvents.getOpenQueueEntries(tx, false);
             logger.info("broadcasting events for run", {
@@ -168,9 +174,9 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
           { newRootSpan: true }
         );
       });
+    } catch (err) {
+      logger.error("broadcasting events for tenant failed", { tenantId }, err);
     }
-  } catch (err) {
-    logger.info("executing event queue run for multi instance and tenant failed", err);
   }
 };
 
@@ -179,23 +185,30 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
   for (const tenantId of tenantIds) {
     const id = cds.utils.uuid();
     let tenantContext;
-    const events = await trace(
-      { id, tenant: tenantId },
-      "fetch-openEvents-and-tokenInfo",
-      async () => {
-        const user = await cds.tx({ tenant: tenantId }, async () => {
-          return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
-        });
-        tenantContext = {
-          tenant: tenantId,
-          user,
-        };
-        return await cds.tx(tenantContext, async (tx) => {
-          return await openEvents.getOpenQueueEntries(tx);
-        });
-      },
-      { newRootSpan: true }
-    );
+    let events;
+    try {
+      events = await trace(
+        { id, tenant: tenantId },
+        "fetch-openEvents-and-authInfo",
+        async () => {
+          const user = await cds.tx({ tenant: tenantId }, async () => {
+            const authInfo = await common.getAuthContext(tenantId);
+            return new cds.User.Privileged({ id: config.userId, authInfo, tokenInfo: authInfo?.token });
+          });
+          tenantContext = {
+            tenant: tenantId,
+            user,
+          };
+          return await cds.tx(tenantContext, async (tx) => {
+            return await openEvents.getOpenQueueEntries(tx);
+          });
+        },
+        { newRootSpan: true }
+      );
+    } catch (err) {
+      cds.log(COMPONENT_NAME).error("fetching open events for tenant failed", { tenantId }, err);
+      continue;
+    }
 
     if (!events.length) {
       continue;
@@ -248,7 +261,8 @@ const _executePeriodicEventsAllTenants = async (tenantIds) => {
   for (const tenantId of tenantIds) {
     try {
       const user = await cds.tx({ tenant: tenantId }, async () => {
-        return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
+        const authInfo = await common.getAuthContext(tenantId);
+        return new cds.User.Privileged({ id: config.userId, authInfo, tokenInfo: authInfo?.token });
       });
       const tenantContext = {
         tenant: tenantId,
@@ -279,7 +293,7 @@ const _singleTenantDb = async () => {
   const id = cds.utils.uuid();
   const events = await trace(
     { id },
-    "fetch-openEvents-and-tokenInfo",
+    "fetch-openEvents-and-authInfo",
     async () => {
       return await cds.tx({}, async (tx) => {
         return await openEvents.getOpenQueueEntries(tx);
@@ -499,6 +513,7 @@ const _checkPeriodicEventsSingleTenantOneTime = async () => {
         tenantScoped: false,
       });
       if (!couldAcquireLock) {
+        logger.info("skipping updating periodic events - lock not acquired");
         return;
       }
       return await cds.tx({}, async (tx) => await periodicEvents.checkAndInsertPeriodicEvents(tx.context));
@@ -519,7 +534,7 @@ const _checkPeriodicEventsSingleTenant = async (context) => {
   try {
     logger.info("executing updating periodic events", {
       tenantId: context.tenant,
-      subdomain: context.user?.tokenInfo?.extAttributes?.zdn,
+      subdomain: context.user?.authInfo?.getSubdomain?.(),
     });
     await periodicEvents.checkAndInsertPeriodicEvents(context);
   } catch (err) {

package/src/shared/cdsHelper.js

@@ -6,10 +6,13 @@ const cds = require("@sap/cds");
 const config = require("../config");
 const common = require("./common");
 const { TenantIdCheckTypes } = require("../constants");
+const { limiter } = require("./common");
 
 const VERROR_CLUSTER_NAME = "ExecuteInNewTransactionError";
 const COMPONENT_NAME = "/eventQueue/cdsHelper";
 
+const CONCURRENCY_AUTH_INFO = 3;
+
 /**
  * Execute logic in a new managed CDS transaction context, auto-handling commit, rollback and error/exception situations.
  * Includes logging of start, end and error situation with additional info object and unique transaction id (txId)
@@ -25,7 +28,8 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
   const logger = cds.log(COMPONENT_NAME);
   let transactionRollbackPromise = Promise.resolve(false);
   try {
-    const user = new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(context.tenant) });
+    const authInfo = await common.getAuthContext(context.tenant);
+    const user = new cds.User.Privileged({ id: config.userId, authInfo, tokenInfo: authInfo?.token });
     if (cds.db.kind === "hana") {
       await cds.tx(
         {
@@ -139,18 +143,31 @@ const getAllTenantIds = async () => {
     return null;
   }
 
-  return response
-    .map((tenant) => tenant.subscribedTenantId ?? tenant.tenant)
-    .reduce(async (result, tenantId) => {
-      result = await result;
-      if (await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId)) {
-        result.push(tenantId);
+  const tenantIds = response.map((tenant) => tenant.subscribedTenantId ?? tenant.tenant);
+  const suspendedTenants = {};
+  if (config.disableProcessingOfSuspendedTenants) {
+    await limiter(CONCURRENCY_AUTH_INFO, tenantIds, async (tenantId) => {
+      const result = await common.getAuthContext(tenantId, { returnError: true });
+      // NOTE: only 404 errors are propagated all others are ignored
+      if (result?.[0]) {
+        suspendedTenants[tenantId] = true;
+        cds.log(COMPONENT_NAME).info("skip event-queue processing, tenant suspended", { tenantId });
       }
-      return result;
-    }, []);
+    });
+  }
+
+  return tenantIds.reduce(async (result, tenantId) => {
+    result = await result;
+    if (!suspendedTenants[tenantId] && (await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId))) {
+      result.push(tenantId);
+    }
+    return result;
+  }, []);
 };
 
-const getAllTenantWithSubdomain = async () => {
+const TENANT_COLUMNS = ["subscribedSubdomain", "createdAt", "modifiedAt"];
+
+const getAllTenantWithMetadata = async () => {
   const response = await _getAllTenantBase();
   if (!response) {
     return null;
@@ -160,10 +177,19 @@ const getAllTenantWithSubdomain = async () => {
     const tenantId = row.subscribedTenantId ?? row.tenant;
     result = await result;
     if (await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId)) {
-      result.push({
-        ID: tenantId,
-        subdomain: row.subscribedSubdomain,
-      });
+      const data = Object.entries(row).reduce(
+        (result, [key, value]) => {
+          if (TENANT_COLUMNS.includes(key)) {
+            result[key] = value;
+          } else {
+            result.metadata[key] = value;
+          }
+          return result;
+        },
+        { metadata: {} }
+      );
+      data.metadata = JSON.stringify(data.metadata);
+      result.push(data);
     }
     return result;
   }, []);
@@ -172,5 +198,5 @@ const getAllTenantWithSubdomain = async () => {
 module.exports = {
   executeInNewTransaction,
   getAllTenantIds,
-  getAllTenantWithSubdomain,
+  getAllTenantWithMetadata,
 };

package/src/shared/common.js

@@ -7,9 +7,11 @@ const xssec = require("@sap/xssec");
 const VError = require("verror");
 
 const config = require("../config");
+const { ExpiringLazyCache } = require("./lazyCache");
 const { TenantIdCheckTypes } = require("../constants");
 
-const MARGIN_AUTH_INFO_EXPIRY = 60 * 1000;
+const EXPIRE_TIME_TENANT_404 = 10 * 60 * 1000; // 10 minutes
+
 const COMPONENT_NAME = "/eventQueue/common";
 
 const arrayToFlatMap = (array, key = "ID") => {
@@ -87,64 +89,61 @@ const processChunkedSync = (inputs, chunkSize, chunkHandler) => {
 
 const hashStringTo32Bit = (value) => crypto.createHash("sha256").update(String(value)).digest("base64").slice(0, 32);
 
-const _getNewTokenInfo = async (tenantId) => {
-  const tokenInfoCache = getTokenInfo._tokenInfoCache;
-  tokenInfoCache[tenantId] = tokenInfoCache[tenantId] ?? {};
+const _getNewAuthContext = async (tenantId) => {
   try {
-    if (!_getNewTokenInfo._xsuaaService) {
-      _getNewTokenInfo._xsuaaService = new xssec.XsuaaService(cds.requires.auth.credentials);
+    if (!_getNewAuthContext._xsuaaService) {
+      _getNewAuthContext._xsuaaService = new xssec.XsuaaService(cds.requires.auth.credentials);
     }
-    const authService = _getNewTokenInfo._xsuaaService;
+    const authService = _getNewAuthContext._xsuaaService;
     const token = await authService.fetchClientCredentialsToken({ zid: tenantId });
     const tokenInfo = new xssec.XsuaaToken(token.access_token);
-    tokenInfoCache[tenantId].expireTs = tokenInfo.getExpirationDate().getTime() - MARGIN_AUTH_INFO_EXPIRY;
-    return tokenInfo;
+    const authInfo = new xssec.XsuaaSecurityContext(authService, tokenInfo);
+    return [tokenInfo.getExpirationDate().getTime() - Date.now(), [null, authInfo]];
   } catch (err) {
-    tokenInfoCache[tenantId] = null;
-    cds.log(COMPONENT_NAME).warn("failed to request tokenInfo", {
+    cds.log(COMPONENT_NAME).warn("failed to request authContext", {
       err: err.message,
       responseCode: err.responseCode,
       responseText: err.responseText,
     });
+
+    if (err.responseCode === 404) {
+      return [EXPIRE_TIME_TENANT_404, [err, null]];
+    }
+    return [0, null];
   }
 };
 
-const getTokenInfo = async (tenantId) => {
-  if (!(await isTenantIdValidCb(TenantIdCheckTypes.getTokenInfo, tenantId))) {
+const getAuthContext = async (tenantId, { returnError = false } = {}) => {
+  if (!(await isTenantIdValidCb(TenantIdCheckTypes.getAuthContext, tenantId))) {
     return null;
   }
 
   if (!cds.requires?.auth?.credentials) {
-    return null; // no credentials not tokenInfo
+    return null; // no credentials not authContext
   }
 
   if (!config.isMultiTenancy) {
     return null; // does only make sense for multi tenancy
   }
 
-  if (!cds.requires?.auth.kind.match(/jwt|xsuaa/i)) {
+  if (!cds.requires?.auth.kind.match(/jwt|xsuaa/i) && !cds.requires?.xsuaa) {
     return null;
   }
 
-  getTokenInfo._tokenInfoCache = getTokenInfo._tokenInfoCache ?? {};
-  const tokenInfoCache = getTokenInfo._tokenInfoCache;
-  // not existing or existing but expired
-  if (
-    !tokenInfoCache[tenantId] ||
-    (tokenInfoCache[tenantId] && tokenInfoCache[tenantId].expireTs && Date.now() > tokenInfoCache[tenantId].expireTs)
-  ) {
-    tokenInfoCache[tenantId] ??= {};
-    tokenInfoCache[tenantId].value = _getNewTokenInfo(tenantId);
-    tokenInfoCache[tenantId].expireTs = null;
+  getAuthContext._cache = getAuthContext._cache ?? new ExpiringLazyCache();
+  const result = await getAuthContext._cache.getSetCb(tenantId, async () => _getNewAuthContext(tenantId));
+  if (returnError) {
+    return result;
+  } else {
+    return result?.[1];
   }
-  return await tokenInfoCache[tenantId].value;
 };
 
 const isTenantIdValidCb = async (checkType, tenantId) => {
   let cb;
   switch (checkType) {
-    case TenantIdCheckTypes.getTokenInfo:
-      cb = config.tenantIdFilterTokenInfo;
+    case TenantIdCheckTypes.getAuthContext:
+      cb = config.tenantIdFilterAuthContext;
       break;
     case TenantIdCheckTypes.eventProcessing:
       cb = config.tenantIdFilterEventProcessing;
@@ -167,10 +166,10 @@ module.exports = {
   isValidDate,
   processChunkedSync,
   hashStringTo32Bit,
-  getTokenInfo,
+  getAuthContext,
   isTenantIdValidCb,
   promiseAllDone,
   __: {
-    clearTokenInfoCache: () => (getTokenInfo._tokenInfoCache = {}),
+    clearAuthContextCache: () => getAuthContext._cache?.clear(),
   },
 };

package/src/shared/distributedLock.js

@@ -108,7 +108,7 @@ const _renewLockRedis = async (context, fullKey, expiryTime, { value = "true" }
 
 const _checkLockExistsRedis = async (context, fullKey) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
-  return await client.get(fullKey);
+  return await client.exists(fullKey);
 };
 
 const _checkLockExistsDb = async (context, fullKey) => {
@@ -121,8 +121,9 @@ const _checkLockExistsDb = async (context, fullKey) => {
 
 const _releaseLockRedis = async (context, fullKey) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
-  await client.del(fullKey);
+  const result = await client.del(fullKey);
   delete existingLocks[fullKey];
+  return result === 1;
 };
 
 const _releaseLockDb = async (context, fullKey) => {
@@ -130,6 +131,7 @@ const _releaseLockDb = async (context, fullKey) => {
     await tx.run(DELETE.from(config.tableNameEventLock).where("code =", fullKey));
   });
   delete existingLocks[fullKey];
+  return true;
 };
 
 const _acquireLockDB = async (
@@ -191,39 +193,36 @@ const _generateKey = (context, tenantScoped, key) => {
 };
 
 const getAllLocksRedis = async () => {
-  const client = await redis.createMainClientAndConnect(config.redisOptions);
-  const batchSize = 500;
-  const results = [];
-  let pipeline = client.multi();
+  const clientOrCluster = await redis.createMainClientAndConnect(config.redisOptions);
   const output = [];
-  let count = 0;
+  const results = [];
 
-  // NOTE: use SCAN because KEYS is not supported for cluster clients
-  for await (const key of client.scanIterator({ MATCH: "EVENT*", COUNT: 1000 })) {
-    const [, tenant, guidOrType, subType] = key.split("##");
-    if (!subType) {
-      continue;
-    }
+  let clients;
+  if (redis.isClusterMode()) {
+    clients = clientOrCluster.masters.map((master) => master.client);
+  } else {
+    clients = [clientOrCluster];
+  }
 
-    output.push({
-      tenant: tenant,
-      type: guidOrType,
-      subType: subType,
-    });
-    pipeline.ttl(key).get(key);
-    count++;
+  // NOTE: use SCAN because KEYS is not supported for cluster clients
+  for (const client of clients) {
+    for await (const key of client.scanIterator({ MATCH: "EVENT*", COUNT: 1000 })) {
+      const [, tenant, guidOrType, subType] = key.split("##");
+      if (!subType) {
+        continue;
+      }
 
-    if (count >= batchSize) {
+      const pipeline = client.multi();
+      output.push({
+        tenant: tenant,
+        type: guidOrType,
+        subType: subType,
+      });
+      pipeline.ttl(key).get(key);
       const replies = await pipeline.exec();
       results.push(...replies);
-      pipeline = client.multi();
-      count = 0;
     }
   }
-  if (count > 0) {
-    const replies = await pipeline.exec();
-    results.push(...replies);
-  }
 
   let counter = 0;
   for (const row of output) {

package/src/shared/lazyCache.js

@@ -0,0 +1,148 @@
+"use strict";
+
+const DEFAULT_SEPARATOR = "##";
+const DEFAULT_EXPIRATION_GAP = 60 * 1000; // 60 seconds
+
+class LazyCache {
+  constructor({ separator = DEFAULT_SEPARATOR } = {}) {
+    this.__data = Object.create(null);
+    this.__separator = separator;
+  }
+  _separator() {
+    return this.__separator;
+  }
+  _data() {
+    return this.__data;
+  }
+  async _dataSettled() {
+    return await Object.entries(this.__data).reduce(async (result, [key, value]) => {
+      (await result)[key] = await value;
+      return result;
+    }, Promise.resolve({}));
+  }
+  _key(keyOrKeys) {
+    return Array.isArray(keyOrKeys) ? keyOrKeys.join(this.__separator) : keyOrKeys;
+  }
+  has(keyOrKeys) {
+    return Object.prototype.hasOwnProperty.call(this.__data, this._key(keyOrKeys));
+  }
+  get(keyOrKeys) {
+    return this.__data[this._key(keyOrKeys)];
+  }
+  set(keyOrKeys, value) {
+    this.__data[this._key(keyOrKeys)] = value;
+    return this;
+  }
+  setCb(keyOrKeys, callback) {
+    const resultOrPromise = callback();
+    return this.set(
+      keyOrKeys,
+      resultOrPromise instanceof Promise
+        ? resultOrPromise.catch((err) => {
+            this.delete(keyOrKeys);
+            return Promise.reject(err);
+          })
+        : resultOrPromise
+    );
+  }
+  getSetCb(keyOrKeys, callback) {
+    const key = this._key(keyOrKeys);
+    if (!this.has(key)) {
+      this.setCb(key, callback);
+    }
+    return this.get(key);
+  }
+  count() {
+    return Object.keys(this.__data).length;
+  }
+  delete(keyOrKeys) {
+    Reflect.deleteProperty(this.__data, this._key(keyOrKeys));
+    return this;
+  }
+  clear() {
+    this.__data = Object.create(null);
+  }
+}
+
+class ExpiringLazyCache extends LazyCache {
+  constructor({ separator = DEFAULT_SEPARATOR, expirationGap = DEFAULT_EXPIRATION_GAP } = {}) {
+    super({ separator });
+    this.__expirationGap = expirationGap;
+  }
+  _expiringGap() {
+    return this.__expirationGap;
+  }
+  _isValid(expirationTime, currentTime = Date.now()) {
+    return expirationTime && currentTime <= expirationTime;
+  }
+  has(keyOrKeys, currentTime = Date.now()) {
+    if (!super.has(keyOrKeys)) {
+      return false;
+    }
+    const [expirationTime] = super.get(keyOrKeys) ?? [];
+    return this._isValid(expirationTime, currentTime);
+  }
+  get(keyOrKeys, currentTime = Date.now()) {
+    const [expirationTime, value] = super.get(keyOrKeys) ?? [];
+    return this._isValid(expirationTime, currentTime) ? value : undefined;
+  }
+  // NOTE the expiration gap is substracted here, because we want to expire a
+  //   little earlier than necessary.
+  // NOTE if the expiration is _less_ than the gap, the value is never valid,
+  //   we still need to call set, because we want getSetCb to always return
+  //   when the callback is used.
+  set(keyOrKeys, expiration, value, currentTime = Date.now()) {
+    return super.set(keyOrKeys, [currentTime + expiration - this.__expirationGap, value]);
+  }
+
+  static _extract(result, extractor) {
+    if (!extractor) {
+      return result;
+    }
+    const { expiration, expiry, value, result: extractedResult } = extractor(result);
+    return [expiration ?? expiry, value ?? extractedResult];
+  }
+
+  // NOTE callback can either return a pair [expiration, value] or use an extractor to extract the right values from
+  //   the callback's result
+  setCb(keyOrKeys, callback, { currentTime = Date.now(), extractor } = {}) {
+    const resultOrPromise = callback();
+    if (!(resultOrPromise instanceof Promise)) {
+      const [expiration, value] = ExpiringLazyCache._extract(resultOrPromise, extractor);
+      return this.set(keyOrKeys, expiration, value, currentTime);
+    }
+    return this.set(
+      keyOrKeys,
+      Infinity,
+      resultOrPromise
+        .catch((err) => {
+          this.delete(keyOrKeys);
+          return Promise.reject(err);
+        })
+        .then((result) => {
+          const [expiration, value] = ExpiringLazyCache._extract(result, extractor);
+          this.set(keyOrKeys, expiration, value, currentTime);
+          return value;
+        })
+    );
+  }
+
+  // NOTE callback can either return a pair [expiration, value] or use an extractor to extract the right values from
+  //   the callback's result
+  getSetCb(keyOrKeys, callback, { currentTime = Date.now(), extractor } = {}) {
+    const key = this._key(keyOrKeys);
+    if (!this.has(key, currentTime) || !super.has(key)) {
+      this.setCb(key, callback, { currentTime, extractor });
+      const [, value] = super.get(key);
+      return value;
+    }
+    return this.get(key, currentTime);
+  }
+}
+
+module.exports = {
+  DEFAULT_EXPIRATION_GAP,
+  DEFAULT_SEPARATOR,
+  LazyCache,
+  ExpiringLazyCache,
+};

package/src/shared/redis.js

@@ -178,6 +178,15 @@ const connectionCheck = async (options) => {
     });
 };
 
+const isClusterMode = () => {
+  if (!("__clusterMode" in isClusterMode)) {
+    const env = getEnvInstance();
+    const { credentials } = env.redisRequires;
+    isClusterMode.__clusterMode = credentials.cluster_mode;
+  }
+  return isClusterMode.__clusterMode;
+};
+
 module.exports = {
   createClientAndConnect,
   createMainClientAndConnect,
@@ -186,4 +195,5 @@ module.exports = {
   closeMainClient,
   closeSubscribeClient,
   connectionCheck,
+  isClusterMode,
 };

package/srv/service/admin-service.cds

@@ -13,14 +13,14 @@ service EventQueueAdminService {
         null as space: String,
         *
   } actions {
-    action setStatusAndAttempts(
-      // TODO: remove tenant as soon as CAP issue is fixed https://github.tools.sap/cap/issues/issues/18445
-      @mandatory
-      tenant: String,
-      status: db.Status,
-      @assert.range: [0,100]
-      attempts: Integer) returns Event;
-  }
+      action setStatusAndAttempts(
+        // TODO: remove tenant as soon as CAP issue is fixed https://github.tools.sap/cap/issues/issues/18445
+        @mandatory
+        tenant: String,
+        status: db.Status,
+        @assert.range: [0,100]
+        attempts: Integer) returns Event;
+    }
 
   @cds.persistence.skip
   @readonly
@@ -32,12 +32,22 @@ service EventQueueAdminService {
     space: String;
     ttl: Integer;
     createdAt: Integer;
-  }
+  } actions {
+      action releaseLock(
+        // TODO: remove tenant as soon as CAP issue is fixed https://github.tools.sap/cap/issues/issues/18445
+        @mandatory
+        tenant: String,
+        @mandatory
+        type: String,
+        @mandatory
+        subType: String) returns Boolean;
+    }
 
    @readonly
    @cds.persistence.skip
    entity Tenant {
       Key ID: String;
       subdomain: String;
+      metadata: String;
    }
 }

package/srv/service/admin-service.js

@@ -5,6 +5,7 @@ const cdsHelper = require("../../src/shared/cdsHelper");
 const { EventProcessingStatus } = require("../../src");
 const config = require("../../src/config");
 const distributedLock = require("../../src/shared/distributedLock");
+const redisPub = require("../../src/redis/redisPub");
 
 module.exports = class AdminService extends cds.ApplicationService {
   async init() {
@@ -61,7 +62,7 @@ module.exports = class AdminService extends cds.ApplicationService {
     });
 
     this.on("READ", Tenant, async () => {
-      const tenants = await cdsHelper.getAllTenantWithSubdomain();
+      const tenants = await cdsHelper.getAllTenantWithMetadata();
       return tenants ?? [];
     });
 
@@ -82,14 +83,27 @@ module.exports = class AdminService extends cds.ApplicationService {
         return req.reject(400, "No status or attempts provided");
       }
 
-      await cds.tx({ tenant, headers: { "z-id": tenant } }, async () => {
+      const event = await cds.tx({ tenant, headers: { "z-id": tenant } }, async () => {
+        const event = await SELECT.one.from(EventDb).where({ ID: req.params[0].ID ?? req.params[0] });
         await UPDATE.entity(EventDb)
           .set(updateData)
           .where({ ID: req.params[0].ID ?? req.params[0] });
+        return event;
+      });
+      redisPub.broadcastEvent(tenant, event).catch(() => {
+        /* ignore errors */
       });
       return await this.send(new cds.Request({ query: req.query, headers: req.headers }));
     });
 
+    this.on("releaseLock", async (req) => {
+      cds.log("eventQueue").info("Releasing event-queue lock", req.data);
+      const { tenant, type, subType } = req.data;
+      return await cds.tx({ tenant }, async (tx) => {
+        return await distributedLock.releaseLock(tx.context, [type, subType].join("##"));
+      });
+    });
+
     await super.init();
   }