@cap-js-community/event-queue 1.10.0-beta.4 → 1.10.0-beta.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.10.0-beta.4",
+  "version": "1.10.0-beta.6",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -44,11 +44,11 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@sap/xssec": "^4.2.4",
-    "cron-parser": "^5.0.0",
+    "@sap/xssec": "^4.5.0",
+    "cron-parser": "^5.1.0",
     "redis": "^4.7.0",
     "verror": "^1.10.1",
-    "yaml": "^2.6.1"
+    "yaml": "^2.7.1"
   },
   "devDependencies": {
     "@cap-js/cds-test": "^0.2.0",

package/src/EventQueueProcessorBase.js

@@ -806,10 +806,10 @@ class EventQueueProcessorBase {
                   currentAttempt: exceededEvent.attempts,
                 }
               );
+              await tx.rollback();
               await executeInNewTransaction(this.__baseContext, "error-hookForExceededEvents", async (tx) =>
                 this.#persistEventQueueStatusForExceeded(tx, [exceededEvent], EventProcessingStatus.Error)
               );
-              await tx.rollback();
             }
           }
         );

package/src/config.js

@@ -24,6 +24,7 @@ const DEFAULT_INCREASE_PRIORITY = true;
 const DEFAULT_KEEP_ALIVE_INTERVAL = 60;
 const DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL = 3.5;
 const DEFAULT_INHERIT_TRACE_CONTEXT = true;
+const DEFAULT_CHECK_FOR_NEXT_CHUNK = true;
 const SUFFIX_PERIODIC = "_PERIODIC";
 const CAP_EVENT_TYPE = "CAP_OUTBOX";
 const CAP_PARALLEL_DEFAULT = 5;
@@ -152,18 +153,50 @@ class Config {
     return !!this.#env.redisRequires?.credentials;
   }
 
+  #parseRegexOrString(str) {
+    const regexLiteralPattern = /^\/((?:\\.|[^\\/])*)\/([gimsuy]*)$/;
+    const match = str.match(regexLiteralPattern);
+
+    if (match) {
+      try {
+        return { type: "regex", value: new RegExp(match[1], match[2]) };
+      } catch {
+        return { type: "string", value: str };
+      }
+    }
+    return { type: "string", value: str };
+  }
+
   shouldBeProcessedInThisApplication(type, subType) {
     const config = this.#eventMap[this.generateKey(type, subType)];
     const appNameConfig = config._appNameMap;
     const appInstanceConfig = config._appInstancesMap;
+    let result = true;
     if (!appNameConfig && !appInstanceConfig) {
-      return true;
+      return result;
     }
 
     if (appNameConfig) {
-      const shouldBeProcessedBasedOnAppName = appNameConfig[this.#env.applicationName];
-      if (!shouldBeProcessedBasedOnAppName) {
-        return false;
+      if (config._appNameContainsRegex) {
+        for (const configKey in appNameConfig) {
+          const config = appNameConfig[configKey];
+          if (config.type === "regex") {
+            result = config.value.test(this.#env.applicationName);
+          } else {
+            const shouldBeProcessedBasedOnAppName = appNameConfig[this.#env.applicationName];
+            if (!shouldBeProcessedBasedOnAppName) {
+              result = config.value === this.#env.applicationName;
+            }
+          }
+          if (result) {
+            break;
+          }
+        }
+      } else {
+        const shouldBeProcessedBasedOnAppName = appNameConfig[this.#env.applicationName];
+        if (!shouldBeProcessedBasedOnAppName) {
+          return false;
+        }
       }
     }
 
@@ -174,7 +207,7 @@ class Config {
       }
     }
 
-    return true;
+    return result;
   }
 
   checkRedisEnabled() {
@@ -335,6 +368,7 @@ class Config {
       this.#config.events.splice(index, 1);
     }
 
+    // NOTE: CAP outbox defaults are injected by cds.requires.outbox
     const eventConfig = this.#sanitizeParamsAdHocEvent({
       type: CAP_EVENT_TYPE,
       subType: serviceName,
@@ -507,6 +541,7 @@ class Config {
     event.increasePriorityOverTime = event.increasePriorityOverTime ?? DEFAULT_INCREASE_PRIORITY;
     event.keepAliveInterval = (event.keepAliveInterval ?? DEFAULT_KEEP_ALIVE_INTERVAL) * 1000;
     event.keepAliveMaxInProgressTime = event.keepAliveInterval * DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL;
+    event.checkForNextChunk = event.checkForNextChunk ?? DEFAULT_CHECK_FOR_NEXT_CHUNK;
   }
 
   #sanitizeParamsBase(config, allowList) {
@@ -527,7 +562,12 @@ class Config {
   }
 
   #basicEventTransformationAfterValidate(event) {
-    event._appNameMap = event.appNames ? Object.fromEntries(new Map(event.appNames.map((a) => [a, true]))) : null;
+    event._appNameMap = event.appNames
+      ? Object.fromEntries(new Map(event.appNames.map((a) => [a, this.#parseRegexOrString(a)])))
+      : null;
+    event._appNameContainsRegex = event.appNames
+      ? event.appNames.some((appName) => this.#parseRegexOrString(appName).type === "regex")
+      : null;
     event._appInstancesMap = event.appInstances
       ? Object.fromEntries(new Map(event.appInstances.map((a) => [a, true])))
       : null;

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -9,6 +9,12 @@ const config = require("../config");
 
 const COMPONENT_NAME = "/eventQueue/outbox/generic";
 
+const EVENT_QUEUE_ACTIONS = {
+  EXCEEDED: "eventQueueRetriesExceeded",
+  CLUSTER: "eventQueueCluster",
+  CHECK_AND_ADJUST: "eventQueueCheckAndAdjustPayload",
+};
+
 class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   constructor(context, eventType, eventSubType, config) {
     super(context, eventType, eventSubType, config);
@@ -21,7 +27,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     this.__srvUnboxed = cds.unboxed(this.__srv);
     const { handlers, clusterRelevant, specificClusterRelevant } = this.__srvUnboxed.handlers.on.reduce(
       (result, handler) => {
-        if (handler.on.startsWith("clusterQueueEntries")) {
+        if (handler.on.startsWith(EVENT_QUEUE_ACTIONS.CLUSTER)) {
           if (handler.on.split(".").length === 2) {
             result.specificClusterRelevant = true;
           } else {
@@ -38,26 +44,16 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     this.__genericClusterHandler = clusterRelevant;
     this.__specificClusterHandler = specificClusterRelevant;
     await this.#setContextUser(this.context, config.userId);
-    return super.getQueueEntriesAndSetToInProgress();
+    return await super.getQueueEntriesAndSetToInProgress();
   }
 
-  // NOTE: issue here: if events are not sorted before it might not be unique here:
-  //       - we have service events
-  //       - we have action specific events
-  // 1. I need to collect all action names
-  // 2. I need to check for which actions a handler exits
-  // 3. For actions with specific existing handler --> call specific action
-  // 4. For all others call generic handler
-  // NOTE: OVERALL idea is that the handler returns the cluster map and MUST not call any baseImpl functions!
-  //      --> structure is a map of { key: { queueEntries: [], payload: {} }
-  // TODO: document that clusterQueueEntries is now async!!!
-  // TODO: validate that return structure is as expected
   async clusterQueueEntries(queueEntriesWithPayloadMap) {
     if (!this.__genericClusterRelevantAndAvailable && !this.__specificClusterRelevantAndAvailable) {
       return super.clusterQueueEntries(queueEntriesWithPayloadMap);
     }
     const { genericClusterEvents, specificClusterEvents } = this.#clusterByAction(queueEntriesWithPayloadMap);
 
+    const clusterMap = {};
     if (Object.keys(genericClusterEvents).length) {
       if (!this.__genericClusterRelevantAndAvailable) {
         for (const actionName in genericClusterEvents) {
@@ -66,39 +62,91 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       } else {
         for (const actionName in genericClusterEvents) {
           const msg = new cds.Request({
-            event: `clusterQueueEntries`,
+            event: EVENT_QUEUE_ACTIONS.CLUSTER,
+            user: this.context.user,
             eventQueue: {
               processor: this,
               clusterByPayloadProperty: (propertyName, cb) =>
-                this.clusterByPayloadProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
+                this.#clusterByPayloadProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
               clusterByEventProperty: (propertyName, cb) =>
-                this.clusterByEventProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
+                this.#clusterByEventProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
               clusterByDataProperty: (propertyName, cb) =>
-                this.clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+                this.#clusterByDataProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
             },
           });
-          const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
-          this.#addToProcessingMap(handlerCluster);
+          const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
+          if (this.#validateCluster(clusterResult)) {
+            Object.assign(clusterMap, clusterResult);
+          } else {
+            this.logger.error(
+              "cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
+              {
+                handler: msg.event,
+                clusterResult: JSON.stringify(clusterResult),
+              }
+            );
+            return super.clusterQueueEntries(queueEntriesWithPayloadMap);
+          }
         }
       }
     }
 
     for (const actionName in specificClusterEvents) {
       const msg = new cds.Request({
-        event: `clusterQueueEntries.${actionName}`,
+        event: `${EVENT_QUEUE_ACTIONS.CLUSTER}.${actionName}`,
+        user: this.context.user,
         eventQueue: {
           processor: this,
           clusterByPayloadProperty: (propertyName, cb) =>
-            this.clusterByPayloadProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            this.#clusterByPayloadProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
           clusterByEventProperty: (propertyName, cb) =>
-            this.clusterByEventProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            this.#clusterByEventProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
           clusterByDataProperty: (propertyName, cb) =>
-            this.clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            this.#clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
         },
       });
-      const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
-      this.#addToProcessingMap(handlerCluster);
+      const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
+      if (this.#validateCluster(clusterResult)) {
+        Object.assign(clusterMap, clusterResult);
+      } else {
+        this.logger.error(
+          "cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
+          {
+            handler: msg.event,
+            clusterResult: JSON.stringify(clusterResult),
+          }
+        );
+        return super.clusterQueueEntries(queueEntriesWithPayloadMap);
+      }
     }
+    this.#addToProcessingMap(clusterMap);
+  }
+
+  #validateCluster(obj) {
+    if (typeof obj !== "object" || obj === null || Array.isArray(obj)) {
+      return false;
+    }
+
+    for (const key of Object.keys(obj)) {
+      const clusterEntry = obj[key];
+      if (typeof clusterEntry !== "object" || clusterEntry === null || Array.isArray(obj)) {
+        return false;
+      }
+
+      if (!Array.isArray(clusterEntry.queueEntries)) {
+        return false;
+      }
+
+      if (
+        typeof clusterEntry.payload !== "object" ||
+        clusterEntry.payload === null ||
+        Array.isArray(clusterEntry.payload)
+      ) {
+        return false;
+      }
+    }
+
+    return true;
   }
 
   clusterBase(queueEntriesWithPayloadMap, propertyName, refCb, cb) {
@@ -139,7 +187,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     return result[key];
   }
 
-  clusterByPayloadProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+  #clusterByPayloadProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
     return this.clusterBase(
       queueEntriesWithPayloadMap,
       propertyName,
@@ -148,7 +196,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     );
   }
 
-  clusterByEventProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+  #clusterByEventProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
     return this.clusterBase(
       queueEntriesWithPayloadMap,
       propertyName,
@@ -157,7 +205,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     );
   }
 
-  clusterByDataProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+  #clusterByDataProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
     return this.clusterBase(
       queueEntriesWithPayloadMap,
       propertyName,
@@ -205,13 +253,13 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   }
 
   #hasEventSpecificClusterHandler(queueEntry) {
-    return !!this.__onHandlers[["clusterQueueEntries", queueEntry.payload.event].join(".")];
+    return !!this.__onHandlers[[EVENT_QUEUE_ACTIONS.CLUSTER, queueEntry.payload.event].join(".")];
   }
 
   async checkEventAndGeneratePayload(queueEntry) {
     const payload = await super.checkEventAndGeneratePayload(queueEntry);
     const { event } = payload;
-    const handlerName = this.#checkHandlerExists("checkEventAndGeneratePayload", event);
+    const handlerName = this.#checkHandlerExists(EVENT_QUEUE_ACTIONS.CHECK_AND_ADJUST, event);
     if (!handlerName) {
       return payload;
     }
@@ -230,18 +278,26 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     }
   }
 
-  // simple here as per entry
   async hookForExceededEvents(exceededEvent) {
-    return await super.hookForExceededEvents(exceededEvent);
+    const { event } = exceededEvent.payload;
+    const handlerName = this.#checkHandlerExists(EVENT_QUEUE_ACTIONS.EXCEEDED, event);
+    if (!handlerName) {
+      return await super.hookForExceededEvents(exceededEvent);
+    }
+
+    const { msg, userId } = this.#buildDispatchData(this.context, exceededEvent.payload, {
+      queueEntries: [exceededEvent],
+    });
+    await this.#setContextUser(this.context, userId, msg);
+    msg.event = handlerName;
+    await this.__srvUnboxed.tx(this.context).send(msg);
   }
 
+  // NOTE: Currently not exposed to CAP service; we wait for a valid use case
   async beforeProcessingEvents() {
     return await super.beforeProcessingEvents();
   }
 
-  // maybe async getter on req.data // only for periodic events
-  // getLastSuccessfulRunTimestamp
-
   #checkHandlerExists(eventQueueFn, event) {
     const specificHandler = this.__onHandlers[[eventQueueFn, event].join(".")];
     if (specificHandler) {

package/src/publishEvent.js

@@ -63,8 +63,8 @@ const publishEvent = async (
       event.context = JSON.stringify({ traceContext: openTelemetry.getCurrentTraceContext() });
     }
 
-    if (eventConfig.timeBucket) {
-      event.startAfter ??= CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
+    if (eventConfig.timeBucket && event.startAfter !== undefined) {
+      event.startAfter = CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
     }
   }
   if (config.insertEventsBeforeCommit && !skipInsertEventsBeforeCommit) {