@cap-js-community/event-queue 1.10.0-beta.3 → 1.10.0-beta.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.10.0-beta.3",
+  "version": "1.10.0-beta.5",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -44,7 +44,7 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@sap/xssec": "^4.2.4",
+    "@sap/xssec": "^4.5.0",
     "cron-parser": "^5.0.0",
     "redis": "^4.7.0",
     "verror": "^1.10.1",

package/src/EventQueueProcessorBase.js

@@ -806,10 +806,10 @@ class EventQueueProcessorBase {
                   currentAttempt: exceededEvent.attempts,
                 }
               );
+              await tx.rollback();
               await executeInNewTransaction(this.__baseContext, "error-hookForExceededEvents", async (tx) =>
                 this.#persistEventQueueStatusForExceeded(tx, [exceededEvent], EventProcessingStatus.Error)
               );
-              await tx.rollback();
             }
           }
         );

package/src/config.js

@@ -24,6 +24,7 @@ const DEFAULT_INCREASE_PRIORITY = true;
 const DEFAULT_KEEP_ALIVE_INTERVAL = 60;
 const DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL = 3.5;
 const DEFAULT_INHERIT_TRACE_CONTEXT = true;
+const DEFAULT_CHECK_FOR_NEXT_CHUNK = true;
 const SUFFIX_PERIODIC = "_PERIODIC";
 const CAP_EVENT_TYPE = "CAP_OUTBOX";
 const CAP_PARALLEL_DEFAULT = 5;
@@ -335,6 +336,7 @@ class Config {
       this.#config.events.splice(index, 1);
     }
 
+    // NOTE: CAP outbox defaults are injected by cds.requires.outbox
     const eventConfig = this.#sanitizeParamsAdHocEvent({
       type: CAP_EVENT_TYPE,
       subType: serviceName,
@@ -507,6 +509,7 @@ class Config {
     event.increasePriorityOverTime = event.increasePriorityOverTime ?? DEFAULT_INCREASE_PRIORITY;
     event.keepAliveInterval = (event.keepAliveInterval ?? DEFAULT_KEEP_ALIVE_INTERVAL) * 1000;
     event.keepAliveMaxInProgressTime = event.keepAliveInterval * DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL;
+    event.checkForNextChunk = event.checkForNextChunk ?? DEFAULT_CHECK_FOR_NEXT_CHUNK;
   }
 
   #sanitizeParamsBase(config, allowList) {

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -38,26 +38,18 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     this.__genericClusterHandler = clusterRelevant;
     this.__specificClusterHandler = specificClusterRelevant;
     await this.#setContextUser(this.context, config.userId);
-    return super.getQueueEntriesAndSetToInProgress();
+    return await super.getQueueEntriesAndSetToInProgress();
   }
 
-  // NOTE: issue here: if events are not sorted before it might not be unique here:
-  //       - we have service events
-  //       - we have action specific events
-  // 1. I need to collect all action names
-  // 2. I need to check for which actions a handler exits
-  // 3. For actions with specific existing handler --> call specific action
-  // 4. For all others call generic handler
-  // NOTE: OVERALL idea is that the handler returns the cluster map and MUST not call any baseImpl functions!
-  //      --> structure is a map of { key: { queueEntries: [], payload: {} }
+  // document structure is a map of { key: { queueEntries: [], payload: {} }
   // TODO: document that clusterQueueEntries is now async!!!
-  // TODO: validate that return structure is as expected
   async clusterQueueEntries(queueEntriesWithPayloadMap) {
     if (!this.__genericClusterRelevantAndAvailable && !this.__specificClusterRelevantAndAvailable) {
       return super.clusterQueueEntries(queueEntriesWithPayloadMap);
     }
     const { genericClusterEvents, specificClusterEvents } = this.#clusterByAction(queueEntriesWithPayloadMap);
 
+    const clusterMap = {};
     if (Object.keys(genericClusterEvents).length) {
       if (!this.__genericClusterRelevantAndAvailable) {
         for (const actionName in genericClusterEvents) {
@@ -67,18 +59,30 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
         for (const actionName in genericClusterEvents) {
           const msg = new cds.Request({
             event: `clusterQueueEntries`,
+            user: this.context.user,
             eventQueue: {
               processor: this,
               clusterByPayloadProperty: (propertyName, cb) =>
-                this.clusterByPayloadProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
+                this.#clusterByPayloadProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
               clusterByEventProperty: (propertyName, cb) =>
-                this.clusterByEventProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
+                this.#clusterByEventProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
               clusterByDataProperty: (propertyName, cb) =>
-                this.clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+                this.#clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
             },
           });
-          const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
-          this.#addToProcessingMap(handlerCluster);
+          const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
+          if (this.#validateCluster(clusterResult)) {
+            Object.assign(clusterMap, clusterResult);
+          } else {
+            this.logger.error(
+              "cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
+              {
+                handler: msg.event,
+                clusterResult: JSON.stringify(clusterResult),
+              }
+            );
+            return super.clusterQueueEntries(queueEntriesWithPayloadMap);
+          }
         }
       }
     }
@@ -86,19 +90,59 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     for (const actionName in specificClusterEvents) {
       const msg = new cds.Request({
         event: `clusterQueueEntries.${actionName}`,
+        user: this.context.user,
         eventQueue: {
           processor: this,
           clusterByPayloadProperty: (propertyName, cb) =>
-            this.clusterByPayloadProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            this.#clusterByPayloadProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
           clusterByEventProperty: (propertyName, cb) =>
-            this.clusterByEventProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            this.#clusterByEventProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
           clusterByDataProperty: (propertyName, cb) =>
-            this.clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            this.#clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
         },
       });
-      const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
-      this.#addToProcessingMap(handlerCluster);
+      const clusterResult = await this.__srvUnboxed.tx(this.context).send(msg);
+      if (this.#validateCluster(clusterResult)) {
+        Object.assign(clusterMap, clusterResult);
+      } else {
+        this.logger.error(
+          "cluster result of handler is not valid. Check the documentation for the expected structure. Continuing without clustering!",
+          {
+            handler: msg.event,
+            clusterResult: JSON.stringify(clusterResult),
+          }
+        );
+        return super.clusterQueueEntries(queueEntriesWithPayloadMap);
+      }
+    }
+    this.#addToProcessingMap(clusterMap);
+  }
+
+  #validateCluster(obj) {
+    if (typeof obj !== "object" || obj === null || Array.isArray(obj)) {
+      return false;
+    }
+
+    for (const key of Object.keys(obj)) {
+      const clusterEntry = obj[key];
+      if (typeof clusterEntry !== "object" || clusterEntry === null || Array.isArray(obj)) {
+        return false;
+      }
+
+      if (!Array.isArray(clusterEntry.queueEntries)) {
+        return false;
+      }
+
+      if (
+        typeof clusterEntry.payload !== "object" ||
+        clusterEntry.payload === null ||
+        Array.isArray(clusterEntry.payload)
+      ) {
+        return false;
+      }
     }
+
+    return true;
   }
 
   clusterBase(queueEntriesWithPayloadMap, propertyName, refCb, cb) {
@@ -112,7 +156,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       for (const clustersKey in clusters) {
         const clusterData = clusters[clustersKey];
         const clusterResult = cb(
-          clustersKey.split("##").shift(),
+          clustersKey.split("##").pop(),
           clusterData.queueEntries.map((entry) => entry.payload.data)
         );
         if (!clusterResult) {
@@ -139,7 +183,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     return result[key];
   }
 
-  clusterByPayloadProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+  #clusterByPayloadProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
     return this.clusterBase(
       queueEntriesWithPayloadMap,
       propertyName,
@@ -148,7 +192,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     );
   }
 
-  clusterByEventProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+  #clusterByEventProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
     return this.clusterBase(
       queueEntriesWithPayloadMap,
       propertyName,
@@ -157,7 +201,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     );
   }
 
-  clusterByDataProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+  #clusterByDataProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
     return this.clusterBase(
       queueEntriesWithPayloadMap,
       propertyName,
@@ -230,18 +274,26 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     }
   }
 
-  // simple here as per entry
   async hookForExceededEvents(exceededEvent) {
-    return await super.hookForExceededEvents(exceededEvent);
+    const { event } = exceededEvent.payload;
+    const handlerName = this.#checkHandlerExists("hookForExceededEvents", event);
+    if (!handlerName) {
+      return await super.hookForExceededEvents(exceededEvent);
+    }
+
+    const { msg, userId } = this.#buildDispatchData(this.context, exceededEvent.payload, {
+      queueEntries: [exceededEvent],
+    });
+    await this.#setContextUser(this.context, userId, msg);
+    msg.event = handlerName;
+    await this.__srvUnboxed.tx(this.context).send(msg);
   }
 
+  // NOTE: Currently not exposed to CAP service; we wait for a valid use case
   async beforeProcessingEvents() {
     return await super.beforeProcessingEvents();
   }
 
-  // maybe async getter on req.data // only for periodic events
-  // getLastSuccessfulRunTimestamp
-
   #checkHandlerExists(eventQueueFn, event) {
     const specificHandler = this.__onHandlers[[eventQueueFn, event].join(".")];
     if (specificHandler) {

package/src/publishEvent.js

@@ -63,8 +63,8 @@ const publishEvent = async (
       event.context = JSON.stringify({ traceContext: openTelemetry.getCurrentTraceContext() });
     }
 
-    if (eventConfig.timeBucket) {
-      event.startAfter ??= CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
+    if (eventConfig.timeBucket && event.startAfter !== undefined) {
+      event.startAfter = CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
     }
   }
   if (config.insertEventsBeforeCommit && !skipInsertEventsBeforeCommit) {