@cap-js-community/event-queue 2.0.0-beta.3 → 2.0.0-beta.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "2.0.0-beta.3",
+  "version": "2.0.0-beta.5",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/EventQueueProcessorBase.js

@@ -24,7 +24,8 @@ const SELECT_LIMIT_EVENTS_PER_TICK = 100;
 const TRIES_FOR_EXCEEDED_EVENTS = 3;
 const EVENT_START_AFTER_HEADROOM = 3 * 1000;
 const SUFFIX_PERIODIC = "_PERIODIC";
-const DEFAULT_RETRY_AFTER = 5 * 60 * 1000;
+
+const ALLOWED_FIELDS_FOR_UPDATE = ["status", "startAfter", "error"];
 
 class EventQueueProcessorBase {
   #eventsWithExceededTries = [];
@@ -38,7 +39,6 @@ class EventQueueProcessorBase {
   #eventConfig;
   #isPeriodic;
   #lastSuccessfulRunTimestamp;
-  #retryFailedAfter;
   #keepAliveRunner;
   #currentKeepAlivePromise = Promise.resolve();
   #etagMap;
@@ -66,7 +66,6 @@ class EventQueueProcessorBase {
     if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
       this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
     }
-    this.#retryFailedAfter = this.#eventConfig.retryFailedAfter ?? DEFAULT_RETRY_AFTER;
     this.__concurrentEventProcessing = this.#eventConfig.multiInstanceProcessing;
     this.__retryAttempts = this.#isPeriodic ? 1 : this.#eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
     this.__selectMaxChunkSize = this.#eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
@@ -345,17 +344,21 @@ class EventQueueProcessorBase {
     }
   }
 
-  #determineAndAddEventStatusToMap(id, processingStatus, statusMap = this.__statusMap) {
+  #determineAndAddEventStatusToMap(id, statusOrUpdateData, statusMap = this.__statusMap) {
+    if (typeof statusOrUpdateData === "number") {
+      statusOrUpdateData = { status: statusOrUpdateData };
+    }
+
     if (!statusMap[id]) {
-      statusMap[id] = processingStatus;
+      statusMap[id] = statusOrUpdateData;
       return;
     }
-    if ([EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(statusMap[id])) {
+    if ([EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(statusMap[id].status)) {
       // NOTE: worst aggregation --> if already error|exceeded keep this state
       return;
     }
-    if (statusMap[id] >= 0) {
-      statusMap[id] = processingStatus;
+    if (statusMap[id].status >= 0) {
+      statusMap[id] = { status: statusOrUpdateData };
     }
   }
 
@@ -396,6 +399,25 @@ class EventQueueProcessorBase {
     );
   }
 
+  #normalizeStatusMap(statusMap) {
+    const originalMap = {};
+    for (const [id, entry] of Object.entries(statusMap)) {
+      originalMap[id] = entry;
+      const result = {};
+      if (typeof entry === "number") {
+        result.status = entry;
+      } else if (typeof entry === "object") {
+        for (const fieldName of ALLOWED_FIELDS_FOR_UPDATE) {
+          if (fieldName in entry) {
+            result[fieldName] = entry[fieldName];
+          }
+        }
+      }
+      statusMap[id] = result;
+    }
+    return originalMap;
+  }
+
   /**
    * This function validates for all selected events one status has been submitted. It's also validated that only for
    * selected events a status has been submitted. Persisting the status of events is done in a dedicated database tx.
@@ -406,98 +428,123 @@ class EventQueueProcessorBase {
       eventType: this.#eventType,
       eventSubType: this.#eventSubType,
     });
+    const originalStatusMap = this.#normalizeStatusMap(statusMap);
     this.#ensureOnlySelectedQueueEntries(statusMap);
     if (!skipChecks) {
       this.#ensureEveryQueueEntryHasStatus();
     }
     this.#ensureEveryStatusIsAllowed(statusMap);
-
-    const { success, failed, exceeded, invalidAttempts } = Object.entries(statusMap).reduce(
-      (result, [queueEntryId, processingStatus]) => {
-        this.__commitedStatusMap[queueEntryId] = processingStatus;
-        delete this.__notCommitedStatusMap[queueEntryId];
-        if (processingStatus === EventProcessingStatus.Open) {
-          result.invalidAttempts.push(queueEntryId);
-        } else if (processingStatus === EventProcessingStatus.Done) {
-          result.success.push(queueEntryId);
-        } else if (processingStatus === EventProcessingStatus.Error) {
-          result.failed.push(queueEntryId);
-        } else if (processingStatus === EventProcessingStatus.Exceeded) {
-          result.exceeded.push(queueEntryId);
-        }
-        return result;
-      },
-      {
-        success: [],
-        failed: [],
-        exceeded: [],
-        invalidAttempts: [],
-      }
-    );
-    if (![success, failed, exceeded, invalidAttempts].some((statusArray) => statusArray.length)) {
-      this.logger.debug("exiting persistEventStatus", {
-        eventType: this.#eventType,
-        eventSubType: this.#eventSubType,
-      });
-      return;
-    }
-
     return await trace(this.baseContext, "persist-event-status", async () => {
       this.logger.debug("persistEventStatus for entries", {
         eventType: this.#eventType,
         eventSubType: this.#eventSubType,
-        invalidAttempts,
-        failed,
-        exceeded,
-        success,
+        statusMap,
       });
-      if (invalidAttempts.length) {
-        await tx.run(
-          UPDATE.entity(this.#config.tableNameEventQueue)
-            .set({
-              status: EventProcessingStatus.Open,
-              lastAttemptTimestamp: new Date().toISOString(),
-              attempts: { "-=": 1 },
-            })
-            .where("ID IN", invalidAttempts)
-        );
-      }
       const ts = new Date().toISOString();
-      const updateTuples = [
-        [success, EventProcessingStatus.Done],
-        [failed, EventProcessingStatus.Error],
-        [exceeded, EventProcessingStatus.Exceeded],
-      ];
-
-      for (const [eventIds, status] of updateTuples) {
-        if (!eventIds.length) {
+      const updateData = Object.entries(statusMap).reduce((result, [id, data]) => {
+        const key = ALLOWED_FIELDS_FOR_UPDATE.map((name) => [name, data[name]])
+          .flat()
+          .join("##");
+
+        result[key] ??= { data, ids: [] };
+        result[key].ids.push(id);
+        return result;
+      }, {});
+
+      for (const { ids, data } of Object.values(updateData)) {
+        if (!("status" in data)) {
+          // TODO: Can this still happen?
+          this.logger.error("can't find status value in return value of event-processing. Setting event to done", {
+            ids,
+            // NOTE: use first id as same return values are clustered
+            eventReturnValue: originalStatusMap[ids[0]],
+          });
           continue;
         }
-        let startAfter;
-        if (status === EventProcessingStatus.Error) {
-          startAfter = new Date(Date.now() + this.#retryFailedAfter);
+
+        for (const id of ids) {
+          this.__commitedStatusMap[id] = data.status;
+          delete this.__notCommitedStatusMap[id];
+        }
+
+        if (![EventProcessingStatus.Open, EventProcessingStatus.Error].includes(data.status)) {
+          delete data.startAfter;
+        }
+
+        if (data.startAfter) {
+          data.startAfter = this.#normalizeDate(data.startAfter);
+        }
+
+        if (data.error) {
+          data.error = this.#error2String(data.error);
+        }
+
+        if (!data.startAfter && [EventProcessingStatus.Error, EventProcessingStatus.Open].includes(data.status)) {
+          data.startAfter = new Date(
+            Date.now() +
+              (EventProcessingStatus.Error ? this.#eventConfig.retryFailedAfter : this.#eventConfig.retryOpenAfter)
+          );
+        }
+
+        if (data.startAfter) {
           this.#eventSchedulerInstance.scheduleEvent(
             this.__context.tenant,
             this.#eventType,
             this.#eventSubType,
             this.#namespace,
-            startAfter
+            data.startAfter
           );
         }
 
         await tx.run(
           UPDATE.entity(this.#config.tableNameEventQueue)
             .set({
-              status: status,
+              ...data,
               lastAttemptTimestamp: ts,
-              ...(status === EventProcessingStatus.Error ? { startAfter: startAfter.toISOString() } : {}),
             })
-            .where("ID IN", eventIds)
+            .where("ID IN", ids)
         );
       }
     });
   }
 
+  #error2String(error) {
+    return JSON.stringify(error, (_, value) => this.#errorReplacer(value));
+  }
+
+  #errorReplacer(value) {
+    if (!(value instanceof Error)) {
+      return value;
+    }
+
+    const plain = {
+      name: value.name,
+      message: value.message,
+      stack: value.stack,
+    };
+
+    for (const key in value) {
+      plain[key] = value[key];
+    }
+
+    return plain;
+  }
+
+  #normalizeDate(value) {
+    if (value instanceof Date) {
+      return value;
+    }
+
+    if (typeof value === "string" || typeof value === "number") {
+      const date = new Date(value);
+      if (!isNaN(date)) {
+        return date;
+      }
+    }
+
+    return null;
+  }
+
   #ensureEveryQueueEntryHasStatus() {
     this.__queueEntries.forEach((queueEntry) => {
       if (
@@ -507,17 +554,17 @@ class EventQueueProcessorBase {
       ) {
         return;
       }
-      this.logger.error("Missing status for selected event entry. Setting status to error", {
+      this.logger.error("Missing status for selected event entry. Setting status to done", {
         eventType: this.#eventType,
         eventSubType: this.#eventSubType,
         queueEntry,
       });
-      this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
+      this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Done);
     });
   }
 
   #ensureEveryStatusIsAllowed(statusMap) {
-    Object.entries(statusMap).forEach(([queueEntryId, status]) => {
+    Object.entries(statusMap).forEach(([queueEntryId, { status }]) => {
       if (
         [
           EventProcessingStatus.Open,
@@ -533,7 +580,7 @@ class EventQueueProcessorBase {
         eventType: this.#eventType,
         eventSubType: this.#eventSubType,
         queueEntryId,
-        status: statusMap[queueEntryId],
+        status: statusMap[queueEntryId].status,
       });
       delete statusMap[queueEntryId];
     });

package/src/config.js

@@ -24,6 +24,7 @@ const CAP_EVENT_TYPE = "CAP_OUTBOX";
 const CAP_PARALLEL_DEFAULT = 5;
 const CAP_MAX_ATTEMPTS_DEFAULT = 5;
 const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
+const DEFAULT_RETRY_AFTER = 5 * 60 * 1000;
 const PRIORITIES = Object.values(Priorities);
 const UTC_DEFAULT = false;
 const USE_CRON_TZ_DEFAULT = true;
@@ -60,6 +61,8 @@ const ALLOWED_EVENT_OPTIONS_AD_HOC = [
   "processAfterCommit",
   "checkForNextChunk",
   "retryFailedAfter",
+  "propagateHeaders",
+  "retryOpenAfter",
   "multiInstanceProcessing",
   "kind",
   "timeBucket",
@@ -435,6 +438,9 @@ class Config {
     event._appInstancesMap = event.appInstances
       ? Object.fromEntries(new Map(event.appInstances.map((a) => [a, true])))
       : null;
+    event.propagateHeaders = event.propagateHeaders ?? [];
+    event.retryFailedAfter = event.retryFailedAfter ?? DEFAULT_RETRY_AFTER;
+    event.retryOpenAfter = event.retryOpenAfter ?? DEFAULT_RETRY_AFTER;
   }
 
   #basicEventValidation(event) {
@@ -751,6 +757,9 @@ class Config {
   }
 
   set useAsCAPOutbox(value) {
+    if (this.#useAsCAPOutbox) {
+      return;
+    }
     this.#useAsCAPOutbox = value;
   }
 
@@ -758,6 +767,14 @@ class Config {
     return this.#useAsCAPOutbox;
   }
 
+  set useAsCAPQueue(value) {
+    this.useAsCAPOutbox = value;
+  }
+
+  get useAsCAPQueue() {
+    return this.useAsCAPOutbox;
+  }
+
   set userId(value) {
     this.#userId = value;
   }

package/src/initialize.js

@@ -35,6 +35,7 @@ const CONFIG_VARS = [
   ["updatePeriodicEvents", true],
   ["thresholdLoggingEventProcessing", 50],
   ["useAsCAPOutbox", false],
+  ["useAsCAPQueue", false],
   ["userId", null],
   ["cleanupLocksAndEventsForDev", false],
   ["redisOptions", {}],
@@ -65,6 +66,7 @@ const CONFIG_VARS = [
  * @param {boolean} [options.updatePeriodicEvents=true] - Automatically update periodic events.
  * @param {number} [options.thresholdLoggingEventProcessing=50] - Threshold for logging event processing time (in milliseconds).
  * @param {boolean} [options.useAsCAPOutbox=false] - Use the event queue as a CAP Outbox.
+ * @param {boolean} [options.useAsCAPQueue=false] - Use the event queue as a CAP Outbox.
  * @param {string} [options.userId=null] - ID of the user initiating the process.
  * @param {boolean} [options.cleanupLocksAndEventsForDev=false] - Cleanup locks and events for development environments.
  * @param {Object} [options.redisOptions={}] - Configuration options for Redis.

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -348,7 +348,13 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       this.logger.error("error processing outboxed service call", err, {
         serviceName: this.eventSubType,
       });
-      return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error]);
+      return queueEntries.map((queueEntry) => [
+        queueEntry.ID,
+        {
+          status: EventProcessingStatus.Error,
+          error: err,
+        },
+      ]);
     }
   }
 
@@ -359,10 +365,48 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       return queueEntries.map((queueEntry) => [queueEntry.ID, result]);
     }
 
+    if (result instanceof Object && !Array.isArray(result)) {
+      return queueEntries.map((queueEntry) => [queueEntry.ID, result]);
+    }
+
     if (!Array.isArray(result)) {
       return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Done]);
     }
 
+    const [firstEntry] = result;
+    if (Array.isArray(firstEntry)) {
+      const [, innerResult] = firstEntry;
+      if (innerResult instanceof Object) {
+        return result;
+      } else {
+        return result.map(([id, status]) => {
+          return [id, { status }];
+        });
+      }
+    } else if (firstEntry instanceof Object) {
+      return result.reduce((result, entry) => {
+        let { ID } = entry;
+
+        if (!ID) {
+          if (queueEntries.length > 1) {
+            throw new Error(
+              "The CAP handler return value does not match the event-queue specification. Please check the documentation"
+            );
+          } else {
+            ID = queueEntries[0].ID;
+          }
+        }
+
+        delete entry.ID;
+        if (!("status" in entry)) {
+          entry.status = EventProcessingStatus.Done;
+        }
+
+        result.push([ID, entry]);
+        return result;
+      }, []);
+    }
+
     const valid = !result.some((entry) => {
       const [, status] = entry;
       return !validStatusValues.includes(status);

package/src/outbox/eventQueueAsOutbox.js

@@ -51,14 +51,16 @@ function outboxed(srv, customOpts) {
       customOpts || {}
     );
     config.addCAPOutboxEventBase(srv.name, outboxOpts);
-    const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.event);
-    if (specificSettings) {
+    const hasSpecificSettings = !!config.getCdsOutboxEventSpecificConfig(srv.name, req.event);
+    if (hasSpecificSettings) {
       outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
     }
-
-    const namespace = (specificSettings ?? outboxOpts).namespace ?? config.namespace;
+    const subType = hasSpecificSettings ? [srv.name, req.event].join(".") : srv.name;
+    const namespace = outboxOpts.namespace ?? config.namespace;
+    outboxOpts = config.getEventConfig(CDS_EVENT_TYPE, subType, namespace);
+    const eventHeaders = getPropagatedHeaders(outboxOpts, req);
     if (["persistent-outbox", "persistent-queue"].includes(outboxOpts.kind)) {
-      await _mapToEventAndPublish(context, srv.name, req, !!specificSettings, namespace);
+      await _mapToEventAndPublish(context, subType, req, eventHeaders, namespace);
       return;
     }
     context.on("succeeded", async () => {
@@ -84,24 +86,35 @@ function unboxed(srv) {
   return srv[UNBOXED] || srv;
 }
 
-const _mapToEventAndPublish = async (context, name, req, actionSpecific, namespace) => {
+const getPropagatedHeaders = (config, req) => {
+  const propagateHeaders = config.propagateHeaders.reduce((headers, headerName) => {
+    if (headerName in req.tx.context.headers) {
+      headers[headerName] = req.tx.context.headers[headerName];
+    }
+    return headers;
+  }, {});
+  return Object.assign(propagateHeaders, req.headers);
+};
+
+const _mapToEventAndPublish = async (context, subType, req, eventHeaders, namespace) => {
   const eventQueueSpecificValues = {};
   for (const header in req.headers ?? {}) {
     for (const field of EVENT_QUEUE_SPECIFIC_FIELDS) {
       if (header.toLocaleLowerCase() === `x-eventqueue-${field.toLocaleLowerCase()}`) {
         eventQueueSpecificValues[field] = req.headers[header];
-        delete req.headers[header];
+        delete eventHeaders[header];
         break;
       }
     }
   }
+
   const event = {
     contextUser: context.user.id,
     ...(req._fromSend || (req.reply && { _fromSend: true })), // send or emit
     ...(req.inbound && { inbound: req.inbound }),
     ...(req.event && { event: req.event }),
     ...(req.data && { data: req.data }),
-    ...(req.headers && { headers: req.headers }),
+    ...(eventHeaders && { headers: eventHeaders }),
     ...(req.query && { query: req.query }),
   };
 
@@ -109,7 +122,7 @@ const _mapToEventAndPublish = async (context, name, req, actionSpecific, namespa
     cds.tx(context),
     {
       type: CDS_EVENT_TYPE,
-      subType: actionSpecific ? [name, req.event].join(".") : name,
+      subType,
       payload: JSON.stringify(event),
       namespace: eventQueueSpecificValues.namespace ?? namespace,
       ...eventQueueSpecificValues,

package/src/shared/redis/client.js

@@ -17,9 +17,9 @@ const subscribeRedisChannel = async (channel, subscribeHandler) => {
   return await redisClient.subscribeChannel(config.redisOptions, channelWithNamespace, subscribeHandler);
 };
 
-const publishMessage = async (channel, message, { addNamespace = true } = {}) => {
+const publishMessage = async (channel, message) => {
   const redisClient = RedisClient.create(REDIS_CLIENT_NAME);
-  const channelWithNamespace = [config.redisNamespace(addNamespace), channel].join("##");
+  const channelWithNamespace = [config.redisNamespace(false), channel].join("##");
   return await redisClient.publishMessage(config.redisOptions, channelWithNamespace, message);
 };
 

package/src/shared/redis/index.js

@@ -27,11 +27,9 @@ const attachRedisUnsubscribeHandler = () => {
 
 const handleUnsubscribe = (tenantId) => {
   if (config.redisEnabled) {
-    client
-      .publishMessage(REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId }), { addNamespace: false })
-      .catch((error) => {
-        cds.log(COMPONENT_NAME).error(`publishing tenant unsubscribe failed. tenantId: ${tenantId}`, error);
-      });
+    client.publishMessage(REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId })).catch((error) => {
+      cds.log(COMPONENT_NAME).error(`publishing tenant unsubscribe failed. tenantId: ${tenantId}`, error);
+    });
   } else {
     config.executeUnsubscribeHandlers(tenantId);
   }