@cap-js-community/event-queue 2.0.4 → 2.1.0-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "2.0.4",
+  "version": "2.1.0-beta.1",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -22,8 +22,6 @@
     "multi-tenancy"
   ],
   "scripts": {
-    "start": "PORT=4005 cds-serve",
-    "watch": "PORT=4005 cds watch",
     "test:unit": "jest --selectProjects unit",
     "test:integration": "jest --selectProjects integration --runInBand",
     "voter:test:integration": "jest --selectProjects integration",
@@ -38,13 +36,12 @@
     "eslint:ci": "eslint .",
     "prettier": "prettier --write --loglevel error .",
     "prettier:ci": "prettier --check .",
-    "prepareRelease": "npm prune --production",
     "docs": "cd docs && bundle exec jekyll serve",
     "docs:install": "cd docs && npx shx rm -rf vendor Gemfile.lock && bundle install",
     "upgrade-lock": "npx shx rm -rf package-lock.json node_modules && npm i --package-lock"
   },
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "dependencies": {
     "@cap-js-community/common": "^0.3.4",
@@ -56,17 +53,16 @@
   "devDependencies": {
     "@actions/core": "^2.0.2",
     "@cap-js/cds-test": "^0.4.1",
-    "@cap-js/db-service": "^2.8.1",
-    "@cap-js/hana": "^2.5.1",
-    "@cap-js/sqlite": "^2.1.2",
+    "@cap-js/db-service": "^2.8.2",
+    "@cap-js/hana": "^2.6.0",
+    "@cap-js/sqlite": "^2.1.3",
     "@opentelemetry/api": "^1.9.0",
-    "@sap/cds": "^9.6.4",
-    "@sap/cds-dk": "^9.6.1",
+    "@sap/cds": "^9.7.0",
+    "@sap/cds-dk": "^9.7.0",
     "eslint": "^8.57.1",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-jest": "^29.12.1",
     "eslint-plugin-node": "^11.1.0",
-    "express": "^4.22.1",
     "jest": "^29.7.0",
     "prettier": "^2.8.8"
   },

package/src/EventQueueProcessorBase.js

@@ -25,7 +25,7 @@ const TRIES_FOR_EXCEEDED_EVENTS = 3;
 const EVENT_START_AFTER_HEADROOM = 3 * 1000;
 const SUFFIX_PERIODIC = "_PERIODIC";
 
-const ALLOWED_FIELDS_FOR_UPDATE = ["status", "startAfter", "error"];
+const ALLOWED_FIELDS_FOR_UPDATE = ["status", "startAfter", "error", "nextData"];
 
 class EventQueueProcessorBase {
   #eventsWithExceededTries = [];
@@ -43,6 +43,7 @@ class EventQueueProcessorBase {
   #currentKeepAlivePromise = Promise.resolve();
   #etagMap;
   #namespace;
+  #nextSagaEvents;
 
   constructor(context, eventType, eventSubType, config) {
     this.__context = context;
@@ -407,7 +408,7 @@ class EventQueueProcessorBase {
       UPDATE.entity(this.#config.tableNameEventQueue)
         .set({
           status: status,
-          ...(error && { error: this.#error2String(error) }),
+          ...(error && { error: this._error2String(error) }),
         })
         .where({
           ID: queueEntryIds,
@@ -468,6 +469,10 @@ class EventQueueProcessorBase {
         return result;
       }, {});
 
+      if (!Object.values(updateData).length) {
+        return;
+      }
+
       for (const { ids, data } of Object.values(updateData)) {
         if (!("status" in data)) {
           // TODO: Can this still happen?
@@ -493,7 +498,7 @@ class EventQueueProcessorBase {
         }
 
         if (data.error) {
-          data.error = this.#error2String(data.error);
+          data.error = this._error2String(data.error);
         }
 
         if (!data.startAfter && [EventProcessingStatus.Error, EventProcessingStatus.Open].includes(data.status)) {
@@ -522,10 +527,15 @@ class EventQueueProcessorBase {
             .where("ID IN", ids)
         );
       }
+
+      if (this.#nextSagaEvents?.length) {
+        await tx.run(INSERT.into(this.#config.tableNameEventQueue).entries(this.#nextSagaEvents));
+        this.#nextSagaEvents = [];
+      }
     });
   }
 
-  #error2String(error) {
+  _error2String(error) {
     return JSON.stringify(error, (_, value) => this.#errorReplacer(value));
   }
 
@@ -1390,6 +1400,10 @@ class EventQueueProcessorBase {
   get allowedFieldsEventHandler() {
     return ALLOWED_FIELDS_FOR_UPDATE;
   }
+
+  set nextSagaEvents(value) {
+    this.#nextSagaEvents = value;
+  }
 }
 
 module.exports = EventQueueProcessorBase;

package/src/config.js

@@ -28,6 +28,8 @@ const DEFAULT_RETRY_AFTER = 5 * 60 * 1000;
 const PRIORITIES = Object.values(Priorities);
 const UTC_DEFAULT = false;
 const USE_CRON_TZ_DEFAULT = true;
+const SAGA_SUCCESS = "#succeeded";
+const SAGA_FAILED = "#failed";
 
 const BASE_TABLES = {
   EVENT: "sap.eventqueue.Event",
@@ -62,6 +64,7 @@ const ALLOWED_EVENT_OPTIONS_AD_HOC = [
   "checkForNextChunk",
   "retryFailedAfter",
   "propagateHeaders",
+  "propagatedContextProperties",
   "retryOpenAfter",
   "multiInstanceProcessing",
   "kind",
@@ -236,7 +239,7 @@ class Config {
 
   executeUnsubscribeHandlers(tenantId) {
     this.#unsubscribedTenants[tenantId] = true;
-    setTimeout(() => delete this.#unsubscribedTenants[tenantId], DELETE_TENANT_BLOCK_AFTER_MS);
+    setTimeout(() => delete this.#unsubscribedTenants[tenantId], DELETE_TENANT_BLOCK_AFTER_MS).unref();
     for (const unsubscribeHandler of this.#unsubscribeHandlers) {
       try {
         unsubscribeHandler(tenantId);
@@ -383,6 +386,20 @@ class Config {
               result.adHoc
             );
             result.adHoc[key] = specificEventConfig;
+            const sagaSuccessKey = [fnName, SAGA_SUCCESS].join("/");
+            if (config.events[sagaSuccessKey]) {
+              const [sagaKey, sagaSpecificEventConfig] = this.addCAPOutboxEventSpecificAction(
+                srvConfig,
+                name,
+                fnName,
+                result.adHoc
+              );
+              result.adHoc[sagaKey] = sagaSpecificEventConfig;
+            } else {
+              const sagaConfig = { ...specificEventConfig };
+              sagaConfig.subType = [sagaConfig.subType, SAGA_SUCCESS].join("/");
+              result.adHoc[[key, SAGA_SUCCESS].join("/")] = sagaConfig;
+            }
           }
         }
         return result;
@@ -405,11 +422,24 @@ class Config {
   getCdsOutboxEventSpecificConfig(serviceName, action) {
     const srv = cds.env.requires[serviceName];
     const config = srv?.outbox ?? srv?.queued;
-    if (config?.events?.[action]) {
+    const specificConfig = config?.events?.[action];
+
+    if (specificConfig) {
       return this.#mixCAPPropertyNamesWithEventQueueNames(config.events[action]);
-    } else {
+    }
+
+    if (!action) {
       return null;
     }
+
+    const [withoutSaga, sagaSuffix] = action.split("/");
+    if ([SAGA_FAILED, SAGA_SUCCESS].includes(sagaSuffix)) {
+      if (config?.events?.[withoutSaga]) {
+        return this.#mixCAPPropertyNamesWithEventQueueNames(config.events[withoutSaga]);
+      }
+    }
+
+    return null;
   }
 
   #mapEnvEvents(events) {
@@ -488,6 +518,7 @@ class Config {
       ? Object.fromEntries(new Map(event.appInstances.map((a) => [a, true])))
       : null;
     event.propagateHeaders = event.propagateHeaders ?? [];
+    event.propagatedContextProperties = event.propagatedContextProperties ?? [];
     event.retryFailedAfter = event.retryFailedAfter ?? DEFAULT_RETRY_AFTER;
     event.retryOpenAfter = event.retryOpenAfter ?? DEFAULT_RETRY_AFTER;
   }

package/src/initialize.js

@@ -123,6 +123,7 @@ const initialize = async (options = {}) => {
     multiTenancyEnabled: config.isMultiTenancy,
     redisEnabled: config.redisEnabled,
     runInterval: config.runInterval,
+    useAsCAPQueue: config.useAsCAPQueue,
   });
   resolveFn();
 };

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -14,6 +14,8 @@ const EVENT_QUEUE_ACTIONS = {
   EXCEEDED: "eventQueueRetriesExceeded",
   CLUSTER: "eventQueueCluster",
   CHECK_AND_ADJUST: "eventQueueCheckAndAdjustPayload",
+  SAGA_SUCCESS: "#succeeded",
+  SAGA_FAILED: "#failed",
 };
 
 class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
@@ -25,7 +27,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   async getQueueEntriesAndSetToInProgress() {
     const { srvName } = config.normalizeSubType(this.eventType, this.eventSubType);
     this.__srv = await cds.connect.to(srvName);
-    this.__srvUnboxed = cds.unboxed(this.__srv);
+    this.__srvUnboxed = cds.unqueued(this.__srv);
     const { handlers, clusterRelevant, specificClusterRelevant } = this.__srvUnboxed.handlers.on.reduce(
       (result, handler) => {
         if (handler.on.startsWith(EVENT_QUEUE_ACTIONS.CLUSTER)) {
@@ -260,7 +262,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   async checkEventAndGeneratePayload(queueEntry) {
     const payload = await super.checkEventAndGeneratePayload(queueEntry);
     const { event } = payload;
-    const handlerName = this.#checkHandlerExists(EVENT_QUEUE_ACTIONS.CHECK_AND_ADJUST, event);
+    const handlerName = this.#checkHandlerExists({ eventQueueFn: EVENT_QUEUE_ACTIONS.CHECK_AND_ADJUST, event });
     if (!handlerName) {
       return payload;
     }
@@ -281,7 +283,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
 
   async hookForExceededEvents(exceededEvent) {
     const { event } = exceededEvent.payload;
-    const handlerName = this.#checkHandlerExists(EVENT_QUEUE_ACTIONS.EXCEEDED, event);
+    const handlerName = this.#checkHandlerExists({ eventQueueFn: EVENT_QUEUE_ACTIONS.EXCEEDED, event });
     if (!handlerName) {
       return await super.hookForExceededEvents(exceededEvent);
     }
@@ -299,14 +301,27 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     return await super.beforeProcessingEvents();
   }
 
-  #checkHandlerExists(eventQueueFn, event) {
-    const specificHandler = this.__onHandlers[[eventQueueFn, event].join(".")];
+  #checkHandlerExists({ eventQueueFn, event, saga } = {}) {
+    if (eventQueueFn) {
+      const specificHandler = this.__onHandlers[[eventQueueFn, event].join(".")];
+      if (specificHandler) {
+        return specificHandler;
+      }
+
+      const genericHandler = this.__onHandlers[eventQueueFn];
+      return genericHandler ?? null;
+    }
+
+    if (event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_SUCCESS)) {
+      [event] = event.split("/");
+    }
+
+    const specificHandler = this.__onHandlers[[event, saga].join("/")];
     if (specificHandler) {
       return specificHandler;
     }
 
-    const genericHandler = this.__onHandlers[eventQueueFn];
-    return genericHandler ?? null;
+    return this.__onHandlers[saga];
   }
 
   async processPeriodicEvent(processContext, key, queueEntry) {
@@ -340,16 +355,17 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   }
 
   async processEvent(processContext, key, queueEntries, payload) {
+    let statusTuple;
+    const { userId, invocationFn, reg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
     try {
-      const { userId, invocationFn, reg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
       await this.#setContextUser(processContext, userId, reg);
       const result = await this.__srvUnboxed.tx(processContext)[invocationFn](reg);
-      return this.#determineResultStatus(result, queueEntries);
+      statusTuple = this.#determineResultStatus(result, queueEntries);
     } catch (err) {
       this.logger.error("error processing outboxed service call", err, {
         serviceName: this.eventSubType,
       });
-      return queueEntries.map((queueEntry) => [
+      statusTuple = queueEntries.map((queueEntry) => [
         queueEntry.ID,
         {
           status: EventProcessingStatus.Error,
@@ -357,13 +373,65 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
         },
       ]);
     }
+
+    await this.#publishFollowupEvents(processContext, reg, statusTuple);
+    return statusTuple;
+  }
+
+  async #publishFollowupEvents(processContext, req, statusTuple) {
+    const succeeded = this.#checkHandlerExists({ event: req.event, saga: EVENT_QUEUE_ACTIONS.SAGA_SUCCESS });
+    const failed = this.#checkHandlerExists({ event: req.event, saga: EVENT_QUEUE_ACTIONS.SAGA_FAILED });
+
+    if (!succeeded && !failed) {
+      return;
+    }
+
+    if (req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_FAILED)) {
+      return;
+    }
+
+    // NOTE: required for #failed because tx is rolledback and new events would not be commmited!
+    const tx = cds.tx(processContext);
+    const nextEvents = tx._eventQueue?.events;
+
+    if (nextEvents?.length) {
+      tx._eventQueue.events = [];
+    }
+
+    for (const [, result] of statusTuple) {
+      const data = result.nextData ?? req.data;
+      if (
+        succeeded &&
+        result.status === EventProcessingStatus.Done &&
+        !req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_SUCCESS)
+      ) {
+        await this.__srv.tx(processContext).send(succeeded, data);
+      }
+
+      if (failed && result.status === EventProcessingStatus.Error) {
+        result.error && (data.error = this._error2String(result.error));
+        await this.__srv.tx(processContext).send(failed, data);
+      }
+
+      delete result.nextData;
+    }
+
+    if (config.insertEventsBeforeCommit) {
+      this.nextSagaEvents = tx._eventQueue?.events;
+    } else {
+      this.nextSagaEvents = tx._eventQueue?.events.filter((event) => JSON.parse(event.payload).event === failed);
+    }
+
+    if (tx._eventQueue) {
+      tx._eventQueue.events = nextEvents ?? [];
+    }
   }
 
   #determineResultStatus(result, queueEntries) {
     const validStatusValues = Object.values(EventProcessingStatus);
     const validStatus = validStatusValues.includes(result);
     if (validStatus) {
-      return queueEntries.map((queueEntry) => [queueEntry.ID, result]);
+      return queueEntries.map((queueEntry) => [queueEntry.ID, { status: result }]);
     }
 
     if (result instanceof Object && !Array.isArray(result)) {
@@ -375,7 +443,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     }
 
     if (!Array.isArray(result)) {
-      return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Done]);
+      return queueEntries.map((queueEntry) => [queueEntry.ID, { status: EventProcessingStatus.Done }]);
     }
 
     const [firstEntry] = result;
@@ -430,7 +498,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     if (valid) {
       return result;
     } else {
-      return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Done]);
+      return queueEntries.map((queueEntry) => [queueEntry.ID, { status: EventProcessingStatus.Done }]);
     }
   }
 }

package/src/outbox/eventQueueAsOutbox.js

@@ -43,8 +43,9 @@ function outboxed(srv, customOpts) {
 
     const outboxOpts = config.getEventConfig(CDS_EVENT_TYPE, subType, srvConfig.namespace);
     const eventHeaders = getPropagatedHeaders(outboxOpts, req);
+    const contextProperties = getPropagatedContextProperties(outboxOpts, req);
     if (["persistent-outbox", "persistent-queue"].includes(outboxOpts.kind)) {
-      await _mapToEventAndPublish(context, subType, req, eventHeaders, srvConfig.namespace);
+      await _mapToEventAndPublish(req, srvConfig.namespace, subType, eventHeaders, contextProperties);
       return;
     }
     context.on("succeeded", async () => {
@@ -80,7 +81,17 @@ const getPropagatedHeaders = (config, req) => {
   return Object.assign(propagateHeaders, req.headers);
 };
 
-const _mapToEventAndPublish = async (context, subType, req, eventHeaders, namespace) => {
+const getPropagatedContextProperties = (config, req) => {
+  return config.propagatedContextProperties.reduce((properties, name) => {
+    if (name in req.tx.context) {
+      properties[name] = req.tx.context[name];
+    }
+    return properties;
+  }, {});
+};
+
+const _mapToEventAndPublish = async (req, namespace, subType, eventHeaders, contextProperties) => {
+  const context = req.context || cds.context;
   const eventQueueSpecificValues = {};
   for (const header in req.headers ?? {}) {
     for (const field of EVENT_QUEUE_SPECIFIC_FIELDS) {
@@ -100,6 +111,7 @@ const _mapToEventAndPublish = async (context, subType, req, eventHeaders, namesp
     ...(req.data && { data: req.data }),
     ...(eventHeaders && { headers: eventHeaders }),
     ...(req.query && { query: req.query }),
+    ...(Object.keys(contextProperties).length && { ...contextProperties }),
   };
 
   await publishEvent(

package/src/publishEvent.js

@@ -76,8 +76,9 @@ const publishEvent = async (
       event.namespace = config.namespace;
     }
   }
+  _addEventsToContext(tx, events);
   if (config.insertEventsBeforeCommit && !skipInsertEventsBeforeCommit) {
-    _registerHandlerAndAddEvents(tx, events, skipBroadcast);
+    _registerHandler(tx, skipBroadcast);
   } else {
     let result;
     tx._skipEventQueueBroadcast = skipBroadcast;
@@ -87,10 +88,7 @@ const publishEvent = async (
   }
 };
 
-const _registerHandlerAndAddEvents = (tx, events, skipBroadcast) => {
-  tx._eventQueue ??= { events: [], handlerRegistered: false };
-  tx._eventQueue.events = tx._eventQueue.events.concat(events);
-
+const _registerHandler = (tx, skipBroadcast) => {
   if (tx._eventQueue.handlerRegistered) {
     return;
   }
@@ -106,6 +104,11 @@ const _registerHandlerAndAddEvents = (tx, events, skipBroadcast) => {
   });
 };
 
+const _addEventsToContext = (tx, events) => {
+  tx._eventQueue ??= { events: [], handlerRegistered: false };
+  tx._eventQueue.events = tx._eventQueue.events.concat(events);
+};
+
 module.exports = {
   publishEvent,
 };

package/src/redis/redisPub.js

@@ -1,7 +1,5 @@
 "use strict";
 
-const { promisify } = require("util");
-
 const cds = require("@sap/cds");
 
 const redis = require("../shared/redis");
@@ -17,7 +15,11 @@ const COMPONENT_NAME = "/eventQueue/redisPub";
 const TRIES_FOR_PUBLISH_PERIODIC_EVENT = 10;
 const SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT = 30 * 1000;
 
-const wait = promisify(setTimeout);
+const wait = async (ms) => {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms).unref();
+  });
+};
 
 /**
  * Broadcasts events to the event queue, either locally or through Redis.

package/src/shared/cdsHelper.js

@@ -130,7 +130,7 @@ const _getAllTenantBase = async () => {
     if (cds.services["cds.xt.SaasProvisioningService"] || cds.services["saas-registry"]) {
       break;
     }
-    await new Promise((resolve) => setTimeout(resolve, 1000));
+    await new Promise((resolve) => setTimeout(resolve, 1000).unref());
   }
 
   const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");