@cap-js-community/event-queue 1.10.0-beta.1 → 1.10.0-beta.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.10.0-beta.1",
+  "version": "1.10.0-beta.3",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/config.js

@@ -10,9 +10,12 @@ const { Priorities } = require("./constants");
 
 const FOR_UPDATE_TIMEOUT = 10;
 const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
-const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
+const REDIS_PREFIX = "EVENT_QUEUE";
+const REDIS_CONFIG_CHANNEL = "CONFIG_CHANNEL";
 const REDIS_OFFBOARD_TENANT_CHANNEL = "REDIS_OFFBOARD_TENANT_CHANNEL";
-const REDIS_CONFIG_BLOCKLIST_CHANNEL = "EVENT_QUEUE_REDIS_CONFIG_BLOCKLIST_CHANNEL";
+const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
+const COMMAND_BLOCK = "EVENT_BLOCK";
+const COMMAND_UNBLOCK = "EVENT_UNBLOCK";
 const COMPONENT_NAME = "/eventQueue/config";
 const MIN_INTERVAL_SEC = 10;
 const DEFAULT_LOAD = 1;
@@ -22,8 +25,6 @@ const DEFAULT_KEEP_ALIVE_INTERVAL = 60;
 const DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL = 3.5;
 const DEFAULT_INHERIT_TRACE_CONTEXT = true;
 const SUFFIX_PERIODIC = "_PERIODIC";
-const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
-const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
 const CAP_EVENT_TYPE = "CAP_OUTBOX";
 const CAP_PARALLEL_DEFAULT = 5;
 const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
@@ -106,6 +107,7 @@ class Config {
   #unsubscribeHandlers = [];
   #unsubscribedTenants = {};
   #cronTimezone;
+  #redisNamespace;
   #publishEventBlockList;
   #crashOnRedisUnavailable;
   #tenantIdFilterTokenInfoCb;
@@ -182,7 +184,7 @@ class Config {
 
   attachConfigChangeHandler() {
     this.#attachBlockListChangeHandler();
-    redis.subscribeRedisChannel(this.#redisOptions, REDIS_CONFIG_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.redisOptions, REDIS_CONFIG_CHANNEL, (messageData) => {
       try {
         const { key, value } = JSON.parse(messageData);
         if (this[key] !== value) {
@@ -199,7 +201,7 @@ class Config {
 
   attachRedisUnsubscribeHandler() {
     this.#logger.info("attached redis handle for unsubscribe events");
-    redis.subscribeRedisChannel(this.#redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, (messageData) => {
       try {
         const { tenantId } = JSON.parse(messageData);
         this.#logger.info("received unsubscribe broadcast event", { tenantId });
@@ -229,7 +231,7 @@ class Config {
   handleUnsubscribe(tenantId) {
     if (this.redisEnabled) {
       redis
-        .publishMessage(this.#redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId }))
+        .publishMessage(this.redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId }))
         .catch((error) => {
           this.#logger.error(`publishing tenant unsubscribe failed. tenantId: ${tenantId}`, error);
         });
@@ -247,13 +249,13 @@ class Config {
       this.#logger.info("redis not connected, config change won't be published", { key, value });
       return;
     }
-    redis.publishMessage(this.#redisOptions, REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
+    redis.publishMessage(this.redisOptions, REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
       this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
     });
   }
 
   #attachBlockListChangeHandler() {
-    redis.subscribeRedisChannel(this.#redisOptions, REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.redisOptions, REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
       try {
         const { command, key, tenant } = JSON.parse(messageData);
         if (command === COMMAND_BLOCK) {
@@ -283,7 +285,7 @@ class Config {
 
     redis
       .publishMessage(
-        this.#redisOptions,
+        this.redisOptions,
         REDIS_CONFIG_BLOCKLIST_CHANNEL,
         JSON.stringify({ command: COMMAND_BLOCK, key, tenant })
       )
@@ -316,7 +318,7 @@ class Config {
 
     redis
       .publishMessage(
-        this.#redisOptions,
+        this.redisOptions,
         REDIS_CONFIG_BLOCKLIST_CHANNEL,
         JSON.stringify({ command: COMMAND_UNBLOCK, key, tenant })
       )
@@ -874,7 +876,18 @@ class Config {
   }
 
   get redisOptions() {
-    return this.#redisOptions;
+    return {
+      ...this.#redisOptions,
+      redisNamespace: `${[REDIS_PREFIX, this.redisNamespace].filter((a) => a).join("_")}`,
+    };
+  }
+
+  set redisNamespace(value) {
+    this.#redisNamespace = value;
+  }
+
+  get redisNamespace() {
+    return this.#redisNamespace;
   }
 
   set insertEventsBeforeCommit(value) {

package/src/initialize.js

@@ -42,6 +42,7 @@ const CONFIG_VARS = [
   ["insertEventsBeforeCommit", true],
   ["enableTelemetry", true],
   ["cronTimezone", null],
+  ["redisNamespace", null],
   ["publishEventBlockList", true],
   ["crashOnRedisUnavailable", false],
 ];

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -57,36 +57,43 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       return super.clusterQueueEntries(queueEntriesWithPayloadMap);
     }
     const { genericClusterEvents, specificClusterEvents } = this.#clusterByAction(queueEntriesWithPayloadMap);
+
     if (Object.keys(genericClusterEvents).length) {
       if (!this.__genericClusterRelevantAndAvailable) {
-        await super.clusterQueueEntries(genericClusterEvents);
+        for (const actionName in genericClusterEvents) {
+          await super.clusterQueueEntries(genericClusterEvents[actionName]);
+        }
       } else {
-        const msg = new cds.Request({
-          event: "clusterQueueEntries",
-          data: { queueEntriesWithPayloadMap: genericClusterEvents },
-          eventQueue: {
-            processor: this,
-            clusterByPayloadProperty: (propertyName) =>
-              EventQueueGenericOutboxHandler.clusterByPayloadProperty(genericClusterEvents, propertyName),
-            clusterByEventProperty: (propertyName) =>
-              EventQueueGenericOutboxHandler.clusterByEventProperty(genericClusterEvents, propertyName),
-          },
-        });
-        const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
-        this.#addToProcessingMap(handlerCluster);
+        for (const actionName in genericClusterEvents) {
+          const msg = new cds.Request({
+            event: `clusterQueueEntries`,
+            eventQueue: {
+              processor: this,
+              clusterByPayloadProperty: (propertyName, cb) =>
+                this.clusterByPayloadProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
+              clusterByEventProperty: (propertyName, cb) =>
+                this.clusterByEventProperty(actionName, genericClusterEvents[actionName], propertyName, cb),
+              clusterByDataProperty: (propertyName, cb) =>
+                this.clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+            },
+          });
+          const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
+          this.#addToProcessingMap(handlerCluster);
+        }
       }
     }
 
     for (const actionName in specificClusterEvents) {
       const msg = new cds.Request({
         event: `clusterQueueEntries.${actionName}`,
-        data: { queueEntriesWithPayloadMap: specificClusterEvents[actionName] },
         eventQueue: {
           processor: this,
-          clusterByPayloadProperty: (propertyName) =>
-            EventQueueGenericOutboxHandler.clusterByPayloadProperty(specificClusterEvents[actionName], propertyName),
-          clusterByEventProperty: (propertyName) =>
-            EventQueueGenericOutboxHandler.clusterByEventProperty(specificClusterEvents[actionName], propertyName),
+          clusterByPayloadProperty: (propertyName, cb) =>
+            this.clusterByPayloadProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+          clusterByEventProperty: (propertyName, cb) =>
+            this.clusterByEventProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
+          clusterByDataProperty: (propertyName, cb) =>
+            this.clusterByDataProperty(actionName, specificClusterEvents[actionName], propertyName, cb),
         },
       });
       const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
@@ -94,26 +101,69 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     }
   }
 
-  static clusterByPayloadProperty(queueEntriesWithPayloadMap, propertyName) {
-    return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
-      result[payload[propertyName]] ??= {
-        queueEntries: [],
-        payload,
-      };
-      result[payload[propertyName]].queueEntries.push(queueEntry);
+  clusterBase(queueEntriesWithPayloadMap, propertyName, refCb, cb) {
+    const clusters = Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
+      const ref = refCb(result, payload, queueEntry);
+      ref.queueEntries.push(queueEntry);
       return result;
     }, {});
+
+    if (cb) {
+      for (const clustersKey in clusters) {
+        const clusterData = clusters[clustersKey];
+        const clusterResult = cb(
+          clustersKey.split("##").shift(),
+          clusterData.queueEntries.map((entry) => entry.payload.data)
+        );
+        if (!clusterResult) {
+          throw new Error("hmm??");
+        }
+        clusterData.payload.data = clusterResult;
+      }
+    }
+    return clusters;
   }
 
-  static clusterByEventProperty(queueEntriesWithPayloadMap, propertyName) {
-    return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
-      result[queueEntry[propertyName]] ??= {
-        queueEntries: [],
-        payload,
-      };
-      result[queueEntry[propertyName]].queueEntries.push(queueEntry);
-      return result;
-    }, {});
+  #resolveRefBase(result, propertyName, actionName, payload, startRef) {
+    const parts = propertyName.split(".");
+    const data = JSON.parse(JSON.stringify(payload.data));
+    let ref = startRef;
+    for (const part of parts) {
+      ref = ref[part];
+    }
+    const key = [actionName, ref].join("##");
+    result[key] ??= {
+      queueEntries: [],
+      payload: { ...payload, data },
+    };
+    return result[key];
+  }
+
+  clusterByPayloadProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+    return this.clusterBase(
+      queueEntriesWithPayloadMap,
+      propertyName,
+      (result, payload) => this.#resolveRefBase(result, propertyName, actionName, payload, payload),
+      cb
+    );
+  }
+
+  clusterByEventProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+    return this.clusterBase(
+      queueEntriesWithPayloadMap,
+      propertyName,
+      (result, payload, queueEntry) => this.#resolveRefBase(result, propertyName, actionName, payload, queueEntry),
+      cb
+    );
+  }
+
+  clusterByDataProperty(actionName, queueEntriesWithPayloadMap, propertyName, cb) {
+    return this.clusterBase(
+      queueEntriesWithPayloadMap,
+      propertyName,
+      (result, payload) => this.#resolveRefBase(result, propertyName, actionName, payload, payload.data),
+      cb
+    );
   }
 
   #clusterByAction(queueEntriesWithPayloadMap) {
@@ -124,7 +174,8 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
           result.specificClusterEvents[clusterData.payload.event] ??= {};
           result.specificClusterEvents[clusterData.payload.event][eventId] = clusterData;
         } else {
-          result.genericClusterEvents[eventId] = clusterData;
+          result.genericClusterEvents[clusterData.payload.event] ??= {};
+          result.genericClusterEvents[clusterData.payload.event][eventId] = clusterData;
         }
         return result;
       },
@@ -169,7 +220,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       queueEntries: [queueEntry],
     });
     msg.event = handlerName;
-    await this.#setContextUser(this.context, userId);
+    await this.#setContextUser(this.context, userId, msg);
     const data = await this.__srvUnboxed.tx(this.context).send(msg);
     if (data) {
       payload.data = data;
@@ -204,7 +255,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   async processPeriodicEvent(processContext, key, queueEntry) {
     const [, action] = this.eventSubType.split(".");
     const msg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
-    await this.#setContextUser(processContext, config.userId);
+    await this.#setContextUser(processContext, config.userId, msg);
     await this.__srvUnboxed.tx(processContext).emit(msg);
   }
 
@@ -219,17 +270,20 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     return { msg, userId, invocationFn };
   }
 
-  async #setContextUser(context, userId) {
+  async #setContextUser(context, userId, data) {
     context.user = new cds.User.Privileged({
       id: userId,
       authInfo: await common.getTokenInfo(this.baseContext.tenant),
     });
+    if (data) {
+      data.user = context.user;
+    }
   }
 
   async processEvent(processContext, key, queueEntries, payload) {
     try {
       const { userId, invocationFn, msg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
-      await this.#setContextUser(processContext, userId);
+      await this.#setContextUser(processContext, userId, msg);
       const result = await this.__srvUnboxed.tx(processContext)[invocationFn](msg);
       return this.#determineResultStatus(result, queueEntries);
     } catch (err) {

package/src/outbox/eventQueueAsOutbox.js

@@ -47,7 +47,8 @@ function outboxed(srv, customOpts) {
       customOpts || {}
     );
     config.addCAPOutboxEventBase(srv.name, outboxOpts);
-    const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.method);
+    // TODO: check req.event ?? req.method
+    const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.event);
     if (specificSettings) {
       outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
     }

package/src/runner/runner.js

@@ -18,10 +18,10 @@ const { runEventCombinationForTenant } = require("./runnerHelper");
 const { trace } = require("../shared/openTelemetry");
 
 const COMPONENT_NAME = "/eventQueue/runner";
-const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
-const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
-const EVENT_QUEUE_RUN_REDIS_CHECK = "EVENT_QUEUE_RUN_REDIS_CHECK";
-const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "EVENT_QUEUE_UPDATE_PERIODIC_EVENTS";
+const EVENT_QUEUE_RUN_ID = "RUN_ID";
+const EVENT_QUEUE_RUN_TS = "RUN_TS";
+const EVENT_QUEUE_RUN_REDIS_CHECK = "RUN_REDIS_CHECK";
+const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "UPDATE_PERIODIC_EVENTS";
 let OFFSET_FIRST_RUN = 10 * 1000;
 
 let tenantIdHash;

package/src/shared/distributedLock.js

@@ -4,7 +4,6 @@ const redis = require("./redis");
 const config = require("../config");
 const cdsHelper = require("./cdsHelper");
 
-const KEY_PREFIX = "EVENT_QUEUE";
 const existingLocks = {};
 const REDIS_COMMAND_OK = "OK";
 const COMPONENT_NAME = "/eventQueue/distributedLock";
@@ -176,10 +175,10 @@ const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", ov
 };
 
 const _generateKey = (context, tenantScoped, key) => {
-  const keyParts = [];
+  const keyParts = [config.redisOptions.redisNamespace];
   tenantScoped && keyParts.push(context.tenant);
   keyParts.push(key);
-  return `${KEY_PREFIX}_${keyParts.join("##")}`;
+  return `${keyParts.join("##")}`;
 };
 
 const shutdownHandler = async () => {

package/src/shared/redis.js

@@ -46,6 +46,7 @@ const _createClientBase = (redisOptions = {}) => {
       password: redisOptions.password ?? options.password ?? credentials.password,
       socket,
     });
+    delete socketOptions.redisNamespace;
     if (credentials.cluster_mode) {
       return redis.createCluster({
         rootNodes: [socketOptions],
@@ -106,9 +107,10 @@ const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClien
         if (client._subscribedChannels[channel]) {
           continue;
         }
-        cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
+        const prefixedChannelName = [options.redisNamespace, channel].join("_");
+        cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel: prefixedChannelName });
         client
-          .subscribe(channel, fn)
+          .subscribe(prefixedChannelName, fn)
           .then(() => {
             client._subscribedChannels ??= {};
             client._subscribedChannels[channel] = 1;
@@ -133,7 +135,7 @@ const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClien
 
 const publishMessage = async (options, channel, message) => {
   const client = await createMainClientAndConnect(options);
-  return await client.publish(channel, message);
+  return await client.publish([options.redisNamespace, channel].join("_"), message);
 };
 
 const closeMainClient = async () => {