@cap-js-community/event-queue 1.10.0-beta.1 → 1.10.0-beta.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.10.0-beta.1",
+  "version": "1.10.0-beta.2",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/config.js

@@ -10,9 +10,12 @@ const { Priorities } = require("./constants");
 
 const FOR_UPDATE_TIMEOUT = 10;
 const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
-const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
+const REDIS_PREFIX = "EVENT_QUEUE";
+const REDIS_CONFIG_CHANNEL = "CONFIG_CHANNEL";
 const REDIS_OFFBOARD_TENANT_CHANNEL = "REDIS_OFFBOARD_TENANT_CHANNEL";
-const REDIS_CONFIG_BLOCKLIST_CHANNEL = "EVENT_QUEUE_REDIS_CONFIG_BLOCKLIST_CHANNEL";
+const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
+const COMMAND_BLOCK = "EVENT_BLOCK";
+const COMMAND_UNBLOCK = "EVENT_UNBLOCK";
 const COMPONENT_NAME = "/eventQueue/config";
 const MIN_INTERVAL_SEC = 10;
 const DEFAULT_LOAD = 1;
@@ -22,8 +25,6 @@ const DEFAULT_KEEP_ALIVE_INTERVAL = 60;
 const DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL = 3.5;
 const DEFAULT_INHERIT_TRACE_CONTEXT = true;
 const SUFFIX_PERIODIC = "_PERIODIC";
-const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
-const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
 const CAP_EVENT_TYPE = "CAP_OUTBOX";
 const CAP_PARALLEL_DEFAULT = 5;
 const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
@@ -106,6 +107,7 @@ class Config {
   #unsubscribeHandlers = [];
   #unsubscribedTenants = {};
   #cronTimezone;
+  #redisNamespace;
   #publishEventBlockList;
   #crashOnRedisUnavailable;
   #tenantIdFilterTokenInfoCb;
@@ -182,7 +184,7 @@ class Config {
 
   attachConfigChangeHandler() {
     this.#attachBlockListChangeHandler();
-    redis.subscribeRedisChannel(this.#redisOptions, REDIS_CONFIG_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.redisOptions, REDIS_CONFIG_CHANNEL, (messageData) => {
       try {
         const { key, value } = JSON.parse(messageData);
         if (this[key] !== value) {
@@ -199,7 +201,7 @@ class Config {
 
   attachRedisUnsubscribeHandler() {
     this.#logger.info("attached redis handle for unsubscribe events");
-    redis.subscribeRedisChannel(this.#redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, (messageData) => {
       try {
         const { tenantId } = JSON.parse(messageData);
         this.#logger.info("received unsubscribe broadcast event", { tenantId });
@@ -229,7 +231,7 @@ class Config {
   handleUnsubscribe(tenantId) {
     if (this.redisEnabled) {
       redis
-        .publishMessage(this.#redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId }))
+        .publishMessage(this.redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId }))
         .catch((error) => {
           this.#logger.error(`publishing tenant unsubscribe failed. tenantId: ${tenantId}`, error);
         });
@@ -247,13 +249,13 @@ class Config {
       this.#logger.info("redis not connected, config change won't be published", { key, value });
       return;
     }
-    redis.publishMessage(this.#redisOptions, REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
+    redis.publishMessage(this.redisOptions, REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
       this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
     });
   }
 
   #attachBlockListChangeHandler() {
-    redis.subscribeRedisChannel(this.#redisOptions, REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
+    redis.subscribeRedisChannel(this.redisOptions, REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
       try {
         const { command, key, tenant } = JSON.parse(messageData);
         if (command === COMMAND_BLOCK) {
@@ -283,7 +285,7 @@ class Config {
 
     redis
       .publishMessage(
-        this.#redisOptions,
+        this.redisOptions,
         REDIS_CONFIG_BLOCKLIST_CHANNEL,
         JSON.stringify({ command: COMMAND_BLOCK, key, tenant })
       )
@@ -316,7 +318,7 @@ class Config {
 
     redis
       .publishMessage(
-        this.#redisOptions,
+        this.redisOptions,
         REDIS_CONFIG_BLOCKLIST_CHANNEL,
         JSON.stringify({ command: COMMAND_UNBLOCK, key, tenant })
       )
@@ -874,7 +876,18 @@ class Config {
   }
 
   get redisOptions() {
-    return this.#redisOptions;
+    return {
+      ...this.#redisOptions,
+      redisNamespace: `${[REDIS_PREFIX, this.redisNamespace].filter((a) => a).join("_")}`,
+    };
+  }
+
+  set redisNamespace(value) {
+    this.#redisNamespace = value;
+  }
+
+  get redisNamespace() {
+    return this.#redisNamespace;
   }
 
   set insertEventsBeforeCommit(value) {

package/src/initialize.js

@@ -42,6 +42,7 @@ const CONFIG_VARS = [
   ["insertEventsBeforeCommit", true],
   ["enableTelemetry", true],
   ["cronTimezone", null],
+  ["redisNamespace", null],
   ["publishEventBlockList", true],
   ["crashOnRedisUnavailable", false],
 ];

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -63,13 +63,11 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       } else {
         const msg = new cds.Request({
           event: "clusterQueueEntries",
-          data: { queueEntriesWithPayloadMap: genericClusterEvents },
           eventQueue: {
             processor: this,
-            clusterByPayloadProperty: (propertyName) =>
-              EventQueueGenericOutboxHandler.clusterByPayloadProperty(genericClusterEvents, propertyName),
-            clusterByEventProperty: (propertyName) =>
-              EventQueueGenericOutboxHandler.clusterByEventProperty(genericClusterEvents, propertyName),
+            clusterByPayloadProperty: (propertyName, cb) =>
+              this.clusterByPayloadProperty(genericClusterEvents, propertyName, cb),
+            clusterByEventProperty: (propertyName) => this.clusterByEventProperty(genericClusterEvents, propertyName),
           },
         });
         const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
@@ -80,13 +78,12 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     for (const actionName in specificClusterEvents) {
       const msg = new cds.Request({
         event: `clusterQueueEntries.${actionName}`,
-        data: { queueEntriesWithPayloadMap: specificClusterEvents[actionName] },
         eventQueue: {
           processor: this,
-          clusterByPayloadProperty: (propertyName) =>
-            EventQueueGenericOutboxHandler.clusterByPayloadProperty(specificClusterEvents[actionName], propertyName),
-          clusterByEventProperty: (propertyName) =>
-            EventQueueGenericOutboxHandler.clusterByEventProperty(specificClusterEvents[actionName], propertyName),
+          clusterByPayloadProperty: (propertyName, cb) =>
+            this.clusterByPayloadProperty(specificClusterEvents[actionName], propertyName, cb),
+          clusterByEventProperty: (propertyName, cb) =>
+            this.clusterByEventProperty(specificClusterEvents[actionName], propertyName, cb),
         },
       });
       const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
@@ -94,26 +91,58 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     }
   }
 
-  static clusterByPayloadProperty(queueEntriesWithPayloadMap, propertyName) {
-    return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
-      result[payload[propertyName]] ??= {
-        queueEntries: [],
-        payload,
-      };
-      result[payload[propertyName]].queueEntries.push(queueEntry);
+  clusterBase(queueEntriesWithPayloadMap, propertyName, refCb, cb) {
+    return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }], index) => {
+      const ref = refCb(result, payload, queueEntry);
+      ref.queueEntries.push(queueEntry);
+      if (cb) {
+        const clusterResult = cb(ref.payload.data, queueEntry.payload.data, index);
+        ref.payload.data ??= clusterResult;
+      }
       return result;
     }, {});
   }
 
-  static clusterByEventProperty(queueEntriesWithPayloadMap, propertyName) {
-    return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
-      result[queueEntry[propertyName]] ??= {
-        queueEntries: [],
-        payload,
-      };
-      result[queueEntry[propertyName]].queueEntries.push(queueEntry);
-      return result;
-    }, {});
+  clusterByPayloadProperty(queueEntriesWithPayloadMap, propertyName, cb) {
+    return this.clusterBase(
+      queueEntriesWithPayloadMap,
+      propertyName,
+      (result, payload) => {
+        const parts = propertyName.split(".");
+        const data = JSON.parse(JSON.stringify(payload.data));
+        let ref = payload;
+        for (const part of parts) {
+          ref = ref[part];
+        }
+        result[ref[propertyName]] ??= {
+          queueEntries: [],
+          payload: { ...payload, data },
+        };
+        return result[ref[propertyName]];
+      },
+      cb
+    );
+  }
+
+  clusterByEventProperty(queueEntriesWithPayloadMap, propertyName, cb) {
+    return this.clusterBase(
+      queueEntriesWithPayloadMap,
+      propertyName,
+      (result, payload, queueEntry) => {
+        const parts = propertyName.split(".");
+        const payloadCopy = JSON.parse(JSON.stringify(payload));
+        let ref = queueEntry;
+        for (const part of parts) {
+          ref = ref[part];
+        }
+        result[queueEntry[propertyName]] ??= {
+          queueEntries: [],
+          payload: payloadCopy,
+        };
+        return result[queueEntry[propertyName]];
+      },
+      cb
+    );
   }
 
   #clusterByAction(queueEntriesWithPayloadMap) {

package/src/outbox/eventQueueAsOutbox.js

@@ -47,7 +47,8 @@ function outboxed(srv, customOpts) {
       customOpts || {}
     );
     config.addCAPOutboxEventBase(srv.name, outboxOpts);
-    const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.method);
+    // TODO: check req.event ?? req.method
+    const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.event);
     if (specificSettings) {
       outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
     }

package/src/runner/runner.js

@@ -18,10 +18,10 @@ const { runEventCombinationForTenant } = require("./runnerHelper");
 const { trace } = require("../shared/openTelemetry");
 
 const COMPONENT_NAME = "/eventQueue/runner";
-const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
-const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
-const EVENT_QUEUE_RUN_REDIS_CHECK = "EVENT_QUEUE_RUN_REDIS_CHECK";
-const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "EVENT_QUEUE_UPDATE_PERIODIC_EVENTS";
+const EVENT_QUEUE_RUN_ID = "RUN_ID";
+const EVENT_QUEUE_RUN_TS = "RUN_TS";
+const EVENT_QUEUE_RUN_REDIS_CHECK = "RUN_REDIS_CHECK";
+const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "UPDATE_PERIODIC_EVENTS";
 let OFFSET_FIRST_RUN = 10 * 1000;
 
 let tenantIdHash;

package/src/shared/distributedLock.js

@@ -4,7 +4,6 @@ const redis = require("./redis");
 const config = require("../config");
 const cdsHelper = require("./cdsHelper");
 
-const KEY_PREFIX = "EVENT_QUEUE";
 const existingLocks = {};
 const REDIS_COMMAND_OK = "OK";
 const COMPONENT_NAME = "/eventQueue/distributedLock";
@@ -176,10 +175,10 @@ const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", ov
 };
 
 const _generateKey = (context, tenantScoped, key) => {
-  const keyParts = [];
+  const keyParts = [config.redisOptions.redisNamespace];
   tenantScoped && keyParts.push(context.tenant);
   keyParts.push(key);
-  return `${KEY_PREFIX}_${keyParts.join("##")}`;
+  return `${keyParts.join("##")}`;
 };
 
 const shutdownHandler = async () => {

package/src/shared/redis.js

@@ -46,6 +46,7 @@ const _createClientBase = (redisOptions = {}) => {
       password: redisOptions.password ?? options.password ?? credentials.password,
       socket,
     });
+    delete socketOptions.redisNamespace;
     if (credentials.cluster_mode) {
       return redis.createCluster({
         rootNodes: [socketOptions],
@@ -106,9 +107,10 @@ const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClien
         if (client._subscribedChannels[channel]) {
           continue;
         }
-        cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
+        const prefixedChannelName = [options.redisNamespace, channel].join("_");
+        cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel: prefixedChannelName });
         client
-          .subscribe(channel, fn)
+          .subscribe(prefixedChannelName, fn)
           .then(() => {
             client._subscribedChannels ??= {};
             client._subscribedChannels[channel] = 1;
@@ -133,7 +135,7 @@ const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClien
 
 const publishMessage = async (options, channel, message) => {
   const client = await createMainClientAndConnect(options);
-  return await client.publish(channel, message);
+  return await client.publish([options.redisNamespace, channel].join("_"), message);
 };
 
 const closeMainClient = async () => {