@cap-js-community/event-queue 1.5.2 → 1.6.0

package/cds-plugin.js

@@ -5,6 +5,6 @@ const cds = require("@sap/cds");
 const eventQueue = require("./src");
 const COMPONENT_NAME = "/eventQueue/plugin";
 
-if (!cds.build?.register && cds.env.eventQueue) {
+if (!cds.build?.register && Object.keys(cds.env.eventQueue ?? {}).length) {
   module.exports = eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.5.2",
+  "version": "1.6.0",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -44,12 +44,12 @@
   },
   "dependencies": {
     "@sap/xssec": "^3.6.1",
-    "redis": "^4.6.14",
+    "redis": "^4.6.15",
     "verror": "^1.10.1",
     "yaml": "^2.4.2"
   },
   "devDependencies": {
-    "@cap-js/hana": "^1.0.0",
+    "@cap-js/hana": "^1.1.0",
     "@cap-js/sqlite": "^1.7.2",
     "@sap/cds": "^7.9.2",
     "@sap/cds-dk": "^7.8.0",

package/src/EventQueueError.js

@@ -18,8 +18,7 @@ const ERROR_CODES = {
   NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
   LOAD_HIGHER_THAN_LIMIT: "LOAD_HIGHER_THAN_LIMIT",
   NOT_ALLOWED_PRIORITY: "NOT_ALLOWED_PRIORITY",
-  SCHEMA_TENANT_MISMATCH: "SCHEMA_TENANT_MISMATCH",
-  GLOBAL_CDS_CONTEXT_MISMATCH: "GLOBAL_CDS_CONTEXT_MISMATCH",
+  APP_NAMES_FORMAT: "APP_NAMES_FORMAT",
 };
 
 const ERROR_CODES_META = {
@@ -69,11 +68,8 @@ const ERROR_CODES_META = {
   [ERROR_CODES.NOT_ALLOWED_PRIORITY]: {
     message: "The supplied priority is not allowed. Only LOW, MEDIUM, HIGH is allowed!",
   },
-  [ERROR_CODES.SCHEMA_TENANT_MISMATCH]: {
-    message: "The db client associated to the tenant context does not match! Processing will be skipped.",
-  },
-  [ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH]: {
-    message: "The global cds context does not match the local cds context.",
+  [ERROR_CODES.APP_NAMES_FORMAT]: {
+    message: "The app names property must be an array and only contain strings.",
   },
 };
 
@@ -244,23 +240,12 @@ class EventQueueError extends VError {
     );
   }
 
-  static dbClientSchemaMismatch(tenantId, dbClientSchema, serviceManagerSchema) {
-    const { message } = ERROR_CODES_META[ERROR_CODES.SCHEMA_TENANT_MISMATCH];
-    return new EventQueueError(
-      {
-        name: ERROR_CODES.SCHEMA_TENANT_MISMATCH,
-        info: { tenantId, dbClientSchema, serviceManagerSchema },
-      },
-      message
-    );
-  }
-
-  static globalCdsContextNotMatchingLocal(globalProperties, localProperties) {
-    const { message } = ERROR_CODES_META[ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH];
+  static appNamesFormat(type, subType, appNames) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.APP_NAMES_FORMAT];
     return new EventQueueError(
       {
-        name: ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH,
-        info: { globalProperties, localProperties },
+        name: ERROR_CODES.APP_NAMES_FORMAT,
+        info: { type, subType, appNames },
       },
       message
     );

package/src/EventQueueProcessorBase.js

@@ -1108,6 +1108,10 @@ class EventQueueProcessorBase {
     return this.#eventType.replace(SUFFIX_PERIODIC, "");
   }
 
+  get rawEventType() {
+    return this.#eventType;
+  }
+
   get eventSubType() {
     return this.#eventSubType;
   }

package/src/config.js

@@ -105,7 +105,12 @@ class Config {
   }
 
   _checkRedisIsBound() {
-    return !!this.#env.getRedisCredentialsFromEnv();
+    return !!this.#env.redisCredentialsFromEnv;
+  }
+
+  shouldBeProcessedInThisApplication(type, subType) {
+    const config = this.#eventMap[this.generateKey(type, subType)];
+    return !config._appNameMap || config._appNameMap[this.#env.applicationName];
   }
 
   checkRedisEnabled() {
@@ -276,8 +281,12 @@ class Config {
       eventOutdatedCheck: config.eventOutdatedCheck,
       checkForNextChunk: config.checkForNextChunk,
       deleteFinishedEventsAfterDays: config.deleteFinishedEventsAfterDays,
+      appNames: config.appNames,
       internalEvent: true,
     };
+    eventConfig._appNameMap = eventConfig.appNames
+      ? Object.fromEntries(new Map(eventConfig.appNames.map((a) => [a, true])))
+      : null;
     this.#config.events.push(eventConfig);
     this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
   }
@@ -317,6 +326,7 @@ class Config {
       event.load = event.load ?? DEFAULT_LOAD;
       event.priority = event.priority ?? DEFAULT_PRIORITY;
       this.validateAdHocEvents(result, event);
+      event._appNameMap = event.appNames ? Object.fromEntries(new Map(event.appNames.map((a) => [a, true]))) : null;
       result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, {});
@@ -326,6 +336,7 @@ class Config {
       event.type = `${event.type}${SUFFIX_PERIODIC}`;
       event.isPeriodic = true;
       this.validatePeriodicConfig(result, event);
+      event._appNameMap = event.appNames ? Object.fromEntries(new Map(event.appNames.map((a) => [a, true]))) : null;
       result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, this.#eventMap);
@@ -355,6 +366,12 @@ class Config {
     if (!config.impl) {
       throw EventQueueError.missingImpl(config.type, config.subType);
     }
+
+    if (config.appNames) {
+      if (!Array.isArray(config.appNames) || config.appNames.some((appName) => typeof appName !== "string")) {
+        throw EventQueueError.appNamesFormat(config.type, config.subType, config.appNames);
+      }
+    }
   }
 
   generateKey(type, subType) {

package/src/initialize.js

@@ -131,22 +131,7 @@ const readConfigFromFile = async (configFilepath) => {
 };
 
 const registerEventProcessors = () => {
-  cds.on("listening", () => {
-    cds.connect
-      .to("cds.xt.DeploymentService")
-      .then((ds) => {
-        cds.log(COMPONENT).info("event-queue unsubscribe handler registered", {
-          redisEnabled: config.redisEnabled,
-        });
-        ds.after("unsubscribe", async (_, req) => {
-          const { tenant } = req.data;
-          config.handleUnsubscribe(tenant);
-        });
-      })
-      .catch(
-        () => {} // ignore errors as the DeploymentService is most of the time only available in the mtx sidecar
-      );
-  });
+  _registerUnsubscribe();
   config.redisEnabled && config.attachRedisUnsubscribeHandler();
 
   if (!config.registerAsEventProcessor) {
@@ -215,6 +200,25 @@ const registerCleanupForDevDb = async () => {
   }
 };
 
+const _registerUnsubscribe = () => {
+  cds.on("listening", () => {
+    cds.connect
+      .to("cds.xt.DeploymentService")
+      .then((ds) => {
+        cds.log(COMPONENT).info("event-queue unsubscribe handler registered", {
+          redisEnabled: config.redisEnabled,
+        });
+        ds.after("unsubscribe", async (_, req) => {
+          const { tenant } = req.data;
+          config.handleUnsubscribe(tenant);
+        });
+      })
+      .catch(
+        () => {} // ignore errors as the DeploymentService is most of the time only available in the mtx sidecar
+      );
+  });
+};
+
 module.exports = {
   initialize,
 };

package/src/outbox/eventQueueAsOutbox.js

@@ -15,9 +15,19 @@ const COMPONENT_NAME = "/eventQueue/eventQueueAsOutbox";
 function outboxed(srv, customOpts) {
   // outbox max. once
   const logger = cds.log(COMPONENT_NAME);
+  const outboxOpts = Object.assign(
+    {},
+    (typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
+    (typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
+    customOpts || {}
+  );
+
   if (!new.target) {
     const former = srv[OUTBOXED];
     if (former) {
+      if (outboxOpts.kind === "persistent-outbox") {
+        config.addCAPOutboxEvent(srv.name, outboxOpts);
+      }
       return former;
     }
   }
@@ -30,14 +40,9 @@ function outboxed(srv, customOpts) {
     Object.defineProperty(srv, OUTBOXED, { value: outboxedSrv });
   }
 
-  const outboxOpts = Object.assign(
-    {},
-    (typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
-    (typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
-    customOpts || {}
-  );
-
-  config.addCAPOutboxEvent(srv.name, outboxOpts);
+  if (outboxOpts.kind === "persistent-outbox") {
+    config.addCAPOutboxEvent(srv.name, outboxOpts);
+  }
   outboxedSrv.handle = async function (req) {
     const context = req.context || cds.context;
     if (outboxOpts.kind === "persistent-outbox") {

package/src/processEventQueue.js

@@ -269,6 +269,7 @@ const _checkEventIsBlocked = async (baseInstance) => {
       });
     }
   } else {
+    // TODO: we should be able to get rid of baseInstance.isPeriodicEvent with rawEventType
     eventBlocked = config.isEventBlocked(
       baseInstance.eventType,
       baseInstance.eventSubType,
@@ -283,11 +284,16 @@ const _checkEventIsBlocked = async (baseInstance) => {
 
   if (eventBlocked) {
     baseInstance.logger.info("skipping run because event is blocked by configuration", {
-      type: baseInstance.eventType,
+      type: baseInstance.rawEventType,
       subType: baseInstance.eventSubType,
       tenantUnsubscribed: config.isTenantUnsubscribed(baseInstance.context.tenant),
     });
   }
+
+  if (!eventBlocked) {
+    eventBlocked = !config.shouldBeProcessedInThisApplication(baseInstance.rawEventType, baseInstance.eventSubType);
+  }
+
   return eventBlocked;
 };
 

package/src/redis/redisPub.js

@@ -87,11 +87,11 @@ const _processLocalWithoutRedis = async (tenantId, events) => {
       };
     }
 
-    return await cds.tx(context, async ({ context }) => {
-      for (const { type, subType } of events) {
+    for (const { type, subType } of events) {
+      await cds.tx(context, async ({ context }) => {
         await runEventCombinationForTenant(context, type, subType, { shouldTrace: true });
-      }
-    });
+      });
+    }
   }
 };
 

package/src/redis/redisSub.js

@@ -35,14 +35,6 @@ const _messageHandlerProcessEvents = async (messageData) => {
       return;
     }
 
-    const user = await cds.tx({ tenant: tenantId }, async () => {
-      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
-    });
-    const tenantContext = {
-      tenant: tenantId,
-      user,
-    };
-
     if (!config.getEventConfig(type, subType)) {
       if (config.isCapOutboxEvent(type)) {
         try {
@@ -64,6 +56,23 @@ const _messageHandlerProcessEvents = async (messageData) => {
       }
     }
 
+    if (!(config.getEventConfig(type, subType) && config.shouldBeProcessedInThisApplication(type, subType))) {
+      logger.debug("event is not configured to be processed on this app-name", {
+        tenantId,
+        type,
+        subType,
+      });
+      return;
+    }
+
+    const user = await cds.tx({ tenant: tenantId }, async () => {
+      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+    });
+    const tenantContext = {
+      tenant: tenantId,
+      user,
+    };
+
     return await cds.tx(tenantContext, async ({ context }) => {
       return await runnerHelper.runEventCombinationForTenant(context, type, subType, { lockId, shouldTrace: true });
     });

package/src/runner/openEvents.js

@@ -29,18 +29,21 @@ const getOpenQueueEntries = async (tx) => {
 
   const result = [];
   for (const { type, subType } of entries) {
-    if (type.startsWith("CAP_OUTBOX")) {
-      if (cds.requires[subType]) {
-        await cds.connect.to(subType).catch(() => {});
-        result.push({ type, subType });
-      } else {
-        const service = await cds.connect.to(subType).catch(() => {});
-        if (service) {
-          result.push({ type, subType });
-        }
-      }
+    if (eventConfig.isCapOutboxEvent(type)) {
+      await cds.connect
+        .to(subType)
+        .then((service) => {
+          if (!service) {
+            return;
+          }
+          cds.outboxed(service);
+          if (eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
+            result.push({ type, subType });
+          }
+        })
+        .catch(() => {});
     } else {
-      if (eventConfig.getEventConfig(type, subType)) {
+      if (eventConfig.getEventConfig(type, subType) && eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
         result.push({ type, subType });
       }
     }

package/src/runner/runner.js

@@ -22,7 +22,7 @@ const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
 const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
 const EVENT_QUEUE_RUN_REDIS_CHECK = "EVENT_QUEUE_RUN_REDIS_CHECK";
 const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "EVENT_QUEUE_UPDATE_PERIODIC_EVENTS";
-const OFFSET_FIRST_RUN = 10 * 1000;
+let OFFSET_FIRST_RUN = 10 * 1000;
 
 let tenantIdHash;
 let singleRunDone;
@@ -160,7 +160,6 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
 
 const _executeEventsAllTenants = async (tenantIds, runId) => {
   const promises = [];
-
   for (const tenantId of tenantIds) {
     const id = cds.utils.uuid();
     let tenantContext;
@@ -254,32 +253,46 @@ const _executePeriodicEventsAllTenants = async (tenantIds) => {
   }
 };
 
-const _singleTenantDb = async (tenantId) => {
-  return Promise.allSettled(
-    eventQueueConfig.allEvents.map(async (eventConfig) => {
+const _singleTenantDb = async () => {
+  const id = cds.utils.uuid();
+  const events = await trace(
+    { id },
+    "fetch-openEvents-and-authInfo",
+    async () => {
+      return await cds.tx({}, async (tx) => {
+        return await openEvents.getOpenQueueEntries(tx);
+      });
+    },
+    { newRootSpan: true }
+  );
+
+  return await Promise.allSettled(
+    events.map(async (openEvent) => {
+      const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
       const label = `${eventConfig.type}_${eventConfig.subType}`;
-      const user = new cds.User.Privileged(config.userId);
-      const tenantContext = {
-        tenant: tenantId,
-        user,
-      };
       return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
-        return await cds.tx(tenantContext, async ({ context }) => {
-          try {
-            const lockId = `${EVENT_QUEUE_RUN_ID}_${label}`;
-            const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-              expiryTime: eventQueueConfig.runInterval * 0.95,
-            });
-            if (!couldAcquireLock) {
-              return;
-            }
-            await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, true);
-          } catch (err) {
-            cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-              tenantId,
-              redisEnabled: eventQueueConfig.redisEnabled,
-            });
-          }
+        return await cds.tx({}, async ({ context }) => {
+          await trace(
+            context,
+            label,
+            async () => {
+              try {
+                const lockId = `${label}`;
+                const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+                  expiryTime: eventQueueConfig.runInterval * 0.95,
+                });
+                if (!couldAcquireLock) {
+                  return;
+                }
+                await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, {
+                  skipWorkerPool: true,
+                });
+              } catch (err) {
+                cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
+              }
+            },
+            { newRootSpan: true }
+          );
         });
       });
     })
@@ -387,7 +400,8 @@ const _multiTenancyPeriodicEvents = async (tenantIds) => {
   }
 };
 
-const _checkPeriodicEventsSingleTenantOneTime = () =>
+const _checkPeriodicEventsSingleTenantOneTime = async () =>
+  eventQueueConfig.updatePeriodicEvents &&
   cds.tx({}, async (tx) => await periodicEvents.checkAndInsertPeriodicEvents(tx.context));
 
 const _checkPeriodicEventsSingleTenant = async (context) => {
@@ -425,5 +439,6 @@ module.exports = {
     _acquireRunId,
     EVENT_QUEUE_RUN_TS,
     clearHash: () => (tenantIdHash = null),
+    setOffsetFirstRun: (value) => (OFFSET_FIRST_RUN = value),
   },
 };

package/src/shared/WorkerQueue.js

@@ -113,7 +113,7 @@ class WorkerQueue {
   }
 
   #checkForNext() {
-    if (!this.#queue.length && this.#runningLoad === this.#concurrencyLimit) {
+    if (!Object.values(this.#queue).some((queue) => queue.length) || this.#runningLoad === this.#concurrencyLimit) {
       return;
     }
 

package/src/shared/env.js

@@ -3,34 +3,42 @@
 let instance;
 
 class Env {
-  #isLocal;
   #vcapServices;
+  #vcapApplication;
 
   constructor() {
     try {
       this.#vcapServices = JSON.parse(process.env.VCAP_SERVICES);
+      this.#vcapApplication = JSON.parse(process.env.VCAP_APPLICATION);
     } catch {
       this.#vcapServices = {};
+      this.#vcapApplication = {};
     }
   }
 
-  getRedisCredentialsFromEnv() {
+  get redisCredentialsFromEnv() {
     return this.#vcapServices["redis-cache"]?.[0]?.credentials;
   }
 
-  set isLocal(value) {
-    this.#isLocal = value;
-  }
-  get isLocal() {
-    return this.#isLocal;
+  get applicationName() {
+    return this.#vcapApplication.application_name;
   }
 
   set vcapServices(value) {
     this.#vcapServices = value;
   }
+
   get vcapServices() {
     return this.#vcapServices;
   }
+
+  set vcapApplication(value) {
+    this.#vcapApplication = value;
+  }
+
+  get vcapApplication() {
+    return this.#vcapApplication;
+  }
 }
 
 module.exports = {

package/src/shared/eventScheduler.js

@@ -47,7 +47,7 @@ class EventScheduler {
   }
 
   clearForTenant(tenantId) {
-    Object.values(this.#eventsByTenants[tenantId]).forEach((timeoutId) => clearTimeout(timeoutId));
+    Object.keys(this.#eventsByTenants[tenantId] ?? []).forEach((timeoutId) => clearTimeout(timeoutId));
   }
 
   calculateOffset(type, subType, startAfter) {
@@ -67,6 +67,14 @@ class EventScheduler {
   clearScheduledEvents() {
     this.#scheduledEvents = {};
   }
+
+  clearEventsByTenants() {
+    this.#eventsByTenants = {};
+  }
+
+  get eventsByTenants() {
+    return this.#eventsByTenants;
+  }
 }
 
 module.exports = {

package/src/shared/redis.js

@@ -30,7 +30,7 @@ const createMainClientAndConnect = (options) => {
 const _createClientBase = (redisOptions) => {
   const env = getEnvInstance();
   try {
-    const credentials = env.getRedisCredentialsFromEnv();
+    const credentials = env.redisCredentialsFromEnv;
     const redisIsCluster = credentials.cluster_mode;
     const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
     if (redisIsCluster) {