@cap-js-community/event-queue 1.5.3 → 1.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.5.3",
+  "version": "1.6.1",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -43,16 +43,16 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@sap/xssec": "^3.6.1",
-    "redis": "^4.6.14",
+    "@sap/xssec": "^4.2.1",
+    "redis": "^4.7.0",
     "verror": "^1.10.1",
-    "yaml": "^2.4.2"
+    "yaml": "^2.5.0"
   },
   "devDependencies": {
-    "@cap-js/hana": "^1.0.0",
-    "@cap-js/sqlite": "^1.7.2",
-    "@sap/cds": "^7.9.2",
-    "@sap/cds-dk": "^7.8.0",
+    "@cap-js/hana": "^1.1.1",
+    "@cap-js/sqlite": "^1.7.3",
+    "@sap/cds": "^8.1.0",
+    "@sap/cds-dk": "^8.1.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jest": "^28.6.0",

package/src/EventQueueError.js

@@ -18,8 +18,7 @@ const ERROR_CODES = {
   NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
   LOAD_HIGHER_THAN_LIMIT: "LOAD_HIGHER_THAN_LIMIT",
   NOT_ALLOWED_PRIORITY: "NOT_ALLOWED_PRIORITY",
-  SCHEMA_TENANT_MISMATCH: "SCHEMA_TENANT_MISMATCH",
-  GLOBAL_CDS_CONTEXT_MISMATCH: "GLOBAL_CDS_CONTEXT_MISMATCH",
+  APP_NAMES_FORMAT: "APP_NAMES_FORMAT",
 };
 
 const ERROR_CODES_META = {
@@ -69,11 +68,8 @@ const ERROR_CODES_META = {
   [ERROR_CODES.NOT_ALLOWED_PRIORITY]: {
     message: "The supplied priority is not allowed. Only LOW, MEDIUM, HIGH is allowed!",
   },
-  [ERROR_CODES.SCHEMA_TENANT_MISMATCH]: {
-    message: "The db client associated to the tenant context does not match! Processing will be skipped.",
-  },
-  [ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH]: {
-    message: "The global cds context does not match the local cds context.",
+  [ERROR_CODES.APP_NAMES_FORMAT]: {
+    message: "The app names property must be an array and only contain strings.",
   },
 };
 
@@ -244,23 +240,12 @@ class EventQueueError extends VError {
     );
   }
 
-  static dbClientSchemaMismatch(tenantId, dbClientSchema, serviceManagerSchema) {
-    const { message } = ERROR_CODES_META[ERROR_CODES.SCHEMA_TENANT_MISMATCH];
-    return new EventQueueError(
-      {
-        name: ERROR_CODES.SCHEMA_TENANT_MISMATCH,
-        info: { tenantId, dbClientSchema, serviceManagerSchema },
-      },
-      message
-    );
-  }
-
-  static globalCdsContextNotMatchingLocal(globalProperties, localProperties) {
-    const { message } = ERROR_CODES_META[ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH];
+  static appNamesFormat(type, subType, appNames) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.APP_NAMES_FORMAT];
     return new EventQueueError(
       {
-        name: ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH,
-        info: { globalProperties, localProperties },
+        name: ERROR_CODES.APP_NAMES_FORMAT,
+        info: { type, subType, appNames },
       },
       message
     );

package/src/EventQueueProcessorBase.js

@@ -163,12 +163,13 @@ class EventQueueProcessorBase {
     );
   }
 
-  logTimeExceeded(iterationCounter) {
-    this.logger.info("Exiting event queue processing as max time exceeded", {
+  logTimeExceededAndPublishContinue(iterationCounter) {
+    this.logger.info("Exiting event queue processing as max time exceeded - but broadcast to trigger processing", {
       eventType: this.#eventType,
       eventSubType: this.#eventSubType,
       iterationCounter,
     });
+    this.#eventSchedulerInstance.scheduleEvent(this.__context.tenant, this.#eventType, this.#eventSubType, new Date());
   }
 
   logStartMessage() {
@@ -1108,6 +1109,10 @@ class EventQueueProcessorBase {
     return this.#eventType.replace(SUFFIX_PERIODIC, "");
   }
 
+  get rawEventType() {
+    return this.#eventType;
+  }
+
   get eventSubType() {
     return this.#eventSubType;
   }

package/src/config.js

@@ -105,7 +105,12 @@ class Config {
   }
 
   _checkRedisIsBound() {
-    return !!this.#env.getRedisCredentialsFromEnv();
+    return !!this.#env.redisCredentialsFromEnv;
+  }
+
+  shouldBeProcessedInThisApplication(type, subType) {
+    const config = this.#eventMap[this.generateKey(type, subType)];
+    return !config._appNameMap || config._appNameMap[this.#env.applicationName];
   }
 
   checkRedisEnabled() {
@@ -276,8 +281,12 @@ class Config {
       eventOutdatedCheck: config.eventOutdatedCheck,
       checkForNextChunk: config.checkForNextChunk,
       deleteFinishedEventsAfterDays: config.deleteFinishedEventsAfterDays,
+      appNames: config.appNames,
       internalEvent: true,
     };
+    eventConfig._appNameMap = eventConfig.appNames
+      ? Object.fromEntries(new Map(eventConfig.appNames.map((a) => [a, true])))
+      : null;
     this.#config.events.push(eventConfig);
     this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
   }
@@ -317,6 +326,7 @@ class Config {
       event.load = event.load ?? DEFAULT_LOAD;
       event.priority = event.priority ?? DEFAULT_PRIORITY;
       this.validateAdHocEvents(result, event);
+      event._appNameMap = event.appNames ? Object.fromEntries(new Map(event.appNames.map((a) => [a, true]))) : null;
       result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, {});
@@ -326,6 +336,7 @@ class Config {
       event.type = `${event.type}${SUFFIX_PERIODIC}`;
       event.isPeriodic = true;
       this.validatePeriodicConfig(result, event);
+      event._appNameMap = event.appNames ? Object.fromEntries(new Map(event.appNames.map((a) => [a, true]))) : null;
       result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, this.#eventMap);
@@ -355,6 +366,12 @@ class Config {
     if (!config.impl) {
       throw EventQueueError.missingImpl(config.type, config.subType);
     }
+
+    if (config.appNames) {
+      if (!Array.isArray(config.appNames) || config.appNames.some((appName) => typeof appName !== "string")) {
+        throw EventQueueError.appNamesFormat(config.type, config.subType, config.appNames);
+      }
+    }
   }
 
   generateKey(type, subType) {

package/src/index.d.ts

@@ -117,6 +117,7 @@ export declare class EventQueueProcessorBase {
   setShouldRollbackTransaction(key: string): void;
   shouldRollbackTransaction(key: string): boolean;
   beforeProcessingEvents(): Promise<void>;
+  addEntryToProcessingMap(key: string, queueEntry: EventEntity, payload: Object): void;
 
   set logger(value: CdsLogger);
   get logger(): CdsLogger;
@@ -214,3 +215,38 @@ declare class Config {
 }
 
 export const config: Config;
+
+export const workerQueue: WorkerQueue;
+
+declare class WorkerQueue {
+  constructor(concurrency: number);
+
+  addToQueue(load: number, label: string, priority?: Priorities, cb?: () => any): Promise<any>;
+
+  _executeFunction(
+    load: number,
+    label: string,
+    cb: () => any,
+    resolve: (value?: unknown) => void,
+    reject: (reason?: any) => void,
+    startTime: bigint,
+    priority: string
+  ): void;
+
+  get runningPromises(): Array<Promise<any>>;
+  get runningLoad(): number;
+
+  static get instance(): WorkerQueue;
+
+  get queue(): Record<
+    string,
+    Array<[number, string, () => any, (value?: unknown) => void, (reason?: any) => void, bigint]>
+  >;
+}
+
+interface Priorities {
+  Low: string;
+  Medium: string;
+  High: string;
+  VeryHigh: string;
+}

package/src/index.js

@@ -10,4 +10,5 @@ module.exports = {
   ...require("./constants"),
   ...require("./publishEvent"),
   EventQueueProcessorBase: require("./EventQueueProcessorBase"),
+  WorkerQueue: require("./shared/WorkerQueue"),
 };

package/src/outbox/eventQueueAsOutbox.js

@@ -15,9 +15,19 @@ const COMPONENT_NAME = "/eventQueue/eventQueueAsOutbox";
 function outboxed(srv, customOpts) {
   // outbox max. once
   const logger = cds.log(COMPONENT_NAME);
+  const outboxOpts = Object.assign(
+    {},
+    (typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
+    (typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
+    customOpts || {}
+  );
+
   if (!new.target) {
     const former = srv[OUTBOXED];
     if (former) {
+      if (outboxOpts.kind === "persistent-outbox") {
+        config.addCAPOutboxEvent(srv.name, outboxOpts);
+      }
       return former;
     }
   }
@@ -30,14 +40,9 @@ function outboxed(srv, customOpts) {
     Object.defineProperty(srv, OUTBOXED, { value: outboxedSrv });
   }
 
-  const outboxOpts = Object.assign(
-    {},
-    (typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
-    (typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
-    customOpts || {}
-  );
-
-  config.addCAPOutboxEvent(srv.name, outboxOpts);
+  if (outboxOpts.kind === "persistent-outbox") {
+    config.addCAPOutboxEvent(srv.name, outboxOpts);
+  }
   outboxedSrv.handle = async function (req) {
     const context = req.context || cds.context;
     if (outboxOpts.kind === "persistent-outbox") {

package/src/processEventQueue.js

@@ -120,7 +120,7 @@ const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime
     return true;
   }
 
-  eventTypeInstance.logTimeExceeded(iterationCounter);
+  eventTypeInstance.logTimeExceededAndPublishContinue(iterationCounter);
   return false;
 };
 
@@ -269,6 +269,7 @@ const _checkEventIsBlocked = async (baseInstance) => {
       });
     }
   } else {
+    // TODO: we should be able to get rid of baseInstance.isPeriodicEvent with rawEventType
     eventBlocked = config.isEventBlocked(
       baseInstance.eventType,
       baseInstance.eventSubType,
@@ -283,11 +284,16 @@ const _checkEventIsBlocked = async (baseInstance) => {
 
   if (eventBlocked) {
     baseInstance.logger.info("skipping run because event is blocked by configuration", {
-      type: baseInstance.eventType,
+      type: baseInstance.rawEventType,
       subType: baseInstance.eventSubType,
       tenantUnsubscribed: config.isTenantUnsubscribed(baseInstance.context.tenant),
     });
   }
+
+  if (!eventBlocked) {
+    eventBlocked = !config.shouldBeProcessedInThisApplication(baseInstance.rawEventType, baseInstance.eventSubType);
+  }
+
   return eventBlocked;
 };
 

package/src/redis/redisSub.js

@@ -35,14 +35,6 @@ const _messageHandlerProcessEvents = async (messageData) => {
       return;
     }
 
-    const user = await cds.tx({ tenant: tenantId }, async () => {
-      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
-    });
-    const tenantContext = {
-      tenant: tenantId,
-      user,
-    };
-
     if (!config.getEventConfig(type, subType)) {
       if (config.isCapOutboxEvent(type)) {
         try {
@@ -64,6 +56,23 @@ const _messageHandlerProcessEvents = async (messageData) => {
       }
     }
 
+    if (!(config.getEventConfig(type, subType) && config.shouldBeProcessedInThisApplication(type, subType))) {
+      logger.debug("event is not configured to be processed on this app-name", {
+        tenantId,
+        type,
+        subType,
+      });
+      return;
+    }
+
+    const user = await cds.tx({ tenant: tenantId }, async () => {
+      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+    });
+    const tenantContext = {
+      tenant: tenantId,
+      user,
+    };
+
     return await cds.tx(tenantContext, async ({ context }) => {
       return await runnerHelper.runEventCombinationForTenant(context, type, subType, { lockId, shouldTrace: true });
     });

package/src/runner/openEvents.js

@@ -29,18 +29,21 @@ const getOpenQueueEntries = async (tx) => {
 
   const result = [];
   for (const { type, subType } of entries) {
-    if (type.startsWith("CAP_OUTBOX")) {
-      if (cds.requires[subType]) {
-        await cds.connect.to(subType).catch(() => {});
-        result.push({ type, subType });
-      } else {
-        const service = await cds.connect.to(subType).catch(() => {});
-        if (service) {
-          result.push({ type, subType });
-        }
-      }
+    if (eventConfig.isCapOutboxEvent(type)) {
+      await cds.connect
+        .to(subType)
+        .then((service) => {
+          if (!service) {
+            return;
+          }
+          cds.outboxed(service);
+          if (eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
+            result.push({ type, subType });
+          }
+        })
+        .catch(() => {});
     } else {
-      if (eventConfig.getEventConfig(type, subType)) {
+      if (eventConfig.getEventConfig(type, subType) && eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
         result.push({ type, subType });
       }
     }

package/src/shared/WorkerQueue.js

@@ -8,7 +8,7 @@ const { Priorities } = require("../constants");
 const SetIntervalDriftSafe = require("./SetIntervalDriftSafe");
 
 const PRIORITIES = Object.values(Priorities).reverse();
-const PRIORITY_MULTIPLICATOR = PRIORITIES.reduce((result, element, index) => {
+const PRIORITY_MULTIPLICATION = PRIORITIES.reduce((result, element, index) => {
   result[element] = index + 1;
   return result;
 }, {});
@@ -152,6 +152,10 @@ class WorkerQueue {
     return this.#queue;
   }
 
+  get runningLoad() {
+    return this.#runningLoad;
+  }
+
   #checkAndLogWaitingTime(startTime, label, priority) {
     const ts = Date.now();
     if (ts - lastLogTs <= 1000) {
@@ -159,7 +163,7 @@ class WorkerQueue {
     }
     lastLogTs = ts;
     const diffMs = Math.round(Number(process.hrtime.bigint() - startTime) / NANO_TO_MS);
-    const priorityMultiplication = PRIORITY_MULTIPLICATOR[priority];
+    const priorityMultiplication = PRIORITY_MULTIPLICATION[priority];
     let logLevel;
     if (diffMs >= THRESHOLD.ERROR * priorityMultiplication) {
       logLevel = "error";

package/src/shared/common.js

@@ -1,15 +1,10 @@
 "use strict";
 
 const crypto = require("crypto");
-const { promisify } = require("util");
 
 const cds = require("@sap/cds");
 const xssec = require("@sap/xssec");
 
-const getAuthTokenAsync = promisify(xssec.requests.requestClientCredentialsToken);
-const getCreateSecurityContextAsync = promisify(xssec.createSecurityContext);
-
-let authInfoCache = {};
 const MARGIN_AUTH_INFO_EXPIRY = 60 * 1000;
 const COMPONENT_NAME = "/eventQueue/common";
 
@@ -74,9 +69,15 @@ const processChunkedSync = (inputs, chunkSize, chunkHandler) => {
 const hashStringTo32Bit = (value) => crypto.createHash("sha256").update(String(value)).digest("base64").slice(0, 32);
 
 const _getNewAuthInfo = async (tenantId) => {
+  const authInfoCache = getAuthInfo._authInfoCache;
+  authInfoCache[tenantId] = authInfoCache[tenantId] ?? {};
   try {
-    const token = await getAuthTokenAsync(null, cds.requires.auth.credentials, null, tenantId);
-    const authInfo = await getCreateSecurityContextAsync(token, cds.requires.auth.credentials);
+    if (!_getNewAuthInfo._xsuaaService) {
+      _getNewAuthInfo._xsuaaService = new xssec.XsuaaService(cds.requires.auth.credentials);
+    }
+    const authService = _getNewAuthInfo._xsuaaService;
+    const token = await authService.fetchClientCredentialsToken();
+    const authInfo = await authService.createSecurityContext(token.access_token);
     authInfoCache[tenantId].expireTs = authInfo.getExpirationDate().getTime() - MARGIN_AUTH_INFO_EXPIRY;
     return authInfo;
   } catch (err) {
@@ -89,7 +90,13 @@ const getAuthInfo = async (tenantId) => {
   if (!cds.requires?.auth?.credentials) {
     return null; // no credentials not authInfo
   }
+  if (!cds.requires?.auth.kind.match(/jwt|xsuaa/i)) {
+    cds.log(COMPONENT_NAME).warn("Only 'jwt' or 'xsuaa' are supported as values for auth.kind.");
+    return null;
+  }
 
+  getAuthInfo._authInfoCache = getAuthInfo._authInfoCache ?? {};
+  const authInfoCache = getAuthInfo._authInfoCache;
   // not existing or existing but expired
   if (
     !authInfoCache[tenantId] ||
@@ -110,6 +117,6 @@ module.exports = {
   hashStringTo32Bit,
   getAuthInfo,
   __: {
-    clearAuthInfoCache: () => (authInfoCache = {}),
+    clearAuthInfoCache: () => (getAuthInfo._authInfoCache = {}),
   },
 };

package/src/shared/env.js

@@ -3,34 +3,42 @@
 let instance;
 
 class Env {
-  #isLocal;
   #vcapServices;
+  #vcapApplication;
 
   constructor() {
     try {
       this.#vcapServices = JSON.parse(process.env.VCAP_SERVICES);
+      this.#vcapApplication = JSON.parse(process.env.VCAP_APPLICATION);
     } catch {
       this.#vcapServices = {};
+      this.#vcapApplication = {};
     }
   }
 
-  getRedisCredentialsFromEnv() {
+  get redisCredentialsFromEnv() {
     return this.#vcapServices["redis-cache"]?.[0]?.credentials;
   }
 
-  set isLocal(value) {
-    this.#isLocal = value;
-  }
-  get isLocal() {
-    return this.#isLocal;
+  get applicationName() {
+    return this.#vcapApplication.application_name;
   }
 
   set vcapServices(value) {
     this.#vcapServices = value;
   }
+
   get vcapServices() {
     return this.#vcapServices;
   }
+
+  set vcapApplication(value) {
+    this.#vcapApplication = value;
+  }
+
+  get vcapApplication() {
+    return this.#vcapApplication;
+  }
 }
 
 module.exports = {

package/src/shared/eventScheduler.js

@@ -47,7 +47,7 @@ class EventScheduler {
   }
 
   clearForTenant(tenantId) {
-    Object.values(this.#eventsByTenants[tenantId]).forEach((timeoutId) => clearTimeout(timeoutId));
+    Object.keys(this.#eventsByTenants[tenantId] ?? []).forEach((timeoutId) => clearTimeout(timeoutId));
   }
 
   calculateOffset(type, subType, startAfter) {
@@ -67,6 +67,14 @@ class EventScheduler {
   clearScheduledEvents() {
     this.#scheduledEvents = {};
   }
+
+  clearEventsByTenants() {
+    this.#eventsByTenants = {};
+  }
+
+  get eventsByTenants() {
+    return this.#eventsByTenants;
+  }
 }
 
 module.exports = {

package/src/shared/redis.js

@@ -30,7 +30,7 @@ const createMainClientAndConnect = (options) => {
 const _createClientBase = (redisOptions) => {
   const env = getEnvInstance();
   try {
-    const credentials = env.getRedisCredentialsFromEnv();
+    const credentials = env.redisCredentialsFromEnv;
     const redisIsCluster = credentials.cluster_mode;
     const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
     if (redisIsCluster) {