@cap-js-community/event-queue 0.2.5 → 1.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "0.2.5",
+  "version": "1.0.0",
   "description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [
@@ -22,7 +22,7 @@
   "scripts": {
     "test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
     "test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
-    "test:all": "npm run test:unit && npm run test:integration",
+    "test": "npm run test:unit && npm run test:integration",
     "test:all:coverage": "jest --runInBand --forceExit --collect-coverage",
     "test:prepare": "npm run build:ci --prefix=./test-integration/_env",
     "test:deploySchema": "node test-integration/_env/srv/hana/deploy.js",
@@ -39,22 +39,22 @@
     "upgrade-lock": "npx shx rm -rf package-lock.json node_modules && npm i --package-lock"
   },
   "engines": {
-    "node": ">=16"
+    "node": ">=18"
   },
   "dependencies": {
-    "redis": "4.6.11",
+    "redis": "4.6.12",
     "verror": "1.10.1",
     "yaml": "2.3.4"
   },
   "devDependencies": {
-    "@sap/cds": "7.4.0",
-    "@sap/cds-dk": "7.4.0",
-    "eslint": "8.54.0",
-    "eslint-config-prettier": "9.0.0",
+    "@sap/cds": "7.5.0",
+    "@sap/cds-dk": "7.5.0",
+    "eslint": "8.56.0",
+    "eslint-config-prettier": "9.1.0",
     "eslint-plugin-jest": "27.6.0",
     "eslint-plugin-node": "11.1.0",
     "express": "4.18.2",
-    "hdb": "0.19.6",
+    "hdb": "0.19.7",
     "jest": "29.7.0",
     "prettier": "2.8.8",
     "sqlite3": "5.1.6"

package/src/EventQueueError.js

@@ -210,12 +210,12 @@ class EventQueueError extends VError {
       message
     );
   }
-  static loadHigherThanLimit(load) {
+  static loadHigherThanLimit(load, label) {
     const { message } = ERROR_CODES_META[ERROR_CODES.LOAD_HIGHER_THAN_LIMIT];
     return new EventQueueError(
       {
         name: ERROR_CODES.LOAD_HIGHER_THAN_LIMIT,
-        info: { load },
+        info: { load, label },
       },
       message
     );

package/src/EventQueueProcessorBase.js

@@ -18,8 +18,6 @@ const DEFAULT_RETRY_ATTEMPTS = 3;
 const DEFAULT_PARALLEL_EVENT_PROCESSING = 1;
 const LIMIT_PARALLEL_EVENT_PROCESSING = 10;
 const SELECT_LIMIT_EVENTS_PER_TICK = 100;
-const DEFAULT_DELETE_FINISHED_EVENTS_AFTER = 0;
-const DAYS_TO_MS = 24 * 60 * 60 * 1000;
 const TRIES_FOR_EXCEEDED_EVENTS = 3;
 const EVENT_START_AFTER_HEADROOM = 3 * 1000;
 
@@ -34,6 +32,7 @@ class EventQueueProcessorBase {
   #eventSchedulerInstance = null;
   #eventConfig;
   #isPeriodic;
+  #lastSuccessfulRunTimestamp;
 
   constructor(context, eventType, eventSubType, config) {
     this.__context = context;
@@ -57,21 +56,12 @@ class EventQueueProcessorBase {
     // NOTE: keep the feature, this might be needed again
     this.__concurrentEventProcessing = false;
     this.__startTime = this.#eventConfig.startTime ?? new Date();
-    this.__retryAttempts = this.#eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
+    this.__retryAttempts = this.#isPeriodic ? 1 : this.#eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
     this.__selectMaxChunkSize = this.#eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
     this.__selectNextChunk = !!this.#eventConfig.checkForNextChunk;
     this.__keepalivePromises = {};
     this.__outdatedCheckEnabled = this.#eventConfig.eventOutdatedCheck ?? true;
     this.__transactionMode = this.#eventConfig.transactionMode ?? TransactionMode.isolated;
-    if (this.#eventConfig.deleteFinishedEventsAfterDays) {
-      this.__deleteFinishedEventsAfter =
-        Number.isInteger(this.#eventConfig.deleteFinishedEventsAfterDays) &&
-        this.#eventConfig.deleteFinishedEventsAfterDays > 0
-          ? this.#eventConfig.deleteFinishedEventsAfterDays
-          : DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
-    } else {
-      this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
-    }
     this.__emptyChunkSelected = false;
     this.__lockAcquired = false;
     this.__txUsageAllowed = true;
@@ -113,11 +103,30 @@ class EventQueueProcessorBase {
   }
 
   startPerformanceTracerEvents() {
-    this.__performanceLoggerEvents = new PerformanceTracer(this.logger, "Processing events");
+    this.__performanceLoggerEvents = new PerformanceTracer(this.logger, "Processing events", {
+      properties: {
+        type: this.eventType,
+        subType: this.eventSubType,
+      },
+    });
+  }
+
+  startPerformanceTracerPeriodicEvents() {
+    this.__performanceLoggerPeriodicEvents = new PerformanceTracer(this.logger, "Processing periodic event", {
+      properties: {
+        type: this.eventType,
+        subType: this.eventSubType,
+      },
+    });
   }
 
   startPerformanceTracerPreprocessing() {
-    this.__performanceLoggerPreprocessing = new PerformanceTracer(this.logger, "Preprocessing events");
+    this.__performanceLoggerPreprocessing = new PerformanceTracer(this.logger, "Preprocessing events", {
+      properties: {
+        type: this.eventType,
+        subType: this.eventSubType,
+      },
+    });
   }
 
   endPerformanceTracerEvents() {
@@ -130,6 +139,16 @@ class EventQueueProcessorBase {
     );
   }
 
+  endPerformanceTracerPeriodicEvents() {
+    this.__performanceLoggerPeriodicEvents?.endPerformanceTrace(
+      { threshold: 50 },
+      {
+        eventType: this.#eventType,
+        eventSubType: this.#eventSubType,
+      }
+    );
+  }
+
   endPerformanceTracerPreprocessing() {
     this.__performanceLoggerPreprocessing?.endPerformanceTrace(
       { threshold: 50 },
@@ -148,10 +167,9 @@ class EventQueueProcessorBase {
     });
   }
 
-  logStartMessage(queueEntries) {
-    // TODO: how to handle custom fields
+  logStartMessage() {
     this.logger.info("Processing queue event", {
-      numberQueueEntries: queueEntries.length,
+      numberClusterEntries: Object.keys(this.eventProcessingMap).length,
       eventType: this.#eventType,
       eventSubType: this.#eventSubType,
     });
@@ -332,11 +350,11 @@ class EventQueueProcessorBase {
     });
   }
 
-  async setPeriodicEventStatus(queueEntryIds) {
+  async setPeriodicEventStatus(queueEntryIds, status) {
     await this.tx.run(
       UPDATE.entity(this.#config.tableNameEventQueue)
         .set({
-          status: EventProcessingStatus.Done,
+          status: status,
         })
         .where({
           ID: queueEntryIds,
@@ -426,28 +444,6 @@ class EventQueueProcessorBase {
     });
   }
 
-  async deleteFinishedEvents(tx) {
-    if (!this.__deleteFinishedEventsAfter) {
-      return;
-    }
-    const deleteCount = await tx.run(
-      DELETE.from(this.#config.tableNameEventQueue).where(
-        "type =",
-        this.#eventType,
-        "AND subType=",
-        this.#eventSubType,
-        "AND lastAttemptTimestamp <=",
-        new Date(Date.now() - this.__deleteFinishedEventsAfter * DAYS_TO_MS).toISOString()
-      )
-    );
-    this.logger.debug("Deleted finished events", {
-      eventType: this.#eventType,
-      eventSubType: this.#eventSubType,
-      deleteFinishedEventsAfter: this.__deleteFinishedEventsAfter,
-      deleteCount,
-    });
-  }
-
   #ensureEveryQueueEntryHasStatus() {
     this.__queueEntries.forEach((queueEntry) => {
       if (queueEntry.ID in this.__statusMap || queueEntry.ID in this.__commitedStatusMap) {
@@ -549,15 +545,27 @@ class EventQueueProcessorBase {
             refDateStartAfter.toISOString(),
             " ) AND ( status =",
             EventProcessingStatus.Open,
-            "OR ( status =",
-            EventProcessingStatus.Error,
-            "AND lastAttemptTimestamp <=",
+            "AND ( lastAttemptTimestamp <=",
             this.__startTime.toISOString(),
-            ") OR ( status =",
-            EventProcessingStatus.InProgress,
-            "AND lastAttemptTimestamp <=",
-            new Date(new Date().getTime() - this.#config.globalTxTimeout).toISOString(),
-            ") )"
+            ...(this.isPeriodicEvent
+              ? [
+                  "OR lastAttemptTimestamp IS NULL ) OR ( status =",
+                  EventProcessingStatus.InProgress,
+                  "AND lastAttemptTimestamp <=",
+                  new Date(new Date().getTime() - this.#config.globalTxTimeout).toISOString(),
+                  ") )",
+                ]
+              : [
+                  "OR lastAttemptTimestamp IS NULL ) OR ( status =",
+                  EventProcessingStatus.Error,
+                  "AND lastAttemptTimestamp <=",
+                  this.__startTime.toISOString(),
+                  ") OR ( status =",
+                  EventProcessingStatus.InProgress,
+                  "AND lastAttemptTimestamp <=",
+                  new Date(new Date().getTime() - this.#config.globalTxTimeout).toISOString(),
+                  ") )",
+                ])
           )
           .orderBy("createdAt", "ID")
       );
@@ -575,9 +583,11 @@ class EventQueueProcessorBase {
         entries,
         refDateStartAfter
       );
-      const eventsForProcessing = exceededTries.concat(openEvents).concat(exceededTriesExceeded);
+      const eventsForProcessing = openEvents
+        .concat(exceededTriesExceeded)
+        .concat(this.#isPeriodic ? [] : exceededTries);
       this.#selectedEventMap = arrayToFlatMap(eventsForProcessing);
-      if (exceededTries.length) {
+      if (!this.#isPeriodic && exceededTries.length) {
         this.#eventsWithExceededTries = exceededTries;
       }
       if (exceededTriesExceeded.length) {
@@ -586,13 +596,30 @@ class EventQueueProcessorBase {
       this.#handleDelayedEvents(delayedEvents);
 
       result = openEvents;
-      this.logger[eventsForProcessing.length ? "info" : "debug"]("Selected event queue entries for processing", {
-        openEvents: openEvents.length,
-        ...(delayedEvents.length && { delayedEvents: delayedEvents.length }),
-        ...(exceededTries.length && { exceededTries: exceededTries.length }),
-        eventType: this.#eventType,
-        eventSubType: this.#eventSubType,
-      });
+      this.logger[eventsForProcessing.length && !this.isPeriodicEvent ? "info" : "debug"](
+        "Selected event queue entries for processing",
+        {
+          openEvents: openEvents.length,
+          ...(delayedEvents.length && { delayedEvents: delayedEvents.length }),
+          ...(exceededTries.length && { exceededTries: exceededTries.length }),
+          eventType: this.#eventType,
+          eventSubType: this.#eventSubType,
+        }
+      );
+
+      if (this.#isPeriodic && exceededTries.length) {
+        await tx.run(
+          UPDATE.entity(this.#config.tableNameEventQueue)
+            .set({
+              status: EventProcessingStatus.Error,
+              lastAttemptTimestamp: new Date(),
+            })
+            .where(
+              "ID IN",
+              exceededTries.map(({ ID }) => ID)
+            )
+        );
+      }
 
       if (!eventsForProcessing.length) {
         this.__emptyChunkSelected = true;
@@ -623,10 +650,26 @@ class EventQueueProcessorBase {
       });
       this.__queueEntries = result;
       this.__queueEntriesMap = arrayToFlatMap(result);
+
+      if (this.#isPeriodic && this.#eventConfig.lastSuccessfulRunTimestamp) {
+        this.#lastSuccessfulRunTimestamp = await this.#selectLastSuccessfulPeriodicTimestamp(tx);
+      }
     });
     return result;
   }
 
+  async #selectLastSuccessfulPeriodicTimestamp() {
+    const entry = await SELECT.one
+      .from(this.#config.tableNameEventQueue)
+      .where({
+        type: this.#eventType,
+        subType: this.#eventSubType,
+        status: EventProcessingStatus.Done,
+      })
+      .columns("max (lastAttemptTimestamp) as lastAttemptsTs");
+    return entry.lastAttemptsTs;
+  }
+
   #handleDelayedEvents(delayedEvents) {
     for (const delayedEvent of delayedEvents) {
       this.#eventSchedulerInstance.scheduleEvent(
@@ -924,10 +967,37 @@ class EventQueueProcessorBase {
         obsoleteEntries.push(queueEntry);
       }
     }
-    await this.setPeriodicEventStatus(obsoleteEntries.map(({ ID }) => ID));
+    await this.setPeriodicEventStatus(
+      obsoleteEntries.map(({ ID }) => ID),
+      EventProcessingStatus.Done
+    );
     return queueEntryToUse;
   }
 
+  /**
+   * Asynchronously gets the timestamp of the last successful run.
+   *
+   * @returns {Promise<string|null>} A Promise that resolves to a string representation of the timestamp
+   * of the last successful run (in ISO 8601 format: YYYY-MM-DDTHH:mm:ss.sss),
+   * or null if there has been no successful run yet.
+   *
+   * @example
+   * const timestamp = await instance.getLastSuccessfulRunTimestamp();
+   * console.log(timestamp);  // Outputs: 2023-12-07T09:15:44.237
+   *
+   * @throws {Error} If an error occurs while fetching the timestamp.
+   */
+  async getLastSuccessfulRunTimestamp() {
+    if (!this.#isPeriodic) {
+      return null;
+    }
+    if (this.#lastSuccessfulRunTimestamp === undefined) {
+      this.#lastSuccessfulRunTimestamp = await this.#selectLastSuccessfulPeriodicTimestamp();
+    }
+
+    return this.#lastSuccessfulRunTimestamp;
+  }
+
   statusMapContainsError(statusMap) {
     return Object.values(statusMap).includes(EventProcessingStatus.Error);
   }

package/src/config.js

@@ -9,9 +9,24 @@ const EventQueueError = require("./EventQueueError");
 const FOR_UPDATE_TIMEOUT = 10;
 const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
 const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
+const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
 const COMPONENT_NAME = "eventQueue/config";
 const MIN_INTERVAL_SEC = 10;
 const DEFAULT_LOAD = 1;
+const SUFFIX_PERIODIC = "_PERIODIC";
+const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
+const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
+
+const BASE_PERIODIC_EVENTS = [
+  {
+    type: "EVENT_QUEUE_BASE",
+    subType: "DELETE_EVENTS",
+    impl: "./housekeeping/EventQueueDeleteEvents",
+    load: 1,
+    interval: 86400, // 1 day,
+    internalEvent: true,
+  },
+];
 
 class Config {
   #logger;
@@ -24,7 +39,7 @@ class Config {
   #instanceLoadLimit;
   #tableNameEventQueue;
   #tableNameEventLock;
-  #isRunnerDeactivated;
+  #isEventQueueActive;
   #configFilePath;
   #processEventsAfterPublish;
   #skipCsnCheck;
@@ -33,6 +48,8 @@ class Config {
   #env;
   #eventMap;
   #updatePeriodicEvents;
+  #blockedPeriodicEvents;
+  #isPeriodicEventBlockedCb;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -42,15 +59,16 @@ class Config {
     this.#runInterval = null;
     this.#redisEnabled = null;
     this.#initialized = false;
-    this.#instanceLoadLimit = null;
+    this.#instanceLoadLimit = 100;
     this.#tableNameEventQueue = null;
     this.#tableNameEventLock = null;
-    this.#isRunnerDeactivated = false;
+    this.#isEventQueueActive = true;
     this.#configFilePath = null;
     this.#processEventsAfterPublish = null;
     this.#skipCsnCheck = null;
     this.#disableRedis = null;
     this.#env = getEnvInstance();
+    this.#blockedPeriodicEvents = {};
   }
 
   getEventConfig(type, subType) {
@@ -70,6 +88,7 @@ class Config {
   }
 
   attachConfigChangeHandler() {
+    this.#attachBlocklistChangeHandler();
     redis.subscribeRedisChannel(REDIS_CONFIG_CHANNEL, (messageData) => {
       try {
         const { key, value } = JSON.parse(messageData);
@@ -78,7 +97,7 @@ class Config {
           this[key] = value;
         }
       } catch (err) {
-        this.#logger.error("could not parse event config change", {
+        this.#logger.error("could not parse event config change", err, {
           messageData,
         });
       }
@@ -95,18 +114,102 @@ class Config {
     });
   }
 
-  get isRunnerDeactivated() {
-    return this.#isRunnerDeactivated;
+  #attachBlocklistChangeHandler() {
+    redis.subscribeRedisChannel(REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
+      try {
+        const { command, key, tenant } = JSON.parse(messageData);
+        if (command === COMMAND_BLOCK) {
+          this.#blockPeriodicEventLocalState(key, tenant);
+        } else {
+          this.#unblockPeriodicEventLocalState(key, tenant);
+        }
+      } catch (err) {
+        this.#logger.error("could not parse event blocklist change", err, {
+          messageData,
+        });
+      }
+    });
+  }
+
+  blockPeriodicEvent(type, subType, tenant = "*") {
+    const typeWithSuffix = `${type}${SUFFIX_PERIODIC}`;
+    const config = this.getEventConfig(typeWithSuffix, subType);
+    if (!config) {
+      return;
+    }
+    const key = this.generateKey(typeWithSuffix, subType);
+    this.#blockPeriodicEventLocalState(key, tenant);
+    if (!this.redisEnabled) {
+      return;
+    }
+
+    redis
+      .publishMessage(REDIS_CONFIG_BLOCKLIST_CHANNEL, JSON.stringify({ command: COMMAND_BLOCK, key, tenant }))
+      .catch((error) => {
+        this.#logger.error(`publishing config block failed key: ${key}`, error);
+      });
+  }
+
+  #blockPeriodicEventLocalState(key, tenant) {
+    this.#blockedPeriodicEvents[key] ??= {};
+    this.#blockedPeriodicEvents[key][tenant] = true;
+    return key;
+  }
+
+  clearPeriodicEventBlockList() {
+    this.#blockedPeriodicEvents = {};
+  }
+
+  unblockPeriodicEvent(type, subType, tenant = "*") {
+    const typeWithSuffix = `${type}${SUFFIX_PERIODIC}`;
+    const key = this.generateKey(typeWithSuffix, subType);
+    const config = this.getEventConfig(typeWithSuffix, subType);
+    if (!config) {
+      return;
+    }
+    this.#unblockPeriodicEventLocalState(key, tenant);
+    if (!this.redisEnabled) {
+      return;
+    }
+
+    redis
+      .publishMessage(REDIS_CONFIG_BLOCKLIST_CHANNEL, JSON.stringify({ command: COMMAND_UNBLOCK, key, tenant }))
+      .catch((error) => {
+        this.#logger.error(`publishing config block failed key: ${key}`, error);
+      });
+  }
+
+  #unblockPeriodicEventLocalState(key, tenant) {
+    const map = this.#blockedPeriodicEvents[key];
+    if (!map) {
+      return;
+    }
+    this.#blockedPeriodicEvents[key][tenant] = false;
+    return key;
+  }
+
+  isPeriodicEventBlocked(type, subType, tenant) {
+    const map = this.#blockedPeriodicEvents[this.generateKey(type, subType)];
+    if (!map) {
+      return false;
+    }
+    const tenantSpecific = map[tenant];
+    const allTenants = map["*"];
+    return tenantSpecific ?? allTenants;
+  }
+
+  get isEventQueueActive() {
+    return this.#isEventQueueActive;
   }
 
-  set isRunnerDeactivated(value) {
-    this.#isRunnerDeactivated = value;
+  set isEventQueueActive(value) {
+    this.#isEventQueueActive = value;
   }
 
   set fileContent(config) {
     this.#config = config;
     config.events = config.events ?? [];
-    config.periodicEvents = config.periodicEvents ?? [];
+    config.periodicEvents = (config.periodicEvents ?? []).concat(BASE_PERIODIC_EVENTS.map((event) => ({ ...event })));
     this.#eventMap = config.events.reduce((result, event) => {
       event.load = event.load ?? DEFAULT_LOAD;
       this.validateAdHocEvents(result, event);
@@ -115,7 +218,6 @@ class Config {
     }, {});
     this.#eventMap = config.periodicEvents.reduce((result, event) => {
       event.load = event.load ?? DEFAULT_LOAD;
-      const SUFFIX_PERIODIC = "_PERIODIC";
       event.type = `${event.type}${SUFFIX_PERIODIC}`;
       event.isPeriodic = true;
       this.validatePeriodicConfig(result, event);
@@ -222,6 +324,14 @@ class Config {
     this.#instanceLoadLimit = value;
   }
 
+  get isPeriodicEventBlockedCb() {
+    return this.#isPeriodicEventBlockedCb;
+  }
+
+  set isPeriodicEventBlockedCb(value) {
+    this.#isPeriodicEventBlockedCb = value;
+  }
+
   get tableNameEventQueue() {
     return this.#tableNameEventQueue;
   }

package/src/housekeeping/EventQueueDeleteEvents.js

@@ -0,0 +1,51 @@
+"use strict";
+
+const EventQueueBaseClass = require("../EventQueueProcessorBase");
+const config = require("../config");
+const eventConfig = require("../config");
+
+const DELETE_DEFAULT_PERIOD_IN_DAYS = 7;
+const DAY_IN_MS = 24 * 60 * 60 * 1000;
+
+class EventQueueDeleteEvents extends EventQueueBaseClass {
+  constructor(context, eventType, eventSubType, config) {
+    super(context, eventType, eventSubType, config);
+  }
+
+  async processPeriodicEvent(processContext, key) {
+    const tx = this.getTxForEventProcessing(key);
+    const deleteByDaysMap = config.events.concat(config.periodicEvents).reduce((result, event) => {
+      if (!event.deleteFinishedEventsAfterDays) {
+        event.deleteFinishedEventsAfterDays = DELETE_DEFAULT_PERIOD_IN_DAYS;
+      }
+      result[event.deleteFinishedEventsAfterDays] ??= [];
+      result[event.deleteFinishedEventsAfterDays].push(event);
+      return result;
+    }, {});
+
+    for (const [deleteAfter, events] of Object.entries(deleteByDaysMap)) {
+      const deleteCount = await tx.run(
+        DELETE.from(eventConfig.tableNameEventQueue).where([
+          { list: [{ ref: ["type"] }, { ref: ["subType"] }] },
+          "IN",
+          {
+            list: events.map((event) => ({
+              list: [{ val: event.type }, { val: event.subType }],
+            })),
+          },
+          "AND",
+          { ref: ["lastAttemptTimestamp"] },
+          "<=",
+          { val: new Date(processContext.timestamp.getTime() - deleteAfter * DAY_IN_MS).toISOString() },
+        ])
+      );
+      this.logger.info("deleted eligible events", {
+        deleteAfterDays: deleteAfter,
+        deleteCount,
+        events: events.map((event) => `${event.type}_${event.subType}`),
+      });
+    }
+  }
+}
+
+module.exports = EventQueueDeleteEvents;

package/src/initialize.js

@@ -27,9 +27,8 @@ const CONFIG_VARS = [
   ["configFilePath", null],
   ["registerAsEventProcessor", true],
   ["processEventsAfterPublish", true],
-  ["isRunnerDeactivated", false],
+  ["isEventQueueActive", true],
   ["runInterval", 5 * 60 * 1000],
-  ["instanceLoadLimit", 20],
   ["tableNameEventQueue", BASE_TABLES.EVENT],
   ["tableNameEventLock", BASE_TABLES.LOCK],
   ["disableRedis", false],
@@ -41,9 +40,8 @@ const initialize = async ({
   configFilePath,
   registerAsEventProcessor,
   processEventsAfterPublish,
-  isRunnerDeactivated,
+  isEventQueueActive,
   runInterval,
-  instanceLoadLimit,
   tableNameEventQueue,
   tableNameEventLock,
   disableRedis,
@@ -52,7 +50,7 @@ const initialize = async ({
 } = {}) => {
   // TODO: initialize check:
   // - content of yaml check
-  // - betweenRuns and instanceLoadLimit
+  // - betweenRuns
 
   if (config.initialized) {
     return;
@@ -63,9 +61,8 @@ const initialize = async ({
     configFilePath,
     registerAsEventProcessor,
     processEventsAfterPublish,
-    isRunnerDeactivated,
+    isEventQueueActive,
     runInterval,
-    instanceLoadLimit,
     tableNameEventQueue,
     tableNameEventLock,
     disableRedis,
@@ -92,7 +89,6 @@ const initialize = async ({
     multiTenancyEnabled: config.isMultiTenancy,
     redisEnabled: config.redisEnabled,
     runInterval: config.runInterval,
-    config: config.instanceLoadLimit,
   });
 };
 

package/src/processEventQueue.js

@@ -5,7 +5,7 @@ const pathLib = require("path");
 const cds = require("@sap/cds");
 
 const config = require("./config");
-const { TransactionMode } = require("./constants");
+const { TransactionMode, EventProcessingStatus } = require("./constants");
 const { limiter } = require("./shared/common");
 
 const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
@@ -20,7 +20,7 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
   try {
     let eventTypeInstance;
     const eventConfig = config.getEventConfig(eventType, eventSubType);
-    const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
+    const [err, EventTypeClass] = resilientRequire(eventConfig);
     if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
       cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
         eventType,
@@ -90,15 +90,6 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
         await eventTypeInstance.persistEventStatus(tx);
       });
       shouldContinue = reevaluateShouldContinue(eventTypeInstance, iterationCounter, startTime);
-      if (!shouldContinue) {
-        await executeInNewTransaction(
-          context,
-          `eventQueue-deleteFinishedEvents-${eventType}##${eventSubType}`,
-          async (tx) => {
-            await eventTypeInstance.deleteFinishedEvents(tx);
-          }
-        );
-      }
     }
   } catch (err) {
     cds.log(COMPONENT_NAME).error("Processing event queue failed with unexpected error.", err, {
@@ -124,11 +115,35 @@ const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime
   return false;
 };
 
-// TODO: don't forget to release lock
 const processPeriodicEvent = async (eventTypeInstance) => {
   let queueEntry;
   let processNext = true;
 
+  const isPeriodicEventBlockedCb = config.isPeriodicEventBlockedCb;
+  const params = [eventTypeInstance.eventType, eventTypeInstance.eventSubType, eventTypeInstance.context.tenant];
+  let eventBlocked = false;
+  if (isPeriodicEventBlockedCb) {
+    try {
+      eventBlocked = await isPeriodicEventBlockedCb(...params);
+    } catch (err) {
+      eventBlocked = true;
+      eventTypeInstance.logger.error("skipping run because periodic event blocked check failed!", err, {
+        type: eventTypeInstance.eventType,
+        subType: eventTypeInstance.eventSubType,
+      });
+    }
+  } else {
+    eventBlocked = config.isPeriodicEventBlocked(...params);
+  }
+
+  if (eventBlocked) {
+    eventTypeInstance.logger.info("skipping run because periodic event is blocked by configuration", {
+      type: eventTypeInstance.eventType,
+      subType: eventTypeInstance.eventSubType,
+    });
+    return;
+  }
+
   try {
     while (processNext) {
       await executeInNewTransaction(
@@ -153,6 +168,7 @@ const processPeriodicEvent = async (eventTypeInstance) => {
         return;
       }
 
+      let status = EventProcessingStatus.Done;
       await executeInNewTransaction(
         eventTypeInstance.context,
         `eventQueue-periodic-process-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
@@ -160,13 +176,17 @@ const processPeriodicEvent = async (eventTypeInstance) => {
           eventTypeInstance.processEventContext = tx.context;
           eventTypeInstance.setTxForEventProcessing(queueEntry.ID, cds.tx(tx.context));
           try {
+            eventTypeInstance.startPerformanceTracerPeriodicEvents();
             await eventTypeInstance.processPeriodicEvent(tx.context, queueEntry.ID, queueEntry);
           } catch (err) {
+            status = EventProcessingStatus.Error;
             eventTypeInstance.handleErrorDuringPeriodicEventProcessing(err, queueEntry);
             throw new TriggerRollback();
+          } finally {
+            eventTypeInstance.endPerformanceTracerPeriodicEvents();
           }
           if (
-            eventTypeInstance.transactionMode !== TransactionMode.alwaysCommit ||
+            eventTypeInstance.transactionMode === TransactionMode.alwaysRollback ||
             eventTypeInstance.shouldRollbackTransaction(queueEntry.ID)
           ) {
             throw new TriggerRollback();
@@ -179,7 +199,7 @@ const processPeriodicEvent = async (eventTypeInstance) => {
         `eventQueue-periodic-setStatus-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
         async (tx) => {
           eventTypeInstance.processEventContext = tx.context;
-          await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID);
+          await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID, status);
         }
       );
     }
@@ -196,6 +216,7 @@ const processPeriodicEvent = async (eventTypeInstance) => {
 const processEventMap = async (eventTypeInstance) => {
   eventTypeInstance.startPerformanceTracerEvents();
   await eventTypeInstance.beforeProcessingEvents();
+  eventTypeInstance.logStartMessage();
   if (eventTypeInstance.commitOnEventLevel) {
     eventTypeInstance.txUsageAllowed = false;
   }
@@ -244,7 +265,6 @@ const processEventMap = async (eventTypeInstance) => {
 
 const _processEvent = async (eventTypeInstance, processContext, key, queueEntries, payload) => {
   try {
-    eventTypeInstance.logStartMessage(queueEntries);
     const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(queueEntries);
     if (eventOutdated) {
       return;
@@ -257,9 +277,11 @@ const _processEvent = async (eventTypeInstance, processContext, key, queueEntrie
   }
 };
 
-const resilientRequire = (path) => {
+const resilientRequire = (eventConfig) => {
   try {
-    const module = require(pathLib.join(process.cwd(), path));
+    const path = eventConfig?.impl;
+    const internal = eventConfig?.internalEvent;
+    const module = require(pathLib.join(internal ? __dirname : process.cwd(), path));
     return [null, module];
   } catch (err) {
     return [err, null];

package/src/redisPubSub.js

@@ -1,9 +1,12 @@
 "use strict";
 
+const cds = require("@sap/cds");
+
 const redis = require("./shared/redis");
 const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
 const config = require("./config");
 const { runEventCombinationForTenant } = require("./runner");
+const { getSubdomainForTenantId } = require("./shared/cdsHelper");
 
 const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
 const COMPONENT_NAME = "eventQueue/redisPubSub";
@@ -26,7 +29,23 @@ const messageHandlerProcessEvents = async (messageData) => {
       type,
       subType,
     });
-    await runEventCombinationForTenant(tenantId, type, subType);
+    if (!config.isEventQueueActive) {
+      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {
+        type,
+        subType,
+      });
+      return;
+    }
+
+    const subdomain = await getSubdomainForTenantId(tenantId);
+    const tenantContext = {
+      tenant: tenantId,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    };
+    return await cds.tx(tenantContext, async ({ context }) => {
+      return await runEventCombinationForTenant(context, type, subType);
+    });
   } catch (err) {
     logger.error("could not parse event information", {
       messageData,
@@ -36,13 +55,32 @@ const messageHandlerProcessEvents = async (messageData) => {
 
 const broadcastEvent = async (tenantId, type, subType) => {
   const logger = cds.log(COMPONENT_NAME);
-  if (!config.redisEnabled) {
-    if (config.registerAsEventProcessor) {
-      await runEventCombinationForTenant(tenantId, type, subType);
-    }
-    return;
-  }
   try {
+    if (!config.isEventQueueActive) {
+      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {
+        type,
+        subType,
+      });
+      return;
+    }
+    if (!config.redisEnabled) {
+      if (config.registerAsEventProcessor) {
+        let context = {};
+        if (tenantId) {
+          const subdomain = await getSubdomainForTenantId(tenantId);
+          context = {
+            // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+            tenant: tenantId,
+            http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+          };
+        }
+
+        return await cds.tx(context, async ({ context }) => {
+          return await runEventCombinationForTenant(context, type, subType);
+        });
+      }
+      return;
+    }
     const result = await checkLockExistsAndReturnValue(
       new cds.EventContext({ tenant: tenantId }),
       [type, subType].join("##")

package/src/runner.js

@@ -37,7 +37,7 @@ const _scheduleFunction = async (singleRunFn, periodicFn) => {
 
   const fnWithRunningCheck = () => {
     const logger = cds.log(COMPONENT_NAME);
-    if (eventQueueConfig.isRunnerDeactivated) {
+    if (!eventQueueConfig.isEventQueueActive) {
       logger.info("runner is deactivated via config variable. Skipping this run.");
       return;
     }
@@ -82,6 +82,9 @@ const _checkAndTriggerPeriodicEventUpdate = (tenantIds) => {
   const hash = hashStringTo32Bit(JSON.stringify(tenantIds));
   if (!tenantIdHash) {
     tenantIdHash = hash;
+    _multiTenancyPeriodicEvents().catch((err) => {
+      cds.log(COMPONENT_NAME).error("Error during triggering updating periodic events!", err);
+    });
     return;
   }
   if (tenantIdHash && tenantIdHash !== hash) {
@@ -94,45 +97,63 @@ const _checkAndTriggerPeriodicEventUpdate = (tenantIds) => {
 
 const _executeEventsAllTenants = (tenantIds, runId) => {
   const events = eventQueueConfig.allEvents;
-  const promises = [];
-  tenantIds.forEach((tenantId) => {
+  const product = tenantIds.reduce((result, tenantId) => {
     events.forEach((event) => {
-      promises.push(
-        WorkerQueue.instance.addToQueue(event.load, async () => {
-          try {
-            const lockId = `${runId}_${event.type}_${event.subType}`;
-            const tenantContext = new cds.EventContext({ tenant: tenantId });
-            const couldAcquireLock = await distributedLock.acquireLock(tenantContext, lockId, {
-              expiryTime: eventQueueConfig.runInterval * 0.95,
-            });
-            if (!couldAcquireLock) {
-              return;
-            }
-            await runEventCombinationForTenant(tenantId, event.type, event.subType, true);
-          } catch (err) {
-            cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-              tenantId,
-            });
+      result.push([tenantId, event]);
+    });
+    return result;
+  }, []);
+
+  return product.map(async ([tenantId, event]) => {
+    const subdomain = await getSubdomainForTenantId(tenantId);
+    const tenantContext = {
+      tenant: tenantId,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    };
+    return await cds.tx(tenantContext, async ({ context }) => {
+      const label = `${event.type}_${event.subType}`;
+      return await WorkerQueue.instance.addToQueue(event.load, label, async () => {
+        try {
+          const lockId = `${runId}_${label}`;
+          const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+            expiryTime: eventQueueConfig.runInterval * 0.95,
+          });
+          if (!couldAcquireLock) {
+            return;
           }
-        })
-      );
+          await runEventCombinationForTenant(context, event.type, event.subType, true);
+        } catch (err) {
+          cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
+            tenantId,
+          });
+        }
+      });
     });
   });
-  return promises;
 };
 
 const _executePeriodicEventsAllTenants = (tenantIds, runId) => {
   tenantIds.forEach((tenantId) => {
-    WorkerQueue.instance.addToQueue(1, async () => {
+    const label = `UPDATE_PERIODIC_EVENTS_${tenantId}`;
+    WorkerQueue.instance.addToQueue(1, label, async () => {
       try {
-        const tenantContext = new cds.EventContext({ tenant: tenantId });
-        const couldAcquireLock = await distributedLock.acquireLock(tenantContext, runId, {
-          expiryTime: eventQueueConfig.runInterval * 0.95,
+        const subdomain = await getSubdomainForTenantId(tenantId);
+        const tenantContext = {
+          tenant: tenantId,
+          // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+          http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+        };
+
+        return await cds.tx(tenantContext, async ({ context }) => {
+          const couldAcquireLock = await distributedLock.acquireLock(context, runId, {
+            expiryTime: eventQueueConfig.runInterval * 0.95,
+          });
+          if (!couldAcquireLock) {
+            return;
+          }
+          await _checkPeriodicEventsSingleTenant(context);
         });
-        if (!couldAcquireLock) {
-          return;
-        }
-        await _checkPeriodicEventsSingleTenant(tenantId);
       } catch (err) {
         cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
           tenantId,
@@ -143,11 +164,19 @@ const _executePeriodicEventsAllTenants = (tenantIds, runId) => {
 };
 
 const _singleTenantDb = async (tenantId) => {
-  const events = eventQueueConfig.allEvents;
-  events.forEach((event) => {
-    WorkerQueue.instance.addToQueue(event.load, async () => {
+  return eventQueueConfig.allEvents.map((event) => {
+    const label = `${event.type}_${event.subType}`;
+    return WorkerQueue.instance.addToQueue(event.load, label, async () => {
       try {
-        await runEventCombinationForTenant(tenantId, event.type, event.subType, true);
+        const context = new cds.EventContext({ tenant: tenantId });
+        const lockId = `${EVENT_QUEUE_RUN_ID}_${label}`;
+        const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+          expiryTime: eventQueueConfig.runInterval * 0.95,
+        });
+        if (!couldAcquireLock) {
+          return;
+        }
+        await runEventCombinationForTenant(context, event.type, event.subType, true);
       } catch (err) {
         cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
           tenantId,
@@ -215,28 +244,23 @@ const _calculateOffsetForFirstRun = async () => {
   return offsetDependingOnLastRun;
 };
 
-const runEventCombinationForTenant = async (tenantId, type, subType, skipWorkerPool) => {
+const runEventCombinationForTenant = async (context, type, subType, skipWorkerPool) => {
   try {
-    const subdomain = await getSubdomainForTenantId(tenantId);
-    const context = new cds.EventContext({
-      tenant: tenantId,
-      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-    });
-    cds.context = context;
     if (skipWorkerPool) {
       return await processEventQueue(context, type, subType);
     } else {
       const config = eventQueueConfig.getEventConfig(type, subType);
+      const label = `${type}_${subType}`;
       return await WorkerQueue.instance.addToQueue(
         config.load,
+        label,
         async () => await processEventQueue(context, type, subType)
       );
     }
   } catch (err) {
     const logger = cds.log(COMPONENT_NAME);
     logger.error("error executing event combination for tenant", err, {
-      tenantId,
+      tenantId: context.tenant,
       type,
       subType,
     });
@@ -266,7 +290,7 @@ const _multiTenancyPeriodicEvents = async () => {
   }
 };
 
-const _checkPeriodicEventsSingleTenant = async (tenantId) => {
+const _checkPeriodicEventsSingleTenant = async (context = {}) => {
   const logger = cds.log(COMPONENT_NAME);
   if (!eventQueueConfig.updatePeriodicEvents || !eventQueueConfig.periodicEvents.length) {
     logger.info("updating of periodic events is disabled or no periodic events configured", {
@@ -276,23 +300,16 @@ const _checkPeriodicEventsSingleTenant = async (tenantId) => {
     return;
   }
   try {
-    const subdomain = await cdsHelper.getSubdomainForTenantId(tenantId);
-    const context = new cds.EventContext({
-      tenant: tenantId,
-      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-    });
-    cds.context = context;
     logger.info("executing updating periotic events", {
-      tenantId,
-      subdomain,
+      tenantId: context.tenant,
+      subdomain: context.http?.req.authInfo.getSubdomain(),
     });
     await cdsHelper.executeInNewTransaction(context, "update-periodic-events", async (tx) => {
       await periodicEvents.checkAndInsertPeriodicEvents(tx.context);
     });
   } catch (err) {
     logger.error("Couldn't update periodic events for tenant! Next try after defined interval.", err, {
-      tenantId,
+      tenantId: context.tenant,
       redisEnabled: eventQueueConfig.redisEnabled,
     });
   }
@@ -304,10 +321,12 @@ module.exports = {
   multiTenancyRedis,
   runEventCombinationForTenant,
   _: {
+    _singleTenantDb,
     _multiTenancyRedis,
     _multiTenancyDb,
     _calculateOffsetForFirstRun,
     _acquireRunId,
     EVENT_QUEUE_RUN_TS,
+    clearHash: () => (tenantIdHash = null),
   },
 };

package/src/shared/PerformanceTracer.js

@@ -17,6 +17,7 @@ class PerformanceTracer {
     this.__start = new Date();
     this.__logger = logger;
     this.__name = name;
+    this.__properties = options.properties ??= {};
     options.startMessage &&
       logger.info("Performance measurement started", {
         name: name,
@@ -64,6 +65,7 @@ class PerformanceTracer {
     this.__logger.info("Performance measurement executed", {
       name: this.__name,
       milliseconds: executionTime,
+      ...this.__properties,
       customFields,
     });
   }

package/src/shared/WorkerQueue.js

@@ -8,9 +8,9 @@ const EventQueueError = require("../EventQueueError");
 const COMPONENT_NAME = "eventQueue/WorkerQueue";
 const NANO_TO_MS = 1e6;
 const THRESHOLD = {
-  INFO: 5 * 1000,
-  WARN: 10 * 1000,
-  ERROR: 15 * 1000,
+  INFO: 35 * 1000,
+  WARN: 55 * 1000,
+  ERROR: 75 * 1000,
 };
 
 class WorkerQueue {
@@ -31,21 +31,21 @@ class WorkerQueue {
     this.#queue = [];
   }
 
-  addToQueue(load, cb) {
+  addToQueue(load, label, cb) {
     if (load > this.#concurrencyLimit) {
-      throw EventQueueError.loadHigherThanLimit(load);
+      throw EventQueueError.loadHigherThanLimit(load, label);
     }
 
     const startTime = process.hrtime.bigint();
     const p = new Promise((resolve, reject) => {
-      this.#queue.push([load, cb, resolve, reject, startTime]);
+      this.#queue.push([load, label, cb, resolve, reject, startTime]);
     });
     this._checkForNext();
     return p;
   }
 
-  _executeFunction(load, cb, resolve, reject, startTime) {
-    this.checkAndLogWaitingTime(startTime);
+  _executeFunction(load, label, cb, resolve, reject, startTime) {
+    this.checkAndLogWaitingTime(startTime, label);
     const promise = Promise.resolve().then(() => cb());
     this.#runningPromises.push(promise);
     this.#runningLoad = this.#runningLoad + load;
@@ -59,7 +59,7 @@ class WorkerQueue {
         resolve(...results);
       })
       .catch((err) => {
-        cds.log(COMPONENT_NAME).error("Error happened in WorkQueue. Errors should be caught before!", err);
+        cds.log(COMPONENT_NAME).error("Error happened in WorkQueue. Errors should be caught before!", err, { label });
         reject(err);
       });
   }
@@ -87,7 +87,7 @@ class WorkerQueue {
     return WorkerQueue.#instance;
   }
 
-  checkAndLogWaitingTime(startTime) {
+  checkAndLogWaitingTime(startTime, label) {
     const diffMs = Math.round(Number(process.hrtime.bigint() - startTime) / NANO_TO_MS);
     let logLevel;
     if (diffMs >= THRESHOLD.ERROR) {
@@ -101,6 +101,7 @@ class WorkerQueue {
     }
     cds.log(COMPONENT_NAME)[logLevel]("Waiting time in worker queue", {
       diffMs,
+      label,
     });
   }
 }

package/src/shared/redis.js

@@ -80,7 +80,7 @@ const subscribeRedisChannel = (channel, subscribeCb) => {
   subscriberChannelClientPromise[channel]
     .then((client) => {
       cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
-      client.subscribe(channel, subscribeCb);
+      client.subscribe(channel, subscribeCb).catch(errorHandlerCreateClient);
     })
     .catch((err) => {
       cds