@cap-js-community/event-queue 1.8.6 → 1.9.0-beta.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.8.6",
+  "version": "1.9.0-beta.0",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/EventQueueProcessorBase.js

@@ -11,8 +11,8 @@ const { arrayToFlatMap } = require("./shared/common");
 const eventScheduler = require("./shared/eventScheduler");
 const eventConfig = require("./config");
 const PerformanceTracer = require("./shared/PerformanceTracer");
-const { broadcastEvent } = require("./redis/redisPub");
 const trace = require("./shared/openTelemetry");
+const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
 
 const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
 const COMPONENT_NAME = "/eventQueue/EventQueueProcessorBase";
@@ -23,7 +23,6 @@ const LIMIT_PARALLEL_EVENT_PROCESSING = 10;
 const SELECT_LIMIT_EVENTS_PER_TICK = 100;
 const TRIES_FOR_EXCEEDED_EVENTS = 3;
 const EVENT_START_AFTER_HEADROOM = 3 * 1000;
-const ETAG_CHECK_AFTER_MIN = 10;
 const SUFFIX_PERIODIC = "_PERIODIC";
 const DEFAULT_RETRY_AFTER = 5 * 60 * 1000;
 
@@ -40,6 +39,9 @@ class EventQueueProcessorBase {
   #isPeriodic;
   #lastSuccessfulRunTimestamp;
   #retryFailedAfter;
+  #keepAliveRunner;
+  #currentKeepAlivePromise = Promise.resolve();
+  #etagMap;
 
   constructor(context, eventType, eventSubType, config) {
     this.__context = context;
@@ -53,6 +55,8 @@ class EventQueueProcessorBase {
     this.__eventProcessingMap = {};
     this.__statusMap = {};
     this.__commitedStatusMap = {};
+    this.__notCommitedStatusMap = {};
+    this.__outdatedEventMap = {};
     this.#eventType = eventType;
     this.#eventSubType = eventSubType;
     this.#eventConfig = config ?? {};
@@ -66,8 +70,6 @@ class EventQueueProcessorBase {
     this.__retryAttempts = this.#isPeriodic ? 1 : this.#eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
     this.__selectMaxChunkSize = this.#eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
     this.__selectNextChunk = !!this.#eventConfig.checkForNextChunk;
-    this.__keepalivePromises = {};
-    this.__outdatedCheckEnabled = this.#eventConfig.eventOutdatedCheck ?? true;
     this.__transactionMode = this.#eventConfig.transactionMode ?? TransactionMode.isolated;
     this.__emptyChunkSelected = false;
     this.__lockAcquired = false;
@@ -75,6 +77,7 @@ class EventQueueProcessorBase {
     this.__txMap = {};
     this.__txRollback = {};
     this.__queueEntries = [];
+    this.#keepAliveRunner = new SetIntervalDriftSafe(this.#eventConfig.keepAliveInterval);
   }
 
   /**
@@ -398,16 +401,17 @@ class EventQueueProcessorBase {
     this.#ensureEveryStatusIsAllowed(statusMap);
 
     const { success, failed, exceeded, invalidAttempts } = Object.entries(statusMap).reduce(
-      (result, [notificationEntityId, processingStatus]) => {
-        this.__commitedStatusMap[notificationEntityId] = processingStatus;
+      (result, [queueEntryId, processingStatus]) => {
+        this.__commitedStatusMap[queueEntryId] = processingStatus;
+        delete this.__notCommitedStatusMap[queueEntryId];
         if (processingStatus === EventProcessingStatus.Open) {
-          result.invalidAttempts.push(notificationEntityId);
+          result.invalidAttempts.push(queueEntryId);
         } else if (processingStatus === EventProcessingStatus.Done) {
-          result.success.push(notificationEntityId);
+          result.success.push(queueEntryId);
         } else if (processingStatus === EventProcessingStatus.Error) {
-          result.failed.push(notificationEntityId);
+          result.failed.push(queueEntryId);
         } else if (processingStatus === EventProcessingStatus.Exceeded) {
-          result.exceeded.push(notificationEntityId);
+          result.exceeded.push(queueEntryId);
         }
         return result;
       },
@@ -483,7 +487,11 @@ class EventQueueProcessorBase {
 
   #ensureEveryQueueEntryHasStatus() {
     this.__queueEntries.forEach((queueEntry) => {
-      if (queueEntry.ID in this.__statusMap || queueEntry.ID in this.__commitedStatusMap) {
+      if (
+        queueEntry.ID in this.__statusMap ||
+        queueEntry.ID in this.__commitedStatusMap ||
+        queueEntry.ID in this.__outdatedEventMap
+      ) {
         return;
       }
       this.logger.error("Missing status for selected event entry. Setting status to error", {
@@ -600,7 +608,7 @@ class EventQueueProcessorBase {
                   "OR lastAttemptTimestamp IS NULL ) OR ( status =",
                   EventProcessingStatus.InProgress,
                   "AND lastAttemptTimestamp <=",
-                  new Date(baseDate - this.#config.globalTxTimeout).toISOString(),
+                  new Date(baseDate - this.#eventConfig.keepAliveMaxInProgressTime).toISOString(),
                   ") )",
                 ]
               : [
@@ -611,7 +619,7 @@ class EventQueueProcessorBase {
                   ") OR ( status =",
                   EventProcessingStatus.InProgress,
                   "AND lastAttemptTimestamp <=",
-                  new Date(baseDate - this.#config.globalTxTimeout).toISOString(),
+                  new Date(baseDate - this.#eventConfig.keepAliveMaxInProgressTime).toISOString(),
                   ") )",
                 ])
           )
@@ -698,6 +706,8 @@ class EventQueueProcessorBase {
       });
       this.__queueEntries = result;
       this.__queueEntriesMap = arrayToFlatMap(result);
+      this.__notCommitedStatusMap = arrayToFlatMap(result);
+      this.#etagMap = Object.fromEntries(result.map((event) => [event.ID, event.lastAttemptTimestamp]));
 
       if (this.#isPeriodic && this.#eventConfig.lastSuccessfulRunTimestamp) {
         this.#lastSuccessfulRunTimestamp = await this.#selectLastSuccessfulPeriodicTimestamp(tx);
@@ -765,7 +775,7 @@ class EventQueueProcessorBase {
     return await trace(this.baseContext, "handle-exceeded-events", async () => {
       for (const exceededEvent of this.#eventsWithExceededTries) {
         await executeInNewTransaction(
-          this.context,
+          this.__baseContext,
           `eventQueue-handleExceededEvents-${this.#eventType}##${this.#eventSubType}`,
           async (tx) => {
             try {
@@ -792,7 +802,7 @@ class EventQueueProcessorBase {
                   currentAttempt: exceededEvent.attempts,
                 }
               );
-              await executeInNewTransaction(this.context, "error-hookForExceededEvents", async (tx) =>
+              await executeInNewTransaction(this.__baseContext, "error-hookForExceededEvents", async (tx) =>
                 this.#persistEventQueueStatusForExceeded(tx, [exceededEvent], EventProcessingStatus.Error)
               );
               await tx.rollback();
@@ -810,7 +820,7 @@ class EventQueueProcessorBase {
         eventSubType: this.#eventSubType,
         queueEntriesIds: this.#eventsWithExceededTries.map(({ ID }) => ID),
       });
-      await executeInNewTransaction(this.context, "exceededTriesExceeded", async (tx) => {
+      await executeInNewTransaction(this.__baseContext, "exceededTriesExceeded", async (tx) => {
         await this.#persistEventQueueStatusForExceeded(tx, this.#exceededTriesExceeded, EventProcessingStatus.Exceeded);
       });
     }
@@ -842,81 +852,89 @@ class EventQueueProcessorBase {
   // eslint-disable-next-line no-unused-vars
   async beforeProcessingEvents() {}
 
-  /**
-   * This function checks if the db records of events have been modified since the selection (beginning of processing)
-   * If the db records are unmodified the field lastAttemptTimestamp of the records is updated to
-   * "send a keep alive signal". This extends the allowed processing time of the events as events which are in progress
-   * for more than 30 minutes (global tx timeout) are selected with the next tick.
-   * If events are outdated/modified these events are not being processed and no status will be persisted.
-   * @return {Promise<boolean>} true if the db record of the event has been modified since selection
-   */
-  async isOutdatedAndKeepalive(queueEntries) {
-    if (!this.__outdatedCheckEnabled || new Date() - this.__startTime <= ETAG_CHECK_AFTER_MIN * 60 * 1000) {
-      return false;
-    }
-    let eventOutdated;
-    const runningChecks = queueEntries.map((queueEntry) => this.__keepalivePromises[queueEntry.ID]).filter((p) => p);
-    if (runningChecks.length === queueEntries.length) {
-      const results = await Promise.allSettled(runningChecks);
-      for (const { value } of results) {
-        if (value) {
-          return true;
-        }
-      }
-      return false;
-    } else if (runningChecks.length) {
-      await Promise.allSettled(runningChecks);
+  async isOutdatedAndKeepalive() {
+    if (this.__keepAliveViolated) {
+      return true;
     }
-    const checkAndUpdatePromise = new Promise((resolve, reject) => {
-      executeInNewTransaction(this.__baseContext, "eventProcessing-isOutdatedAndKeepalive", async (tx) => {
-        const queueEntriesFresh = await tx.run(
-          SELECT.from(this.#config.tableNameEventQueue)
-            .forUpdate({ wait: this.#config.forUpdateTimeout })
-            .where(
-              "ID IN",
-              queueEntries.map(({ ID }) => ID)
-            )
-            .columns("ID", "lastAttemptTimestamp")
-        );
-        eventOutdated = queueEntriesFresh.some((queueEntryFresh) => {
-          const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
-          return queueEntry?.lastAttemptTimestamp !== queueEntryFresh.lastAttemptTimestamp;
-        });
-        let newTs = new Date().toISOString();
-        if (!eventOutdated) {
-          await tx.run(
-            UPDATE.entity(this.#config.tableNameEventQueue)
-              .set("lastAttemptTimestamp =", newTs)
-              .where(
-                "ID IN",
-                queueEntries.map(({ ID }) => ID)
-              )
+  }
+
+  continuesKeepAlive() {
+    this.#keepAliveRunner.run(async () => {
+      await this.#currentKeepAlivePromise;
+      this.#currentKeepAlivePromise = executeInNewTransaction(this.__baseContext, "keepAlive", async (tx) => {
+        await trace(tx.context, "keepAlive", async () => {
+          const ids = Object.values(this.__notCommitedStatusMap).map(({ ID }) => ID);
+          if (!ids.length) {
+            return;
+          }
+          this.logger.info("keep alive triggered for events", { numberOfEvents: ids.length });
+          await this.#renewDistributedLock();
+
+          // we make sure to always keep alive the global event lock; but we do not modify the events itself anymore
+          if (this.__keepAliveViolated) {
+            return;
+          }
+
+          const events = await tx.run(
+            SELECT.from(this.#config.tableNameEventQueue)
+              .forUpdate({ wait: this.#config.forUpdateTimeout })
+              .where("ID IN", ids, "AND status =", EventProcessingStatus.InProgress)
+              .columns("ID", "lastAttemptTimestamp")
           );
-        } else {
-          newTs = null;
-          this.logger.warn("event data has been modified. Processing skipped.", {
-            eventType: this.#eventType,
-            eventSubType: this.#eventSubType,
-            queueEntriesIds: queueEntries.map(({ ID }) => ID),
-          });
-          queueEntries.forEach(({ ID: queueEntryId }) => delete this.__queueEntriesMap[queueEntryId]);
-        }
-        this.__queueEntries = Object.values(this.__queueEntriesMap);
-        queueEntriesFresh.forEach((queueEntryFresh) => {
-          if (this.__queueEntriesMap[queueEntryFresh.ID]) {
-            const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
-            if (newTs) {
-              queueEntry.lastAttemptTimestamp = newTs;
+
+          const newTs = new Date().toISOString();
+          const outdatedEvents = [];
+          const validEventIds = [];
+          for (const event of events) {
+            const etag = this.#etagMap[event.ID];
+            if (etag !== event.lastAttemptTimestamp) {
+              outdatedEvents.push(event);
+              this.__keepAliveViolated = true;
+            } else {
+              validEventIds.push(event.ID);
+              this.#etagMap[event.ID] = newTs;
             }
           }
-          delete this.__keepalivePromises[queueEntryFresh.ID];
+
+          if (outdatedEvents.length) {
+            // NOTE: we stop right here something is really off
+            outdatedEvents.forEach(({ ID: queueEntryId }) => {
+              delete this.__queueEntriesMap[queueEntryId];
+              this.__outdatedEventMap[queueEntryId] = 1;
+            });
+
+            this.logger.warn(
+              "Event data has been modified on the database. Further processing skipped. Parallel running events might have already commited status!",
+              {
+                eventType: this.#eventType,
+                eventSubType: this.#eventSubType,
+                queueEntriesIds: outdatedEvents.map(({ ID }) => ID),
+              }
+            );
+          }
+
+          if (validEventIds.length) {
+            await tx.run(
+              UPDATE.entity(this.#config.tableNameEventQueue)
+                .set("lastAttemptTimestamp =", newTs)
+                .where("ID IN", validEventIds)
+            );
+            // NOTE: update internal map after tx is successfully commited!
+            tx.context.on("succeeded", () => {
+              for (const event of events) {
+                const etag = this.#etagMap[event.ID];
+                if (etag === event.lastAttemptTimestamp) {
+                  this.#etagMap[event.ID] = newTs;
+                }
+              }
+            });
+          }
         });
-        resolve(eventOutdated);
-      }).catch(reject);
+      }).catch((err) => {
+        this.logger.error("keep alive handling failed!", err);
+      });
+      await this.#currentKeepAlivePromise;
     });
-
-    queueEntries.forEach((queueEntry) => (this.__keepalivePromises[queueEntry.ID] = checkAndUpdatePromise));
-    return await checkAndUpdatePromise;
   }
 
   async acquireDistributedLock() {
@@ -926,9 +944,9 @@ class EventQueueProcessorBase {
 
     return await trace(this.baseContext, "acquire-lock", async () => {
       const lockAcquired = await distributedLock.acquireLock(
-        this.context,
+        this.__context,
         [this.#eventType, this.#eventSubType].join("##"),
-        { keepTrackOfLock: true }
+        { keepTrackOfLock: true, expiryTime: this.#eventConfig.keepAliveMaxInProgressTime }
       );
       if (!lockAcquired) {
         this.logger.debug("no lock available, exit processing", {
@@ -942,6 +960,26 @@ class EventQueueProcessorBase {
     });
   }
 
+  async #renewDistributedLock() {
+    if (this.concurrentEventProcessing) {
+      return true;
+    }
+
+    const lockAcquired = await distributedLock.renewLock(
+      this.__context,
+      [this.#eventType, this.#eventSubType].join("##"),
+      { expiryTime: this.#eventConfig.keepAliveMaxInProgressTime }
+    );
+    if (!lockAcquired) {
+      this.logger.error("renewing redis lock failed!", {
+        type: this.#eventType,
+        subType: this.#eventSubType,
+      });
+      return false;
+    }
+    return true;
+  }
+
   async handleReleaseLock() {
     if (!this.__lockAcquired) {
       return;
@@ -1085,16 +1123,12 @@ class EventQueueProcessorBase {
     this.__processTx = null;
   }
 
-  broadCastEvent() {
-    setTimeout(() => {
-      broadcastEvent(this.__baseContext.tenant, { type: this.#eventType, subType: this.#eventSubType }).catch((err) => {
-        this.logger.error("could not execute scheduled event", err, {
-          tenantId: this.__baseContext.tenant,
-          type: this.#eventType,
-          subType: this.#eventSubType,
-        });
-      });
-    }, 1000).unref();
+  stopKeepAlive() {
+    this.#keepAliveRunner.stop();
+  }
+
+  get keepAlivePromise() {
+    return this.#currentKeepAlivePromise;
   }
 
   get logger() {

package/src/config.js

@@ -17,6 +17,9 @@ const COMPONENT_NAME = "/eventQueue/config";
 const MIN_INTERVAL_SEC = 10;
 const DEFAULT_LOAD = 1;
 const DEFAULT_PRIORITY = Priorities.Medium;
+const DEFAULT_INCREASE_PRIORITY = true;
+const DEFAULT_KEEP_ALIVE_INTERVAL_MIN = 1;
+const DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL = 3.5;
 const SUFFIX_PERIODIC = "_PERIODIC";
 const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
 const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
@@ -301,14 +304,13 @@ class Config {
     const eventConfig = {
       type: CAP_EVENT_TYPE,
       subType: serviceName,
-      load: config.load ?? DEFAULT_LOAD,
+      load: config.load,
       impl: "./outbox/EventQueueGenericOutboxHandler",
       selectMaxChunkSize: config.chunkSize,
       parallelEventProcessing: config.parallelEventProcessing ?? (config.parallel && CAP_PARALLEL_DEFAULT),
       retryAttempts: config.maxAttempts,
       transactionMode: config.transactionMode,
       processAfterCommit: config.processAfterCommit,
-      eventOutdatedCheck: config.eventOutdatedCheck,
       checkForNextChunk: config.checkForNextChunk,
       deleteFinishedEventsAfterDays: config.deleteFinishedEventsAfterDays,
       appNames: config.appNames,
@@ -317,9 +319,12 @@ class Config {
       retryFailedAfter: config.retryFailedAfter,
       priority: config.priority,
       multiInstanceProcessing: config.multiInstanceProcessing,
+      increasePriorityOverTime: config.increasePriorityOverTime,
+      keepAliveInterval: config.keepAliveInterval,
       internalEvent: true,
     };
 
+    this.#basicEventTransformation(eventConfig);
     this.#basicEventTransformationAfterValidate(eventConfig);
     this.#config.events.push(eventConfig);
     this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
@@ -355,7 +360,10 @@ class Config {
   set fileContent(config) {
     this.#config = config;
     config.events = config.events ?? [];
-    config.periodicEvents = (config.periodicEvents ?? []).concat(BASE_PERIODIC_EVENTS.map((event) => ({ ...event })));
+    const shouldIncludeBaseEvents = cds.env.profiles.includes("production") || cds.env.profiles.includes("test");
+    config.periodicEvents = (config.periodicEvents ?? []).concat(
+      (shouldIncludeBaseEvents ? BASE_PERIODIC_EVENTS : []).map((event) => ({ ...event }))
+    );
     this.#eventMap = config.events.reduce((result, event) => {
       this.#basicEventTransformation(event);
       this.#validateAdHocEvents(result, event);
@@ -364,7 +372,6 @@ class Config {
       return result;
     }, {});
     this.#eventMap = config.periodicEvents.reduce((result, event) => {
-      event.priority = event.priority ?? DEFAULT_PRIORITY;
       event.type = `${event.type}${SUFFIX_PERIODIC}`;
       event.isPeriodic = true;
       this.#basicEventTransformation(event);
@@ -378,6 +385,9 @@ class Config {
   #basicEventTransformation(event) {
     event.load = event.load ?? DEFAULT_LOAD;
     event.priority = event.priority ?? DEFAULT_PRIORITY;
+    event.increasePriorityOverTime = event.increasePriorityOverTime ?? DEFAULT_INCREASE_PRIORITY;
+    event.keepAliveInterval = (event.keepAliveInterval ?? DEFAULT_KEEP_ALIVE_INTERVAL_MIN) * 60 * 1000;
+    event.keepAliveMaxInProgressTime = event.keepAliveInterval * DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL;
   }
 
   #basicEventTransformationAfterValidate(event) {

package/src/index.d.ts

@@ -78,7 +78,6 @@ export type EventConfig = {
   retryAttempts: number | undefined;
   transactionMode: string | undefined;
   processAfterCommit: boolean | undefined;
-  eventOutdatedCheck: boolean | undefined;
   checkForNextChunk: boolean | undefined;
   deleteFinishedEventsAfterDays: number | undefined;
   appNames: string[] | undefined;
@@ -269,7 +268,13 @@ export const workerQueue: WorkerQueue;
 declare class WorkerQueue {
   constructor(concurrency: number);
 
-  addToQueue(load: number, label: string, priority?: Priorities, cb?: () => any): Promise<any>;
+  addToQueue(
+    load: number,
+    label: string,
+    priority: Priorities,
+    increasePriorityOverTime: boolean,
+    cb: () => any
+  ): Promise<any>;
 
   _executeFunction(
     load: number,

package/src/processEventQueue.js

@@ -208,6 +208,7 @@ const processEventMap = async (instance) => {
   if (instance.commitOnEventLevel) {
     instance.txUsageAllowed = false;
   }
+  instance.continuesKeepAlive();
   await limiter(
     instance.parallelEventProcessing,
     Object.entries(instance.eventProcessingMap),
@@ -241,10 +242,12 @@ const processEventMap = async (instance) => {
       instance.handleErrorTx(err);
     })
     .finally(() => {
+      instance.stopKeepAlive();
       instance.clearEventProcessingContext();
       if (instance.commitOnEventLevel) {
         instance.txUsageAllowed = true;
       }
+      return instance.keepAlivePromise;
     });
   instance.endPerformanceTracerEvents();
 };
@@ -313,7 +316,8 @@ const _processEvent = async (eventTypeInstance, processContext, key, queueEntrie
   try {
     const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(queueEntries);
     if (eventOutdated) {
-      return;
+      // NOTE: return empty status map to comply with the interface
+      return {};
     }
     eventTypeInstance.setTxForEventProcessing(key, cds.tx(processContext));
     const statusTuple = await eventTypeInstance.processEvent(processContext, key, queueEntries, payload);

package/src/runner/runner.js

@@ -194,33 +194,39 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
       events.map(async (openEvent) => {
         const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
         const label = `${eventConfig.type}_${eventConfig.subType}`;
-        return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
-          return await cds.tx(tenantContext, async ({ context }) => {
-            await trace(
-              context,
-              label,
-              async () => {
-                try {
-                  const lockId = `${runId}_${label}`;
-                  const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-                    expiryTime: eventQueueConfig.runInterval * 0.95,
-                  });
-                  if (!couldAcquireLock) {
-                    return;
+        return await WorkerQueue.instance.addToQueue(
+          eventConfig.load,
+          label,
+          eventConfig.priority,
+          eventConfig.increasePriorityOverTime,
+          async () => {
+            return await cds.tx(tenantContext, async ({ context }) => {
+              await trace(
+                context,
+                label,
+                async () => {
+                  try {
+                    const lockId = `${runId}_${label}`;
+                    const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+                      expiryTime: eventQueueConfig.runInterval * 0.95,
+                    });
+                    if (!couldAcquireLock) {
+                      return;
+                    }
+                    await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, {
+                      skipWorkerPool: true,
+                    });
+                  } catch (err) {
+                    cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
+                      tenantId,
+                    });
                   }
-                  await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, {
-                    skipWorkerPool: true,
-                  });
-                } catch (err) {
-                  cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-                    tenantId,
-                  });
-                }
-              },
-              { newRootSpan: true }
-            );
-          });
-        });
+                },
+                { newRootSpan: true }
+              );
+            });
+          }
+        );
       })
     );
   }
@@ -275,33 +281,39 @@ const _singleTenantDb = async () => {
     events.map(async (openEvent) => {
       const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
       const label = `${eventConfig.type}_${eventConfig.subType}`;
-      return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
-        return await cds.tx({}, async ({ context }) => {
-          await trace(
-            context,
-            label,
-            async () => {
-              try {
-                const lockId = `${label}`;
-                const couldAcquireLock = eventConfig.multiInstanceProcessing
-                  ? true
-                  : await distributedLock.acquireLock(context, lockId, {
-                      expiryTime: eventQueueConfig.runInterval * 0.95,
-                    });
-                if (!couldAcquireLock) {
-                  return;
+      return await WorkerQueue.instance.addToQueue(
+        eventConfig.load,
+        label,
+        eventConfig.priority,
+        eventConfig.increasePriorityOverTime,
+        async () => {
+          return await cds.tx({}, async ({ context }) => {
+            await trace(
+              context,
+              label,
+              async () => {
+                try {
+                  const lockId = `${label}`;
+                  const couldAcquireLock = eventConfig.multiInstanceProcessing
+                    ? true
+                    : await distributedLock.acquireLock(context, lockId, {
+                        expiryTime: eventQueueConfig.runInterval * 0.95,
+                      });
+                  if (!couldAcquireLock) {
+                    return;
+                  }
+                  await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, {
+                    skipWorkerPool: true,
+                  });
+                } catch (err) {
+                  cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
                 }
-                await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, {
-                  skipWorkerPool: true,
-                });
-              } catch (err) {
-                cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
-              }
-            },
-            { newRootSpan: true }
-          );
-        });
-      });
+              },
+              { newRootSpan: true }
+            );
+          });
+        }
+      );
     })
   );
 };

package/src/runner/runnerHelper.js

@@ -23,6 +23,7 @@ const runEventCombinationForTenant = async (context, type, subType, { skipWorker
         eventConfig.load,
         label,
         eventConfig.priority,
+        eventConfig.increasePriorityOverTime,
         AsyncResource.bind(async () => {
           const _exec = async () => {
             if (!eventConfig.multiInstanceProcessing && lockId) {

package/src/shared/SetIntervalDriftSafe.js

@@ -8,6 +8,8 @@ class SetIntervalDriftSafe {
   #expectedCycleTime = 0;
   #nextTickScheduledFor;
   #logger;
+  #shouldRun = true;
+  #lastTimeoutId;
 
   constructor(interval) {
     this.#interval = interval;
@@ -16,6 +18,10 @@ class SetIntervalDriftSafe {
   }
 
   run(fn) {
+    if (!this.#shouldRun) {
+      return;
+    }
+
     const now = Date.now();
     if (this.#expectedCycleTime === 0) {
       this.#expectedCycleTime = now + this.#interval;
@@ -24,10 +30,19 @@ class SetIntervalDriftSafe {
       this.#expectedCycleTime += this.#interval;
     }
     this.#nextTickScheduledFor = now + this.#adjustedInterval;
-    setTimeout(() => {
+    const timeoutId = setTimeout(() => {
+      if (!this.#shouldRun) {
+        return;
+      }
       this.run(fn);
       fn();
     }, this.#adjustedInterval).unref();
+    this.#lastTimeoutId = Number(timeoutId);
+  }
+
+  stop() {
+    this.#shouldRun = false;
+    clearTimeout(this.#lastTimeoutId);
   }
 }
 

package/src/shared/WorkerQueue.js

@@ -52,7 +52,7 @@ class WorkerQueue {
     runner.run(this.#adjustPriority.bind(this));
   }
 
-  addToQueue(load, label, priority = Priorities.Medium, cb) {
+  addToQueue(load, label, priority = Priorities.Medium, increasePriorityOverTime, cb) {
     if (load > this.#concurrencyLimit) {
       throw EventQueueError.loadHigherThanLimit(load, label);
     }
@@ -63,7 +63,7 @@ class WorkerQueue {
 
     const startTime = process.hrtime.bigint();
     const p = new Promise((resolve, reject) => {
-      this.#queue[priority].push([load, label, cb, resolve, reject, startTime]);
+      this.#queue[priority].push([load, label, cb, resolve, reject, increasePriorityOverTime, startTime]);
     });
     this.#checkForNext();
     return p;
@@ -78,7 +78,12 @@ class WorkerQueue {
       const nextPriority = priorityValues[i + 1];
       for (let i = 0; i < this.queue[priority].length; i++) {
         const queueEntry = this.queue[priority][i];
-        const startTime = queueEntry[6] ?? queueEntry[5];
+        // NOTE: index 5 - increasingPrioOverTime
+        if (!queueEntry[5]) {
+          continue;
+        }
+        // NOTE: index 6 original time; index 7 shifted time
+        const startTime = queueEntry[7] ?? queueEntry[6];
         if (Math.round(Number(checkTime - startTime) / NANO_TO_MS) > INCREASE_PRIORITY_AFTER * MIN_TO_MS) {
           const [entry] = this.queue[priority].splice(i, 1);
           entry.push(checkTime);
@@ -88,7 +93,7 @@ class WorkerQueue {
     }
   }
 
-  _executeFunction(load, label, cb, resolve, reject, startTime, priority) {
+  _executeFunction(priority, load, label, cb, resolve, reject, skipIncreasingPrioOverTime, startTime) {
     this.#checkAndLogWaitingTime(startTime, label, priority);
     const promise = Promise.resolve().then(() => cb());
     this.#runningPromises.push(promise);
@@ -123,7 +128,7 @@ class WorkerQueue {
         const [load] = this.#queue[priority][i];
         if (this.#runningLoad + load <= this.#concurrencyLimit) {
           const [args] = this.#queue[priority].splice(i, 1);
-          this._executeFunction(...args, priority);
+          this._executeFunction(priority, ...args);
           entryFound = true;
           break;
         }

package/src/shared/distributedLock.js

@@ -22,6 +22,15 @@ const acquireLock = async (
   }
 };
 
+const renewLock = async (context, key, { tenantScoped = true, expiryTime = config.globalTxTimeout } = {}) => {
+  const fullKey = _generateKey(context, tenantScoped, key);
+  if (config.redisEnabled) {
+    return await _renewLockRedis(context, fullKey, expiryTime);
+  } else {
+    return await _acquireLockDB(context, fullKey, expiryTime, { overrideValue: true });
+  }
+};
+
 const setValueWithExpire = async (
   context,
   key,
@@ -79,6 +88,15 @@ const _acquireLockRedis = async (
   return isOk;
 };
 
+const _renewLockRedis = async (context, fullKey, expiryTime, { value = "true" } = {}) => {
+  const client = await redis.createMainClientAndConnect(config.redisOptions);
+  const result = await client.set(fullKey, value, {
+    PX: Math.round(expiryTime),
+    XX: true,
+  });
+  return result === REDIS_COMMAND_OK;
+};
+
 const _checkLockExistsRedis = async (context, fullKey) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
   return await client.get(fullKey);
@@ -136,7 +154,7 @@ const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", ov
               createdAt: new Date().toISOString(),
               value,
             })
-            .where("code =", currentEntry.code)
+            .where("code =", fullKey)
         );
         result = true;
       } else {
@@ -178,4 +196,5 @@ module.exports = {
   checkLockExistsAndReturnValue,
   setValueWithExpire,
   shutdownHandler,
+  renewLock,
 };