@cap-js-community/event-queue 1.5.1 → 1.5.3

package/cds-plugin.js

@@ -5,6 +5,6 @@ const cds = require("@sap/cds");
 const eventQueue = require("./src");
 const COMPONENT_NAME = "/eventQueue/plugin";
 
-if (!cds.build?.register && cds.env.eventQueue) {
+if (!cds.build?.register && Object.keys(cds.env.eventQueue ?? {}).length) {
   module.exports = eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -49,13 +49,13 @@
     "yaml": "^2.4.2"
   },
   "devDependencies": {
-    "@cap-js/hana": "^0.4.0",
-    "@cap-js/sqlite": "^1.7.1",
+    "@cap-js/hana": "^1.0.0",
+    "@cap-js/sqlite": "^1.7.2",
     "@sap/cds": "^7.9.2",
     "@sap/cds-dk": "^7.8.0",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-plugin-jest": "^27.9.0",
+    "eslint-plugin-jest": "^28.6.0",
     "eslint-plugin-node": "^11.1.0",
     "express": "^4.18.2",
     "hdb": "^0.19.7",

package/src/initialize.js

@@ -131,22 +131,7 @@ const readConfigFromFile = async (configFilepath) => {
 };
 
 const registerEventProcessors = () => {
-  cds.on("listening", () => {
-    cds.connect
-      .to("cds.xt.DeploymentService")
-      .then((ds) => {
-        cds.log(COMPONENT).info("event-queue unsubscribe handler registered", {
-          redisEnabled: config.redisEnabled,
-        });
-        ds.after("unsubscribe", async (_, req) => {
-          const { tenant } = req.data;
-          config.handleUnsubscribe(tenant);
-        });
-      })
-      .catch(
-        () => {} // ignore errors as the DeploymentService is most of the time only available in the mtx sidecar
-      );
-  });
+  _registerUnsubscribe();
   config.redisEnabled && config.attachRedisUnsubscribeHandler();
 
   if (!config.registerAsEventProcessor) {
@@ -215,6 +200,25 @@ const registerCleanupForDevDb = async () => {
   }
 };
 
+const _registerUnsubscribe = () => {
+  cds.on("listening", () => {
+    cds.connect
+      .to("cds.xt.DeploymentService")
+      .then((ds) => {
+        cds.log(COMPONENT).info("event-queue unsubscribe handler registered", {
+          redisEnabled: config.redisEnabled,
+        });
+        ds.after("unsubscribe", async (_, req) => {
+          const { tenant } = req.data;
+          config.handleUnsubscribe(tenant);
+        });
+      })
+      .catch(
+        () => {} // ignore errors as the DeploymentService is most of the time only available in the mtx sidecar
+      );
+  });
+};
+
 module.exports = {
   initialize,
 };

package/src/redis/redisPub.js

@@ -9,6 +9,7 @@ const { checkLockExistsAndReturnValue } = require("../shared/distributedLock");
 const config = require("../config");
 const common = require("../shared/common");
 const { runEventCombinationForTenant } = require("../runner/runnerHelper");
+const trace = require("../shared/openTelemetry");
 
 const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
 const COMPONENT_NAME = "/eventQueue/redisPub";
@@ -19,67 +20,53 @@ const wait = promisify(setTimeout);
 
 const broadcastEvent = async (tenantId, events) => {
   const logger = cds.log(COMPONENT_NAME);
+
+  if (!config.isEventQueueActive) {
+    cds.log(COMPONENT_NAME).info("event-queue is deactivated, broadcasting is skipped!");
+    return;
+  }
+
   events = Array.isArray(events) ? events : [events];
   try {
-    if (!config.isEventQueueActive) {
-      cds.log(COMPONENT_NAME).info("Skipping processing because runner is deactivated!", {});
-      return;
-    }
     if (!config.redisEnabled) {
-      if (config.registerAsEventProcessor) {
-        let context = {};
-        if (tenantId) {
-          const user = await cds.tx({ tenant: tenantId }, async () => {
-            return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
-          });
-          context = {
-            tenant: tenantId,
-            user,
-          };
-        }
-
-        return await cds.tx(context, async ({ context }) => {
-          for (const { type, subType } of events) {
-            await runEventCombinationForTenant(context, type, subType, { shouldTrace: true });
-          }
-        });
-      }
+      await _processLocalWithoutRedis(tenantId, events);
       return;
     }
-    for (const { type, subType } of events) {
-      const eventConfig = config.getEventConfig(type, subType);
-      for (let i = 0; i < TRIES_FOR_PUBLISH_PERIODIC_EVENT; i++) {
-        const result = await checkLockExistsAndReturnValue(
-          new cds.EventContext({ tenant: tenantId }),
-          [type, subType].join("##")
-        );
-        if (result) {
-          logger.debug("skip publish redis event as no lock is available", {
-            type,
-            subType,
-            index: i,
-            isPeriodic: eventConfig.isPeriodic,
-            waitInterval: SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT,
-          });
-          if (!eventConfig.isPeriodic) {
+    await cds.tx({ tenant: tenantId }, async ({ context }) => {
+      await trace(context, "broadcast-inserted-events", async () => {
+        for (const { type, subType } of events) {
+          const eventConfig = config.getEventConfig(type, subType);
+          for (let i = 0; i < TRIES_FOR_PUBLISH_PERIODIC_EVENT; i++) {
+            const result = await checkLockExistsAndReturnValue(context, [type, subType].join("##"));
+            if (result) {
+              logger.debug("skip publish redis event as no lock is available", {
+                type,
+                subType,
+                index: i,
+                isPeriodic: eventConfig.isPeriodic,
+                waitInterval: SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT,
+              });
+              if (!eventConfig.isPeriodic) {
+                break;
+              }
+              await wait(SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT);
+              continue;
+            }
+            logger.debug("publishing redis event", {
+              tenantId,
+              type,
+              subType,
+            });
+            await redis.publishMessage(
+              config.redisOptions,
+              EVENT_MESSAGE_CHANNEL,
+              JSON.stringify({ lockId: cds.utils.uuid(), tenantId, type, subType })
+            );
             break;
           }
-          await wait(SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT);
-          continue;
         }
-        logger.debug("publishing redis event", {
-          tenantId,
-          type,
-          subType,
-        });
-        await redis.publishMessage(
-          config.redisOptions,
-          EVENT_MESSAGE_CHANNEL,
-          JSON.stringify({ lockId: cds.utils.uuid(), tenantId, type, subType })
-        );
-        break;
-      }
-    }
+      });
+    });
   } catch (err) {
     logger.error("publish events failed!", err, {
       tenantId,
@@ -87,6 +74,27 @@ const broadcastEvent = async (tenantId, events) => {
   }
 };
 
+const _processLocalWithoutRedis = async (tenantId, events) => {
+  if (config.registerAsEventProcessor) {
+    let context = {};
+    if (tenantId) {
+      const user = await cds.tx({ tenant: tenantId }, async () => {
+        return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+      });
+      context = {
+        tenant: tenantId,
+        user,
+      };
+    }
+
+    for (const { type, subType } of events) {
+      await cds.tx(context, async ({ context }) => {
+        await runEventCombinationForTenant(context, type, subType, { shouldTrace: true });
+      });
+    }
+  }
+};
+
 module.exports = {
   broadcastEvent,
 };

package/src/runner/runner.js

@@ -22,7 +22,7 @@ const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
 const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
 const EVENT_QUEUE_RUN_REDIS_CHECK = "EVENT_QUEUE_RUN_REDIS_CHECK";
 const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "EVENT_QUEUE_UPDATE_PERIODIC_EVENTS";
-const OFFSET_FIRST_RUN = 10 * 1000;
+let OFFSET_FIRST_RUN = 10 * 1000;
 
 let tenantIdHash;
 let singleRunDone;
@@ -160,7 +160,6 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
 
 const _executeEventsAllTenants = async (tenantIds, runId) => {
   const promises = [];
-
   for (const tenantId of tenantIds) {
     const id = cds.utils.uuid();
     let tenantContext;
@@ -254,32 +253,46 @@ const _executePeriodicEventsAllTenants = async (tenantIds) => {
   }
 };
 
-const _singleTenantDb = async (tenantId) => {
-  return Promise.allSettled(
-    eventQueueConfig.allEvents.map(async (eventConfig) => {
+const _singleTenantDb = async () => {
+  const id = cds.utils.uuid();
+  const events = await trace(
+    { id },
+    "fetch-openEvents-and-authInfo",
+    async () => {
+      return await cds.tx({}, async (tx) => {
+        return await openEvents.getOpenQueueEntries(tx);
+      });
+    },
+    { newRootSpan: true }
+  );
+
+  return await Promise.allSettled(
+    events.map(async (openEvent) => {
+      const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
       const label = `${eventConfig.type}_${eventConfig.subType}`;
-      const user = new cds.User.Privileged(config.userId);
-      const tenantContext = {
-        tenant: tenantId,
-        user,
-      };
       return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
-        return await cds.tx(tenantContext, async ({ context }) => {
-          try {
-            const lockId = `${EVENT_QUEUE_RUN_ID}_${label}`;
-            const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-              expiryTime: eventQueueConfig.runInterval * 0.95,
-            });
-            if (!couldAcquireLock) {
-              return;
-            }
-            await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, true);
-          } catch (err) {
-            cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-              tenantId,
-              redisEnabled: eventQueueConfig.redisEnabled,
-            });
-          }
+        return await cds.tx({}, async ({ context }) => {
+          await trace(
+            context,
+            label,
+            async () => {
+              try {
+                const lockId = `${label}`;
+                const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+                  expiryTime: eventQueueConfig.runInterval * 0.95,
+                });
+                if (!couldAcquireLock) {
+                  return;
+                }
+                await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, {
+                  skipWorkerPool: true,
+                });
+              } catch (err) {
+                cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
+              }
+            },
+            { newRootSpan: true }
+          );
         });
       });
     })
@@ -387,7 +400,8 @@ const _multiTenancyPeriodicEvents = async (tenantIds) => {
   }
 };
 
-const _checkPeriodicEventsSingleTenantOneTime = () =>
+const _checkPeriodicEventsSingleTenantOneTime = async () =>
+  eventQueueConfig.updatePeriodicEvents &&
   cds.tx({}, async (tx) => await periodicEvents.checkAndInsertPeriodicEvents(tx.context));
 
 const _checkPeriodicEventsSingleTenant = async (context) => {
@@ -425,5 +439,6 @@ module.exports = {
     _acquireRunId,
     EVENT_QUEUE_RUN_TS,
     clearHash: () => (tenantIdHash = null),
+    setOffsetFirstRun: (value) => (OFFSET_FIRST_RUN = value),
   },
 };

package/src/shared/WorkerQueue.js

@@ -113,7 +113,7 @@ class WorkerQueue {
   }
 
   #checkForNext() {
-    if (!this.#queue.length && this.#runningLoad === this.#concurrencyLimit) {
+    if (!Object.values(this.#queue).some((queue) => queue.length) || this.#runningLoad === this.#concurrencyLimit) {
       return;
     }
 

package/src/shared/eventScheduler.js

@@ -28,9 +28,11 @@ class EventScheduler {
       delaySeconds: (date.getTime() - Date.now()) / 1000,
     });
     this.#eventsByTenants[tenantId] ??= {};
-    let timeoutId;
     const timeout = setTimeout(() => {
-      delete this.#eventsByTenants[tenantId][timeoutId];
+      // NOTE: needed due to a bug in node.js which is leaking memory; will be fixed in v22.4.0
+      // https://github.com/nodejs/node/pull/53337/files
+      clearTimeout(timeout);
+      delete this.#eventsByTenants[tenantId][timeout];
       delete this.#scheduledEvents[key];
       redisPub.broadcastEvent(tenantId, { type, subType }).catch((err) => {
         cds.log(COMPONENT_NAME).error("could not execute scheduled event", err, {
@@ -41,11 +43,7 @@ class EventScheduler {
         });
       });
     }, relative).unref();
-    // Convert the timeout object to a primitive timeout id to avoid circular dependencies between the callback of setTimeout
-    // and the closure. The usage of the timeout object in the callback, leads to a deadlock for the garbage collector
-    // as the timeout object has a reference to the callback of setTimeout.
-    timeoutId = String(timeout);
-    this.#eventsByTenants[tenantId][timeoutId] = true;
+    this.#eventsByTenants[tenantId][timeout] = true;
   }
 
   clearForTenant(tenantId) {