@cap-js-community/event-queue 2.1.0-beta.2 → 2.1.0

package/db/Event.cds

@@ -16,7 +16,7 @@ entity Event: cuid {
     type: String not null;
     subType: String not null;
     referenceEntity: String;
-    referenceEntityKey: UUID;
+    referenceEntityKey: String;
     status: Status default 0 not null;
     payload: LargeString;
     attempts: Integer default 0 not null;
@@ -24,6 +24,6 @@ entity Event: cuid {
     createdAt: Timestamp @cds.on.insert : $now;
     startAfter: Timestamp;
     context: LargeString;
-    error: String;
+    error: LargeString;
     namespace: String default 'default';
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "2.1.0-beta.2",
+  "version": "2.1.0",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -44,11 +44,11 @@
     "node": ">=20"
   },
   "dependencies": {
-    "@cap-js-community/common": "^0.3.4",
-    "@sap/xssec": "^4.12.2",
+    "@cap-js-community/common": "^0.4.0",
+    "@sap/xssec": "^4.13.0",
     "cron-parser": "^5.5.0",
     "verror": "^1.10.1",
-    "yaml": "^2.8.2"
+    "yaml": "^2.8.3"
   },
   "devDependencies": {
     "@actions/core": "^2.0.2",
@@ -56,9 +56,9 @@
     "@cap-js/db-service": "^2.8.2",
     "@cap-js/hana": "^2.6.0",
     "@cap-js/sqlite": "^2.1.3",
-    "@opentelemetry/api": "^1.9.0",
-    "@sap/cds": "^9.7.0",
-    "@sap/cds-dk": "^9.7.0",
+    "@opentelemetry/api": "^1.9.1",
+    "@sap/cds": "^9.8.4",
+    "@sap/cds-dk": "^9.8.2",
     "eslint": "^8.57.1",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-jest": "^29.12.1",
@@ -100,8 +100,7 @@
     "requires": {
       "xsuaa-eventQueue": {
         "vcap": {
-          "label": "xsuaa",
-          "plan": "application"
+          "label": "xsuaa"
         }
       },
       "redis-eventQueue": {

package/src/config.js

@@ -30,6 +30,7 @@ const UTC_DEFAULT = false;
 const USE_CRON_TZ_DEFAULT = true;
 const SAGA_SUCCESS = "#succeeded";
 const SAGA_FAILED = "#failed";
+const SAGA_DONE = "#done";
 
 const BASE_TABLES = {
   EVENT: "sap.eventqueue.Event",
@@ -107,6 +108,7 @@ class Config {
   #redisOptions;
   #insertEventsBeforeCommit;
   #enableTelemetry;
+  #collectEventQueueMetrics;
   #unsubscribeHandlers = [];
   #unsubscribedTenants = {};
   #cronTimezone;
@@ -386,19 +388,20 @@ class Config {
               result.adHoc
             );
             result.adHoc[key] = specificEventConfig;
-            const sagaSuccessKey = [fnName, SAGA_SUCCESS].join("/");
-            if (config.events[sagaSuccessKey]) {
-              const [sagaKey, sagaSpecificEventConfig] = this.addCAPOutboxEventSpecificAction(
-                srvConfig,
-                name,
-                fnName,
-                result.adHoc
-              );
-              result.adHoc[sagaKey] = sagaSpecificEventConfig;
-            } else {
-              const sagaConfig = { ...specificEventConfig };
-              sagaConfig.subType = [sagaConfig.subType, SAGA_SUCCESS].join("/");
-              result.adHoc[[key, SAGA_SUCCESS].join("/")] = sagaConfig;
+            for (const sagaSuffix of [SAGA_SUCCESS, SAGA_DONE, SAGA_FAILED]) {
+              if (config.events[sagaSuffix]) {
+                const [adHocKey, sagaSpecificEventConfig] = this.addCAPOutboxEventSpecificAction(
+                  srvConfig,
+                  name,
+                  fnName,
+                  result.adHoc
+                );
+                result.adHoc[adHocKey] = sagaSpecificEventConfig;
+              } else {
+                const sagaConfig = { ...specificEventConfig };
+                sagaConfig.subType = [sagaConfig.subType, sagaSuffix].join("/");
+                result.adHoc[[key, sagaSuffix].join("/")] = sagaConfig;
+              }
             }
           }
         }
@@ -433,7 +436,7 @@ class Config {
     }
 
     const [withoutSaga, sagaSuffix] = action.split("/");
-    if ([SAGA_FAILED, SAGA_SUCCESS].includes(sagaSuffix)) {
+    if ([SAGA_FAILED, SAGA_SUCCESS, SAGA_DONE].includes(sagaSuffix)) {
       if (config?.events?.[withoutSaga]) {
         return this.#mixCAPPropertyNamesWithEventQueueNames(config.events[withoutSaga]);
       }
@@ -901,6 +904,14 @@ class Config {
     return this.#enableTelemetry;
   }
 
+  set collectEventQueueMetrics(value) {
+    this.#collectEventQueueMetrics = value;
+  }
+
+  get collectEventQueueMetrics() {
+    return this.#collectEventQueueMetrics;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }

package/src/dbHandler.js

@@ -4,11 +4,14 @@ const cds = require("@sap/cds");
 
 const redisPub = require("./redis/redisPub");
 const config = require("./config");
+const eventQueueStats = require("./shared/eventQueueStats");
+const { EventProcessingStatus } = require("./constants");
 
 const COMPONENT_NAME = "/eventQueue/dbHandler";
 const registeredHandlers = {
   eventQueueDbHandler: false,
   beforeDbHandler: false,
+  updateDbHandler: false,
 };
 
 const registerEventQueueDbHandler = (dbService) => {
@@ -26,36 +29,114 @@ const registerEventQueueDbHandler = (dbService) => {
     req.tx._.eventQueuePublishEvents = req.tx._.eventQueuePublishEvents ?? {};
     const eventQueuePublishEvents = req.tx._.eventQueuePublishEvents;
     const data = Array.isArray(req.query.INSERT.entries) ? req.query.INSERT.entries : [req.query.INSERT.entries];
-    const eventCombinations = Object.keys(
-      data.reduce((result, event) => {
-        const key = [event.type, event.subType, event.namespace].join("##");
-        if (
-          !config.hasEventAfterCommitFlag(event.type, event.subType, event.namespace) ||
-          eventQueuePublishEvents[key]
-        ) {
-          return result;
-        }
+
+    req.tx._.eventQueueStatsOpenCount = (req.tx._.eventQueueStatsOpenCount ?? 0) + data.length;
+    const newCombinations = data.reduce((result, event) => {
+      const key = [event.type, event.subType, event.namespace].join("##");
+      if (config.hasEventAfterCommitFlag(event.type, event.subType, event.namespace) && !eventQueuePublishEvents[key]) {
         eventQueuePublishEvents[key] = true;
-        result[key] = true;
-        return result;
-      }, {})
-    );
+        result.push(key);
+      }
+      return result;
+    }, []);
 
-    eventCombinations.length &&
+    req.tx._.eventQueueBroadcastCombinations ??= [];
+    req.tx._.eventQueueBroadcastCombinations.push(...newCombinations);
+    if (!req.tx._.eventQueueSucceededHandlerRegistered) {
+      req.tx._.eventQueueSucceededHandlerRegistered = true;
       req.on("succeeded", () => {
-        const events = eventCombinations.map((eventCombination) => {
-          const [type, subType, namespace] = eventCombination.split("##");
-          return { type, subType, namespace };
-        });
-
-        redisPub.broadcastEvent(req.tenant, events).catch((err) => {
-          cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
-            tenant: req.tenant,
-            events,
+        if (config.collectEventQueueMetrics && config.redisEnabled && req.tx._.eventQueueStatsOpenCount) {
+          eventQueueStats
+            .incrementCounters(req.tenant, eventQueueStats.StatusField.Pending, req.tx._.eventQueueStatsOpenCount)
+            .catch((err) => {
+              cds.log(COMPONENT_NAME).error("db handler failure during updating event stats", err, {
+                tenant: req.tenant,
+              });
+            });
+        }
+        const combinations = req.tx._.eventQueueBroadcastCombinations;
+        if (combinations.length) {
+          const events = combinations.map((combination) => {
+            const [type, subType, namespace] = combination.split("##");
+            return { type, subType, namespace };
           });
-        });
+          redisPub.broadcastEvent(req.tenant, events).catch((err) => {
+            cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
+              tenant: req.tenant,
+              events,
+            });
+          });
+        }
       });
+    }
   });
+
+  if (!registeredHandlers.updateDbHandler) {
+    if (!config.collectEventQueueMetrics || !config.redisEnabled) {
+      return;
+    }
+    registeredHandlers.updateDbHandler = true;
+    dbService.after("UPDATE", def, (count, req) => {
+      const newStatus = req.query.UPDATE?.data?.status;
+      if (newStatus == null) {
+        return;
+      }
+
+      req.tx._ = req.tx._ ?? {};
+      req.tx._.eventQueueStatsPendingDelta = req.tx._.eventQueueStatsPendingDelta ?? 0;
+      req.tx._.eventQueueStatsInProgressDelta = req.tx._.eventQueueStatsInProgressDelta ?? 0;
+
+      if (newStatus === EventProcessingStatus.InProgress) {
+        req.tx._.eventQueueStatsPendingDelta -= count;
+        req.tx._.eventQueueStatsInProgressDelta += count;
+      } else if (newStatus === EventProcessingStatus.Error) {
+        req.tx._.eventQueueStatsInProgressDelta -= count;
+        req.tx._.eventQueueStatsPendingDelta += count;
+      } else if (
+        newStatus === EventProcessingStatus.Done ||
+        newStatus === EventProcessingStatus.Exceeded ||
+        newStatus === EventProcessingStatus.Suspended
+      ) {
+        req.tx._.eventQueueStatsInProgressDelta -= count;
+      }
+
+      if (!req.tx._.eventQueueUpdateSucceededHandlerRegistered) {
+        req.tx._.eventQueueUpdateSucceededHandlerRegistered = true;
+        req.on("succeeded", () => {
+          if (!config.redisEnabled) {
+            return;
+          }
+          const pendingDelta = req.tx._.eventQueueStatsPendingDelta;
+          const inProgressDelta = req.tx._.eventQueueStatsInProgressDelta;
+          const ops = [];
+
+          if (pendingDelta !== 0) {
+            ops.push(
+              eventQueueStats.adjustTenantCounter(req.tenant, eventQueueStats.StatusField.Pending, pendingDelta),
+              eventQueueStats.adjustGlobalCounter(eventQueueStats.StatusField.Pending, pendingDelta)
+            );
+          }
+          if (inProgressDelta !== 0) {
+            ops.push(
+              eventQueueStats.adjustTenantCounter(req.tenant, eventQueueStats.StatusField.InProgress, inProgressDelta),
+              eventQueueStats.adjustGlobalCounter(eventQueueStats.StatusField.InProgress, inProgressDelta)
+            );
+          }
+          Promise.allSettled(ops).then((results) => {
+            for (const result of results) {
+              if (result.status === "rejected") {
+                cds
+                  .log(COMPONENT_NAME)
+                  .error("db handler failure during updating event stats on update", result.reason, {
+                    tenant: req.tenant,
+                  });
+              }
+            }
+          });
+        });
+      }
+    });
+  }
 };
 
 module.exports = {

package/src/initialize.js

@@ -17,6 +17,7 @@ const { getAllTenantIds } = require("./shared/cdsHelper");
 const { EventProcessingStatus } = require("./constants");
 const distributedLock = require("./shared/distributedLock");
 const EventQueueError = require("./EventQueueError");
+const { initMetrics } = require("./shared/openTelemetry");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -49,6 +50,7 @@ const CONFIG_VARS = [
   ["disableProcessingOfSuspendedTenants", true],
   ["namespace", "default"],
   ["processingNamespaces", ["default"]],
+  ["collectEventQueueMetrics", false],
 ];
 
 /**
@@ -78,6 +80,7 @@ const CONFIG_VARS = [
  * @param {string} [options.crashOnRedisUnavailable=true] - If enabled an error is thrown if the redis connection check is not successful
  * @param {string} [options.namespace=default] - Default namespace in which events are published
  * @param {string} [options.processingNamespaces=[default]] - Namespaces which the application processes
+ * @param {boolean} [options.collectEventQueueMetrics=false] - Enable collection of event queue metrics (pending/inProgress counters) stored in Redis and exposed via OpenTelemetry gauges.
  */
 const initialize = async (options = {}) => {
   if (config.initialized) {
@@ -125,6 +128,7 @@ const initialize = async (options = {}) => {
     runInterval: config.runInterval,
     useAsCAPQueue: config.useAsCAPQueue,
   });
+  initMetrics();
   resolveFn();
 };
 

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -16,8 +16,18 @@ const EVENT_QUEUE_ACTIONS = {
   CHECK_AND_ADJUST: "eventQueueCheckAndAdjustPayload",
   SAGA_SUCCESS: "#succeeded",
   SAGA_FAILED: "#failed",
+  SAGA_DONE: "#done",
 };
 
+const PROPAGATE_EVENT_QUEUE_ENTRIES = [
+  "ID",
+  "lastAttempTimestamp",
+  "payload",
+  "referenceEntity",
+  "referenceEntityKey",
+  "status",
+];
+
 class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   constructor(context, eventType, eventSubType, config) {
     super(context, eventType, eventSubType, config);
@@ -312,7 +322,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       return genericHandler ?? null;
     }
 
-    if (event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_SUCCESS)) {
+    if (event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_SUCCESS) || event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_DONE)) {
       [event] = event.split("/");
     }
 
@@ -334,11 +344,23 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   #buildDispatchData(payload, { key, queueEntries } = {}) {
     const { useEventQueueUser } = this.eventConfig;
     const userId = useEventQueueUser ? config.userId : payload.contextUser;
+    let triggerEvent;
+
+    if (payload.data?.triggerEvent) {
+      try {
+        triggerEvent = JSON.parse(payload.data.triggerEvent);
+      } catch (err) {
+        this.logger.error("[saga] error parsing triggering event data", err);
+      } finally {
+        delete payload.data.triggerEvent;
+      }
+    }
+
     const req = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
     const invocationFn = payload._fromSend ? "send" : "emit";
     delete req._fromSend;
     delete req.contextUser;
-    req.eventQueue = { processor: this, key, queueEntries, payload };
+    req.eventQueue = { processor: this, key, queueEntries, payload, triggerEvent };
 
     if (this.eventConfig.propagateContextProperties?.length && this.transactionMode === "isolated" && cds.context) {
       for (const prop of this.eventConfig.propagateContextProperties) {
@@ -362,11 +384,11 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   }
 
   async processEvent(processContext, key, queueEntries, payload) {
-    let statusTuple;
+    let statusTuple, result;
     const { userId, invocationFn, req } = this.#buildDispatchData(payload, { key, queueEntries });
     try {
       await this.#setContextUser(processContext, userId, req);
-      const result = await this.__srvUnboxed.tx(processContext)[invocationFn](req);
+      result = await this.__srvUnboxed.tx(processContext)[invocationFn](req);
       statusTuple = this.#determineResultStatus(result, queueEntries);
     } catch (err) {
       this.logger.error("error processing outboxed service call", err, {
@@ -381,19 +403,20 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       ]);
     }
 
-    await this.#publishFollowupEvents(processContext, req, statusTuple);
+    await this.#publishFollowupEvents(processContext, req, statusTuple, result);
     return statusTuple;
   }
 
-  async #publishFollowupEvents(processContext, req, statusTuple) {
+  async #publishFollowupEvents(processContext, req, statusTuple, triggerEventResult) {
     const succeeded = this.#checkHandlerExists({ event: req.event, saga: EVENT_QUEUE_ACTIONS.SAGA_SUCCESS });
     const failed = this.#checkHandlerExists({ event: req.event, saga: EVENT_QUEUE_ACTIONS.SAGA_FAILED });
+    const done = this.#checkHandlerExists({ event: req.event, saga: EVENT_QUEUE_ACTIONS.SAGA_DONE });
 
-    if (!succeeded && !failed) {
+    if (!succeeded && !failed && !done) {
       return;
     }
 
-    if (req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_FAILED)) {
+    if (req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_FAILED) || req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_DONE)) {
       return;
     }
 
@@ -405,6 +428,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       tx._eventQueue.events = [];
     }
 
+    const queued = cds.queued(this.__srv);
     for (const [, result] of statusTuple) {
       const data = result.nextData ?? req.data;
       if (
@@ -412,12 +436,41 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
         result.status === EventProcessingStatus.Done &&
         !req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_SUCCESS)
       ) {
-        await this.__srv.tx(processContext).send(succeeded, data);
+        if (statusTuple.length === 1 && req.eventQueue.queueEntries.length === 1) {
+          const triggerEventPropagate = { triggerEventResult };
+          const [triggerEvent] = req.eventQueue.queueEntries;
+          for (const propertyName of PROPAGATE_EVENT_QUEUE_ENTRIES) {
+            triggerEventPropagate[propertyName] = triggerEvent[propertyName];
+          }
+          data.triggerEvent = JSON.stringify(triggerEventPropagate);
+        }
+
+        await queued.tx(processContext).send(succeeded, data);
       }
 
       if (failed && result.status === EventProcessingStatus.Error) {
         result.error && (data.error = this._error2String(result.error));
-        await this.__srv.tx(processContext).send(failed, data);
+        if (statusTuple.length === 1 && req.eventQueue.queueEntries.length === 1) {
+          const triggerEventPropagate = { triggerEventResult };
+          const [triggerEvent] = req.eventQueue.queueEntries;
+          for (const propertyName of PROPAGATE_EVENT_QUEUE_ENTRIES) {
+            triggerEventPropagate[propertyName] = triggerEvent[propertyName];
+          }
+          data.triggerEvent = JSON.stringify(triggerEventPropagate);
+        }
+        await queued.tx(processContext).send(failed, data);
+      }
+
+      if (done && !req.event.endsWith(EVENT_QUEUE_ACTIONS.SAGA_SUCCESS)) {
+        if (statusTuple.length === 1 && req.eventQueue.queueEntries.length === 1) {
+          const triggerEventPropagate = { triggerEventResult };
+          const [triggerEvent] = req.eventQueue.queueEntries;
+          for (const propertyName of PROPAGATE_EVENT_QUEUE_ENTRIES) {
+            triggerEventPropagate[propertyName] = triggerEvent[propertyName];
+          }
+          data.triggerEvent = JSON.stringify(triggerEventPropagate);
+        }
+        await queued.tx(processContext).send(done, data);
       }
 
       delete result.nextData;
@@ -426,7 +479,11 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
     if (config.insertEventsBeforeCommit) {
       this.nextSagaEvents = tx._eventQueue?.events;
     } else {
-      this.nextSagaEvents = tx._eventQueue?.events.filter((event) => JSON.parse(event.payload).event === failed);
+      const hasError = statusTuple.some(([, result]) => result.status === EventProcessingStatus.Error);
+      this.nextSagaEvents = tx._eventQueue?.events.filter((event) => {
+        const eventName = JSON.parse(event.payload).event;
+        return eventName === failed || (hasError && eventName === done);
+      });
     }
 
     if (tx._eventQueue) {

package/src/outbox/eventQueueAsOutbox.js

@@ -11,6 +11,8 @@ const CDS_EVENT_TYPE = "CAP_OUTBOX";
 const COMPONENT_NAME = "/eventQueue/eventQueueAsOutbox";
 const EVENT_QUEUE_SPECIFIC_FIELDS = ["startAfter", "referenceEntity", "referenceEntityKey", "namespace"];
 
+const TO_COPY = ["inbound", "event", "data", "queue", "results", "method", "path", "params", "entity", "service"];
+
 function outboxed(srv, customOpts) {
   if (!(new.target || customOpts)) {
     const former = srv[OUTBOXED];
@@ -102,18 +104,27 @@ const _mapToEventAndPublish = async (req, namespace, subType, eventHeaders, cont
       }
     }
   }
-
   const event = {
     contextUser: context.user.id,
     ...(req._fromSend || (req.reply && { _fromSend: true })), // send or emit
-    ...(req.inbound && { inbound: req.inbound }),
-    ...(req.event && { event: req.event }),
-    ...(req.data && { data: req.data }),
     ...(eventHeaders && { headers: eventHeaders }),
-    ...(req.query && { query: req.query }),
     ...(Object.keys(contextProperties).length && { ...contextProperties }),
   };
 
+  for (const prop of TO_COPY) {
+    if (req[prop]) {
+      event[prop] = req[prop];
+    }
+  }
+
+  if (req.query) {
+    event.query = typeof req.query.flat === "function" ? req.query.flat() : req.query;
+    delete event.query._target;
+    delete event.query.__target;
+    delete event.query.target;
+    delete event.data; // `req.data` should be a getter to whatever is in `req.query`
+  }
+
   await publishEvent(
     cds.tx(context),
     {

package/src/runner/openEvents.js

@@ -29,12 +29,12 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
         new Date(startTime.getTime() - 30 * MS_IN_DAYS).toISOString(),
         ")"
       )
-      .columns("type", "subType", "namespace")
+      .columns("count(ID) as count", "type", "subType", "namespace")
       .groupBy("type", "subType", "namespace")
   );
 
   const result = [];
-  for (const { type, subType, namespace } of entries) {
+  for (const { type, subType, namespace, count } of entries) {
     if (config.isCapOutboxEvent(type)) {
       const { srvName, actionName } = config.normalizeSubType(type, subType);
       try {
@@ -48,14 +48,14 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
             config.addCAPServiceWithoutEnvConfig(subType, service);
           }
           if (config.shouldBeProcessedInThisApplication(type, subType, namespace)) {
-            result.push({ namespace, type, subType });
+            result.push({ namespace, type, subType, count });
           }
         }
       } catch {
         /* ignore catch */
       } finally {
         if (!filterAppSpecificEvents) {
-          result.push({ namespace, type, subType });
+          result.push({ namespace, type, subType, count });
         }
       }
     } else {
@@ -64,10 +64,10 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
           config.getEventConfig(type, subType, namespace) &&
           config.shouldBeProcessedInThisApplication(type, subType, namespace)
         ) {
-          result.push({ namespace, type, subType });
+          result.push({ namespace, type, subType, count });
         }
       } else {
-        result.push({ namespace, type, subType });
+        result.push({ namespace, type, subType, count });
       }
     }
   }

package/src/runner/runner.js

@@ -14,6 +14,7 @@ const common = require("../shared/common");
 const config = require("../config");
 const redisPub = require("../redis/redisPub");
 const openEvents = require("./openEvents");
+const eventQueueStats = require("../shared/eventQueueStats");
 const { runEventCombinationForTenant } = require("./runnerHelper");
 const { trace } = require("../shared/openTelemetry");
 
@@ -141,7 +142,7 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
   } catch (err) {
     logger.error("executing event queue run for multi instance and tenant failed", err);
   }
-
+  const tenantCounts = {};
   for (const tenantId of tenantIds) {
     try {
       await cds.tx({ tenant: tenantId }, async (tx) => {
@@ -160,6 +161,18 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
               tenantId,
               entries: entries.length,
             });
+            tenantCounts[tenantId] = entries;
+            const pendingByNamespace = Object.fromEntries(config.processingNamespaces.map((name) => [name, 0]));
+            for (const entry of entries) {
+              pendingByNamespace[entry.namespace] = (pendingByNamespace[entry.namespace] ?? 0) + entry.count;
+            }
+            if (config.collectEventQueueMetrics) {
+              for (const [namespace, count] of Object.entries(pendingByNamespace)) {
+                eventQueueStats
+                  .setTenantCounter(tenantId, namespace, eventQueueStats.StatusField.Pending, count)
+                  .catch((err) => logger.error("updating tenant stats failed", err, { tenantId, namespace }));
+              }
+            }
             if (!entries.length) {
               return;
             }
@@ -178,6 +191,19 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
       logger.error("broadcasting events for tenant failed", { tenantId }, err);
     }
   }
+  const globalPendingByNamespace = Object.fromEntries(config.processingNamespaces.map((namespace) => [namespace, 0]));
+  for (const tenantEntries of Object.values(tenantCounts)) {
+    for (const entry of tenantEntries) {
+      globalPendingByNamespace[entry.namespace] = (globalPendingByNamespace[entry.namespace] ?? 0) + entry.count;
+    }
+  }
+  if (config.collectEventQueueMetrics) {
+    for (const [namespace, count] of Object.entries(globalPendingByNamespace)) {
+      eventQueueStats
+        .setGlobalCounter(namespace, eventQueueStats.StatusField.Pending, count)
+        .catch((err) => logger.error("updating global stats failed", err, { namespace }));
+    }
+  }
 };
 
 const _executeEventsAllTenants = async (tenantIds) => {
@@ -367,6 +393,17 @@ const _singleTenantRedis = async () => {
           logger.info("broadcasting events for run", {
             entries: entries.length,
           });
+          const pendingByNamespace = Object.fromEntries(config.processingNamespaces.map((name) => [name, 0]));
+          for (const entry of entries) {
+            pendingByNamespace[entry.namespace] = (pendingByNamespace[entry.namespace] ?? 0) + entry.count;
+          }
+          if (config.collectEventQueueMetrics) {
+            for (const [namespace, count] of Object.entries(pendingByNamespace)) {
+              eventQueueStats
+                .setGlobalCounter(namespace, eventQueueStats.StatusField.Pending, count)
+                .catch((err) => logger.error("updating global stats failed", err, { namespace }));
+            }
+          }
           if (!entries.length) {
             return;
           }

package/src/shared/eventQueueStats.js

@@ -0,0 +1,209 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const redis = require("./redis");
+const config = require("../config");
+
+const COMPONENT_NAME = "/eventQueue/eventQueueStats";
+
+const StatusField = {
+  Pending: "pending",
+  InProgress: "inProgress",
+};
+
+const _tenantKey = (tenantId) => `${config.redisNamespace(true)}##stats##tenant##${tenantId}`;
+const _globalKey = () => `${config.redisNamespace(true)}##stats##global`;
+const _keyPrefix = (namespace) => `${config.redisNamespace(false)}##${namespace}`;
+
+/**
+ * Atomically adjusts a tenant's event counter for the given status field.
+ *
+ * @param {string} tenantId
+ * @param {string} field - one of StatusField.*
+ * @param {number} increment - positive to increment, negative to decrement
+ */
+const adjustTenantCounter = async (tenantId, field, increment) => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    await client.hIncrBy(_tenantKey(tenantId), field, increment);
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to adjust tenant stats counter", err, { tenantId, field, increment });
+  }
+};
+
+/**
+ * Atomically adjusts the global event counter for the given status field.
+ * Also updates the `updatedAt` timestamp on the global hash.
+ *
+ * @param {string} field - one of StatusField.*
+ * @param {number} increment - positive to increment, negative to decrement
+ */
+const adjustGlobalCounter = async (field, increment) => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    await client.hIncrBy(_globalKey(), field, increment);
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to adjust global stats counter", err, { field, increment });
+  }
+};
+
+/**
+ * Increments a tenant counter and the matching global counter in a single call.
+ *
+ * @param {string} tenantId
+ * @param {string} field - one of StatusField.*
+ * @param {number} [increment=1]
+ */
+const incrementCounters = async (tenantId, field, increment = 1) => {
+  await Promise.allSettled([adjustTenantCounter(tenantId, field, increment), adjustGlobalCounter(field, increment)]);
+};
+
+/**
+ * Decrements a tenant counter and the matching global counter in a single call.
+ *
+ * @param {string} tenantId
+ * @param {string} field - one of StatusField.*
+ * @param {number} [decrement=1]
+ */
+const decrementCounters = async (tenantId, field, decrement = 1) => {
+  await Promise.allSettled([adjustTenantCounter(tenantId, field, -decrement), adjustGlobalCounter(field, -decrement)]);
+};
+
+/**
+ * Returns the current stats hash for a single tenant.
+ * All counter values are returned as integers; missing fields default to 0.
+ *
+ * @param {string} tenantId
+ * @returns {Promise<{pending: number, inProgress: number}>}
+ */
+const getTenantStats = async (tenantId) => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    const raw = await client.hGetAll(_tenantKey(tenantId));
+    return _parseCounterHash(raw);
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to read tenant stats", err, { tenantId });
+    return _emptyCounters();
+  }
+};
+
+/**
+ * Returns the current global stats hash.
+ * All counter values are returned as integers; missing fields default to 0.
+ *
+ * @returns {Promise<{pending: number, inProgress: number}>}
+ */
+const getGlobalStats = async () => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    const raw = await client.hGetAll(_globalKey());
+    return _parseCounterHash(raw);
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to read global stats", err);
+    return _emptyCounters();
+  }
+};
+
+/**
+ * Deletes the stats hash for a specific tenant.
+ * Intended for use during tenant offboarding. It does not adjust the global stats still will be fixed with the next global run
+ *
+ * @param {string} tenantId
+ */
+const setTenantCounter = async (tenantId, namespace, field, value) => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    await client.hSet(`${_keyPrefix(namespace)}##stats##tenant##${tenantId}`, field, value);
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to set tenant stats counter", err, { tenantId, namespace, field, value });
+  }
+};
+
+const setGlobalCounter = async (namespace, field, value) => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    await client.hSet(`${_keyPrefix(namespace)}##stats##global`, field, value);
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to set global stats counter", err, { namespace, field, value });
+  }
+};
+
+const getAllNamespaceStats = async () => {
+  const namespaces = config.processingNamespaces;
+  const client = await redis.createMainClientAndConnect();
+  const results = await Promise.allSettled(
+    namespaces.map(async (namespace) => {
+      const raw = await client.hGetAll(`${_keyPrefix(namespace)}##stats##global`);
+      return { namespace, stats: _parseCounterHash(raw) };
+    })
+  );
+  const out = {};
+  for (const result of results) {
+    if (result.status === "fulfilled") {
+      out[result.value.namespace] = result.value.stats;
+    } else {
+      cds.log(COMPONENT_NAME).error("failed to read namespace stats", result.reason);
+    }
+  }
+  return out;
+};
+
+const deleteTenantStats = async (tenantId) => {
+  try {
+    const client = await redis.createMainClientAndConnect();
+    await client.del(_tenantKey(tenantId));
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to delete tenant stats", err, { tenantId });
+  }
+};
+
+/**
+ * Resets the inProgress counter to 0 for all processing namespaces (global + all tenants).
+ * Called on instance startup to clean up stale counts left by a previous crash.
+ */
+const resetInProgressCounters = async () => {
+  try {
+    const clientOrCluster = await redis.createMainClientAndConnect();
+    const clients = redis.isClusterMode() ? clientOrCluster.masters.map((master) => master.client) : [clientOrCluster];
+
+    const globalOps = config.processingNamespaces.map((namespace) =>
+      clientOrCluster.hSet(`${_keyPrefix(namespace)}##stats##global`, StatusField.InProgress, 0)
+    );
+    await Promise.allSettled(globalOps);
+
+    // NOTE: use SCAN because KEYS is not supported for cluster clients
+    for (const client of clients) {
+      for await (const key of client.scanIterator({ MATCH: "*##stats##tenant##*", COUNT: 1000 })) {
+        await client.hSet(key, StatusField.InProgress, 0);
+      }
+    }
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to reset inProgress counters on startup", err);
+  }
+};
+
+const _parseCounterHash = (raw) => ({
+  [StatusField.Pending]: raw[StatusField.Pending] != null ? parseInt(raw[StatusField.Pending]) : 0,
+  [StatusField.InProgress]: raw[StatusField.InProgress] != null ? parseInt(raw[StatusField.InProgress]) : 0,
+});
+
+const _emptyCounters = () => ({
+  [StatusField.Pending]: 0,
+  [StatusField.InProgress]: 0,
+});
+
+module.exports = {
+  StatusField,
+  incrementCounters,
+  decrementCounters,
+  adjustTenantCounter,
+  adjustGlobalCounter,
+  setTenantCounter,
+  setGlobalCounter,
+  getAllNamespaceStats,
+  getTenantStats,
+  getGlobalStats,
+  deleteTenantStats,
+  resetInProgressCounters,
+};

package/src/shared/openTelemetry.js

@@ -12,9 +12,14 @@ const cds = require("@sap/cds");
 const otel = _resilientRequire("@opentelemetry/api");
 
 const config = require("../config");
+const eventQueueStats = require("./eventQueueStats");
+const { getEnvInstance } = require("./env");
 
 const COMPONENT_NAME = "/shared/openTelemetry";
 
+let _statsSnapshot = null;
+let _metricsInitialized = false;
+
 const trace = async (context, label, fn, { attributes = {}, newRootSpan = false, traceContext } = {}) => {
   if (!config.enableTelemetry || !otel) {
     return fn();
@@ -110,4 +115,77 @@ const getCurrentTraceContext = () => {
   return carrier;
 };
 
-module.exports = { trace, getCurrentTraceContext };
+const _refreshStats = async () => {
+  try {
+    const namespaces = await eventQueueStats.getAllNamespaceStats();
+    _statsSnapshot = { namespaces, lastRefreshedAt: Date.now() };
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("failed to refresh queue stats for metrics", err);
+  }
+};
+
+const initMetrics = () => {
+  if (
+    _metricsInitialized ||
+    !config.collectEventQueueMetrics ||
+    !config.enableTelemetry ||
+    !config.redisEnabled ||
+    !config.registerAsEventProcessor ||
+    (getEnvInstance().applicationInstance !== undefined && getEnvInstance().applicationInstance !== 0) ||
+    !otel?.metrics
+  ) {
+    return;
+  }
+  const meterProvider = otel.metrics.getMeterProvider?.();
+  if (!meterProvider) {
+    return;
+  }
+
+  _metricsInitialized = true;
+
+  eventQueueStats
+    .resetInProgressCounters()
+    .catch((err) => cds.log(COMPONENT_NAME).error("failed to reset inProgress counters", err));
+
+  const meter = otel.metrics.getMeter("@cap-js-community/event-queue");
+
+  const pendingGauge = meter.createObservableGauge("cap.event_queue.jobs.pending", {
+    description: "Current number of jobs waiting to be processed.",
+    unit: "1",
+  });
+  const inProgressGauge = meter.createObservableGauge("cap.event_queue.jobs.in_progress", {
+    description: "Current number of jobs actively being processed by workers.",
+    unit: "1",
+  });
+  const refreshAgeGauge = meter.createObservableGauge("cap.event_queue.stats.refresh_age", {
+    description: "Age of the most recent queue statistics snapshot.",
+    unit: "s",
+  });
+
+  _statsSnapshot = {
+    lastRefreshedAt: Date.now(),
+    namespaces: Object.fromEntries(
+      config.processingNamespaces.map((namespace) => [namespace, { pending: 0, inProgress: 0 }])
+    ),
+  };
+  _refreshStats();
+
+  meter.addBatchObservableCallback(
+    (observableResult) => {
+      if (!_statsSnapshot) {
+        return;
+      }
+      observableResult.observe(refreshAgeGauge, (Date.now() - _statsSnapshot.lastRefreshedAt) / 1000);
+      for (const [namespace, stats] of Object.entries(_statsSnapshot.namespaces)) {
+        const attrs = { "queue.namespace": namespace };
+        observableResult.observe(pendingGauge, stats.pending, attrs);
+        observableResult.observe(inProgressGauge, stats.inProgress, attrs);
+      }
+    },
+    [pendingGauge, inProgressGauge, refreshAgeGauge]
+  );
+
+  setInterval(_refreshStats, 30_000).unref();
+};
+
+module.exports = { trace, getCurrentTraceContext, initMetrics };