@cap-js-community/event-queue 2.0.0-beta.8 → 2.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "2.0.0-beta.8",
+  "version": "2.0.1",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/EventQueueProcessorBase.js

@@ -296,7 +296,7 @@ class EventQueueProcessorBase {
     const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
     const errorHandler = (error) => {
       queueEntries.forEach((queueEntry) =>
-        this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
+        this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, { statusMap })
       );
       this.logger.error(
         "The supplied status tuple doesn't have the required structure. Setting all entries to error.",
@@ -317,7 +317,7 @@ class EventQueueProcessorBase {
 
     try {
       queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
-        this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
+        this.#determineAndAddEventStatusToMap(id, processingStatus, { statusMap })
       );
     } catch (error) {
       errorHandler(error);
@@ -344,11 +344,15 @@ class EventQueueProcessorBase {
     }
   }
 
-  #determineAndAddEventStatusToMap(id, statusOrUpdateData, statusMap = this.__statusMap) {
+  #determineAndAddEventStatusToMap(id, statusOrUpdateData, { statusMap = this.__statusMap, error } = {}) {
     if (typeof statusOrUpdateData === "number") {
       statusOrUpdateData = { status: statusOrUpdateData };
     }
 
+    if (error) {
+      statusOrUpdateData.error = error;
+    }
+
     if (!statusMap[id]) {
       statusMap[id] = statusOrUpdateData;
       return;
@@ -376,9 +380,17 @@ class EventQueueProcessorBase {
       }
     );
     queueEntries.forEach((queueEntry) =>
-      this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error)
+      this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, { error })
+    );
+    return Object.fromEntries(
+      queueEntries.map((queueEntry) => [
+        queueEntry.ID,
+        {
+          status: EventProcessingStatus.Error,
+          error,
+        },
+      ])
     );
-    return Object.fromEntries(queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error]));
   }
 
   handleErrorDuringPeriodicEventProcessing(error, queueEntry) {
@@ -389,11 +401,12 @@ class EventQueueProcessorBase {
     });
   }
 
-  async setPeriodicEventStatus(queueEntryIds, status) {
+  async setPeriodicEventStatus(queueEntryIds, status, error) {
     await this.tx.run(
       UPDATE.entity(this.#config.tableNameEventQueue)
         .set({
           status: status,
+          ...(error && { error: this.#error2String(error) }),
         })
         .where({
           ID: queueEntryIds,
@@ -1018,7 +1031,7 @@ class EventQueueProcessorBase {
       const lockAcquired = await distributedLock.acquireLock(
         this.__context,
         [this.#namespace, this.#eventType, this.#eventSubType].join("##"),
-        { keepTrackOfLock: true, expiryTime: this.#eventConfig.keepAliveMaxInProgressTime * 1000 }
+        { keepTrackOfLock: true, expiryTime: this.#eventConfig.keepAliveMaxInProgressTime * 1000, skipNamespace: true }
       );
       if (!lockAcquired) {
         this.logger.debug("no lock available, exit processing", {
@@ -1061,7 +1074,8 @@ class EventQueueProcessorBase {
       await trace(this.baseContext, "release-lock", async () => {
         await distributedLock.releaseLock(
           this.context,
-          [this.#namespace, this.#eventType, this.#eventSubType].join("##")
+          [this.#namespace, this.#eventType, this.#eventSubType].join("##"),
+          { skipNamespace: true }
         );
       });
     } catch (err) {

package/src/processEventQueue.js

@@ -154,6 +154,7 @@ const processPeriodicEvent = async (context, eventTypeInstance) => {
       }
 
       let status = EventProcessingStatus.Done;
+      let error;
       await executeInNewTransaction(
         eventTypeInstance.context,
         `eventQueue-periodic-process-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
@@ -166,6 +167,7 @@ const processPeriodicEvent = async (context, eventTypeInstance) => {
               eventTypeInstance.startPerformanceTracerPeriodicEvents();
               await eventTypeInstance.processPeriodicEvent(tx.context, queueEntry.ID, queueEntry);
             } catch (err) {
+              error = err;
               status = EventProcessingStatus.Error;
               eventTypeInstance.handleErrorDuringPeriodicEventProcessing(err, queueEntry);
               await tx.rollback();
@@ -190,7 +192,7 @@ const processPeriodicEvent = async (context, eventTypeInstance) => {
         async (tx) => {
           await trace(eventTypeInstance.context, "periodic-event-set-status", async () => {
             eventTypeInstance.processEventContext = tx.context;
-            await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID, status);
+            await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID, status, error);
           });
         }
       );

package/src/runner/runner.js

@@ -180,7 +180,7 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
   }
 };
 
-const _executeEventsAllTenants = async (tenantIds, runId) => {
+const _executeEventsAllTenants = async (tenantIds) => {
   const promises = [];
   for (const tenantId of tenantIds) {
     const id = cds.utils.uuid();
@@ -230,13 +230,6 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
                 label,
                 async () => {
                   try {
-                    const lockId = `${runId}_${label}`;
-                    const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-                      expiryTime: eventQueueConfig.runInterval * 0.95,
-                    });
-                    if (!couldAcquireLock) {
-                      return;
-                    }
                     await runEventCombinationForTenant(
                       context,
                       eventConfig.type,
@@ -308,50 +301,40 @@ const _singleTenantDb = async () => {
     { newRootSpan: true }
   );
 
-  return await Promise.allSettled(
-    events.map(async (openEvent) => {
-      const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType, openEvent.namespace);
-      const label = [openEvent.namespace, eventConfig.type, eventConfig.subType].join("##");
-      return await WorkerQueue.instance.addToQueue(
-        eventConfig.load,
-        label,
-        eventConfig.priority,
-        eventConfig.increasePriorityOverTime,
-        async () => {
-          return await cds.tx({}, async ({ context }) => {
-            await trace(
-              context,
-              label,
-              async () => {
-                try {
-                  const couldAcquireLock = eventConfig.multiInstanceProcessing
-                    ? true
-                    : await distributedLock.acquireLock(context, label, {
-                        expiryTime: eventQueueConfig.runInterval * 0.95,
-                      });
-                  if (!couldAcquireLock) {
-                    return;
+  for (const openEvent of events) {
+    const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType, openEvent.namespace);
+    const label = [openEvent.namespace, eventConfig.type, eventConfig.subType].join("##");
+    await WorkerQueue.instance.addToQueue(
+      eventConfig.load,
+      label,
+      eventConfig.priority,
+      eventConfig.increasePriorityOverTime,
+      async () => {
+        return await cds.tx({}, async ({ context }) => {
+          await trace(
+            context,
+            label,
+            async () => {
+              try {
+                await runEventCombinationForTenant(
+                  context,
+                  eventConfig.type,
+                  eventConfig.subType,
+                  openEvent.namespace,
+                  {
+                    skipWorkerPool: true,
                   }
-                  await runEventCombinationForTenant(
-                    context,
-                    eventConfig.type,
-                    eventConfig.subType,
-                    openEvent.namespace,
-                    {
-                      skipWorkerPool: true,
-                    }
-                  );
-                } catch (err) {
-                  cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
-                }
-              },
-              { newRootSpan: true }
-            );
-          });
-        }
-      );
-    })
-  );
+                );
+              } catch (err) {
+                cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
+              }
+            },
+            { newRootSpan: true }
+          );
+        });
+      }
+    );
+  }
 };
 
 const _singleTenantRedis = async () => {
@@ -467,7 +450,7 @@ const _multiTenancyDb = async () => {
     logger.info("executing event queue run for single instance and multi tenant");
     const tenantIds = await cdsHelper.getAllTenantIds();
     await _checkPeriodicEventUpdate(tenantIds);
-    return await _executeEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
+    return await _executeEventsAllTenants(tenantIds);
   } catch (err) {
     logger.error("Couldn't fetch tenant ids for event queue processing! Next try after defined interval.", err);
   }

package/src/shared/common.js

@@ -165,6 +165,14 @@ const isTenantIdValidCb = async (checkType, tenantId) => {
   }
 };
 
+const cleanUndefined = (input) =>
+  Object.entries(input).reduce((acc, [key, value]) => {
+    if (value !== undefined) {
+      acc[key] = value;
+    }
+    return acc;
+  }, {});
+
 module.exports = {
   arrayToFlatMap,
   limiter,
@@ -174,6 +182,7 @@ module.exports = {
   getAuthContext,
   isTenantIdValidCb,
   promiseAllDone,
+  cleanUndefined,
   __: {
     clearAuthContextCache: () => getAuthContext._cache?.clear(),
   },

package/src/shared/distributedLock.js

@@ -11,9 +11,9 @@ const COMPONENT_NAME = "/eventQueue/distributedLock";
 const acquireLock = async (
   context,
   key,
-  { tenantScoped = true, expiryTime = config.globalTxTimeout, keepTrackOfLock = false } = {}
+  { tenantScoped = true, expiryTime = config.globalTxTimeout, keepTrackOfLock = false, skipNamespace = false } = {}
 ) => {
-  const fullKey = _generateKey(context, tenantScoped, key);
+  const fullKey = _generateKey(context, tenantScoped, key, skipNamespace);
   if (config.redisEnabled) {
     return await _acquireLockRedis(context, fullKey, expiryTime, { keepTrackOfLock });
   } else {
@@ -51,8 +51,8 @@ const setValueWithExpire = async (
   }
 };
 
-const releaseLock = async (context, key, { tenantScoped = true } = {}) => {
-  const fullKey = _generateKey(context, tenantScoped, key);
+const releaseLock = async (context, key, { tenantScoped = true, skipNamespace = false } = {}) => {
+  const fullKey = _generateKey(context, tenantScoped, key, skipNamespace);
   if (config.redisEnabled) {
     return await _releaseLockRedis(context, fullKey);
   } else {
@@ -194,8 +194,8 @@ const _acquireLockDB = async (
   return result;
 };
 
-const _generateKey = (context, tenantScoped, key) => {
-  const keyParts = [config.redisNamespace()];
+const _generateKey = (context, tenantScoped, key, skipNamespace) => {
+  const keyParts = [config.redisNamespace(!skipNamespace)];
   tenantScoped && keyParts.push(context.tenant);
   keyParts.push(key);
   return `${keyParts.join("##")}`;

package/srv/service/admin-service.cds

@@ -42,4 +42,17 @@ service EventQueueAdminService {
         @mandatory
         subType: String) returns Boolean;
     }
+
+      action publishEvent(
+          @mandatory
+          tenants: array of String,
+          @mandatory
+          type: String,
+          @mandatory
+          subType: String,
+          referenceEntity: String,
+          referenceEntityKey: String,
+          @open
+          payload: {},
+          startAfter: String);
 }

package/srv/service/admin-service.js

@@ -1,26 +1,35 @@
 "use strict";
 
 const cds = require("@sap/cds");
-const { EventProcessingStatus } = require("../../src");
-const config = require("../../src/config");
+
+const eventQueue = require("../../src");
 const distributedLock = require("../../src/shared/distributedLock");
 const redisPub = require("../../src/redis/redisPub");
+const publishEventHelper = require("./publishEventHelper");
+const commonHelper = require("../../src/shared/common");
+
+const COMPONENT_NAME = "/eventQueue/admin";
 
 module.exports = class AdminService extends cds.ApplicationService {
   async init() {
     const { Event: EventService, Lock: LockService } = this.entities;
     const { Event: EventDb } = cds.db.entities("sap.eventqueue");
+    const { publishEvent } = this.actions;
     const { landscape, space } = this.getLandscapeAndSpace();
 
     this.before("*", (req) => {
-      if (!config.enableAdminService) {
+      if (!eventQueue.config.enableAdminService) {
         req.reject(403, "Admin service is disabled by configuration");
       }
 
+      if (req.event === publishEvent.name.split(".")[1]) {
+        return;
+      }
+
       const headers = Object.assign({}, req.headers, req.req?.headers);
       const tenant = headers["z-id"] ?? req.data.tenant;
 
-      if (config.isMultiTenancy && tenant == null) {
+      if (eventQueue.config.isMultiTenancy && tenant == null) {
         req.reject(400, "Missing tenant ID in request header (z-id)");
       }
       req.headers ??= {};
@@ -46,7 +55,7 @@ module.exports = class AdminService extends cds.ApplicationService {
     });
 
     this.on("READ", LockService, async () => {
-      if (!config.redisEnabled) {
+      if (!eventQueue.config.redisEnabled) {
         return [];
       }
       const locks = await distributedLock.getAllLocksRedis();
@@ -59,14 +68,14 @@ module.exports = class AdminService extends cds.ApplicationService {
 
     this.on("setStatusAndAttempts", async (req) => {
       const tenant = req.headers["z-id"];
-      cds.log("eventQueue").info("Restarting processing for event queue");
+      cds.log(COMPONENT_NAME).info("Restarting processing for event queue");
       const updateData = {};
 
       if (Number.isInteger(req.data.attempts)) {
         updateData.attempts = req.data.attempts;
       }
 
-      if (Object.values(EventProcessingStatus).includes(req.data.status)) {
+      if (Object.values(eventQueue.EventProcessingStatus).includes(req.data.status)) {
         updateData.status = req.data.status;
       }
 
@@ -95,6 +104,32 @@ module.exports = class AdminService extends cds.ApplicationService {
       });
     });
 
+    this.on(publishEvent, async (req) => {
+      const logger = cds.log(COMPONENT_NAME);
+      try {
+        const { type, subType, referenceEntity, referenceEntityKey, payload, startAfter } = req.data;
+        const tenants = await publishEventHelper.resolveTenantInfos(req);
+        const eventOptions = commonHelper.cleanUndefined({
+          type,
+          subType,
+          referenceEntity,
+          referenceEntityKey,
+          payload,
+          startAfter,
+        });
+        const publishInfo = { count: tenants.length, type, subType, tenants: req.data.tenants };
+        logger.info("publishing event for tenant(s)", publishInfo);
+        for (const tenant of tenants) {
+          await cds.tx({ tenant }, async (tx) => {
+            await eventQueue.publishEvent(tx, { ...eventOptions });
+          });
+        }
+        logger.info("finished publishing event for tenant(s)", publishInfo);
+      } catch (err) {
+        logger.error("error publishing event", err);
+      }
+    });
+
     await super.init();
   }
 

package/srv/service/publishEventHelper.js

@@ -0,0 +1,52 @@
+"use strict";
+
+const cdsHelper = require("../../src/shared/cdsHelper");
+
+/**
+ * @typedef TenantInfo
+ * @type object
+ * @property {string} tenantId
+ * @property {string} subdomain
+ */
+/**
+ * For service action interfaces, with a tenants = [...] data field this code
+ * resolves the inputs into a tenantInfos array.
+ *
+ * Inputs:
+ * - ["all"]   give me all tenants
+ * - ["self"]  give me just the context tenant
+ * - []        give me no tenant
+ * - ["<tenantId1>", "<tenantId2>", "<subdomain3>", "<subdomain4>",...]
+ *             give me those tenants where either tenantId or subdomain corresponds to the given inputs
+ *
+ * @returns {Promise<Array<TenantInfo>>} returns an array of {@link TenantInfo} objects.
+ */
+const resolveTenantInfos = async (context, { sortByTenantId = true } = {}) => {
+  let result = await _resolveTenantInfos(context);
+  if (sortByTenantId) {
+    result = result.sort(_orderByTenantId);
+  }
+  return result;
+};
+
+const _resolveTenantInfos = async (context) => {
+  if (!Array.isArray(context.data.tenants)) {
+    return [];
+  }
+  for (const tenant of context.data.tenants) {
+    if (tenant === "self") {
+      return [context.tenant];
+    }
+    if (["all", "*"].includes(tenant)) {
+      return await cdsHelper.getAllTenantIds();
+    }
+  }
+  const contextTenantsMap = Object.fromEntries(context.data.tenants.map((tenant) => [tenant, true]));
+  return (await cdsHelper.getAllTenantIds()).filter((tenantId) => contextTenantsMap[tenantId]);
+};
+
+const _orderByTenantId = (a, b) => a.localeCompare(b);
+
+module.exports = {
+  resolveTenantInfos,
+};