@cap-js-community/event-queue 1.4.1 → 1.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [
@@ -50,8 +50,8 @@
   "devDependencies": {
     "@cap-js/hana": "^0.1.0",
     "@cap-js/sqlite": "^1.5.0",
-    "@sap/cds": "^7.7.0",
-    "@sap/cds-dk": "^7.5.1",
+    "@sap/cds": "^7.8.0",
+    "@sap/cds-dk": "^7.8.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jest": "^27.9.0",

package/src/EventQueueProcessorBase.js

@@ -279,22 +279,33 @@ class EventQueueProcessorBase {
       eventSubType: this.#eventSubType,
     });
     const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
-    try {
-      queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
-        this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
-      );
-    } catch (error) {
+    const errorHandler = (error) => {
       queueEntries.forEach((queueEntry) =>
         this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
       );
       this.logger.error(
         "The supplied status tuple doesn't have the required structure. Setting all entries to error.",
-        error,
-        {
-          eventType: this.#eventType,
-          eventSubType: this.#eventSubType,
-        }
+        ...[
+          error,
+          {
+            eventType: this.#eventType,
+            eventSubType: this.#eventSubType,
+          },
+        ].filter((a) => a)
+      );
+    };
+
+    if (!queueEntryProcessingStatusTuple) {
+      errorHandler();
+      return statusMap;
+    }
+
+    try {
+      queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
+        this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
       );
+    } catch (error) {
+      errorHandler(error);
     }
     return statusMap;
   }

package/src/config.js

@@ -66,6 +66,7 @@ class Config {
   #userId;
   #cleanupLocksAndEventsForDev;
   #redisOptions;
+  #insertEventsBeforeCommit;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -494,6 +495,14 @@ class Config {
     return this.#redisOptions;
   }
 
+  set insertEventsBeforeCommit(value) {
+    this.#insertEventsBeforeCommit = value;
+  }
+
+  get insertEventsBeforeCommit() {
+    return this.#insertEventsBeforeCommit;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }

package/src/dbHandler.js

@@ -6,8 +6,17 @@ const { broadcastEvent } = require("./redis/redisPub");
 const config = require("./config");
 
 const COMPONENT_NAME = "/eventQueue/dbHandler";
+const registeredHandlers = {
+  eventQueueDbHandler: false,
+  beforeDbHandler: false,
+};
 
 const registerEventQueueDbHandler = (dbService) => {
+  if (registeredHandlers.eventQueueDbHandler) {
+    return;
+  }
+
+  registeredHandlers.eventQueueDbHandler = true;
   const def = dbService.model.definitions[config.tableNameEventQueue];
   dbService.after("CREATE", def, (_, req) => {
     if (req.tx._skipEventQueueBroadcase) {
@@ -46,6 +55,21 @@ const registerEventQueueDbHandler = (dbService) => {
   });
 };
 
+const registerBeforeDbHandler = (dbService) => {
+  if (!config.insertEventsBeforeCommit || registeredHandlers.beforeDbHandler) {
+    return;
+  }
+
+  registeredHandlers.beforeDbHandler = true;
+  dbService.before("COMMIT", async (req) => {
+    if (req.context._eventQueueEvents?.length) {
+      await cds.tx(req).run(INSERT.into(config.tableNameEventQueue).entries(req.context._eventQueueEvents));
+      req.context._eventQueueEvents = null;
+    }
+  });
+};
+
 module.exports = {
   registerEventQueueDbHandler,
+  registerBeforeDbHandler,
 };

package/src/initialize.js

@@ -15,6 +15,7 @@ const redis = require("./shared/redis");
 const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 const { getAllTenantIds } = require("./shared/cdsHelper");
 const { EventProcessingStatus } = require("./constants");
+const distributedLock = require("./shared/distributedLock");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -34,6 +35,7 @@ const CONFIG_VARS = [
   ["userId", null],
   ["cleanupLocksAndEventsForDev", false],
   ["redisOptions", {}],
+  ["insertEventsBeforeCommit", false],
 ];
 
 const initialize = async ({
@@ -49,6 +51,7 @@ const initialize = async ({
   userId,
   cleanupLocksAndEventsForDev,
   redisOptions,
+  insertEventsBeforeCommit,
 } = {}) => {
   if (config.initialized) {
     return;
@@ -67,7 +70,8 @@ const initialize = async ({
     useAsCAPOutbox,
     userId,
     cleanupLocksAndEventsForDev,
-    redisOptions
+    redisOptions,
+    insertEventsBeforeCommit
   );
 
   const logger = cds.log(COMPONENT);
@@ -77,6 +81,7 @@ const initialize = async ({
   cds.on("connect", (service) => {
     if (service.name === "db") {
       config.processEventsAfterPublish && dbHandler.registerEventQueueDbHandler(service);
+      config.insertEventsBeforeCommit && dbHandler.registerBeforeDbHandler(service);
       config.cleanupLocksAndEventsForDev && registerCleanupForDevDb().catch(() => {});
       initFinished.then(registerEventProcessors);
     }
@@ -166,6 +171,7 @@ const mixConfigVarsWithEnv = (...args) => {
 
 const registerCdsShutdown = () => {
   cds.on("shutdown", async () => {
+    await distributedLock.shutdownHandler();
     await Promise.allSettled([redis.closeMainClient(), closeSubscribeClient()]);
   });
 };

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -23,6 +23,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       delete msg._fromSend;
       delete msg.contextUser;
       processContext.user = new cds.User.Privileged(userId);
+      processContext._eventQueue = { processor: this, key, queueEntries, payload };
       await cds.unboxed(service).tx(processContext)[invocationFn](msg);
     } catch (err) {
       status = EventProcessingStatus.Error;

package/src/publishEvent.js

@@ -45,10 +45,16 @@ const publishEvent = async (tx, events, skipBroadcast = false) => {
       throw EventQueueError.manuelPeriodicEventInsert(type, subType);
     }
   }
-  tx._skipEventQueueBroadcase = skipBroadcast;
-  const result = await tx.run(INSERT.into(config.tableNameEventQueue).entries(eventsForProcessing));
-  tx._skipEventQueueBroadcase = false;
-  return result;
+
+  if (config.insertEventsBeforeCommit) {
+    tx.context._eventQueueEvents ??= [];
+    tx.context._eventQueueEvents = tx.context._eventQueueEvents.concat(events);
+  } else {
+    tx._skipEventQueueBroadcase = skipBroadcast;
+    const result = await tx.run(INSERT.into(config.tableNameEventQueue).entries(eventsForProcessing));
+    tx._skipEventQueueBroadcase = false;
+    return result;
+  }
 };
 
 module.exports = {

package/src/runner/openEvents.js

@@ -31,6 +31,7 @@ const getOpenQueueEntries = async (tx) => {
   for (const { type, subType } of entries) {
     if (type.startsWith("CAP_OUTBOX")) {
       if (cds.requires[subType]) {
+        await cds.connect.to(subType).catch(() => {});
         result.push({ type, subType });
       } else {
         const service = await cds.connect.to(subType).catch(() => {});

package/src/shared/distributedLock.js

@@ -6,6 +6,10 @@ const cdsHelper = require("./cdsHelper");
 
 const KEY_PREFIX = "EVENT_QUEUE";
 
+const existingLocks = {};
+
+const REDIS_COMMAND_OK = "OK";
+
 const acquireLock = async (context, key, { tenantScoped = true, expiryTime = config.globalTxTimeout } = {}) => {
   const fullKey = _generateKey(context, tenantScoped, key);
   if (config.redisEnabled) {
@@ -59,7 +63,11 @@ const _acquireLockRedis = async (context, fullKey, expiryTime, { value = "true",
     PX: expiryTime,
     ...(overrideValue ? null : { NX: true }),
   });
-  return result === "OK";
+  const isOk = result === REDIS_COMMAND_OK;
+  if (isOk) {
+    existingLocks[fullKey] = 1;
+  }
+  return isOk;
 };
 
 const _checkLockExistsRedis = async (context, fullKey) => {
@@ -78,6 +86,7 @@ const _checkLockExistsDb = async (context, fullKey) => {
 const _releaseLockRedis = async (context, fullKey) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
   await client.del(fullKey);
+  delete existingLocks[fullKey];
 };
 
 const _releaseLockDb = async (context, fullKey) => {
@@ -133,9 +142,14 @@ const _generateKey = (context, tenantScoped, key) => {
   return `${KEY_PREFIX}_${keyParts.join("##")}`;
 };
 
+const shutdownHandler = async () => {
+  await Promise.allSettled(Object.keys(existingLocks).map((key) => _releaseLockRedis(null, key)));
+};
+
 module.exports = {
   acquireLock,
   releaseLock,
   checkLockExistsAndReturnValue,
   setValueWithExpire,
+  shutdownHandler,
 };