@cap-js-community/event-queue 0.1.49 → 0.1.51

package/src/config.js

@@ -23,7 +23,6 @@ class Config {
     this.__tableNameEventLock = null;
     this.__vcapServices = this._parseVcapServices();
     this.__isRunnerDeactivated = false;
-    this.__eventsForAutomaticRun = null;
     this.__configFilePath = null;
     this.__processEventsAfterPublish = null;
     this.__skipCsnCheck = null;
@@ -34,9 +33,7 @@ class Config {
   }
 
   hasEventAfterCommitFlag(type, subType) {
-    return (
-      this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true
-    );
+    return this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
   }
 
   _checkRedisIsBound() {

package/src/constants.js

@@ -8,4 +8,9 @@ module.exports = {
     Error: 3,
     Exceeded: 4,
   },
+  TransactionMode: {
+    isolated: "isolated",
+    alwaysCommit: "alwaysCommit",
+    alwaysRollback: "alwaysRollback",
+  },
 };

package/src/dbHandler.js

@@ -14,10 +14,7 @@ const registerEventQueueDbHandler = (dbService) => {
     const eventCombinations = Object.keys(
       data.reduce((result, event) => {
         const key = [event.type, event.subType].join("##");
-        if (
-          !configInstance.hasEventAfterCommitFlag(event.type, event.subType) ||
-          eventQueuePublishEvents[key]
-        ) {
+        if (!configInstance.hasEventAfterCommitFlag(event.type, event.subType) || eventQueuePublishEvents[key]) {
           return result;
         }
         eventQueuePublishEvents[key] = true;

package/src/initialize.js

@@ -56,26 +56,13 @@ const initialize = async ({
   );
 
   const logger = cds.log(COMPONENT);
-  configInstance.fileContent = await readConfigFromFile(
-    configInstance.configFilePath
-  );
+  configInstance.fileContent = await readConfigFromFile(configInstance.configFilePath);
   configInstance.calculateIsRedisEnabled();
 
   const dbService = await cds.connect.to("db");
-  await (cds.model
-    ? Promise.resolve()
-    : new Promise((resolve) => cds.on("serving", resolve)));
+  await (cds.model ? Promise.resolve() : new Promise((resolve) => cds.on("serving", resolve)));
   !configInstance.skipCsnCheck && (await csnCheck());
   if (configInstance.processEventsAfterPublish) {
-    // TODO: remove this as soon as CDS fixes the current plugin model issues --> cds 7
-    if (BASE_TABLES.EVENT === configInstance.tableNameEventQueue) {
-      cds.db.model.definitions[BASE_TABLES.EVENT] =
-        cds.model.definitions[BASE_TABLES.EVENT];
-    }
-    if (BASE_TABLES.LOCK === configInstance.tableNameEventLock) {
-      cds.db.model.definitions[BASE_TABLES.LOCK] =
-        cds.model.definitions[BASE_TABLES.LOCK];
-    }
     dbHandler.registerEventQueueDbHandler(dbService);
   }
 
@@ -161,8 +148,7 @@ const checkCustomTable = (baseCsn, customCsn) => {
 
     if (
       customCsn.elements[columnName].type !== "cds.Association" &&
-      customCsn.elements[columnName].type !==
-        baseCsn.elements[columnName].type &&
+      customCsn.elements[columnName].type !== baseCsn.elements[columnName].type &&
       columnName === "status" &&
       customCsn.elements[columnName].type !== "cds.Integer"
     ) {
@@ -183,32 +169,18 @@ const mixConfigVarsWithEnv = (
 ) => {
   const configInstance = getConfigInstance();
 
-  configInstance.configFilePath =
-    configFilePath ?? cds.env.eventQueue?.configFilePath;
+  configInstance.configFilePath = configFilePath ?? cds.env.eventQueue?.configFilePath;
   configInstance.registerAsEventProcessor =
-    registerAsEventProcessor ??
-    cds.env.eventQueue?.registerAsEventProcessor ??
-    true;
+    registerAsEventProcessor ?? cds.env.eventQueue?.registerAsEventProcessor ?? true;
   configInstance.processEventsAfterPublish =
-    processEventsAfterPublish ??
-    cds.env.eventQueue?.processEventsAfterPublish ??
-    true;
-  configInstance.runInterval =
-    runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
+    processEventsAfterPublish ?? cds.env.eventQueue?.processEventsAfterPublish ?? true;
+  configInstance.runInterval = runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
   configInstance.parallelTenantProcessing =
-    parallelTenantProcessing ??
-    cds.env.eventQueue?.parallelTenantProcessing ??
-    5;
+    parallelTenantProcessing ?? cds.env.eventQueue?.parallelTenantProcessing ?? 5;
   configInstance.tableNameEventQueue =
-    tableNameEventQueue ??
-    cds.env.eventQueue?.tableNameEventQueue ??
-    BASE_TABLES.EVENT;
-  configInstance.tableNameEventLock =
-    tableNameEventLock ??
-    cds.env.eventQueue?.tableNameEventLock ??
-    BASE_TABLES.LOCK;
-  configInstance.skipCsnCheck =
-    skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
+    tableNameEventQueue ?? cds.env.eventQueue?.tableNameEventQueue ?? BASE_TABLES.EVENT;
+  configInstance.tableNameEventLock = tableNameEventLock ?? cds.env.eventQueue?.tableNameEventLock ?? BASE_TABLES.LOCK;
+  configInstance.skipCsnCheck = skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
 };
 
 module.exports = {

package/src/processEventQueue.js

@@ -5,13 +5,11 @@ const pathLib = require("path");
 const cds = require("@sap/cds");
 
 const { getConfigInstance } = require("./config");
+const { TransactionMode } = require("./constants");
 const { limiter, Funnel } = require("./shared/common");
 
 const EventQueueBase = require("./EventQueueProcessorBase");
-const {
-  executeInNewTransaction,
-  TriggerRollback,
-} = require("./shared/cdsHelper");
+const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
 
 const COMPONENT_NAME = "eventQueue/processEventQueue";
 const MAX_EXECUTION_TIME = 5 * 60 * 1000;
@@ -21,47 +19,24 @@ const eventQueueRunner = async (context, events) => {
   const funnel = new Funnel();
   await Promise.allSettled(
     events.map((event) =>
-      funnel.run(event.load, async () =>
-        processEventQueue(context, event.type, event.subType, startTime)
-      )
+      funnel.run(event.load, async () => processEventQueue(context, event.type, event.subType, startTime))
     )
   );
 };
 
-const processEventQueue = async (
-  context,
-  eventType,
-  eventSubType,
-  startTime = new Date()
-) => {
+const processEventQueue = async (context, eventType, eventSubType, startTime = new Date()) => {
   let iterationCounter = 0;
   let shouldContinue = true;
   let baseInstance;
   try {
     let eventTypeInstance;
-    const eventConfig = getConfigInstance().getEventConfig(
-      eventType,
-      eventSubType
-    );
+    const eventConfig = getConfigInstance().getEventConfig(eventType, eventSubType);
     const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
-    if (
-      !eventConfig ||
-      err ||
-      !(typeof EventTypeClass.constructor === "function")
-    ) {
-      await EventQueueBase.handleMissingTypeImplementation(
-        context,
-        eventType,
-        eventSubType
-      );
+    if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
+      await EventQueueBase.handleMissingTypeImplementation(context, eventType, eventSubType);
       return;
     }
-    baseInstance = new EventTypeClass(
-      context,
-      eventType,
-      eventSubType,
-      eventConfig
-    );
+    baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
     const continueProcessing = await baseInstance.handleDistributedLock();
     if (!continueProcessing) {
       return;
@@ -69,118 +44,79 @@ const processEventQueue = async (
     eventConfig.startTime = startTime;
     while (shouldContinue) {
       iterationCounter++;
-      await executeInNewTransaction(
-        context,
-        `eventQueue-pre-processing-${eventType}##${eventSubType}`,
-        async (tx) => {
-          eventTypeInstance = new EventTypeClass(
-            tx.context,
-            eventType,
-            eventSubType,
-            eventConfig
-          );
-          const queueEntries =
-            await eventTypeInstance.getQueueEntriesAndSetToInProgress();
-          eventTypeInstance.startPerformanceTracerPreprocessing();
-          for (const queueEntry of queueEntries) {
-            try {
-              eventTypeInstance.modifyQueueEntry(queueEntry);
-              const payload =
-                await eventTypeInstance.checkEventAndGeneratePayload(
-                  queueEntry
-                );
-              if (payload === null) {
-                eventTypeInstance.setStatusToDone(queueEntry);
-                continue;
-              }
-              if (payload === undefined) {
-                eventTypeInstance.handleInvalidPayloadReturned(queueEntry);
-                continue;
-              }
-              eventTypeInstance.addEventWithPayloadForProcessing(
-                queueEntry,
-                payload
-              );
-            } catch (err) {
-              eventTypeInstance.handleErrorDuringProcessing(err, queueEntry);
+      await executeInNewTransaction(context, `eventQueue-pre-processing-${eventType}##${eventSubType}`, async (tx) => {
+        eventTypeInstance = new EventTypeClass(tx.context, eventType, eventSubType, eventConfig);
+        const queueEntries = await eventTypeInstance.getQueueEntriesAndSetToInProgress();
+        eventTypeInstance.startPerformanceTracerPreprocessing();
+        for (const queueEntry of queueEntries) {
+          try {
+            eventTypeInstance.modifyQueueEntry(queueEntry);
+            const payload = await eventTypeInstance.checkEventAndGeneratePayload(queueEntry);
+            if (payload === null) {
+              eventTypeInstance.setStatusToDone(queueEntry);
+              continue;
             }
+            if (payload === undefined) {
+              eventTypeInstance.handleInvalidPayloadReturned(queueEntry);
+              continue;
+            }
+            eventTypeInstance.addEventWithPayloadForProcessing(queueEntry, payload);
+          } catch (err) {
+            eventTypeInstance.handleErrorDuringProcessing(err, queueEntry);
           }
-          throw new TriggerRollback();
         }
-      );
-      eventTypeInstance.exceededEvents.length &&
-        (await executeInNewTransaction(
-          context,
-          `eventQueue-handleExceededEvents-${eventType}##${eventSubType}`,
-          async (tx) => {
-            eventTypeInstance.processEventContext = tx.context;
-            await eventTypeInstance.handleExceededEvents(
-              eventTypeInstance.exceededEvents
-            );
-          }
-        ));
+        throw new TriggerRollback();
+      });
+      await eventTypeInstance.handleExceededEvents();
       if (!eventTypeInstance) {
         return;
       }
       eventTypeInstance.endPerformanceTracerPreprocessing();
       if (Object.keys(eventTypeInstance.queueEntriesWithPayloadMap).length) {
+        await executeInNewTransaction(context, `eventQueue-processing-${eventType}##${eventSubType}`, async (tx) => {
+          eventTypeInstance.processEventContext = tx.context;
+          try {
+            eventTypeInstance.clusterQueueEntries();
+            await processEventMap(eventTypeInstance);
+          } catch (err) {
+            eventTypeInstance.handleErrorDuringClustering(err);
+          }
+          if (
+            eventTypeInstance.transactionMode !== TransactionMode.alwaysCommit ||
+            Object.entries(eventTypeInstance.eventProcessingMap).some(([key]) =>
+              eventTypeInstance.shouldRollbackTransaction(key)
+            )
+          ) {
+            throw new TriggerRollback();
+          }
+        });
+      }
+      await executeInNewTransaction(context, `eventQueue-persistStatus-${eventType}##${eventSubType}`, async (tx) => {
+        await eventTypeInstance.persistEventStatus(tx);
+      });
+      shouldContinue = reevaluateShouldContinue(eventTypeInstance, iterationCounter, startTime);
+      if (!shouldContinue) {
         await executeInNewTransaction(
           context,
-          `eventQueue-processing-${eventType}##${eventSubType}`,
+          `eventQueue-deleteFinishedEvents-${eventType}##${eventSubType}`,
           async (tx) => {
-            eventTypeInstance.processEventContext = tx.context;
-            try {
-              eventTypeInstance.clusterQueueEntries();
-              await processEventMap(eventTypeInstance);
-            } catch (err) {
-              eventTypeInstance.handleErrorDuringClustering(err);
-            }
-            if (
-              eventTypeInstance.shouldTriggerRollback ||
-              Object.entries(eventTypeInstance.eventProcessingMap).some(
-                ([key]) => eventTypeInstance.shouldRollbackTransaction(key)
-              )
-            ) {
-              throw new TriggerRollback();
-            }
+            await eventTypeInstance.deleteFinishedEvents(tx);
           }
         );
       }
-      await executeInNewTransaction(
-        context,
-        `eventQueue-persistStatus-${eventType}##${eventSubType}`,
-        async (tx) => {
-          await eventTypeInstance.persistEventStatus(tx);
-        }
-      );
-      shouldContinue = reevaluateShouldContinue(
-        eventTypeInstance,
-        iterationCounter,
-        startTime
-      );
     }
   } catch (err) {
-    cds
-      .log(COMPONENT_NAME)
-      .error(
-        "Processing event queue failed with unexpected error. Error:",
-        err,
-        {
-          eventType,
-          eventSubType,
-        }
-      );
+    cds.log(COMPONENT_NAME).error("Processing event queue failed with unexpected error. Error:", err, {
+      eventType,
+      eventSubType,
+    });
   } finally {
     await baseInstance?.handleReleaseLock();
   }
 };
 
-const reevaluateShouldContinue = (
-  eventTypeInstance,
-  iterationCounter,
-  startTime
-) => {
-  if (!eventTypeInstance.getSelectNextChunk()) {
+const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime) => {
+  if (!eventTypeInstance.selectNextChunk) {
     return false; // no select next chunk configured for this event
   }
   if (eventTypeInstance.emptyChunkSelected) {
@@ -209,13 +145,7 @@ const processEventMap = async (eventTypeInstance) => {
           eventTypeInstance.baseContext,
           `eventQueue-processEvent-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
           async (tx) => {
-            statusMap = await _processEvent(
-              eventTypeInstance,
-              tx.context,
-              key,
-              queueEntries,
-              payload
-            );
+            statusMap = await _processEvent(eventTypeInstance, tx.context, key, queueEntries, payload);
             if (
               eventTypeInstance.statusMapContainsError(statusMap) ||
               eventTypeInstance.shouldRollbackTransaction(key)
@@ -236,13 +166,7 @@ const processEventMap = async (eventTypeInstance) => {
           }
         );
       } else {
-        await _processEvent(
-          eventTypeInstance,
-          eventTypeInstance.context,
-          key,
-          queueEntries,
-          payload
-        );
+        await _processEvent(eventTypeInstance, eventTypeInstance.context, key, queueEntries, payload);
       }
     }
   ).finally(() => {
@@ -254,28 +178,15 @@ const processEventMap = async (eventTypeInstance) => {
   eventTypeInstance.endPerformanceTracerEvents();
 };
 
-const _processEvent = async (
-  eventTypeInstance,
-  processContext,
-  key,
-  queueEntries,
-  payload
-) => {
+const _processEvent = async (eventTypeInstance, processContext, key, queueEntries, payload) => {
   try {
     eventTypeInstance.logStartMessage(queueEntries);
-    const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(
-      queueEntries
-    );
+    const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(queueEntries);
     if (eventOutdated) {
       return;
     }
     eventTypeInstance.setTxForEventProcessing(key, cds.tx(processContext));
-    const statusTuple = await eventTypeInstance.processEvent(
-      processContext,
-      key,
-      queueEntries,
-      payload
-    );
+    const statusTuple = await eventTypeInstance.processEvent(processContext, key, queueEntries, payload);
     return eventTypeInstance.setEventStatus(queueEntries, statusTuple);
   } catch (err) {
     return eventTypeInstance.handleErrorDuringProcessing(err, queueEntries);

package/src/publishEvent.js

@@ -15,9 +15,7 @@ const publishEvent = async (tx, events) => {
       throw EventQueueError.unknownEventType(type, subType);
     }
   }
-  await tx.run(
-    INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing)
-  );
+  return await tx.run(INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing));
 };
 
 module.exports = {

package/src/redisPubSub.js

@@ -22,15 +22,11 @@ const initEventQueueRedisSubscribe = () => {
 
 const subscribeRedisClient = () => {
   const errorHandlerCreateClient = (err) => {
-    cds
-      .log(COMPONENT_NAME)
-      .error("error from redis client for pub/sub failed", err);
+    cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
     subscriberClientPromise = null;
     setTimeout(subscribeRedisClient, 5 * 1000).unref();
   };
-  subscriberClientPromise = redis.createClientAndConnect(
-    errorHandlerCreateClient
-  );
+  subscriberClientPromise = redis.createClientAndConnect(errorHandlerCreateClient);
   subscriberClientPromise
     .then((client) => {
       cds.log(COMPONENT_NAME).info("subscribe redis client connected");
@@ -39,10 +35,7 @@ const subscribeRedisClient = () => {
     .catch((err) => {
       cds
         .log(COMPONENT_NAME)
-        .error(
-          "error from redis client for pub/sub failed during startup - trying to reconnect",
-          err
-        );
+        .error("error from redis client for pub/sub failed during startup - trying to reconnect", err);
     });
 };
 
@@ -68,9 +61,7 @@ const messageHandlerProcessEvents = async (messageData) => {
     type,
     subType,
   });
-  getWorkerPoolInstance().addToQueue(async () =>
-    processEventQueue(context, type, subType)
-  );
+  getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
 };
 
 const publishEvent = async (tenantId, type, subType) => {
@@ -89,9 +80,7 @@ const publishEvent = async (tenantId, type, subType) => {
   };
   try {
     if (!publishClient) {
-      publishClient = await redis.createClientAndConnect(
-        errorHandlerCreateClient
-      );
+      publishClient = await redis.createClientAndConnect(errorHandlerCreateClient);
       logger.info("publish redis client connected");
     }
 
@@ -108,10 +97,7 @@ const publishEvent = async (tenantId, type, subType) => {
       type,
       subType,
     });
-    await publishClient.publish(
-      MESSAGE_CHANNEL,
-      JSON.stringify({ tenantId, type, subType })
-    );
+    await publishClient.publish(MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
   } catch (err) {
     logger.error(`publish event failed with error: ${err.toString()}`, {
       tenantId,