@cap-js-community/event-queue 1.2.4 → 1.2.6

package/README.md

@@ -28,7 +28,6 @@ For detailed information check out the [documentation](https://cap-js-community.
 {
   "cds": {
     "eventQueue": {
-      "plugin": true,
       "configFilePath": "./srv/eventQueueConfig.yml"
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [
@@ -49,15 +49,15 @@
   "devDependencies": {
     "@sap/cds": "^7.5.3",
     "@sap/cds-dk": "^7.5.1",
-    "eslint": "8.56.0",
-    "eslint-config-prettier": "9.1.0",
-    "eslint-plugin-jest": "27.6.3",
-    "eslint-plugin-node": "11.1.0",
-    "express": "4.18.2",
-    "hdb": "0.19.7",
-    "jest": "29.7.0",
-    "prettier": "2.8.8",
-    "sqlite3": "5.1.7"
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-jest": "^27.6.3",
+    "eslint-plugin-node": "^11.1.0",
+    "express": "^4.18.2",
+    "hdb": "^0.19.7",
+    "jest": "^29.7.0",
+    "prettier": "^2.8.8",
+    "sqlite3": "^5.1.7"
   },
   "homepage": "https://cap-js-community.github.io/event-queue/",
   "repository": {

package/src/EventQueueError.js

@@ -18,6 +18,7 @@ const ERROR_CODES = {
   NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
   LOAD_HIGHER_THAN_LIMIT: "LOAD_HIGHER_THAN_LIMIT",
   SCHEMA_TENANT_MISMATCH: "SCHEMA_TENANT_MISMATCH",
+  GLOBAL_CDS_CONTEXT_MISSMATCH: "GLOBAL_CDS_CONTEXT_MISSMATCH",
 };
 
 const ERROR_CODES_META = {
@@ -67,6 +68,9 @@ const ERROR_CODES_META = {
   [ERROR_CODES.SCHEMA_TENANT_MISMATCH]: {
     message: "The db client associated to the tenant context does not match! Processing will be skipped.",
   },
+  [ERROR_CODES.GLOBAL_CDS_CONTEXT_MISSMATCH]: {
+    message: "The global cds context does not match the local cds context.",
+  },
 };
 
 class EventQueueError extends VError {
@@ -235,6 +239,17 @@ class EventQueueError extends VError {
       message
     );
   }
+
+  static globalCdsContextNotMatchingLocal(globalProperties, localProperties) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.GLOBAL_CDS_CONTEXT_MISSMATCH];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.GLOBAL_CDS_CONTEXT_MISSMATCH,
+        info: { globalProperties, localProperties },
+      },
+      message
+    );
+  }
 }
 
 module.exports = EventQueueError;

package/src/EventQueueProcessorBase.js

@@ -70,6 +70,8 @@ class EventQueueProcessorBase {
     this.__txMap = {};
     this.__txRollback = {};
     this.__queueEntries = [];
+
+    this.#checkGlobalContextToLocalContext();
   }
 
   /**
@@ -533,7 +535,9 @@ class EventQueueProcessorBase {
   async getQueueEntriesAndSetToInProgress() {
     let result = [];
     const refDateStartAfter = new Date(Date.now() + this.#config.runInterval * 1.2);
+    this.#checkGlobalContextToLocalContext();
     await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
+      this.#checkGlobalContextToLocalContext();
       await this.checkTxConsistency(tx);
       const entries = await tx.run(
         SELECT.from(this.#config.tableNameEventQueue)
@@ -692,7 +696,7 @@ class EventQueueProcessorBase {
     } catch (err) {
       errorHandler(err);
     }
-    if (txSchema !== serviceManagerSchema) {
+    if (serviceManagerSchema && txSchema !== serviceManagerSchema) {
       const err = EventQueueError.dbClientSchemaMismatch(tx.context.tenant, txSchema, serviceManagerSchema);
       errorHandler(err);
       throw err;
@@ -724,6 +728,48 @@ class EventQueueProcessorBase {
     return entry.lastAttemptsTs;
   }
 
+  #checkGlobalContextToLocalContext() {
+    if (!this.#config.enableTxConsistencyCheck) {
+      return;
+    }
+    if (this.__context.tenant !== cds.context.tenant) {
+      throw EventQueueError.globalCdsContextNotMatchingLocal(
+        JSON.stringify(
+          {
+            correlationId: cds.context.id,
+            tenantId: cds.context.tenant,
+            timestamp: cds.context.timestamp,
+            base: JSON.stringify(
+              {
+                correlationId: cds.context.context?.id,
+                tenantId: cds.context.context?.tenant,
+                timestamp: cds.context.context?.timestamp,
+              },
+              null,
+              2
+            ),
+          },
+          null,
+          2
+        ),
+        JSON.stringify(
+          {
+            correlationId: this.__context.id,
+            tenantId: this.__context.tenant,
+            timestamp: this.__context.timestamp,
+            base: JSON.stringify({
+              correlationId: this.__context.context?.id,
+              tenantId: this.__context.context?.tenant,
+              timestamp: this.__context.context?.timestamp,
+            }),
+          },
+          null,
+          2
+        )
+      );
+    }
+  }
+
   #handleDelayedEvents(delayedEvents) {
     for (const delayedEvent of delayedEvents) {
       this.#eventSchedulerInstance.scheduleEvent(

package/src/config.js

@@ -57,6 +57,7 @@ class Config {
   #useAsCAPOutbox;
   #userId;
   #enableTxConsistencyCheck;
+  #cleanupLocksAndEventsForDev;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -340,6 +341,9 @@ class Config {
   }
 
   set runInterval(value) {
+    if (!Number.isInteger(value) || value <= 10 * 1000) {
+      throw EventQueueError.invalidInterval();
+    }
     this.#runInterval = value;
   }
 
@@ -471,6 +475,14 @@ class Config {
     return this.#enableTxConsistencyCheck;
   }
 
+  set cleanupLocksAndEventsForDev(value) {
+    this.#cleanupLocksAndEventsForDev = value;
+  }
+
+  get cleanupLocksAndEventsForDev() {
+    return this.#cleanupLocksAndEventsForDev;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }

package/src/initialize.js

@@ -14,6 +14,8 @@ const config = require("./config");
 const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
 const { closeMainClient } = require("./shared/redis");
 const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
+const { getAllTenantIds } = require("./shared/cdsHelper");
+const { EventProcessingStatus } = require("./constants");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -38,6 +40,7 @@ const CONFIG_VARS = [
   ["useAsCAPOutbox", false],
   ["userId", null],
   ["enableTxConsistencyCheck", false],
+  ["registerCleanupForDev", true],
 ];
 
 const initialize = async ({
@@ -55,11 +58,8 @@ const initialize = async ({
   useAsCAPOutbox,
   userId,
   enableTxConsistencyCheck,
+  cleanupLocksAndEventsForDev,
 } = {}) => {
-  // TODO: initialize check:
-  // - content of yaml check
-  // - betweenRuns
-
   if (config.initialized) {
     return;
   }
@@ -79,7 +79,8 @@ const initialize = async ({
     thresholdLoggingEventProcessing,
     useAsCAPOutbox,
     userId,
-    enableTxConsistencyCheck
+    enableTxConsistencyCheck,
+    cleanupLocksAndEventsForDev
   );
 
   const logger = cds.log(COMPONENT);
@@ -90,6 +91,7 @@ const initialize = async ({
     cds.on("connect", (service) => {
       if (service.name === "db") {
         dbHandler.registerEventQueueDbHandler(service);
+        config.cleanupLocksAndEventsForDev && registerCleanupForDevDb().catch(() => {});
       }
     });
   }
@@ -108,20 +110,28 @@ const initialize = async ({
 };
 
 const readConfigFromFile = async (configFilepath) => {
-  const fileData = await readFileAsync(configFilepath);
-  if (/\.ya?ml$/i.test(configFilepath)) {
-    return yaml.parse(fileData.toString());
-  }
-  if (/\.json$/i.test(configFilepath)) {
-    return JSON.parse(fileData.toString());
+  try {
+    const fileData = await readFileAsync(configFilepath);
+    if (/\.ya?ml$/i.test(configFilepath)) {
+      return yaml.parse(fileData.toString());
+    }
+    if (/\.json$/i.test(configFilepath)) {
+      return JSON.parse(fileData.toString());
+    }
+
+    throw new VError(
+      {
+        name: VERROR_CLUSTER_NAME,
+        info: { configFilepath },
+      },
+      "configFilepath with unsupported extension, allowed extensions are .yaml and .json"
+    );
+  } catch (err) {
+    if (config.useAsCAPOutbox) {
+      return {};
+    }
+    throw err;
   }
-  throw new VError(
-    {
-      name: VERROR_CLUSTER_NAME,
-      info: { configFilepath },
-    },
-    "configFilepath with unsupported extension, allowed extensions are .yaml and .json"
-  );
 };
 
 const registerEventProcessors = () => {
@@ -149,9 +159,11 @@ const monkeyPatchCAPOutbox = () => {
   if (config.useAsCAPOutbox) {
     Object.defineProperty(cds, "outboxed", {
       get: () => eventQueueAsOutbox.outboxed,
+      configurable: true,
     });
     Object.defineProperty(cds, "unboxed", {
       get: () => eventQueueAsOutbox.unboxed,
+      configurable: true,
     });
   }
 };
@@ -213,6 +225,25 @@ const registerCdsShutdown = () => {
   });
 };
 
+const registerCleanupForDevDb = async () => {
+  const profile = cds.env.profiles.find((profile) => profile === "development");
+  if (!profile) {
+    return;
+  }
+
+  const tenantIds = await getAllTenantIds();
+  for (const tenantId of tenantIds) {
+    await cds.tx({ tenant: tenantId }, async (tx) => {
+      await tx.run(DELETE.from(config.tableNameEventLock));
+      await tx.run(
+        UPDATE.entity(config.tableNameEventQueue).where({ status: EventProcessingStatus.InProgress }).set({
+          status: EventProcessingStatus.Error,
+        })
+      );
+    });
+  }
+};
+
 module.exports = {
   initialize,
 };

package/src/runner.js

@@ -26,9 +26,9 @@ let singleRunDone;
 
 const singleTenant = () => _scheduleFunction(_checkPeriodicEventsSingleTenant, _singleTenantDb);
 
-const multiTenancyDb = () => _scheduleFunction(_multiTenancyPeriodicEvents, _multiTenancyDb);
+const multiTenancyDb = () => _scheduleFunction(async () => {}, _multiTenancyDb);
 
-const multiTenancyRedis = () => _scheduleFunction(_multiTenancyPeriodicEvents, _multiTenancyRedis);
+const multiTenancyRedis = () => _scheduleFunction(async () => {}, _multiTenancyRedis);
 
 const _scheduleFunction = async (singleRunFn, periodicFn) => {
   const logger = cds.log(COMPONENT_NAME);
@@ -69,7 +69,7 @@ const _multiTenancyRedis = async () => {
   const emptyContext = new cds.EventContext({});
   logger.info("executing event queue run for multi instance and tenant");
   const tenantIds = await cdsHelper.getAllTenantIds();
-  _checkAndTriggerPeriodicEventUpdate(tenantIds);
+  await _checkPeriodicEventUpdate(tenantIds);
 
   const runId = await _acquireRunId(emptyContext);
 
@@ -78,22 +78,21 @@ const _multiTenancyRedis = async () => {
     return;
   }
 
-  return _executeEventsAllTenants(tenantIds, runId);
+  return await _executeEventsAllTenants(tenantIds, runId);
 };
 
-const _checkAndTriggerPeriodicEventUpdate = (tenantIds) => {
+const _checkPeriodicEventUpdate = async (tenantIds) => {
   const hash = hashStringTo32Bit(JSON.stringify(tenantIds));
   if (!tenantIdHash) {
     tenantIdHash = hash;
-    _multiTenancyPeriodicEvents().catch((err) => {
+    return await _multiTenancyPeriodicEvents(tenantIds).catch((err) => {
       cds.log(COMPONENT_NAME).error("Error during triggering updating periodic events!", err);
     });
-    return;
   }
   if (tenantIdHash && tenantIdHash !== hash) {
     tenantIdHash = hash;
     cds.log(COMPONENT_NAME).info("tenant id hash changed, triggering updating periodic events!");
-    _multiTenancyPeriodicEvents().catch((err) => {
+    return await _multiTenancyPeriodicEvents(tenantIds).catch((err) => {
       cds.log(COMPONENT_NAME).error("Error during triggering updating periodic events!", err);
     });
   }
@@ -108,91 +107,103 @@ const _executeEventsAllTenants = (tenantIds, runId) => {
     return result;
   }, []);
 
-  return product.map(async ([tenantId, event]) => {
-    const subdomain = await getSubdomainForTenantId(tenantId);
-    const user = new cds.User.Privileged(config.userId);
-    const tenantContext = {
-      tenant: tenantId,
-      user,
-      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-    };
-    return await cds.tx(tenantContext, async ({ context }) => {
+  return Promise.allSettled(
+    product.map(async ([tenantId, event]) => {
+      const subdomain = await getSubdomainForTenantId(tenantId);
+      const user = new cds.User.Privileged(config.userId);
+      const tenantContext = {
+        tenant: tenantId,
+        user,
+        // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+        http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+      };
       const label = `${event.type}_${event.subType}`;
       return await WorkerQueue.instance.addToQueue(event.load, label, async () => {
+        return await cds.tx(tenantContext, async ({ context }) => {
+          try {
+            const lockId = `${runId}_${label}`;
+            const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+              expiryTime: eventQueueConfig.runInterval * 0.95,
+            });
+            if (!couldAcquireLock) {
+              return;
+            }
+            await runEventCombinationForTenant(context, event.type, event.subType, true);
+          } catch (err) {
+            cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
+              tenantId,
+            });
+          }
+        });
+      });
+    })
+  );
+};
+
+const _executePeriodicEventsAllTenants = async (tenantIds, runId) => {
+  return await Promise.allSettled(
+    tenantIds.map(async (tenantId) => {
+      const label = `UPDATE_PERIODIC_EVENTS_${tenantId}`;
+      return await WorkerQueue.instance.addToQueue(1, label, async () => {
         try {
-          const lockId = `${runId}_${label}`;
-          const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-            expiryTime: eventQueueConfig.runInterval * 0.95,
+          const subdomain = await getSubdomainForTenantId(tenantId);
+          const user = new cds.User.Privileged(config.userId);
+          const tenantContext = {
+            tenant: tenantId,
+            user,
+            // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+            http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+          };
+
+          return await cds.tx(tenantContext, async ({ context }) => {
+            const couldAcquireLock = await distributedLock.acquireLock(context, runId, {
+              expiryTime: eventQueueConfig.runInterval * 0.95,
+            });
+            if (!couldAcquireLock) {
+              return;
+            }
+            await _checkPeriodicEventsSingleTenant(context);
           });
-          if (!couldAcquireLock) {
-            return;
-          }
-          await runEventCombinationForTenant(context, event.type, event.subType, true);
         } catch (err) {
           cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
             tenantId,
           });
         }
       });
-    });
-  });
+    })
+  );
 };
 
-const _executePeriodicEventsAllTenants = (tenantIds, runId) => {
-  tenantIds.forEach((tenantId) => {
-    const label = `UPDATE_PERIODIC_EVENTS_${tenantId}`;
-    WorkerQueue.instance.addToQueue(1, label, async () => {
-      try {
-        const subdomain = await getSubdomainForTenantId(tenantId);
-        const user = new cds.User.Privileged(config.userId);
-        const tenantContext = {
-          tenant: tenantId,
-          user,
-          // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-          http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-        };
-
+const _singleTenantDb = async (tenantId) => {
+  return Promise.allSettled(
+    eventQueueConfig.allEvents.map(async (event) => {
+      const label = `${event.type}_${event.subType}`;
+      const user = new cds.User.Privileged(config.userId);
+      const tenantContext = {
+        tenant: tenantId,
+        user,
+      };
+      return await WorkerQueue.instance.addToQueue(event.load, label, async () => {
         return await cds.tx(tenantContext, async ({ context }) => {
-          const couldAcquireLock = await distributedLock.acquireLock(context, runId, {
-            expiryTime: eventQueueConfig.runInterval * 0.95,
-          });
-          if (!couldAcquireLock) {
-            return;
+          try {
+            const lockId = `${EVENT_QUEUE_RUN_ID}_${label}`;
+            const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
+              expiryTime: eventQueueConfig.runInterval * 0.95,
+            });
+            if (!couldAcquireLock) {
+              return;
+            }
+            await runEventCombinationForTenant(context, event.type, event.subType, true);
+          } catch (err) {
+            cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
+              tenantId,
+              redisEnabled: eventQueueConfig.redisEnabled,
+            });
           }
-          await _checkPeriodicEventsSingleTenant(context);
         });
-      } catch (err) {
-        cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-          tenantId,
-        });
-      }
-    });
-  });
-};
-
-const _singleTenantDb = async (tenantId) => {
-  return eventQueueConfig.allEvents.map((event) => {
-    const label = `${event.type}_${event.subType}`;
-    return WorkerQueue.instance.addToQueue(event.load, label, async () => {
-      try {
-        const context = new cds.EventContext({ tenant: tenantId });
-        const lockId = `${EVENT_QUEUE_RUN_ID}_${label}`;
-        const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
-          expiryTime: eventQueueConfig.runInterval * 0.95,
-        });
-        if (!couldAcquireLock) {
-          return;
-        }
-        await runEventCombinationForTenant(context, event.type, event.subType, true);
-      } catch (err) {
-        cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
-          tenantId,
-          redisEnabled: eventQueueConfig.redisEnabled,
-        });
-      }
-    });
-  });
+      });
+    })
+  );
 };
 
 const _acquireRunId = async (context) => {
@@ -280,19 +291,19 @@ const _multiTenancyDb = async () => {
   try {
     logger.info("executing event queue run for single instance and multi tenant");
     const tenantIds = await cdsHelper.getAllTenantIds();
-    _checkAndTriggerPeriodicEventUpdate(tenantIds);
-    return _executeEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
+    await _checkPeriodicEventUpdate(tenantIds);
+    return await _executeEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
   } catch (err) {
     logger.error("Couldn't fetch tenant ids for event queue processing! Next try after defined interval.", err);
   }
 };
 
-const _multiTenancyPeriodicEvents = async () => {
+const _multiTenancyPeriodicEvents = async (tenantIds) => {
   const logger = cds.log(COMPONENT_NAME);
   try {
     logger.info("executing event queue update periodic events");
-    const tenantIds = await cdsHelper.getAllTenantIds();
-    _executePeriodicEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_PERIODIC_EVENT);
+    tenantIds = tenantIds ?? (await cdsHelper.getAllTenantIds());
+    return await _executePeriodicEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_PERIODIC_EVENT);
   } catch (err) {
     logger.error("Couldn't fetch tenant ids for updating periodic event processing!", err);
   }