@cap-js-community/event-queue 0.1.58 → 0.2.1

package/src/config.js

@@ -13,38 +13,56 @@ const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
 const COMPONENT_NAME = "eventQueue/config";
 
 class Config {
+  #logger;
+  #config;
+  #forUpdateTimeout;
+  #globalTxTimeout;
+  #runInterval;
+  #redisEnabled;
+  #initialized;
+  #parallelTenantProcessing;
+  #tableNameEventQueue;
+  #tableNameEventLock;
+  #isRunnerDeactivated;
+  #configFilePath;
+  #processEventsAfterPublish;
+  #skipCsnCheck;
+  #disableRedis;
+  #env;
+  #eventMap;
   constructor() {
-    this.__logger = cds.log(COMPONENT_NAME);
-    this.__config = null;
-    this.__forUpdateTimeout = FOR_UPDATE_TIMEOUT;
-    this.__globalTxTimeout = GLOBAL_TX_TIMEOUT;
-    this.__runInterval = null;
-    this.__redisEnabled = null;
-    this.__initialized = false;
-    this.__parallelTenantProcessing = null;
-    this.__tableNameEventQueue = null;
-    this.__tableNameEventLock = null;
-    this.__isRunnerDeactivated = false;
-    this.__configFilePath = null;
-    this.__processEventsAfterPublish = null;
-    this.__skipCsnCheck = null;
-    this.__env = getEnvInstance();
+    this.#logger = cds.log(COMPONENT_NAME);
+    this.#config = null;
+    this.#forUpdateTimeout = FOR_UPDATE_TIMEOUT;
+    this.#globalTxTimeout = GLOBAL_TX_TIMEOUT;
+    this.#runInterval = null;
+    this.#redisEnabled = null;
+    this.#initialized = false;
+    this.#parallelTenantProcessing = null;
+    this.#tableNameEventQueue = null;
+    this.#tableNameEventLock = null;
+    this.#isRunnerDeactivated = false;
+    this.#configFilePath = null;
+    this.#processEventsAfterPublish = null;
+    this.#skipCsnCheck = null;
+    this.#disableRedis = null;
+    this.#env = getEnvInstance();
   }
 
   getEventConfig(type, subType) {
-    return this.__eventMap[[type, subType].join("##")];
+    return this.#eventMap[[type, subType].join("##")];
   }
 
   hasEventAfterCommitFlag(type, subType) {
-    return this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
+    return this.#eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
   }
 
   _checkRedisIsBound() {
-    return !!this.__env.getRedisCredentialsFromEnv();
+    return !!this.#env.getRedisCredentialsFromEnv();
   }
 
   checkRedisEnabled() {
-    this.__redisEnabled = this._checkRedisIsBound() && this.__env.isOnCF;
+    this.#redisEnabled = !this.#disableRedis && this._checkRedisIsBound() && this.#env.isOnCF;
   }
 
   attachConfigChangeHandler() {
@@ -52,11 +70,11 @@ class Config {
       try {
         const { key, value } = JSON.parse(messageData);
         if (this[key] !== value) {
-          this.__logger.info("received config change", { key, value });
+          this.#logger.info("received config change", { key, value });
           this[key] = value;
         }
       } catch (err) {
-        this.__logger.error("could not parse event config change", {
+        this.#logger.error("could not parse event config change", {
           messageData,
         });
       }
@@ -65,124 +83,132 @@ class Config {
 
   publishConfigChange(key, value) {
     if (!this.redisEnabled) {
-      this.__logger.info("redis not connected, config change won't be published", { key, value });
+      this.#logger.info("redis not connected, config change won't be published", { key, value });
       return;
     }
     redis.publishMessage(REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
-      this.__logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
+      this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
     });
   }
 
   get isRunnerDeactivated() {
-    return this.__isRunnerDeactivated;
+    return this.#isRunnerDeactivated;
   }
 
   set isRunnerDeactivated(value) {
-    this.__isRunnerDeactivated = value;
+    this.#isRunnerDeactivated = value;
   }
 
   set fileContent(config) {
-    this.__config = config;
-    this.__eventMap = config.events.reduce((result, event) => {
+    this.#config = config;
+    this.#eventMap = config.events.reduce((result, event) => {
       result[[event.type, event.subType].join("##")] = event;
       return result;
     }, {});
   }
 
   get fileContent() {
-    return this.__config;
+    return this.#config;
   }
 
   get events() {
-    return this.__config.events;
+    return this.#config.events;
   }
 
   get forUpdateTimeout() {
-    return this.__forUpdateTimeout;
+    return this.#forUpdateTimeout;
   }
 
   get globalTxTimeout() {
-    return this.__globalTxTimeout;
+    return this.#globalTxTimeout;
   }
 
   set forUpdateTimeout(value) {
-    this.__forUpdateTimeout = value;
+    this.#forUpdateTimeout = value;
   }
 
   set globalTxTimeout(value) {
-    this.__globalTxTimeout = value;
+    this.#globalTxTimeout = value;
   }
 
   get runInterval() {
-    return this.__runInterval;
+    return this.#runInterval;
   }
 
   set runInterval(value) {
-    this.__runInterval = value;
+    this.#runInterval = value;
   }
 
   get redisEnabled() {
-    return this.__redisEnabled;
+    return this.#redisEnabled;
   }
 
   set redisEnabled(value) {
-    this.__redisEnabled = value;
+    this.#redisEnabled = value;
   }
 
   get initialized() {
-    return this.__initialized;
+    return this.#initialized;
   }
 
   set initialized(value) {
-    this.__initialized = value;
+    this.#initialized = value;
   }
 
   get parallelTenantProcessing() {
-    return this.__parallelTenantProcessing;
+    return this.#parallelTenantProcessing;
   }
 
   set parallelTenantProcessing(value) {
-    this.__parallelTenantProcessing = value;
+    this.#parallelTenantProcessing = value;
   }
 
   get tableNameEventQueue() {
-    return this.__tableNameEventQueue;
+    return this.#tableNameEventQueue;
   }
 
   set tableNameEventQueue(value) {
-    this.__tableNameEventQueue = value;
+    this.#tableNameEventQueue = value;
   }
 
   get tableNameEventLock() {
-    return this.__tableNameEventLock;
+    return this.#tableNameEventLock;
   }
 
   set tableNameEventLock(value) {
-    this.__tableNameEventLock = value;
+    this.#tableNameEventLock = value;
   }
 
   set configFilePath(value) {
-    this.__configFilePath = value;
+    this.#configFilePath = value;
   }
 
   get configFilePath() {
-    return this.__configFilePath;
+    return this.#configFilePath;
   }
 
   set processEventsAfterPublish(value) {
-    this.__processEventsAfterPublish = value;
+    this.#processEventsAfterPublish = value;
   }
 
   get processEventsAfterPublish() {
-    return this.__processEventsAfterPublish;
+    return this.#processEventsAfterPublish;
   }
 
   set skipCsnCheck(value) {
-    this.__skipCsnCheck = value;
+    this.#skipCsnCheck = value;
   }
 
   get skipCsnCheck() {
-    return this.__skipCsnCheck;
+    return this.#skipCsnCheck;
+  }
+
+  set disableRedis(value) {
+    this.#disableRedis = value;
+  }
+
+  get disableRedis() {
+    return this.#disableRedis;
   }
 
   get isMultiTenancy() {

package/src/dbHandler.js

@@ -1,8 +1,12 @@
 "use strict";
 
-const { publishEvent } = require("./redisPubSub");
+const cds = require("@sap/cds");
+
+const { broadcastEvent } = require("./redisPubSub");
 const config = require("./config");
 
+const COMPONENT_NAME = "eventQueue/dbHandler";
+
 const registerEventQueueDbHandler = (dbService) => {
   const configInstance = config.getConfigInstance();
   const def = dbService.model.definitions[configInstance.tableNameEventQueue];
@@ -26,7 +30,12 @@ const registerEventQueueDbHandler = (dbService) => {
     eventCombinations.length &&
       req.on("succeeded", () => {
         for (const eventCombination of eventCombinations) {
-          publishEvent(req.tenant, ...eventCombination.split("##"));
+          broadcastEvent(req.tenant, ...eventCombination.split("##")).catch((err) => {
+            cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
+              tenant: req.tenant,
+              eventCombination,
+            });
+          });
         }
       });
   });

package/src/initialize.js

@@ -5,7 +5,6 @@ const fs = require("fs");
 const path = require("path");
 
 const cds = require("@sap/cds");
-
 const yaml = require("yaml");
 const VError = require("verror");
 
@@ -13,7 +12,8 @@ const EventQueueError = require("./EventQueueError");
 const runner = require("./runner");
 const dbHandler = require("./dbHandler");
 const { getConfigInstance } = require("./config");
-const { initEventQueueRedisSubscribe } = require("./redisPubSub");
+const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
+const { closeMainClient } = require("./shared/redis");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -23,6 +23,18 @@ const BASE_TABLES = {
   EVENT: "sap.eventqueue.Event",
   LOCK: "sap.eventqueue.Lock",
 };
+const CONFIG_VARS = [
+  ["configFilePath", null],
+  ["registerAsEventProcessor", true],
+  ["processEventsAfterPublish", true],
+  ["isRunnerDeactivated", false],
+  ["runInterval", 5 * 60 * 1000],
+  ["parallelTenantProcessing", 5],
+  ["tableNameEventQueue", BASE_TABLES.EVENT],
+  ["tableNameEventLock", BASE_TABLES.LOCK],
+  ["disableRedis", false],
+  ["skipCsnCheck", false],
+];
 
 const initialize = async ({
   configFilePath,
@@ -33,6 +45,7 @@ const initialize = async ({
   parallelTenantProcessing,
   tableNameEventQueue,
   tableNameEventLock,
+  disableRedis,
   skipCsnCheck,
 } = {}) => {
   // TODO: initialize check:
@@ -54,6 +67,7 @@ const initialize = async ({
     parallelTenantProcessing,
     tableNameEventQueue,
     tableNameEventLock,
+    disableRedis,
     skipCsnCheck
   );
 
@@ -69,6 +83,7 @@ const initialize = async ({
   }
 
   registerEventProcessors();
+  registerCdsShutdown();
   logger.info("event queue initialized", {
     registerAsEventProcessor: configInstance.registerAsEventProcessor,
     multiTenancyEnabled: configInstance.isMultiTenancy,
@@ -160,32 +175,18 @@ const checkCustomTable = (baseCsn, customCsn) => {
   }
 };
 
-const mixConfigVarsWithEnv = (
-  configFilePath,
-  registerAsEventProcessor,
-  processEventsAfterPublish,
-  isRunnerDeactivated,
-  runInterval,
-  parallelTenantProcessing,
-  tableNameEventQueue,
-  tableNameEventLock,
-  skipCsnCheck
-) => {
+const mixConfigVarsWithEnv = (...args) => {
   const configInstance = getConfigInstance();
+  CONFIG_VARS.forEach(([configName, defaultValue], index) => {
+    const configValue = args[index];
+    configInstance[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
+  });
+};
 
-  configInstance.configFilePath = configFilePath ?? cds.env.eventQueue?.configFilePath;
-  configInstance.registerAsEventProcessor =
-    registerAsEventProcessor ?? cds.env.eventQueue?.registerAsEventProcessor ?? true;
-  configInstance.isRunnerDeactivated = isRunnerDeactivated ?? cds.env.eventQueue?.isRunnerDeactivated ?? false;
-  configInstance.processEventsAfterPublish =
-    processEventsAfterPublish ?? cds.env.eventQueue?.processEventsAfterPublish ?? true;
-  configInstance.runInterval = runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
-  configInstance.parallelTenantProcessing =
-    parallelTenantProcessing ?? cds.env.eventQueue?.parallelTenantProcessing ?? 5;
-  configInstance.tableNameEventQueue =
-    tableNameEventQueue ?? cds.env.eventQueue?.tableNameEventQueue ?? BASE_TABLES.EVENT;
-  configInstance.tableNameEventLock = tableNameEventLock ?? cds.env.eventQueue?.tableNameEventLock ?? BASE_TABLES.LOCK;
-  configInstance.skipCsnCheck = skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
+const registerCdsShutdown = () => {
+  cds.on("shutdown", async () => {
+    await Promise.allSettled([closeMainClient(), closeSubscribeClient()]);
+  });
 };
 
 module.exports = {

package/src/processEventQueue.js

@@ -8,7 +8,6 @@ const { getConfigInstance } = require("./config");
 const { TransactionMode } = require("./constants");
 const { limiter, Funnel } = require("./shared/common");
 
-const EventQueueBase = require("./EventQueueProcessorBase");
 const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
 
 const COMPONENT_NAME = "eventQueue/processEventQueue";
@@ -33,7 +32,10 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
     const eventConfig = getConfigInstance().getEventConfig(eventType, eventSubType);
     const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
     if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
-      await EventQueueBase.handleMissingTypeImplementation(context, eventType, eventSubType);
+      cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
+        eventType,
+        eventSubType,
+      });
       return;
     }
     baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);

package/src/publishEvent.js

@@ -1,19 +1,45 @@
 "use strict";
 
 const config = require("./config");
+const common = require("./shared/common");
 const EventQueueError = require("./EventQueueError");
 
+/**
+ * Asynchronously publishes a series of events to the event queue.
+ *
+ * @param {Transaction} tx - The transaction object to be used for database operations.
+ * @param {Array|Object} events - An array of event objects or a single event object. Each event object should match the Event table structure:
+ *   {
+ *     type: String, // Event type. This is a required field.
+ *     subType: String, // Event subtype. This is a required field.
+ *     referenceEntity: String, // Reference entity associated with the event.
+ *     referenceEntityKey: UUID, // UUID key of the reference entity.
+ *     status: Status, // Status of the event, defaults to 0.
+ *     payload: LargeString, // Payload of the event.
+ *     attempts: Integer, // The number of attempts made, defaults to 0.
+ *     lastAttemptTimestamp: Timestamp, // Timestamp of the last attempt.
+ *     createdAt: Timestamp, // Timestamp of event creation. This field is automatically set on insert.
+ *     startAfter: Timestamp, // Timestamp indicating when the event should start after.
+ *   }
+ * @throws {EventQueueError} Throws an error if the configuration is not initialized.
+ * @throws {EventQueueError} Throws an error if the event type is unknown.
+ * @throws {EventQueueError} Throws an error if the startAfter field is not a valid date.
+ * @returns {Promise} Returns a promise which resolves to the result of the database insert operation.
+ */
 const publishEvent = async (tx, events) => {
   const configInstance = config.getConfigInstance();
   if (!configInstance.initialized) {
     throw EventQueueError.notInitialized();
   }
   const eventsForProcessing = Array.isArray(events) ? events : [events];
-  for (const { type, subType } of eventsForProcessing) {
+  for (const { type, subType, startAfter } of eventsForProcessing) {
     const eventConfig = configInstance.getEventConfig(type, subType);
     if (!eventConfig) {
       throw EventQueueError.unknownEventType(type, subType);
     }
+    if (startAfter && !common.isValidDate(startAfter)) {
+      throw EventQueueError.malformedDate(startAfter);
+    }
   }
   return await tx.run(INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing));
 };

package/src/redisPubSub.js

@@ -1,11 +1,9 @@
 "use strict";
 
 const redis = require("./shared/redis");
-const { processEventQueue } = require("./processEventQueue");
-const { getSubdomainForTenantId } = require("./shared/cdsHelper");
 const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
 const config = require("./config");
-const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
+const { runEventCombinationForTenant } = require("./runner");
 
 const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
 const COMPONENT_NAME = "eventQueue/redisPubSub";
@@ -23,19 +21,12 @@ const messageHandlerProcessEvents = async (messageData) => {
   const logger = cds.log(COMPONENT_NAME);
   try {
     const { tenantId, type, subType } = JSON.parse(messageData);
-    const subdomain = await getSubdomainForTenantId(tenantId);
-    const context = new cds.EventContext({
-      tenant: tenantId,
-      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-    });
-    cds.context = context;
     logger.debug("received redis event", {
       tenantId,
       type,
       subType,
     });
-    getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
+    await runEventCombinationForTenant(tenantId, type, subType);
   } catch (err) {
     logger.error("could not parse event information", {
       messageData,
@@ -43,12 +34,12 @@ const messageHandlerProcessEvents = async (messageData) => {
   }
 };
 
-const publishEvent = async (tenantId, type, subType) => {
+const broadcastEvent = async (tenantId, type, subType) => {
   const logger = cds.log(COMPONENT_NAME);
   const configInstance = config.getConfigInstance();
   if (!configInstance.redisEnabled) {
     if (configInstance.registerAsEventProcessor) {
-      await _handleEventInternally(tenantId, type, subType);
+      await runEventCombinationForTenant(tenantId, type, subType);
     }
     return;
   }
@@ -76,22 +67,19 @@ const publishEvent = async (tenantId, type, subType) => {
   }
 };
 
-const _handleEventInternally = async (tenantId, type, subType) => {
-  cds.log(COMPONENT_NAME).info("processEventQueue internally", {
-    tenantId,
-    type,
-    subType,
-  });
-  const subdomain = await getSubdomainForTenantId(tenantId);
-  const context = new cds.EventContext({
-    tenant: tenantId,
-    // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-    http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-  });
-  getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
+const closeSubscribeClient = async () => {
+  try {
+    const client = await subscriberClientPromise;
+    if (client?.quit) {
+      await client.quit();
+    }
+  } catch (err) {
+    // ignore errors during shutdown
+  }
 };
 
 module.exports = {
   initEventQueueRedisSubscribe,
-  publishEvent,
+  broadcastEvent,
+  closeSubscribeClient,
 };

package/src/runner.js

@@ -3,11 +3,12 @@
 const { randomUUID } = require("crypto");
 
 const eventQueueConfig = require("./config");
-const { eventQueueRunner } = require("./processEventQueue");
+const { eventQueueRunner, processEventQueue } = require("./processEventQueue");
 const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
 const cdsHelper = require("./shared/cdsHelper");
 const distributedLock = require("./shared/distributedLock");
 const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
+const { getSubdomainForTenantId } = require("./shared/cdsHelper");
 
 const COMPONENT_NAME = "eventQueue/runner";
 const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
@@ -190,10 +191,31 @@ const _calculateOffsetForFirstRun = async () => {
   return offsetDependingOnLastRun;
 };
 
+const runEventCombinationForTenant = async (tenantId, type, subType) => {
+  try {
+    const subdomain = await getSubdomainForTenantId(tenantId);
+    const context = new cds.EventContext({
+      tenant: tenantId,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    });
+    cds.context = context;
+    getWorkerPoolInstance().addToQueue(async () => await processEventQueue(context, type, subType));
+  } catch (err) {
+    const logger = cds.log(COMPONENT_NAME);
+    logger.error("error executing event combination for tenant", err, {
+      tenantId,
+      type,
+      subType,
+    });
+  }
+};
+
 module.exports = {
   singleTenant,
   multiTenancyDb,
   multiTenancyRedis,
+  runEventCombinationForTenant,
   _: {
     _multiTenancyRedis,
     _multiTenancyDb,

package/src/shared/EventScheduler.js

@@ -0,0 +1,53 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const { broadcastEvent } = require("../redisPubSub");
+
+const COMPONENT_NAME = "eventQueue/shared/EventScheduler";
+
+let instance;
+class EventScheduler {
+  #scheduledEvents = {};
+  constructor() {}
+
+  scheduleEvent(tenantId, type, subType, startAfter) {
+    const startAfterSeconds = startAfter.getSeconds();
+    const secondsUntilNextTen = 10 - (startAfterSeconds % 10);
+    const roundUpDate = new Date(startAfter.getTime() + secondsUntilNextTen * 1000);
+    const key = [tenantId, type, subType, roundUpDate.toISOString()].join("##");
+    if (this.#scheduledEvents[key]) {
+      return; // event combination already scheduled
+    }
+    this.#scheduledEvents[key] = true;
+    cds.log(COMPONENT_NAME).info("scheduling event queue run for delayed event", {
+      type,
+      subType,
+      delaySeconds: (roundUpDate.getTime() - Date.now()) / 1000,
+    });
+    setTimeout(() => {
+      delete this.#scheduledEvents[key];
+      broadcastEvent(tenantId, type, subType).catch((err) => {
+        cds.log(COMPONENT_NAME).error("could not execute scheduled event", err, {
+          tenantId,
+          type,
+          subType,
+          scheduledFor: roundUpDate.toISOString(),
+        });
+      });
+    }, roundUpDate.getTime() - Date.now()).unref();
+  }
+
+  clearScheduledEvents() {
+    this.#scheduledEvents = {};
+  }
+}
+
+module.exports = {
+  getInstance: () => {
+    if (!instance) {
+      instance = new EventScheduler();
+    }
+    return instance;
+  },
+};

package/src/shared/common.js

@@ -107,4 +107,15 @@ const limiter = async (limit, payloads, iterator) => {
   return Promise.allSettled(returnPromises);
 };
 
-module.exports = { arrayToFlatMap, Funnel, limiter };
+const isValidDate = (value) => {
+  if (typeof value === "string") {
+    const date = Date.parse(value);
+    return !isNaN(date);
+  } else if (value instanceof Date) {
+    return !isNaN(value.getTime());
+  } else {
+    return false;
+  }
+};
+
+module.exports = { arrayToFlatMap, Funnel, limiter, isValidDate };