@cap-js-community/event-queue 0.2.1 → 0.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [

package/src/EventQueueError.js

@@ -12,6 +12,10 @@ const ERROR_CODES = {
   MISSING_ELEMENT_IN_TABLE: "MISSING_ELEMENT_IN_TABLE",
   TYPE_MISMATCH_TABLE: "TYPE_MISMATCH_TABLE",
   NO_VALID_DATE: "NO_VALID_DATE",
+  INVALID_INTERVAL: "INVALID_INTERVAL",
+  MISSING_IMPL: "MISSING_IMPL",
+  DUPLICATE_EVENT_REGISTRATION: "DUPLICATE_EVENT_REGISTRATION",
+  NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
 };
 
 const ERROR_CODES_META = {
@@ -43,6 +47,18 @@ const ERROR_CODES_META = {
   [ERROR_CODES.NO_VALID_DATE]: {
     message: "One or more events contain a date in a malformed format.",
   },
+  [ERROR_CODES.INVALID_INTERVAL]: {
+    message: "Invalid interval, the value needs to greater than 10 seconds.",
+  },
+  [ERROR_CODES.MISSING_IMPL]: {
+    message: "Missing path to event class implementation.",
+  },
+  [ERROR_CODES.DUPLICATE_EVENT_REGISTRATION]: {
+    message: "Duplicate event registration, check the uniqueness of type and subType.",
+  },
+  [ERROR_CODES.NO_MANUEL_INSERT_OF_PERIODIC]: {
+    message: "Periodic events are managed by the framework and are not allowed to insert manually.",
+  },
 };
 
 class EventQueueError extends VError {
@@ -146,6 +162,50 @@ class EventQueueError extends VError {
       message
     );
   }
+
+  static invalidInterval(type, subType, interval) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.INVALID_INTERVAL];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.INVALID_INTERVAL,
+        info: { type, subType, interval },
+      },
+      message
+    );
+  }
+
+  static missingImpl(type, subType) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.MISSING_IMPL];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.MISSING_IMPL,
+        info: { type, subType },
+      },
+      message
+    );
+  }
+
+  static duplicateEventRegistration(type, subType) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.DUPLICATE_EVENT_REGISTRATION];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.DUPLICATE_EVENT_REGISTRATION,
+        info: { type, subType },
+      },
+      message
+    );
+  }
+
+  static manuelPeriodicEventInsert(type, subType) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.NO_MANUEL_INSERT_OF_PERIODIC];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.NO_MANUEL_INSERT_OF_PERIODIC,
+        info: { type, subType },
+      },
+      message
+    );
+  }
 }
 
 module.exports = EventQueueError;

package/src/EventQueueProcessorBase.js

@@ -7,8 +7,8 @@ const { EventProcessingStatus, TransactionMode } = require("./constants");
 const distributedLock = require("./shared/distributedLock");
 const EventQueueError = require("./EventQueueError");
 const { arrayToFlatMap } = require("./shared/common");
-const eventScheduler = require("./shared/EventScheduler");
-const eventQueueConfig = require("./config");
+const eventScheduler = require("./shared/eventScheduler");
+const eventConfig = require("./config");
 const PerformanceTracer = require("./shared/PerformanceTracer");
 
 const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
@@ -31,37 +31,43 @@ class EventQueueProcessorBase {
   #eventType = null;
   #eventSubType = null;
   #config = null;
+  #eventSchedulerInstance = null;
+  #eventConfig;
+  #isPeriodic;
 
   constructor(context, eventType, eventSubType, config) {
     this.__context = context;
     this.__baseContext = context;
     this.__tx = cds.tx(context);
     this.__baseLogger = cds.log(COMPONENT_NAME);
+    this.#eventSchedulerInstance = eventScheduler.getInstance();
+    this.#config = eventConfig;
+    this.#isPeriodic = this.#config.isPeriodicEvent(eventType, eventSubType);
     this.__logger = null;
     this.__eventProcessingMap = {};
     this.__statusMap = {};
     this.__commitedStatusMap = {};
     this.#eventType = eventType;
     this.#eventSubType = eventSubType;
-    this.__eventConfig = config ?? {};
-    this.__parallelEventProcessing = this.__eventConfig.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
+    this.#eventConfig = config ?? {};
+    this.__parallelEventProcessing = this.#eventConfig.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
     if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
       this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
     }
     // NOTE: keep the feature, this might be needed again
     this.__concurrentEventProcessing = false;
-    this.__startTime = this.__eventConfig.startTime ?? new Date();
-    this.__retryAttempts = this.__eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
-    this.__selectMaxChunkSize = this.__eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
-    this.__selectNextChunk = !!this.__eventConfig.checkForNextChunk;
+    this.__startTime = this.#eventConfig.startTime ?? new Date();
+    this.__retryAttempts = this.#eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
+    this.__selectMaxChunkSize = this.#eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
+    this.__selectNextChunk = !!this.#eventConfig.checkForNextChunk;
     this.__keepalivePromises = {};
-    this.__outdatedCheckEnabled = this.__eventConfig.eventOutdatedCheck ?? true;
-    this.__transactionMode = this.__eventConfig.transactionMode ?? TransactionMode.isolated;
-    if (this.__eventConfig.deleteFinishedEventsAfterDays) {
+    this.__outdatedCheckEnabled = this.#eventConfig.eventOutdatedCheck ?? true;
+    this.__transactionMode = this.#eventConfig.transactionMode ?? TransactionMode.isolated;
+    if (this.#eventConfig.deleteFinishedEventsAfterDays) {
       this.__deleteFinishedEventsAfter =
-        Number.isInteger(this.__eventConfig.deleteFinishedEventsAfterDays) &&
-        this.__eventConfig.deleteFinishedEventsAfterDays > 0
-          ? this.__eventConfig.deleteFinishedEventsAfterDays
+        Number.isInteger(this.#eventConfig.deleteFinishedEventsAfterDays) &&
+        this.#eventConfig.deleteFinishedEventsAfterDays > 0
+          ? this.#eventConfig.deleteFinishedEventsAfterDays
           : DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
     } else {
       this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
@@ -71,7 +77,6 @@ class EventQueueProcessorBase {
     this.__txUsageAllowed = true;
     this.__txMap = {};
     this.__txRollback = {};
-    this.#config = eventQueueConfig.getConfigInstance();
     this.__queueEntries = [];
   }
 
@@ -303,6 +308,29 @@ class EventQueueProcessorBase {
     return Object.fromEntries(queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error]));
   }
 
+  handleErrorDuringPeriodicEventProcessing(error, queueEntry) {
+    this.logger.error(
+      `Caught error during event periodic processing. Please catch your promises/exceptions. Error: ${error}`,
+      {
+        eventType: this.#eventType,
+        eventSubType: this.#eventSubType,
+        queueEntryId: queueEntry.ID,
+      }
+    );
+  }
+
+  async setPeriodicEventStatus(queueEntryIds) {
+    await this.tx.run(
+      UPDATE.entity(this.#config.tableNameEventQueue)
+        .set({
+          status: EventProcessingStatus.Done,
+        })
+        .where({
+          ID: queueEntryIds,
+        })
+    );
+  }
+
   /**
    * This function validates for all selected events one status has been submitted. It's also validated that only for
    * selected events a status has been submitted. Persisting the status of events is done in a dedicated database tx.
@@ -587,9 +615,8 @@ class EventQueueProcessorBase {
   }
 
   #handleDelayedEvents(delayedEvents) {
-    const eventSchedulerInstance = eventScheduler.getInstance();
     for (const delayedEvent of delayedEvents) {
-      eventSchedulerInstance.scheduleEvent(
+      this.#eventSchedulerInstance.scheduleEvent(
         this.__context.tenant,
         this.#eventType,
         this.#eventSubType,
@@ -812,6 +839,77 @@ class EventQueueProcessorBase {
     }
   }
 
+  async scheduleNextPeriodEvent(queueEntry) {
+    const interval = this.#eventConfig.interval;
+    const newEvent = {
+      type: this.#eventType,
+      subType: this.#eventSubType,
+      startAfter: new Date(new Date(queueEntry.startAfter).getTime() + interval * 1000),
+    };
+    const { relative } = this.#eventSchedulerInstance.calculateOffset(
+      this.#eventType,
+      this.#eventSubType,
+      newEvent.startAfter
+    );
+
+    // more than one interval behind - shift tick to keep up
+    if (relative < 0 && Math.abs(relative) >= this.#eventConfig.interval * 1000) {
+      newEvent.startAfter = new Date(Date.now() + 5 * 1000);
+      this.logger.info("interval adjusted because shifted more than one interval", {
+        eventType: this.#eventType,
+        eventSubType: this.#eventSubType,
+        newStartAfter: newEvent.startAfter,
+      });
+    }
+
+    this.tx._skipEventQueueBroadcase = true;
+    await this.tx.run(INSERT.into(this.#config.tableNameEventQueue).entries({ ...newEvent }));
+    this.tx._skipEventQueueBroadcase = false;
+    if (interval < this.#config.runInterval) {
+      this.#handleDelayedEvents([newEvent]);
+      const { relative: relativeAfterSchedule } = this.#eventSchedulerInstance.calculateOffset(
+        this.#eventType,
+        this.#eventSubType,
+        newEvent.startAfter
+      );
+      // next tick is already behind schedule --> execute direct
+      if (relativeAfterSchedule <= 0) {
+        this.logger.info("running behind schedule - executing next tick immediately", {
+          eventType: this.#eventType,
+          eventSubType: this.#eventSubType,
+          newStartAfter: newEvent.startAfter,
+        });
+        return true;
+      }
+    }
+  }
+
+  async handleDuplicatedPeriodicEventEntry(queueEntries) {
+    this.logger.error("More than one open events for the same configuration which is not allowed!", {
+      eventType: this.#eventType,
+      eventSubType: this.#eventSubType,
+      queueEntriesIds: queueEntries.map(({ ID }) => ID),
+    });
+
+    let queueEntryToUse;
+    const obsoleteEntries = [];
+    for (const queueEntry of queueEntries) {
+      if (!queueEntryToUse) {
+        queueEntryToUse = queueEntry;
+        continue;
+      }
+
+      if (queueEntryToUse.startAfter <= queueEntry.queueEntry) {
+        obsoleteEntries.push(queueEntryToUse);
+        queueEntryToUse = queueEntry;
+      } else {
+        obsoleteEntries.push(queueEntry);
+      }
+    }
+    await this.setPeriodicEventStatus(obsoleteEntries.map(({ ID }) => ID));
+    return queueEntryToUse;
+  }
+
   statusMapContainsError(statusMap) {
     return Object.values(statusMap).includes(EventProcessingStatus.Error);
   }
@@ -920,6 +1018,10 @@ class EventQueueProcessorBase {
   setTxForEventProcessing(key, tx) {
     this.__txMap[key] = tx;
   }
+
+  get isPeriodicEvent() {
+    return this.#eventConfig.isPeriodic;
+  }
 }
 
 module.exports = EventQueueProcessorBase;

package/src/config.js

@@ -4,13 +4,13 @@ const cds = require("@sap/cds");
 
 const { getEnvInstance } = require("./shared/env");
 const redis = require("./shared/redis");
-
-let instance;
+const EventQueueError = require("./EventQueueError");
 
 const FOR_UPDATE_TIMEOUT = 10;
 const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
 const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
 const COMPONENT_NAME = "eventQueue/config";
+const MIN_INTERVAL_SEC = 10;
 
 class Config {
   #logger;
@@ -30,6 +30,8 @@ class Config {
   #disableRedis;
   #env;
   #eventMap;
+  #updatePeriodicEvents;
+  static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
     this.#config = null;
@@ -50,11 +52,11 @@ class Config {
   }
 
   getEventConfig(type, subType) {
-    return this.#eventMap[[type, subType].join("##")];
+    return this.#eventMap[this.generateKey(type, subType)];
   }
 
   hasEventAfterCommitFlag(type, subType) {
-    return this.#eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
+    return this.#eventMap[this.generateKey(type, subType)]?.processAfterCommit ?? true;
   }
 
   _checkRedisIsBound() {
@@ -101,10 +103,51 @@ class Config {
 
   set fileContent(config) {
     this.#config = config;
+    config.events = config.events ?? [];
+    config.periodicEvents = config.periodicEvents ?? [];
     this.#eventMap = config.events.reduce((result, event) => {
+      this.validateAdHocEvents(result, event);
       result[[event.type, event.subType].join("##")] = event;
       return result;
     }, {});
+    this.#eventMap = config.periodicEvents.reduce((result, event) => {
+      const SUFFIX_PERIODIC = "_PERIODIC";
+      event.type = `${event.type}${SUFFIX_PERIODIC}`;
+      event.isPeriodic = true;
+      this.validatePeriodicConfig(result, event);
+      result[[event.type, event.subType].join("##")] = event;
+      return result;
+    }, this.#eventMap);
+  }
+
+  validatePeriodicConfig(eventMap, config) {
+    const key = this.generateKey(config.type, config.subType);
+    if (eventMap[key] && eventMap[key].isPeriodic) {
+      throw EventQueueError.duplicateEventRegistration(config.type, config.subType);
+    }
+
+    if (!config.interval || config.interval <= MIN_INTERVAL_SEC) {
+      throw EventQueueError.invalidInterval(config.type, config.subType, config.interval);
+    }
+
+    if (!config.impl) {
+      throw EventQueueError.missingImpl(config.type, config.subType);
+    }
+  }
+
+  validateAdHocEvents(eventMap, config) {
+    const key = this.generateKey(config.type, config.subType);
+    if (eventMap[key] && !eventMap[key].isPeriodic) {
+      throw EventQueueError.duplicateEventRegistration(config.type, config.subType);
+    }
+
+    if (!config.impl) {
+      throw EventQueueError.missingImpl(config.type, config.subType);
+    }
+  }
+
+  generateKey(type, subType) {
+    return [type, subType].join("##");
   }
 
   get fileContent() {
@@ -115,6 +158,18 @@ class Config {
     return this.#config.events;
   }
 
+  get periodicEvents() {
+    return this.#config.periodicEvents;
+  }
+
+  isPeriodicEvent(type, subType) {
+    return this.#eventMap[this.generateKey(type, subType)]?.isPeriodic;
+  }
+
+  get allEvents() {
+    return this.#config.events.concat(this.#config.periodicEvents);
+  }
+
   get forUpdateTimeout() {
     return this.#forUpdateTimeout;
   }
@@ -211,18 +266,29 @@ class Config {
     return this.#disableRedis;
   }
 
+  set updatePeriodicEvents(value) {
+    this.#updatePeriodicEvents = value;
+  }
+
+  get updatePeriodicEvents() {
+    return this.#updatePeriodicEvents;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }
-}
 
-const getConfigInstance = () => {
-  if (!instance) {
-    instance = new Config();
+  /**
+    @return { Config }
+  **/
+  static get instance() {
+    if (!Config.#instance) {
+      Config.#instance = new Config();
+    }
+    return Config.#instance;
   }
-  return instance;
-};
+}
+
+const instance = Config.instance;
 
-module.exports = {
-  getConfigInstance,
-};
+module.exports = instance;

package/src/dbHandler.js

@@ -8,9 +8,11 @@ const config = require("./config");
 const COMPONENT_NAME = "eventQueue/dbHandler";
 
 const registerEventQueueDbHandler = (dbService) => {
-  const configInstance = config.getConfigInstance();
-  const def = dbService.model.definitions[configInstance.tableNameEventQueue];
+  const def = dbService.model.definitions[config.tableNameEventQueue];
   dbService.after("CREATE", def, (_, req) => {
+    if (req.tx._skipEventQueueBroadcase) {
+      return;
+    }
     req.tx._ = req.tx._ ?? {};
     req.tx._.eventQueuePublishEvents = req.tx._.eventQueuePublishEvents ?? {};
     const eventQueuePublishEvents = req.tx._.eventQueuePublishEvents;
@@ -18,7 +20,7 @@ const registerEventQueueDbHandler = (dbService) => {
     const eventCombinations = Object.keys(
       data.reduce((result, event) => {
         const key = [event.type, event.subType].join("##");
-        if (!configInstance.hasEventAfterCommitFlag(event.type, event.subType) || eventQueuePublishEvents[key]) {
+        if (!config.hasEventAfterCommitFlag(event.type, event.subType) || eventQueuePublishEvents[key]) {
           return result;
         }
         eventQueuePublishEvents[key] = true;

package/src/index.js

@@ -11,7 +11,7 @@
 
 module.exports = {
   ...require("./initialize"),
-  ...require("./config"),
+  config: require("./config"),
   ...require("./processEventQueue"),
   ...require("./dbHandler"),
   ...require("./constants"),

package/src/initialize.js

@@ -11,7 +11,7 @@ const VError = require("verror");
 const EventQueueError = require("./EventQueueError");
 const runner = require("./runner");
 const dbHandler = require("./dbHandler");
-const { getConfigInstance } = require("./config");
+const config = require("./config");
 const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
 const { closeMainClient } = require("./shared/redis");
 
@@ -34,6 +34,7 @@ const CONFIG_VARS = [
   ["tableNameEventLock", BASE_TABLES.LOCK],
   ["disableRedis", false],
   ["skipCsnCheck", false],
+  ["updatePeriodicEvents", true],
 ];
 
 const initialize = async ({
@@ -47,16 +48,16 @@ const initialize = async ({
   tableNameEventLock,
   disableRedis,
   skipCsnCheck,
+  updatePeriodicEvents,
 } = {}) => {
   // TODO: initialize check:
   // - content of yaml check
   // - betweenRuns and parallelTenantProcessing
 
-  const configInstance = getConfigInstance();
-  if (configInstance.initialized) {
+  if (config.initialized) {
     return;
   }
-  configInstance.initialized = true;
+  config.initialized = true;
 
   mixConfigVarsWithEnv(
     configFilePath,
@@ -68,28 +69,29 @@ const initialize = async ({
     tableNameEventQueue,
     tableNameEventLock,
     disableRedis,
-    skipCsnCheck
+    skipCsnCheck,
+    updatePeriodicEvents
   );
 
   const logger = cds.log(COMPONENT);
-  configInstance.fileContent = await readConfigFromFile(configInstance.configFilePath);
-  configInstance.checkRedisEnabled();
+  config.fileContent = await readConfigFromFile(config.configFilePath);
+  config.checkRedisEnabled();
 
   const dbService = await cds.connect.to("db");
   await (cds.model ? Promise.resolve() : new Promise((resolve) => cds.on("serving", resolve)));
-  !configInstance.skipCsnCheck && (await csnCheck());
-  if (configInstance.processEventsAfterPublish) {
+  !config.skipCsnCheck && (await csnCheck());
+  if (config.processEventsAfterPublish) {
     dbHandler.registerEventQueueDbHandler(dbService);
   }
 
   registerEventProcessors();
   registerCdsShutdown();
   logger.info("event queue initialized", {
-    registerAsEventProcessor: configInstance.registerAsEventProcessor,
-    multiTenancyEnabled: configInstance.isMultiTenancy,
-    redisEnabled: configInstance.redisEnabled,
-    runInterval: configInstance.runInterval,
-    parallelTenantProcessing: configInstance.parallelTenantProcessing,
+    registerAsEventProcessor: config.registerAsEventProcessor,
+    multiTenancyEnabled: config.isMultiTenancy,
+    redisEnabled: config.redisEnabled,
+    runInterval: config.runInterval,
+    config: config.parallelTenantProcessing,
   });
 };
 
@@ -111,42 +113,38 @@ const readConfigFromFile = async (configFilepath) => {
 };
 
 const registerEventProcessors = () => {
-  const configInstance = getConfigInstance();
-
-  if (!configInstance.registerAsEventProcessor) {
+  if (!config.registerAsEventProcessor) {
     return;
   }
 
-  if (!configInstance.isMultiTenancy) {
-    runner.singleTenant();
+  const errorHandler = (err) => cds.log(COMPONENT).error("error during init runner", err);
+
+  if (!config.isMultiTenancy) {
+    runner.singleTenant().catch(errorHandler);
     return;
   }
 
-  if (configInstance.redisEnabled) {
+  if (config.redisEnabled) {
     initEventQueueRedisSubscribe();
-    configInstance.attachConfigChangeHandler();
-    runner.multiTenancyRedis();
+    config.attachConfigChangeHandler();
+    runner.multiTenancyRedis().catch(errorHandler);
   } else {
-    runner.multiTenancyDb();
+    runner.multiTenancyDb().catch(errorHandler);
   }
 };
 
 const csnCheck = async () => {
-  const configInstance = getConfigInstance();
-  const eventCsn = cds.model.definitions[configInstance.tableNameEventQueue];
+  const eventCsn = cds.model.definitions[config.tableNameEventQueue];
   if (!eventCsn) {
-    throw EventQueueError.missingTableInCsn(configInstance.tableNameEventQueue);
+    throw EventQueueError.missingTableInCsn(config.tableNameEventQueue);
   }
 
-  const lockCsn = cds.model.definitions[configInstance.tableNameEventLock];
+  const lockCsn = cds.model.definitions[config.tableNameEventLock];
   if (!lockCsn) {
-    throw EventQueueError.missingTableInCsn(configInstance.tableNameEventLock);
+    throw EventQueueError.missingTableInCsn(config.tableNameEventLock);
   }
 
-  if (
-    configInstance.tableNameEventQueue === BASE_TABLES.EVENT &&
-    configInstance.tableNameEventLock === BASE_TABLES.LOCK
-  ) {
+  if (config.tableNameEventQueue === BASE_TABLES.EVENT && config.tableNameEventLock === BASE_TABLES.LOCK) {
     return; // no need to check base tables
   }
 
@@ -176,10 +174,9 @@ const checkCustomTable = (baseCsn, customCsn) => {
 };
 
 const mixConfigVarsWithEnv = (...args) => {
-  const configInstance = getConfigInstance();
   CONFIG_VARS.forEach(([configName, defaultValue], index) => {
     const configValue = args[index];
-    configInstance[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
+    config[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
   });
 };