@cap-js-community/event-queue 1.9.0-beta.2 → 1.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.9.0-beta.2",
+  "version": "1.9.1",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -79,6 +79,17 @@
         "registerAsEventProcessor": false,
         "updatePeriodicEvents": false,
         "insertEventsBeforeCommit": false
+      },
+      "periodicEvents": {
+        "[production]": {
+          "EVENT_QUEUE_BASE/DELETE_EVENTS": {
+            "priority": "low",
+            "impl": "./housekeeping/EventQueueDeleteEvents",
+            "load": 20,
+            "interval": 86400,
+            "internalEvent": true
+          }
+        }
       }
     },
     "requires": {

package/src/EventQueueError.js

@@ -18,6 +18,8 @@ const ERROR_CODES = {
   NO_INTERVAL_OR_CRON: "NO_INTERVAL_OR_CRON",
   INTERVAL_AND_CRON: "INTERVAL_AND_CRON",
   MISSING_IMPL: "MISSING_IMPL",
+  MISSING_TYPE: "MISSING_TYPE",
+  MISSING_SUBTYPE: "MISSING_SUBTYPE",
   DUPLICATE_EVENT_REGISTRATION: "DUPLICATE_EVENT_REGISTRATION",
   NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
   LOAD_HIGHER_THAN_LIMIT: "LOAD_HIGHER_THAN_LIMIT",
@@ -62,6 +64,12 @@ const ERROR_CODES_META = {
   [ERROR_CODES.MISSING_IMPL]: {
     message: "Missing path to event class implementation.",
   },
+  [ERROR_CODES.MISSING_TYPE]: {
+    message: "Missing type event implementation.",
+  },
+  [ERROR_CODES.MISSING_SUBTYPE]: {
+    message: "Missing subtype event implementation.",
+  },
   [ERROR_CODES.DUPLICATE_EVENT_REGISTRATION]: {
     message: "Duplicate event registration, check the uniqueness of type and subType.",
   },
@@ -265,6 +273,28 @@ class EventQueueError extends VError {
     );
   }
 
+  static missingType(config) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.MISSING_TYPE];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.MISSING_TYPE,
+        info: { config },
+      },
+      message
+    );
+  }
+
+  static missingSubType(config) {
+    const { message } = ERROR_CODES_META[ERROR_CODES.MISSING_SUBTYPE];
+    return new EventQueueError(
+      {
+        name: ERROR_CODES.MISSING_SUBTYPE,
+        info: { config },
+      },
+      message
+    );
+  }
+
   static duplicateEventRegistration(type, subType) {
     const { message } = ERROR_CODES_META[ERROR_CODES.DUPLICATE_EVENT_REGISTRATION];
     return new EventQueueError(

package/src/EventQueueProcessorBase.js

@@ -929,6 +929,7 @@ class EventQueueProcessorBase {
               }
             });
           }
+          this.logger.info("keep alive finished!", { numberOfEvents: ids.length });
         });
       }).catch((err) => {
         this.logger.error("keep alive handling failed!", err);
@@ -971,7 +972,7 @@ class EventQueueProcessorBase {
       { expiryTime: this.#eventConfig.keepAliveMaxInProgressTime }
     );
     if (!lockAcquired) {
-      this.logger.error("renewing redis lock failed!", {
+      this.logger.error("renewing distributed lock failed!", {
         type: this.#eventType,
         subType: this.#eventSubType,
       });

package/src/config.js

@@ -30,18 +30,6 @@ const PRIORITIES = Object.values(Priorities);
 const UTC_DEFAULT = false;
 const USE_CRON_TZ_DEFAULT = true;
 
-const BASE_PERIODIC_EVENTS = [
-  {
-    type: "EVENT_QUEUE_BASE",
-    subType: "DELETE_EVENTS",
-    priority: Priorities.Low,
-    impl: "./housekeeping/EventQueueDeleteEvents",
-    load: 20,
-    interval: 86400, // 1 day,
-    internalEvent: true,
-  },
-];
-
 const BASE_TABLES = {
   EVENT: "sap.eventqueue.Event",
   LOCK: "sap.eventqueue.Lock",
@@ -82,6 +70,8 @@ class Config {
   #crashOnRedisUnavailable;
   #tenantIdFilterTokenInfoCb;
   #tenantIdFilterEventProcessingCb;
+  #configEvents;
+  #configPeriodicEvents;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -357,13 +347,33 @@ class Config {
     this.#isEventQueueActive = value;
   }
 
+  mixFileContentWithEnv(fileContent) {
+    fileContent.events ??= [];
+    fileContent.periodicEvents ??= [];
+    const events = this.#configEvents ?? {};
+    const periodicEvents = this.#configPeriodicEvents ?? {};
+    fileContent.events = fileContent.events.concat(this.#mapEnvEvents(events));
+    fileContent.periodicEvents = fileContent.periodicEvents.concat(this.#mapEnvEvents(periodicEvents));
+    this.fileContent = fileContent;
+  }
+
+  #mapEnvEvents(events) {
+    return Object.entries(events)
+      .map(([key, event]) => {
+        if (!event) {
+          return;
+        }
+        const [type, subType] = key.split("/");
+        event.type ??= type;
+        event.subType ??= subType;
+        return { ...event };
+      })
+      .filter((a) => a);
+  }
+
   set fileContent(config) {
-    this.#config = config;
     config.events = config.events ?? [];
-    const shouldIncludeBaseEvents = cds.env.profiles.includes("production") || cds.env.profiles.includes("test");
-    config.periodicEvents = (config.periodicEvents ?? []).concat(
-      (shouldIncludeBaseEvents ? BASE_PERIODIC_EVENTS : []).map((event) => ({ ...event }))
-    );
+    config.periodicEvents = config.periodicEvents ?? [];
     this.#eventMap = config.events.reduce((result, event) => {
       this.#basicEventTransformation(event);
       this.#validateAdHocEvents(result, event);
@@ -380,6 +390,7 @@ class Config {
       result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, this.#eventMap);
+    this.#config = config;
   }
 
   #basicEventTransformation(event) {
@@ -402,6 +413,14 @@ class Config {
       throw EventQueueError.missingImpl(event.type, event.subType);
     }
 
+    if (!event.type) {
+      throw EventQueueError.missingType(event);
+    }
+
+    if (!event.subType) {
+      throw EventQueueError.missingSubType(event);
+    }
+
     if (event.appNames) {
       if (!Array.isArray(event.appNames) || event.appNames.some((appName) => typeof appName !== "string")) {
         throw EventQueueError.appNamesFormat(event.type, event.subType, event.appNames);
@@ -506,6 +525,18 @@ class Config {
     return this.#config.events;
   }
 
+  set configEvents(value) {
+    this.#configEvents = JSON.parse(JSON.stringify(value));
+  }
+
+  get hasConfigEvents() {
+    return !!(Object.keys(this.#configEvents ?? {}).length || Object.keys(this.#configPeriodicEvents ?? {}).length);
+  }
+
+  set configPeriodicEvents(value) {
+    this.#configPeriodicEvents = JSON.parse(JSON.stringify(value));
+  }
+
   get periodicEvents() {
     return this.#config.periodicEvents;
   }

package/src/initialize.js

@@ -26,6 +26,8 @@ const TIMEOUT_SHUTDOWN = 2500;
 
 const CONFIG_VARS = [
   ["configFilePath", null],
+  ["events", null, "configEvents"],
+  ["periodicEvents", null, "configPeriodicEvents"],
   ["registerAsEventProcessor", true],
   ["processEventsAfterPublish", true],
   ["isEventQueueActive", true],
@@ -49,6 +51,8 @@ const CONFIG_VARS = [
  *
  * @param {Object} options - The configuration options.
  * @param {string} [options.configFilePath=null] - Path to the configuration file.
+ * @param {string} [options.events={}] - Options to allow events in the configuration.
+ * @param {string} [options.periodicEvents={}] - Options to allow periodicEvents in the configuration.
  * @param {boolean} [options.registerAsEventProcessor=true] - Register the instance as an event processor.
  * @param {boolean} [options.processEventsAfterPublish=true] - Process events immediately after publishing.
  * @param {boolean} [options.isEventQueueActive=true] - Flag to activate/deactivate the event queue.
@@ -97,7 +101,8 @@ const initialize = async (options = {}) => {
       throw EventQueueError.redisConnectionFailure();
     }
   }
-  config.fileContent = await readConfigFromFile(config.configFilePath);
+  const fileContent = await readConfigFromFile(config.configFilePath);
+  config.mixFileContentWithEnv(fileContent);
 
   monkeyPatchCAPOutbox();
   registerCdsShutdown();
@@ -129,7 +134,7 @@ const readConfigFromFile = async (configFilepath) => {
       "configFilepath with unsupported extension, allowed extensions are .yaml and .json"
     );
   } catch (err) {
-    if (config.useAsCAPOutbox) {
+    if (!configFilepath && (config.useAsCAPOutbox || config.hasConfigEvents)) {
       return {};
     }
     throw err;
@@ -178,9 +183,9 @@ const monkeyPatchCAPOutbox = () => {
 };
 
 const mixConfigVarsWithEnv = (options) => {
-  CONFIG_VARS.forEach(([configName, defaultValue]) => {
+  CONFIG_VARS.forEach(([configName, defaultValue, mappingName]) => {
     const configValue = options[configName];
-    config[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
+    config[mappingName ?? configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
   });
 };
 

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -16,7 +16,6 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
   }
 
   async processEvent(processContext, key, queueEntries, payload) {
-    let status = EventProcessingStatus.Done;
     try {
       const service = await cds.connect.to(this.eventSubType);
       const { useEventQueueUser } = this.eventConfig;
@@ -30,14 +29,37 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
         authInfo: await common.getTokenInfo(processContext.tenant),
       });
       processContext._eventQueue = { processor: this, key, queueEntries, payload };
-      await cds.unboxed(service).tx(processContext)[invocationFn](msg);
+      const result = await cds.unboxed(service).tx(processContext)[invocationFn](msg);
+      return this.#determineResultStatus(result, queueEntries);
     } catch (err) {
-      status = EventProcessingStatus.Error;
       this.logger.error("error processing outboxed service call", err, {
         serviceName: this.eventSubType,
       });
+      return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error]);
+    }
+  }
+
+  #determineResultStatus(result, queueEntries) {
+    const validStatusValues = Object.values(EventProcessingStatus);
+    const validStatus = validStatusValues.includes(result);
+    if (validStatus) {
+      return queueEntries.map((queueEntry) => [queueEntry.ID, result]);
+    }
+
+    if (!Array.isArray(result)) {
+      return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Done]);
+    }
+
+    const valid = !result.some((entry) => {
+      const [, status] = entry;
+      return !validStatusValues.includes(status);
+    });
+
+    if (valid) {
+      return result;
+    } else {
+      return queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Done]);
     }
-    return queueEntries.map((queueEntry) => [queueEntry.ID, status]);
   }
 }
 

package/src/periodicEvents.js

@@ -150,8 +150,14 @@ const _insertPeriodEvents = async (tx, events, now) => {
   processChunkedSync(eventsToBeInserted, CHUNK_SIZE_INSERT_PERIODIC_EVENTS, (chunk) => {
     logger.info(`${counter}/${chunks} | inserting chunk of changed or new periodic events`, {
       events: chunk.map(({ type, subType, startAfter }) => {
-        const { interval } = eventConfig.getEventConfig(type, subType);
-        return { type, subType, interval, ...(startAfter && { startAfter }) };
+        const { interval, cron } = eventConfig.getEventConfig(type, subType);
+        return {
+          type,
+          subType,
+          ...(startAfter && { startAfter }),
+          ...(interval && { interval }),
+          ...(cron && { cron }),
+        };
       }),
     });
     counter++;

package/src/processEventQueue.js

@@ -20,7 +20,7 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
   try {
     let eventTypeInstance;
     const eventConfig = config.getEventConfig(eventType, eventSubType);
-    const [err, EventTypeClass] = resilientRequire(eventConfig);
+    const [err, EventTypeClass] = await resilientRequire(eventConfig);
     if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
       cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
         eventType,
@@ -331,12 +331,24 @@ const _processEvent = async (eventTypeInstance, processContext, key, queueEntrie
   }
 };
 
-const resilientRequire = (eventConfig) => {
+const resilientRequire = async (eventConfig) => {
   try {
     const path = eventConfig?.impl;
     const internal = eventConfig?.internalEvent;
-    const module = require(pathLib.join(internal ? __dirname : process.cwd(), path));
-    return [null, module];
+    const filePath = pathLib.join(internal ? __dirname : process.cwd(), path);
+    const fileExtension = pathLib.extname(filePath);
+    switch (fileExtension) {
+      case ".js":
+        return [null, require(filePath)];
+      case ".mjs":
+        return [null, (await import(`file://${filePath}`)).default];
+      case "":
+        try {
+          return [null, require(filePath)];
+        } catch {
+          return [null, (await import(`file://${filePath}`)).default];
+        }
+    }
   } catch (err) {
     return [err, null];
   }

package/src/runner/runner.js

@@ -51,7 +51,10 @@ const _scheduleFunction = async (singleRunFn, periodicFn) => {
     }
     if (!singleRunDone) {
       singleRunDone = true;
-      singleRunFn().catch(() => (singleRunDone = false));
+      singleRunFn()
+        .then(periodicFn)
+        .catch(() => (singleRunDone = false));
+      return;
     }
     return periodicFn();
   };

package/src/shared/common.js

@@ -123,7 +123,6 @@ const getTokenInfo = async (tenantId) => {
   }
 
   if (!cds.requires?.auth.kind.match(/jwt|xsuaa/i)) {
-    cds.log(COMPONENT_NAME).warn("Only 'jwt' or 'xsuaa' are supported as values for auth.kind.");
     return null;
   }
 

package/src/shared/distributedLock.js

@@ -90,10 +90,20 @@ const _acquireLockRedis = async (
 
 const _renewLockRedis = async (context, fullKey, expiryTime, { value = "true" } = {}) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
-  const result = await client.set(fullKey, value, {
+  let result = await client.set(fullKey, value, {
     PX: Math.round(expiryTime),
     XX: true,
   });
+
+  if (result !== REDIS_COMMAND_OK) {
+    const readResult = await client.get(fullKey);
+    if (!readResult) {
+      result = await client.set(fullKey, value, {
+        PX: Math.round(expiryTime),
+      });
+    }
+  }
+
   return result === REDIS_COMMAND_OK;
 };