@cap-js-community/event-queue 1.0.3 → 1.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "files": [
@@ -47,17 +47,17 @@
     "yaml": "2.3.4"
   },
   "devDependencies": {
-    "@sap/cds": "7.5.1",
-    "@sap/cds-dk": "7.5.0",
+    "@sap/cds": "7.5.2",
+    "@sap/cds-dk": "7.5.1",
     "eslint": "8.56.0",
     "eslint-config-prettier": "9.1.0",
-    "eslint-plugin-jest": "27.6.1",
+    "eslint-plugin-jest": "27.6.2",
     "eslint-plugin-node": "11.1.0",
     "express": "4.18.2",
     "hdb": "0.19.7",
     "jest": "29.7.0",
     "prettier": "2.8.8",
-    "sqlite3": "5.1.7-rc.0"
+    "sqlite3": "5.1.7"
   },
   "homepage": "https://cap-js-community.github.io/event-queue/",
   "repository": {

package/src/EventQueueProcessorBase.js

@@ -12,7 +12,7 @@ const eventConfig = require("./config");
 const PerformanceTracer = require("./shared/PerformanceTracer");
 
 const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
-const COMPONENT_NAME = "eventQueue/EventQueueProcessorBase";
+const COMPONENT_NAME = "/eventQueue/EventQueueProcessorBase";
 
 const DEFAULT_RETRY_ATTEMPTS = 3;
 const DEFAULT_PARALLEL_EVENT_PROCESSING = 1;
@@ -869,7 +869,7 @@ class EventQueueProcessorBase {
     return await checkAndUpdatePromise;
   }
 
-  async handleDistributedLock() {
+  async acquireDistributedLock() {
     if (this.concurrentEventProcessing) {
       return true;
     }

package/src/config.js

@@ -10,12 +10,15 @@ const FOR_UPDATE_TIMEOUT = 10;
 const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
 const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
 const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
-const COMPONENT_NAME = "eventQueue/config";
+const COMPONENT_NAME = "/eventQueue/config";
 const MIN_INTERVAL_SEC = 10;
 const DEFAULT_LOAD = 1;
 const SUFFIX_PERIODIC = "_PERIODIC";
 const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
 const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
+const CAP_EVENT_TYPE = "CAP_OUTBOX";
+
+const CAP_PARALLEL_DEFAULT = 5;
 
 const BASE_PERIODIC_EVENTS = [
   {
@@ -51,6 +54,7 @@ class Config {
   #blockedPeriodicEvents;
   #isPeriodicEventBlockedCb;
   #thresholdLoggingEventProcessing;
+  #useAsCAPOutbox;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -76,6 +80,10 @@ class Config {
     return this.#eventMap[this.generateKey(type, subType)];
   }
 
+  isCapOutboxEvent(type) {
+    return type === CAP_EVENT_TYPE;
+  }
+
   hasEventAfterCommitFlag(type, subType) {
     return this.#eventMap[this.generateKey(type, subType)]?.processAfterCommit ?? true;
   }
@@ -180,6 +188,30 @@ class Config {
       });
   }
 
+  addCAPOutboxEvent(serviceName, config) {
+    if (this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)]) {
+      return;
+    }
+
+    const eventConfig = {
+      type: CAP_EVENT_TYPE,
+      subType: serviceName,
+      load: config.load ?? DEFAULT_LOAD,
+      impl: "./outbox/EventQueueGenericOutboxHandler",
+      selectMaxChunkSize: config.chunkSize,
+      parallelEventProcessing: config.parallelEventProcessing ?? (config.parallel && CAP_PARALLEL_DEFAULT),
+      retryAttempts: config.maxAttempts,
+      transactionMode: config.transactionMode,
+      processAfterCommit: config.processAfterCommit,
+      eventOutdatedCheck: config.eventOutdatedCheck,
+      checkForNextChunk: config.checkForNextChunk,
+      deleteFinishedEventsAfterDays: config.deleteFinishedEventsAfterDays,
+      internalEvent: true,
+    };
+    this.#config.events.push(eventConfig);
+    this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
+  }
+
   #unblockPeriodicEventLocalState(key, tenant) {
     const map = this.#blockedPeriodicEvents[key];
     if (!map) {
@@ -214,7 +246,7 @@ class Config {
     this.#eventMap = config.events.reduce((result, event) => {
       event.load = event.load ?? DEFAULT_LOAD;
       this.validateAdHocEvents(result, event);
-      result[[event.type, event.subType].join("##")] = event;
+      result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, {});
     this.#eventMap = config.periodicEvents.reduce((result, event) => {
@@ -222,7 +254,7 @@ class Config {
       event.type = `${event.type}${SUFFIX_PERIODIC}`;
       event.isPeriodic = true;
       this.validatePeriodicConfig(result, event);
-      result[[event.type, event.subType].join("##")] = event;
+      result[this.generateKey(event.type, event.subType)] = event;
       return result;
     }, this.#eventMap);
   }
@@ -257,6 +289,14 @@ class Config {
     return [type, subType].join("##");
   }
 
+  removeEvent(type, subType) {
+    const index = this.#config.events.findIndex((event) => event.type === "CAP_OUTBOX");
+    if (index >= 0) {
+      this.#config.events.splice(index, 1);
+    }
+    delete this.#eventMap[this.generateKey(type, subType)];
+  }
+
   get fileContent() {
     return this.#config;
   }
@@ -405,6 +445,14 @@ class Config {
     return this.#thresholdLoggingEventProcessing;
   }
 
+  set useAsCAPOutbox(value) {
+    this.#useAsCAPOutbox = value;
+  }
+
+  get useAsCAPOutbox() {
+    return this.#useAsCAPOutbox;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }

package/src/dbHandler.js

@@ -5,7 +5,7 @@ const cds = require("@sap/cds");
 const { broadcastEvent } = require("./redisPubSub");
 const config = require("./config");
 
-const COMPONENT_NAME = "eventQueue/dbHandler";
+const COMPONENT_NAME = "/eventQueue/dbHandler";
 
 const registerEventQueueDbHandler = (dbService) => {
   const def = dbService.model.definitions[config.tableNameEventQueue];

package/src/initialize.js

@@ -14,6 +14,7 @@ const dbHandler = require("./dbHandler");
 const config = require("./config");
 const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
 const { closeMainClient } = require("./shared/redis");
+const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -35,6 +36,7 @@ const CONFIG_VARS = [
   ["skipCsnCheck", false],
   ["updatePeriodicEvents", true],
   ["thresholdLoggingEventProcessing", 50],
+  ["useAsCAPOutbox", false],
 ];
 
 const initialize = async ({
@@ -49,6 +51,7 @@ const initialize = async ({
   skipCsnCheck,
   updatePeriodicEvents,
   thresholdLoggingEventProcessing,
+  useAsCAPOutbox,
 } = {}) => {
   // TODO: initialize check:
   // - content of yaml check
@@ -70,7 +73,8 @@ const initialize = async ({
     disableRedis,
     skipCsnCheck,
     updatePeriodicEvents,
-    thresholdLoggingEventProcessing
+    thresholdLoggingEventProcessing,
+    useAsCAPOutbox
   );
 
   const logger = cds.log(COMPONENT);
@@ -84,6 +88,7 @@ const initialize = async ({
     dbHandler.registerEventQueueDbHandler(dbService);
   }
 
+  monkeyPatchCAPOutbox();
   registerEventProcessors();
   registerCdsShutdown();
   logger.info("event queue initialized", {
@@ -133,6 +138,17 @@ const registerEventProcessors = () => {
   }
 };
 
+const monkeyPatchCAPOutbox = () => {
+  if (config.useAsCAPOutbox) {
+    Object.defineProperty(cds, "outboxed", {
+      get: () => eventQueueAsOutbox.outboxed,
+    });
+    Object.defineProperty(cds, "unboxed", {
+      get: () => eventQueueAsOutbox.unboxed,
+    });
+  }
+};
+
 const csnCheck = async () => {
   const eventCsn = cds.model.definitions[config.tableNameEventQueue];
   if (!eventCsn) {

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -0,0 +1,37 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const EventQueueBaseClass = require("../EventQueueProcessorBase");
+const { EventProcessingStatus } = require("../constants");
+
+const COMPONENT_NAME = "/eventQueue/outbox/generic";
+
+class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
+  constructor(context, eventType, eventSubType, config) {
+    super(context, eventType, eventSubType, config);
+    this.logger = cds.log(`${COMPONENT_NAME}/${eventSubType}`);
+  }
+
+  async processEvent(processContext, key, queueEntries, payload) {
+    let status = EventProcessingStatus.Done;
+    try {
+      const service = await cds.connect.to(this.eventSubType);
+      const userId = payload.contextUser;
+      const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
+      const invocationFn = payload._fromSend ? "send" : "emit";
+      delete msg._fromSend;
+      delete msg.contextUser;
+      processContext.user = new cds.User.Privileged(userId);
+      await cds.unboxed(service).tx(processContext)[invocationFn](msg);
+    } catch (err) {
+      status = EventProcessingStatus.Error;
+      this.logger.error("error processing outboxed service call", err, {
+        serviceName: this.eventSubType,
+      });
+    }
+    return queueEntries.map((queueEntry) => [queueEntry.ID, status]);
+  }
+}
+
+module.exports = EventQueueGenericOutboxHandler;

package/src/outbox/eventQueueAsOutbox.js

@@ -0,0 +1,110 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const { publishEvent } = require("../publishEvent");
+const config = require("../config");
+
+const OUTBOXED = Symbol("outboxed");
+const UNBOXED = Symbol("unboxed");
+
+const CDS_EVENT_TYPE = "CAP_OUTBOX";
+
+const COMPONENT_NAME = "/eventQueue/eventQueueAsOutbox";
+
+function outboxed(srv, customOpts) {
+  // outbox max. once
+  const logger = cds.log(COMPONENT_NAME);
+  if (!new.target) {
+    const former = srv[OUTBOXED];
+    if (former) {
+      return former;
+    }
+  }
+
+  const originalSrv = srv[UNBOXED] || srv;
+  const outboxedSrv = Object.create(originalSrv);
+  outboxedSrv[UNBOXED] = originalSrv;
+
+  if (!new.target) {
+    Object.defineProperty(srv, OUTBOXED, { value: outboxedSrv });
+  }
+
+  const outboxOpts = Object.assign(
+    {},
+    (typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
+    (typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
+    customOpts || {}
+  );
+
+  config.addCAPOutboxEvent(srv.name, outboxOpts);
+  outboxedSrv.handle = async function (req) {
+    const context = req.context || cds.context;
+    if (outboxOpts.kind === "persistent-outbox") {
+      config.addCAPOutboxEvent(srv.name, outboxOpts);
+      await _mapToEventAndPublish(context, srv.name, req);
+      return;
+    }
+    context.on("succeeded", async () => {
+      try {
+        if (req.reply) {
+          await originalSrv.send(req);
+        } else {
+          await originalSrv.emit(req);
+        }
+      } catch (err) {
+        logger.error("In memory processing failed", { event: req.event, cause: err });
+        if (isUnrecoverable(originalSrv, err) && outboxOpts.crashOnError !== false) {
+          cds.exit(1);
+        }
+      }
+    });
+  };
+
+  return outboxedSrv;
+}
+
+function unboxed(srv) {
+  return srv[UNBOXED] || srv;
+}
+
+const _mapToEventAndPublish = async (context, name, msg) => {
+  const event = {
+    contextUser: context.user.id,
+    ...(msg._fromSend || (msg.reply && { _fromSend: true })), // send or emit
+    ...(msg.inbound && { inbound: msg.inbound }),
+    ...(msg.event && { event: msg.event }),
+    ...(msg.data && { data: msg.data }),
+    ...(msg.headers && { headers: msg.headers }),
+    ...(msg.query && { query: msg.query }),
+  };
+
+  await publishEvent(cds.tx(context), {
+    type: CDS_EVENT_TYPE,
+    subType: name,
+    payload: JSON.stringify(event),
+  });
+};
+
+const isUnrecoverable = (service, error) => {
+  let unrecoverable = service.isUnrecoverableError && service.isUnrecoverableError(error);
+  if (unrecoverable === undefined) {
+    unrecoverable = error.unrecoverable;
+  }
+  return unrecoverable || isStandardError(error);
+};
+
+const isStandardError = (err) => {
+  return (
+    err instanceof TypeError ||
+    err instanceof ReferenceError ||
+    err instanceof SyntaxError ||
+    err instanceof RangeError ||
+    err instanceof URIError
+  );
+};
+
+module.exports = {
+  outboxed,
+  unboxed,
+};

package/src/periodicEvents.js

@@ -6,7 +6,7 @@ const { EventProcessingStatus } = require("./constants");
 const { processChunkedSync } = require("./shared/common");
 const eventConfig = require("./config");
 
-const COMPONENT_NAME = "eventQueue/periodicEvents";
+const COMPONENT_NAME = "/eventQueue/periodicEvents";
 const CHUNK_SIZE_INSERT_PERIODIC_EVENTS = 4;
 
 const checkAndInsertPeriodicEvents = async (context) => {

package/src/processEventQueue.js

@@ -10,7 +10,7 @@ const { limiter } = require("./shared/common");
 
 const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
 
-const COMPONENT_NAME = "eventQueue/processEventQueue";
+const COMPONENT_NAME = "/eventQueue/processEventQueue";
 const MAX_EXECUTION_TIME = 5 * 60 * 1000;
 
 const processEventQueue = async (context, eventType, eventSubType, startTime = new Date()) => {
@@ -29,7 +29,7 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
       return;
     }
     baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
-    const continueProcessing = await baseInstance.handleDistributedLock();
+    const continueProcessing = await baseInstance.acquireDistributedLock();
     if (!continueProcessing) {
       return;
     }

package/src/redisPubSub.js

@@ -5,11 +5,11 @@ const cds = require("@sap/cds");
 const redis = require("./shared/redis");
 const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
 const config = require("./config");
-const { runEventCombinationForTenant } = require("./runner");
+const runner = require("./runner");
 const { getSubdomainForTenantId } = require("./shared/cdsHelper");
 
 const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
-const COMPONENT_NAME = "eventQueue/redisPubSub";
+const COMPONENT_NAME = "/eventQueue/redisPubSub";
 
 let subscriberClientPromise;
 
@@ -17,10 +17,10 @@ const initEventQueueRedisSubscribe = () => {
   if (subscriberClientPromise || !config.redisEnabled) {
     return;
   }
-  redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL, messageHandlerProcessEvents);
+  redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL, _messageHandlerProcessEvents);
 };
 
-const messageHandlerProcessEvents = async (messageData) => {
+const _messageHandlerProcessEvents = async (messageData) => {
   const logger = cds.log(COMPONENT_NAME);
   try {
     const { tenantId, type, subType } = JSON.parse(messageData);
@@ -43,8 +43,30 @@ const messageHandlerProcessEvents = async (messageData) => {
       // NOTE: we need this because of logging otherwise logs would not contain the subdomain
       http: { req: { authInfo: { getSubdomain: () => subdomain } } },
     };
+
+    if (!config.getEventConfig(type, subType)) {
+      if (config.isCapOutboxEvent(type)) {
+        try {
+          const service = await cds.connect.to(subType);
+          cds.outboxed(service);
+        } catch (err) {
+          logger.error("could not connect to outboxed service", err, {
+            type,
+            subType,
+          });
+          return;
+        }
+      } else {
+        logger.error("cannot find configuration for published event. Event won't be processed", {
+          type,
+          subType,
+        });
+        return;
+      }
+    }
+
     return await cds.tx(tenantContext, async ({ context }) => {
-      return await runEventCombinationForTenant(context, type, subType);
+      return await runner.runEventCombinationForTenant(context, type, subType);
     });
   } catch (err) {
     logger.error("could not parse event information", {
@@ -76,7 +98,7 @@ const broadcastEvent = async (tenantId, type, subType) => {
         }
 
         return await cds.tx(context, async ({ context }) => {
-          return await runEventCombinationForTenant(context, type, subType);
+          return await runner.runEventCombinationForTenant(context, type, subType);
         });
       }
       return;
@@ -122,4 +144,7 @@ module.exports = {
   initEventQueueRedisSubscribe,
   broadcastEvent,
   closeSubscribeClient,
+  __: {
+    _messageHandlerProcessEvents,
+  },
 };

package/src/runner.js

@@ -12,7 +12,7 @@ const { getSubdomainForTenantId } = require("./shared/cdsHelper");
 const periodicEvents = require("./periodicEvents");
 const { hashStringTo32Bit } = require("./shared/common");
 
-const COMPONENT_NAME = "eventQueue/runner";
+const COMPONENT_NAME = "/eventQueue/runner";
 const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
 const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
 const EVENT_QUEUE_RUN_PERIODIC_EVENT = "EVENT_QUEUE_RUN_PERIODIC_EVENT";
@@ -321,7 +321,7 @@ module.exports = {
   multiTenancyDb,
   multiTenancyRedis,
   runEventCombinationForTenant,
-  _: {
+  __: {
     _singleTenantDb,
     _multiTenancyRedis,
     _multiTenancyDb,

package/src/shared/WorkerQueue.js

@@ -5,7 +5,7 @@ const cds = require("@sap/cds");
 const config = require("../config");
 const EventQueueError = require("../EventQueueError");
 
-const COMPONENT_NAME = "eventQueue/WorkerQueue";
+const COMPONENT_NAME = "/eventQueue/WorkerQueue";
 const NANO_TO_MS = 1e6;
 const THRESHOLD = {
   INFO: 35 * 1000,

package/src/shared/cdsHelper.js

@@ -8,7 +8,7 @@ const config = require("../config");
 const subdomainCache = {};
 
 const VERROR_CLUSTER_NAME = "ExecuteInNewTransactionError";
-const COMPONENT_NAME = "eventQueue/cdsHelper";
+const COMPONENT_NAME = "/eventQueue/cdsHelper";
 
 /**
  * Execute logic in a new managed CDS transaction context, auto-handling commit, rollback and error/exception situations.

package/src/shared/eventScheduler.js

@@ -5,7 +5,7 @@ const cds = require("@sap/cds");
 const { broadcastEvent } = require("../redisPubSub");
 const config = require("./../config");
 
-const COMPONENT_NAME = "eventQueue/shared/eventScheduler";
+const COMPONENT_NAME = "/eventQueue/shared/eventScheduler";
 
 let instance;
 class EventScheduler {

package/src/shared/redis.js

@@ -5,7 +5,7 @@ const redis = require("redis");
 const { getEnvInstance } = require("./env");
 const EventQueueError = require("../EventQueueError");
 
-const COMPONENT_NAME = "eventQueue/shared/redis";
+const COMPONENT_NAME = "/eventQueue/shared/redis";
 
 let mainClientPromise;
 const subscriberChannelClientPromise = {};