@cap-js-community/event-queue 1.4.1 → 1.4.3

package/package.json

@@ -1,8 +1,9 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
+  "types": "src/index.d.ts",
   "files": [
     "src",
     "db",
@@ -50,8 +51,8 @@
   "devDependencies": {
     "@cap-js/hana": "^0.1.0",
     "@cap-js/sqlite": "^1.5.0",
-    "@sap/cds": "^7.7.0",
-    "@sap/cds-dk": "^7.5.1",
+    "@sap/cds": "^7.8.0",
+    "@sap/cds-dk": "^7.8.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jest": "^27.9.0",

package/src/EventQueueProcessorBase.js

@@ -22,6 +22,7 @@ const SELECT_LIMIT_EVENTS_PER_TICK = 100;
 const TRIES_FOR_EXCEEDED_EVENTS = 3;
 const EVENT_START_AFTER_HEADROOM = 3 * 1000;
 const ETAG_CHECK_AFTER_MIN = 10;
+const SUFFIX_PERIODIC = "_PERIODIC";
 
 class EventQueueProcessorBase {
   #eventsWithExceededTries = [];
@@ -279,22 +280,33 @@ class EventQueueProcessorBase {
       eventSubType: this.#eventSubType,
     });
     const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
-    try {
-      queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
-        this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
-      );
-    } catch (error) {
+    const errorHandler = (error) => {
       queueEntries.forEach((queueEntry) =>
         this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
       );
       this.logger.error(
         "The supplied status tuple doesn't have the required structure. Setting all entries to error.",
-        error,
-        {
-          eventType: this.#eventType,
-          eventSubType: this.#eventSubType,
-        }
+        ...[
+          error,
+          {
+            eventType: this.#eventType,
+            eventSubType: this.#eventSubType,
+          },
+        ].filter((a) => a)
+      );
+    };
+
+    if (!queueEntryProcessingStatusTuple) {
+      errorHandler();
+      return statusMap;
+    }
+
+    try {
+      queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
+        this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
       );
+    } catch (error) {
+      errorHandler(error);
     }
     return statusMap;
   }
@@ -1084,7 +1096,7 @@ class EventQueueProcessorBase {
   }
 
   get eventType() {
-    return this.#eventType;
+    return this.#eventType.replace(SUFFIX_PERIODIC, "");
   }
 
   get eventSubType() {

package/src/config.js

@@ -66,6 +66,7 @@ class Config {
   #userId;
   #cleanupLocksAndEventsForDev;
   #redisOptions;
+  #insertEventsBeforeCommit;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -241,8 +242,8 @@ class Config {
     return key;
   }
 
-  isEventBlocked(type, subType, tenant) {
-    const map = this.#blockedEvents[this.generateKey(type, subType)];
+  isEventBlocked(type, subType, isPeriodicEvent, tenant) {
+    const map = this.#blockedEvents[this.generateKey(`${type}${isPeriodicEvent ? SUFFIX_PERIODIC : ""}`, subType)];
     if (!map) {
       return false;
     }
@@ -494,6 +495,14 @@ class Config {
     return this.#redisOptions;
   }
 
+  set insertEventsBeforeCommit(value) {
+    this.#insertEventsBeforeCommit = value;
+  }
+
+  get insertEventsBeforeCommit() {
+    return this.#insertEventsBeforeCommit;
+  }
+
   get isMultiTenancy() {
     return !!cds.requires.multitenancy;
   }

package/src/dbHandler.js

@@ -6,8 +6,17 @@ const { broadcastEvent } = require("./redis/redisPub");
 const config = require("./config");
 
 const COMPONENT_NAME = "/eventQueue/dbHandler";
+const registeredHandlers = {
+  eventQueueDbHandler: false,
+  beforeDbHandler: false,
+};
 
 const registerEventQueueDbHandler = (dbService) => {
+  if (registeredHandlers.eventQueueDbHandler) {
+    return;
+  }
+
+  registeredHandlers.eventQueueDbHandler = true;
   const def = dbService.model.definitions[config.tableNameEventQueue];
   dbService.after("CREATE", def, (_, req) => {
     if (req.tx._skipEventQueueBroadcase) {
@@ -46,6 +55,21 @@ const registerEventQueueDbHandler = (dbService) => {
   });
 };
 
+const registerBeforeDbHandler = (dbService) => {
+  if (!config.insertEventsBeforeCommit || registeredHandlers.beforeDbHandler) {
+    return;
+  }
+
+  registeredHandlers.beforeDbHandler = true;
+  dbService.before("COMMIT", async (req) => {
+    if (req.context._eventQueueEvents?.length) {
+      await cds.tx(req).run(INSERT.into(config.tableNameEventQueue).entries(req.context._eventQueueEvents));
+      req.context._eventQueueEvents = null;
+    }
+  });
+};
+
 module.exports = {
   registerEventQueueDbHandler,
+  registerBeforeDbHandler,
 };

package/src/index.d.ts

@@ -0,0 +1,138 @@
+import * as cds from "@sap/cds";
+
+export declare const EventProcessingStatus: {
+  Open: 0;
+  InProgress: 1;
+  Done: 2;
+  Error: 3;
+  Exceeded: 4;
+};
+
+declare type EventProcessingStatusKeysType = keyof typeof EventProcessingStatus;
+export declare type EventProcessingStatusType = (typeof EventProcessingStatus)[EventProcessingStatusKeysType];
+
+export declare const TransactionMode: {
+  isolated: "isolated";
+  alwaysCommit: "alwaysCommit";
+  alwaysRollback: "alwaysRollback";
+};
+
+export declare type TransactionModeType = keyof typeof TransactionMode;
+
+export declare const Priorities: {
+  Low: "low";
+  Medium: "medium";
+  High: "high";
+  VeryHigh: "veryHigh";
+};
+export declare type PrioritiesType = keyof typeof Priorities;
+
+interface RedisOptions {
+  host: string;
+  port: number;
+}
+
+interface InitializeParams {
+  configFilePath: string;
+  registerAsEventProcessor?: boolean;
+  processEventsAfterPublish?: boolean;
+  isEventQueueActive?: boolean;
+  runInterval?: number;
+  disableRedis?: boolean;
+  updatePeriodicEvents?: boolean;
+  thresholdLoggingEventProcessing?: number;
+  useAsCAPOutbox?: boolean;
+  userId?: string | null;
+  cleanupLocksAndEventsForDev?: boolean;
+  redisOptions?: RedisOptions;
+  insertEventsBeforeCommit?: boolean;
+}
+
+type CdsLogger = ReturnType<typeof cds.log>;
+
+export function initialize(params: InitializeParams): Promise<void>;
+
+type EventConfigType = {
+  type: string;
+  subType: string;
+  priority: string;
+  impl: string;
+  load: number;
+  interval: number;
+  internalEvent: boolean;
+  isPeriodic: boolean;
+};
+
+// Define Status Type
+type Status = 0 | 1 | 2 | 3 | 4;
+
+// Define Event Entity Type
+interface EventEntity {
+  type: string;
+  subType: string;
+  referenceEntity?: string;
+  referenceEntityKey?: string;
+  status: Status;
+  payload?: string;
+  attempts: number;
+  lastAttemptTimestamp?: string;
+  createdAt: string;
+  startAfter?: string;
+}
+
+interface EventEntityPublish {
+  type: string;
+  subType: string;
+  referenceEntity?: string;
+  referenceEntityKey?: string;
+  payload?: string;
+}
+
+interface QueueEntriesPayloadMap {
+  [key: string]: {
+    queueEntry: EventEntity;
+    payload: Object;
+  };
+}
+
+export declare class EventQueueProcessorBase {
+  constructor(context: cds.EventContext, eventType: string, eventSubType: string, config: EventConfigType);
+
+  processEvent(
+    processContext: cds.EventContext,
+    key: string,
+    queueEntries: EventEntity[],
+    payload: Object
+  ): Promise<Array<[string, EventProcessingStatusType]>>;
+
+  processPeriodicEvent(processContext: cds.EventContext, key: string, queueEntry: EventEntity): Promise<undefined>;
+  checkEventAndGeneratePayload(queueEntry: EventEntity): Promise<Object>;
+  addEventWithPayloadForProcessing(queueEntry: EventEntity, payload: Object): void;
+  clusterQueueEntries(queueEntriesWithPayloadMap: Object): void;
+  hookForExceededEvents(exceededEvent: EventEntity): Promise<void>;
+  clusterQueueEntries(queueEntriesWithPayloadMap: QueueEntriesPayloadMap): void;
+  getLastSuccessfulRunTimestamp(): Promise<string | null>;
+  getContextForEventProcessing(key: string): cds.EventContext;
+  getTxForEventProcessing(key: string): cds.Transaction;
+  setShouldRollbackTransaction(key: string): void;
+  shouldRollbackTransaction(key: string): boolean;
+
+  set logger(value: CdsLogger);
+  get logger(): CdsLogger;
+  get tx(): cds.Transaction;
+  get context(): cds.EventContext;
+  get isPeriodicEvent(): boolean;
+}
+
+export function publishEvent(
+  tx: cds.Transaction,
+  events: EventEntityPublish[] | EventEntityPublish,
+  skipBroadcast?: boolean
+): Promise<any>;
+
+export function processEventQueue(
+  context: cds.EventContext,
+  eventType: string,
+  eventSubType: string,
+  startTime: Date
+): Promise<any>;

package/src/index.js

@@ -1,6 +1,5 @@
 "use strict";
 
-// TODO: how to deal with fatal logs
 // TODO: add tests for config --> similar to csn check
 
 module.exports = {

package/src/initialize.js

@@ -15,6 +15,7 @@ const redis = require("./shared/redis");
 const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
 const { getAllTenantIds } = require("./shared/cdsHelper");
 const { EventProcessingStatus } = require("./constants");
+const distributedLock = require("./shared/distributedLock");
 
 const readFileAsync = promisify(fs.readFile);
 
@@ -34,6 +35,7 @@ const CONFIG_VARS = [
   ["userId", null],
   ["cleanupLocksAndEventsForDev", false],
   ["redisOptions", {}],
+  ["insertEventsBeforeCommit", false],
 ];
 
 const initialize = async ({
@@ -49,6 +51,7 @@ const initialize = async ({
   userId,
   cleanupLocksAndEventsForDev,
   redisOptions,
+  insertEventsBeforeCommit,
 } = {}) => {
   if (config.initialized) {
     return;
@@ -67,7 +70,8 @@ const initialize = async ({
     useAsCAPOutbox,
     userId,
     cleanupLocksAndEventsForDev,
-    redisOptions
+    redisOptions,
+    insertEventsBeforeCommit
   );
 
   const logger = cds.log(COMPONENT);
@@ -77,6 +81,7 @@ const initialize = async ({
   cds.on("connect", (service) => {
     if (service.name === "db") {
       config.processEventsAfterPublish && dbHandler.registerEventQueueDbHandler(service);
+      config.insertEventsBeforeCommit && dbHandler.registerBeforeDbHandler(service);
       config.cleanupLocksAndEventsForDev && registerCleanupForDevDb().catch(() => {});
       initFinished.then(registerEventProcessors);
     }
@@ -166,6 +171,7 @@ const mixConfigVarsWithEnv = (...args) => {
 
 const registerCdsShutdown = () => {
   cds.on("shutdown", async () => {
+    await distributedLock.shutdownHandler();
     await Promise.allSettled([redis.closeMainClient(), closeSubscribeClient()]);
   });
 };

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -23,6 +23,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       delete msg._fromSend;
       delete msg.contextUser;
       processContext.user = new cds.User.Privileged(userId);
+      processContext._eventQueue = { processor: this, key, queueEntries, payload };
       await cds.unboxed(service).tx(processContext)[invocationFn](msg);
     } catch (err) {
       status = EventProcessingStatus.Error;

package/src/processEventQueue.js

@@ -263,6 +263,7 @@ const _checkEventIsBlocked = async (baseInstance) => {
     eventBlocked = config.isEventBlocked(
       baseInstance.eventType,
       baseInstance.eventSubType,
+      baseInstance.isPeriodicEvent,
       baseInstance.context.tenant
     );
   }

package/src/publishEvent.js

@@ -45,10 +45,16 @@ const publishEvent = async (tx, events, skipBroadcast = false) => {
       throw EventQueueError.manuelPeriodicEventInsert(type, subType);
     }
   }
-  tx._skipEventQueueBroadcase = skipBroadcast;
-  const result = await tx.run(INSERT.into(config.tableNameEventQueue).entries(eventsForProcessing));
-  tx._skipEventQueueBroadcase = false;
-  return result;
+
+  if (config.insertEventsBeforeCommit) {
+    tx.context._eventQueueEvents ??= [];
+    tx.context._eventQueueEvents = tx.context._eventQueueEvents.concat(events);
+  } else {
+    tx._skipEventQueueBroadcase = skipBroadcast;
+    const result = await tx.run(INSERT.into(config.tableNameEventQueue).entries(eventsForProcessing));
+    tx._skipEventQueueBroadcase = false;
+    return result;
+  }
 };
 
 module.exports = {

package/src/redis/redisPub.js

@@ -29,7 +29,9 @@ const broadcastEvent = async (tenantId, events) => {
       if (config.registerAsEventProcessor) {
         let context = {};
         if (tenantId) {
-          const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+          const user = await cds.tx({ tenant: tenantId }, async () => {
+            return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+          });
           context = {
             tenant: tenantId,
             user,

package/src/redis/redisSub.js

@@ -35,7 +35,9 @@ const _messageHandlerProcessEvents = async (messageData) => {
       return;
     }
 
-    const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+    const user = await cds.tx({ tenant: tenantId }, async () => {
+      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+    });
     const tenantContext = {
       tenant: tenantId,
       user,

package/src/runner/openEvents.js

@@ -31,6 +31,7 @@ const getOpenQueueEntries = async (tx) => {
   for (const { type, subType } of entries) {
     if (type.startsWith("CAP_OUTBOX")) {
       if (cds.requires[subType]) {
+        await cds.connect.to(subType).catch(() => {});
         result.push({ type, subType });
       } else {
         const service = await cds.connect.to(subType).catch(() => {});

package/src/runner/runner.js

@@ -111,6 +111,7 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
 
     for (const tenantId of tenantIds) {
       await cds.tx({ tenant: tenantId }, async (tx) => {
+        tx.context.user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
         const entries = await openEvents.getOpenQueueEntries(tx);
         logger.info("broadcasting events for run", {
           tenantId,
@@ -136,7 +137,9 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
   const promises = [];
 
   for (const tenantId of tenantIds) {
-    const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+    const user = await cds.tx({ tenant: tenantId }, async () => {
+      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+    });
     const tenantContext = {
       tenant: tenantId,
       user,
@@ -180,7 +183,9 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
 const _executePeriodicEventsAllTenants = async (tenantIds) => {
   for (const tenantId of tenantIds) {
     try {
-      const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+      const user = await cds.tx({ tenant: tenantId }, async () => {
+        return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+      });
       const tenantContext = {
         tenant: tenantId,
         user,

package/src/shared/distributedLock.js

@@ -6,6 +6,10 @@ const cdsHelper = require("./cdsHelper");
 
 const KEY_PREFIX = "EVENT_QUEUE";
 
+const existingLocks = {};
+
+const REDIS_COMMAND_OK = "OK";
+
 const acquireLock = async (context, key, { tenantScoped = true, expiryTime = config.globalTxTimeout } = {}) => {
   const fullKey = _generateKey(context, tenantScoped, key);
   if (config.redisEnabled) {
@@ -59,7 +63,11 @@ const _acquireLockRedis = async (context, fullKey, expiryTime, { value = "true",
     PX: expiryTime,
     ...(overrideValue ? null : { NX: true }),
   });
-  return result === "OK";
+  const isOk = result === REDIS_COMMAND_OK;
+  if (isOk) {
+    existingLocks[fullKey] = 1;
+  }
+  return isOk;
 };
 
 const _checkLockExistsRedis = async (context, fullKey) => {
@@ -78,6 +86,7 @@ const _checkLockExistsDb = async (context, fullKey) => {
 const _releaseLockRedis = async (context, fullKey) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
   await client.del(fullKey);
+  delete existingLocks[fullKey];
 };
 
 const _releaseLockDb = async (context, fullKey) => {
@@ -133,9 +142,14 @@ const _generateKey = (context, tenantScoped, key) => {
   return `${KEY_PREFIX}_${keyParts.join("##")}`;
 };
 
+const shutdownHandler = async () => {
+  await Promise.allSettled(Object.keys(existingLocks).map((key) => _releaseLockRedis(null, key)));
+};
+
 module.exports = {
   acquireLock,
   releaseLock,
   checkLockExistsAndReturnValue,
   setValueWithExpire,
+  shutdownHandler,
 };