@cap-js-community/event-queue 1.7.3 → 1.8.0

package/db/Event.cds

@@ -9,6 +9,7 @@ type Status: Integer enum {
     Done = 2;
     Error = 3;
     Exceeded = 4;
+    Suspended = 5;
 }
 
 entity Event: cuid {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.7.3",
+  "version": "1.8.0",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
@@ -47,13 +47,13 @@
     "cron-parser": "^4.9.0",
     "redis": "^4.7.0",
     "verror": "^1.10.1",
-    "yaml": "^2.5.1"
+    "yaml": "^2.6.1"
   },
   "devDependencies": {
     "@cap-js/hana": "^1.3.0",
     "@cap-js/sqlite": "^1.7.3",
-    "@sap/cds": "^8.3.0",
-    "@sap/cds-dk": "^8.3.0",
+    "@sap/cds": "^8.4.2",
+    "@sap/cds-dk": "^8.4.2",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jest": "^28.6.0",

package/src/config.js

@@ -78,6 +78,7 @@ class Config {
   #cronTimezone;
   #publishEventBlockList;
   #crashOnRedisUnavailable;
+  #tenantIdFilterCb;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -519,6 +520,14 @@ class Config {
     this.#crashOnRedisUnavailable = value;
   }
 
+  get tenantIdFilterCb() {
+    return this.#tenantIdFilterCb;
+  }
+
+  set tenantIdFilterCb(value) {
+    this.#tenantIdFilterCb = value;
+  }
+
   set globalTxTimeout(value) {
     this.#globalTxTimeout = value;
   }

package/src/constants.js

@@ -7,6 +7,7 @@ module.exports = {
     Done: 2,
     Error: 3,
     Exceeded: 4,
+    Suspended: 5,
   },
   TransactionMode: {
     isolated: "isolated",
@@ -19,4 +20,8 @@ module.exports = {
     High: "high",
     VeryHigh: "veryHigh",
   },
+  TenantIdCheckTypes: {
+    getAllTenantIds: "getAllTenantIds",
+    getTokenInfo: "getTokenInfo",
+  },
 };

package/src/initialize.js

@@ -41,6 +41,7 @@ const CONFIG_VARS = [
   ["cronTimezone", null],
   ["publishEventBlockList", true],
   ["crashOnRedisUnavailable", false],
+  ["tenantIdFilterCb", null],
 ];
 
 /**
@@ -64,6 +65,7 @@ const CONFIG_VARS = [
  * @param {string} [options.cronTimezone=null] - Default timezone for cron jobs.
  * @param {string} [options.publishEventBlockList=true] - If redis is available event blocklist is distributed to all application instances
  * @param {string} [options.crashOnRedisUnavailable=true] - If enabled an error is thrown if the redis connection check is not successful
+ * @param {function} [options.tenantIdFilterCb=null] - Allows to set customer filter function to filter the tenants ids which should be processed in the event-queue
  */
 const initialize = async (options = {}) => {
   if (config.initialized) {

package/src/outbox/EventQueueGenericOutboxHandler.js

@@ -27,7 +27,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
       delete msg.contextUser;
       processContext.user = new cds.User.Privileged({
         id: userId,
-        authInfo: await common.getAuthInfo(processContext.tenant),
+        authInfo: await common.getTokenInfo(processContext.tenant),
       });
       processContext._eventQueue = { processor: this, key, queueEntries, payload };
       await cds.unboxed(service).tx(processContext)[invocationFn](msg);

package/src/redis/redisPub.js

@@ -109,7 +109,7 @@ const _processLocalWithoutRedis = async (tenantId, events) => {
     let context = {};
     if (tenantId) {
       const user = await cds.tx({ tenant: tenantId }, async () => {
-        return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+        return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
       });
       context = {
         tenant: tenantId,

package/src/redis/redisSub.js

@@ -41,14 +41,14 @@ const _messageHandlerProcessEvents = async (messageData) => {
           const service = await cds.connect.to(subType);
           cds.outboxed(service);
         } catch (err) {
-          logger.error("could not connect to outboxed service", err, {
+          logger.warn("could not connect to outboxed service", err, {
             type,
             subType,
           });
           return;
         }
       } else {
-        logger.error("cannot find configuration for published event. Event won't be processed", {
+        logger.warn("cannot find configuration for published event. Event won't be processed", {
           type,
           subType,
         });
@@ -66,7 +66,7 @@ const _messageHandlerProcessEvents = async (messageData) => {
     }
 
     const user = await cds.tx({ tenant: tenantId }, async () => {
-      return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+      return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
     });
     const tenantContext = {
       tenant: tenantId,

package/src/runner/openEvents.js

@@ -5,6 +5,8 @@ const cds = require("@sap/cds");
 const eventConfig = require("../config");
 const { EventProcessingStatus } = require("../constants");
 
+const MS_IN_DAYS = 24 * 60 * 60 * 1000;
+
 const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
   const startTime = new Date();
   const refDateStartAfter = new Date(startTime.getTime() + eventConfig.runInterval * 1.2);
@@ -21,7 +23,11 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
         EventProcessingStatus.InProgress,
         "AND lastAttemptTimestamp <=",
         new Date(startTime.getTime() - eventConfig.globalTxTimeout).toISOString(),
-        ") )"
+        ") ) AND (createdAt >=",
+        new Date(startTime.getTime() - 30 * MS_IN_DAYS).toISOString(),
+        " OR startAfter >=",
+        new Date(startTime.getTime() - 30 * MS_IN_DAYS).toISOString(),
+        ")"
       )
       .columns("type", "subType")
       .groupBy("type", "subType")
@@ -30,9 +36,13 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
   const result = [];
   for (const { type, subType } of entries) {
     if (eventConfig.isCapOutboxEvent(type)) {
-      await cds.connect
+      cds.connect
         .to(subType)
         .then((service) => {
+          if (!filterAppSpecificEvents) {
+            return; // will be done in finally
+          }
+
           if (!service) {
             return;
           }
@@ -45,7 +55,12 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
             result.push({ type, subType });
           }
         })
-        .catch(() => {});
+        .catch(() => {})
+        .finally(() => {
+          if (!filterAppSpecificEvents) {
+            result.push({ type, subType });
+          }
+        });
     } else {
       if (filterAppSpecificEvents) {
         if (
@@ -55,7 +70,7 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
           result.push({ type, subType });
         }
       } else {
-        eventConfig.getEventConfig(type, subType) && result.push({ type, subType });
+        result.push({ type, subType });
       }
     }
   }

package/src/runner/runner.js

@@ -134,7 +134,7 @@ const _executeEventsAllTenantsRedis = async (tenantIds) => {
           async () => {
             tx.context.user = new cds.User.Privileged({
               id: config.userId,
-              authInfo: await common.getAuthInfo(tenantId),
+              authInfo: await common.getTokenInfo(tenantId),
             });
             const entries = await openEvents.getOpenQueueEntries(tx, false);
             logger.info("broadcasting events for run", {
@@ -168,10 +168,10 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
     let tenantContext;
     const events = await trace(
       { id, tenant: tenantId },
-      "fetch-openEvents-and-authInfo",
+      "fetch-openEvents-and-tokenInfo",
       async () => {
         const user = await cds.tx({ tenant: tenantId }, async () => {
-          return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+          return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
         });
         tenantContext = {
           tenant: tenantId,
@@ -229,7 +229,7 @@ const _executePeriodicEventsAllTenants = async (tenantIds) => {
   for (const tenantId of tenantIds) {
     try {
       const user = await cds.tx({ tenant: tenantId }, async () => {
-        return new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
+        return new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(tenantId) });
       });
       const tenantContext = {
         tenant: tenantId,
@@ -260,7 +260,7 @@ const _singleTenantDb = async () => {
   const id = cds.utils.uuid();
   const events = await trace(
     { id },
-    "fetch-openEvents-and-authInfo",
+    "fetch-openEvents-and-tokenInfo",
     async () => {
       return await cds.tx({}, async (tx) => {
         return await openEvents.getOpenQueueEntries(tx);
@@ -328,7 +328,7 @@ const _singleTenantRedis = async () => {
       "get-openEvents-and-publish",
       async () => {
         return await cds.tx({}, async (tx) => {
-          const entries = await openEvents.getOpenQueueEntries(tx);
+          const entries = await openEvents.getOpenQueueEntries(tx, false);
           logger.info("broadcasting events for run", {
             entries: entries.length,
           });
@@ -491,7 +491,7 @@ const _checkPeriodicEventsSingleTenant = async (context) => {
   try {
     logger.info("executing updating periodic events", {
       tenantId: context.tenant,
-      subdomain: context.user?.authInfo?.getSubdomain(),
+      subdomain: context.user?.tokenInfo?.extAttributes?.zdn,
     });
     await periodicEvents.checkAndInsertPeriodicEvents(context);
   } catch (err) {

package/src/shared/SetIntervalDriftSafe.js

@@ -2,8 +2,6 @@
 
 const COMPONENT = "eventQueue/SetIntervalDriftSafe";
 
-const ALLOWED_SHIFT_IN_PROCENT = 0.1;
-
 class SetIntervalDriftSafe {
   #adjustedInterval;
   #interval;
@@ -21,12 +19,6 @@ class SetIntervalDriftSafe {
     const now = Date.now();
     if (this.#expectedCycleTime === 0) {
       this.#expectedCycleTime = now + this.#interval;
-    } else if (
-      Math.abs(now + this.#interval - this.#nextTickScheduledFor - this.#interval) >
-      this.#interval * ALLOWED_SHIFT_IN_PROCENT
-    ) {
-      this.#logger.log("overlapping ticks, skipping this run");
-      return;
     } else {
       this.#adjustedInterval = this.#interval - (now - this.#expectedCycleTime);
       this.#expectedCycleTime += this.#interval;

package/src/shared/cdsHelper.js

@@ -5,6 +5,7 @@ const cds = require("@sap/cds");
 
 const config = require("../config");
 const common = require("./common");
+const { TenantIdCheckTypes } = require("../constants");
 
 const VERROR_CLUSTER_NAME = "ExecuteInNewTransactionError";
 const COMPONENT_NAME = "/eventQueue/cdsHelper";
@@ -23,7 +24,7 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
   const parameters = Array.isArray(args) ? args : [args];
   const logger = cds.log(COMPONENT_NAME);
   try {
-    const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(context.tenant) });
+    const user = new cds.User.Privileged({ id: config.userId, tokenInfo: await common.getTokenInfo(context.tenant) });
     if (cds.db.kind === "hana") {
       await cds.tx(
         {
@@ -116,11 +117,9 @@ const getAllTenantIds = async () => {
   const response = await ssp.get("/tenant");
   return response
     .map((tenant) => tenant.subscribedTenantId ?? tenant.tenant)
-    .filter((tenantId) => !isFakeTenant(tenantId));
+    .filter((tenantId) => common.isTenantIdValidCb(TenantIdCheckTypes.getAllTenantIds, tenantId));
 };
 
-const isFakeTenant = (tenantId) => /00000000-0000-4000-8000-\d{12}/.test(tenantId);
-
 module.exports = {
   executeInNewTransaction,
   TriggerRollback,

package/src/shared/common.js

@@ -4,6 +4,8 @@ const crypto = require("crypto");
 
 const cds = require("@sap/cds");
 const xssec = require("@sap/xssec");
+const config = require("../config");
+const { TenantIdCheckTypes } = require("../constants");
 
 const MARGIN_AUTH_INFO_EXPIRY = 60 * 1000;
 const COMPONENT_NAME = "/eventQueue/common";
@@ -68,49 +70,57 @@ const processChunkedSync = (inputs, chunkSize, chunkHandler) => {
 
 const hashStringTo32Bit = (value) => crypto.createHash("sha256").update(String(value)).digest("base64").slice(0, 32);
 
-const _getNewAuthInfo = async (tenantId) => {
-  const authInfoCache = getAuthInfo._authInfoCache;
-  authInfoCache[tenantId] = authInfoCache[tenantId] ?? {};
+const _getNewTokenInfo = async (tenantId) => {
+  const tokenInfoCache = getTokenInfo._tokenInfoCache;
+  tokenInfoCache[tenantId] = tokenInfoCache[tenantId] ?? {};
   try {
-    if (!_getNewAuthInfo._xsuaaService) {
-      _getNewAuthInfo._xsuaaService = new xssec.XsuaaService(cds.requires.auth.credentials);
+    if (!_getNewTokenInfo._xsuaaService) {
+      _getNewTokenInfo._xsuaaService = new xssec.XsuaaService(cds.requires.auth.credentials);
     }
-    const authService = _getNewAuthInfo._xsuaaService;
+    const authService = _getNewTokenInfo._xsuaaService;
     const token = await authService.fetchClientCredentialsToken({ zid: tenantId });
-    const authInfo = await authService.createSecurityContext(token.access_token);
-    authInfoCache[tenantId].expireTs = authInfo.getExpirationDate().getTime() - MARGIN_AUTH_INFO_EXPIRY;
-    return authInfo;
+    const tokenInfo = new xssec.XsuaaToken(token.access_token);
+    tokenInfoCache[tenantId].expireTs = tokenInfo.getExpirationDate().getTime() - MARGIN_AUTH_INFO_EXPIRY;
+    return tokenInfo;
   } catch (err) {
-    authInfoCache[tenantId] = null;
-    cds.log(COMPONENT_NAME).warn("failed to request authInfo", err);
+    tokenInfoCache[tenantId] = null;
+    cds.log(COMPONENT_NAME).warn("failed to request tokenInfo", err);
   }
 };
 
-const getAuthInfo = async (tenantId) => {
-  if (!tenantId) {
+const getTokenInfo = async (tenantId) => {
+  if (!isTenantIdValidCb(TenantIdCheckTypes.getTokenInfo, tenantId)) {
     return null;
   }
 
   if (!cds.requires?.auth?.credentials) {
-    return null; // no credentials not authInfo
+    return null; // no credentials not tokenInfo
   }
   if (!cds.requires?.auth.kind.match(/jwt|xsuaa/i)) {
     cds.log(COMPONENT_NAME).warn("Only 'jwt' or 'xsuaa' are supported as values for auth.kind.");
     return null;
   }
 
-  getAuthInfo._authInfoCache = getAuthInfo._authInfoCache ?? {};
-  const authInfoCache = getAuthInfo._authInfoCache;
+  getTokenInfo._tokenInfoCache = getTokenInfo._tokenInfoCache ?? {};
+  const tokenInfoCache = getTokenInfo._tokenInfoCache;
   // not existing or existing but expired
   if (
-    !authInfoCache[tenantId] ||
-    (authInfoCache[tenantId] && authInfoCache[tenantId].expireTs && Date.now() > authInfoCache[tenantId].expireTs)
+    !tokenInfoCache[tenantId] ||
+    (tokenInfoCache[tenantId] && tokenInfoCache[tenantId].expireTs && Date.now() > tokenInfoCache[tenantId].expireTs)
   ) {
-    authInfoCache[tenantId] ??= {};
-    authInfoCache[tenantId].value = _getNewAuthInfo(tenantId);
-    authInfoCache[tenantId].expireTs = null;
+    tokenInfoCache[tenantId] ??= {};
+    tokenInfoCache[tenantId].value = _getNewTokenInfo(tenantId);
+    tokenInfoCache[tenantId].expireTs = null;
+  }
+  return await tokenInfoCache[tenantId].value;
+};
+
+const isTenantIdValidCb = (checkType, tenantId) => {
+  if (config.tenantIdFilterCb) {
+    return config.tenantIdFilterCb(checkType, tenantId);
+  } else {
+    return true;
   }
-  return await authInfoCache[tenantId].value;
 };
 
 module.exports = {
@@ -119,8 +129,9 @@ module.exports = {
   isValidDate,
   processChunkedSync,
   hashStringTo32Bit,
-  getAuthInfo,
+  getTokenInfo,
+  isTenantIdValidCb,
   __: {
-    clearAuthInfoCache: () => (getAuthInfo._authInfoCache = {}),
+    clearTokenInfoCache: () => (getTokenInfo._tokenInfoCache = {}),
   },
 };

package/src/shared/distributedLock.js

@@ -60,7 +60,7 @@ const checkLockExistsAndReturnValue = async (context, key, { tenantScoped = true
 const _acquireLockRedis = async (context, fullKey, expiryTime, { value = "true", overrideValue = false } = {}) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
   const result = await client.set(fullKey, value, {
-    PX: expiryTime,
+    PX: Math.round(expiryTime),
     ...(overrideValue ? null : { NX: true }),
   });
   const isOk = result === REDIS_COMMAND_OK;
@@ -117,7 +117,10 @@ const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", ov
             .where("code =", fullKey)
         );
       }
-      if (overrideValue || (currentEntry && new Date(currentEntry.createdAt).getTime() + expiryTime <= Date.now())) {
+      if (
+        overrideValue ||
+        (currentEntry && new Date(currentEntry.createdAt).getTime() + Math.round(expiryTime) <= Date.now())
+      ) {
         await tx.run(
           UPDATE.entity(config.tableNameEventLock)
             .set({

package/src/shared/eventScheduler.js

@@ -51,19 +51,10 @@ class EventScheduler {
   }
 
   calculateOffset(type, subType, startAfter) {
-    const eventConfig = config.getEventConfig(type, subType);
-    const scheduleWithoutDelay = config.isPeriodicEvent(type, subType) && eventConfig.interval < 30 * 1000;
-    const date = scheduleWithoutDelay ? startAfter : this.calculateFutureTime(startAfter, 10);
-
+    const date = startAfter;
     return { date, relative: date.getTime() - Date.now() };
   }
 
-  calculateFutureTime(date, seoncds) {
-    const startAfterSeconds = date.getSeconds();
-    const secondsUntil = seoncds - (startAfterSeconds % seoncds);
-    return new Date(date.getTime() + secondsUntil * 1000);
-  }
-
   clearScheduledEvents() {
     this.#scheduledEvents = {};
   }