@cap-js-community/event-queue 1.10.3 → 1.10.5

package/index.cds

@@ -1 +1,2 @@
 using from './db';
+using from './srv';

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.10.3",
+  "version": "1.10.5",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
   "files": [
     "src",
+    "srv",
     "db",
     "cds-plugin.js",
     "index.cds"
@@ -21,6 +22,8 @@
     "multi-tenancy"
   ],
   "scripts": {
+    "start": "PORT=4005 cds-serve",
+    "watch": "PORT=4005 cds watch",
     "test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
     "test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
     "voter:test:integration": "jest --testPathIgnorePatterns=\"/test/\" --forceExit",
@@ -45,7 +48,7 @@
   },
   "dependencies": {
     "@sap/xssec": "^4.6.0",
-    "cron-parser": "^5.1.0",
+    "cron-parser": "^5.2.0",
     "redis": "^4.7.0",
     "verror": "^1.10.1",
     "yaml": "^2.7.1"

package/src/EventQueueProcessorBase.js

@@ -76,7 +76,7 @@ class EventQueueProcessorBase {
     this.__txMap = {};
     this.__txRollback = {};
     this.__queueEntries = [];
-    this.#keepAliveRunner = new SetIntervalDriftSafe(this.#eventConfig.keepAliveInterval);
+    this.#keepAliveRunner = new SetIntervalDriftSafe(this.#eventConfig.keepAliveInterval * 1000);
   }
 
   /**
@@ -174,7 +174,12 @@ class EventQueueProcessorBase {
       eventSubType: this.#eventSubType,
       iterationCounter,
     });
-    this.#eventSchedulerInstance.scheduleEvent(this.__context.tenant, this.#eventType, this.#eventSubType, new Date());
+    this.#eventSchedulerInstance.scheduleEvent(
+      this.__context.tenant,
+      this.#eventType,
+      this.#eventSubType,
+      new Date(Date.now() + 5 * 1000) // add some offset to make sure all locks are released
+    );
   }
 
   logStartMessage() {
@@ -612,7 +617,7 @@ class EventQueueProcessorBase {
                   "OR lastAttemptTimestamp IS NULL ) OR ( status =",
                   EventProcessingStatus.InProgress,
                   "AND lastAttemptTimestamp <=",
-                  new Date(baseDate - this.#eventConfig.keepAliveMaxInProgressTime).toISOString(),
+                  new Date(baseDate - this.#eventConfig.keepAliveMaxInProgressTime * 1000).toISOString(),
                   ") )",
                 ]
               : [
@@ -623,7 +628,7 @@ class EventQueueProcessorBase {
                   ") OR ( status =",
                   EventProcessingStatus.InProgress,
                   "AND lastAttemptTimestamp <=",
-                  new Date(baseDate - this.#eventConfig.keepAliveMaxInProgressTime).toISOString(),
+                  new Date(baseDate - this.#eventConfig.keepAliveMaxInProgressTime * 1000).toISOString(),
                   ") )",
                 ])
           )
@@ -863,7 +868,7 @@ class EventQueueProcessorBase {
   }
 
   continuesKeepAlive() {
-    if (Date.now() - this.lockAcquiredTime.getTime() >= this.#eventConfig.keepAliveInterval) {
+    if (Date.now() - this.lockAcquiredTime.getTime() >= this.#eventConfig.keepAliveInterval * 1000) {
       trace(this.baseContext, "keepAlive-between-iterations", async () => {
         await this.#renewDistributedLock();
       }).catch((err) => this.logger.error("renewing lock between intervals failed!", err));
@@ -956,7 +961,7 @@ class EventQueueProcessorBase {
       const lockAcquired = await distributedLock.acquireLock(
         this.__context,
         [this.#eventType, this.#eventSubType].join("##"),
-        { keepTrackOfLock: true, expiryTime: this.#eventConfig.keepAliveMaxInProgressTime }
+        { keepTrackOfLock: true, expiryTime: this.#eventConfig.keepAliveMaxInProgressTime * 1000 }
       );
       if (!lockAcquired) {
         this.logger.debug("no lock available, exit processing", {
@@ -978,7 +983,7 @@ class EventQueueProcessorBase {
     const lockAcquired = await distributedLock.renewLock(
       this.__context,
       [this.#eventType, this.#eventSubType].join("##"),
-      { expiryTime: this.#eventConfig.keepAliveMaxInProgressTime }
+      { expiryTime: this.#eventConfig.keepAliveMaxInProgressTime * 1000 }
     );
     if (!lockAcquired) {
       this.logger.error("renewing distributed lock failed!", {

package/src/config.js

@@ -117,6 +117,7 @@ class Config {
   #tenantIdFilterEventProcessingCb;
   #configEvents;
   #configPeriodicEvents;
+  #enableAdminService;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -549,7 +550,7 @@ class Config {
     event.load = event.load ?? DEFAULT_LOAD;
     event.priority = event.priority ?? DEFAULT_PRIORITY;
     event.increasePriorityOverTime = event.increasePriorityOverTime ?? DEFAULT_INCREASE_PRIORITY;
-    event.keepAliveInterval = (event.keepAliveInterval ?? DEFAULT_KEEP_ALIVE_INTERVAL) * 1000;
+    event.keepAliveInterval = event.keepAliveInterval ?? DEFAULT_KEEP_ALIVE_INTERVAL;
     event.keepAliveMaxInProgressTime = event.keepAliveInterval * DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL;
     event.checkForNextChunk = event.checkForNextChunk ?? DEFAULT_CHECK_FOR_NEXT_CHUNK;
   }
@@ -638,18 +639,17 @@ class Config {
       let cron;
 
       // NOTE: logic is as follows:
-      // - if event.utc is true --> always use UTC
+      // - if event.utc is true --> always use UTC (default is false)
       // - if event.useCronTimezone is false OR event.cronTimezone is not defined --> use UTC as well
       // - if event.utc is not true AND event.cronTimezone is set AND event.useCronTimezone is NOT set to false use event.cronTimezone
       event.utc = event.utc ?? UTC_DEFAULT;
-
-      if (!event.cronTimezone) {
+      if (!this.cronTimezone) {
         event.useCronTimezone = false;
       } else {
         event.useCronTimezone = event.useCronTimezone ?? USE_CRON_TZ_DEFAULT;
       }
 
-      event.tz = event.utc || !event.useCronTimezone ? "UTC" : event.cronTimezone;
+      event.tz = event.utc || !event.useCronTimezone ? "UTC" : this.cronTimezone;
 
       try {
         cron = CronExpressionParser.parse(event.cron);
@@ -972,6 +972,18 @@ class Config {
     return this.#eventMap;
   }
 
+  get developmentMode() {
+    return cds.env.profiles.find((profile) => profile === "development");
+  }
+
+  get enableAdminService() {
+    return this.#enableAdminService;
+  }
+
+  set enableAdminService(value) {
+    this.#enableAdminService = value;
+  }
+
   /**
     @return { Config }
   **/

package/src/index.d.ts

@@ -257,6 +257,8 @@ declare class Config {
   get insertEventsBeforeCommit(): any;
   set enableTelemetry(value: any);
   get enableTelemetry(): any;
+  set enableAdminService(value: any);
+  get enableAdminService(): any;
   get isMultiTenancy(): boolean;
 }
 

package/src/initialize.js

@@ -46,6 +46,7 @@ const CONFIG_VARS = [
   ["redisNamespace", null],
   ["publishEventBlockList", true],
   ["crashOnRedisUnavailable", false],
+  ["enableAdminService", false],
 ];
 
 /**
@@ -87,7 +88,7 @@ const initialize = async (options = {}) => {
       "Event queue initialization skipped: no configFilePath provided, and event queue is not configured as a CAP outbox."
     );
   }
-
+  _disableAdminService();
   const redisEnabled = config.checkRedisEnabled();
   let resolveFn;
   let initFinished = new Promise((resolve) => (resolveFn = resolve));
@@ -182,6 +183,14 @@ const monkeyPatchCAPOutbox = () => {
       get: () => eventQueueAsOutbox.unboxed,
       configurable: true,
     });
+    Object.defineProperty(cds, "queued", {
+      get: () => eventQueueAsOutbox.outboxed,
+      configurable: true,
+    });
+    Object.defineProperty(cds, "unqueued", {
+      get: () => eventQueueAsOutbox.unboxed,
+      configurable: true,
+    });
   }
 };
 
@@ -193,34 +202,31 @@ const mixConfigVarsWithEnv = (options) => {
 };
 
 const registerCdsShutdown = () => {
-  const isTestProfile = cds.env.profiles.find((profile) => profile.includes("test"));
-  if (isTestProfile || !config.redisEnabled) {
-    return;
-  }
   cds.on("shutdown", async () => {
     return await new Promise((resolve) => {
-      const timeoutRef = setTimeout(() => {
+      let timeoutRef;
+      timeoutRef = setTimeout(() => {
         clearTimeout(timeoutRef);
         cds.log(COMPONENT).info("shutdown timeout reached - some locks might not have been released!");
         resolve();
       }, TIMEOUT_SHUTDOWN);
-      distributedLock.shutdownHandler().then(() =>
-        Promise.allSettled([redis.closeMainClient(), redis.closeSubscribeClient()]).then((result) => {
+      distributedLock.shutdownHandler().then(() => {
+        Promise.allSettled(
+          config.redisEnabled ? [redis.closeMainClient(), redis.closeSubscribeClient()] : [Promise.resolve()]
+        ).then((result) => {
           clearTimeout(timeoutRef);
           resolve(result);
-        })
-      );
+        });
+      });
     });
   });
 };
 
 const registerCleanupForDevDb = async () => {
-  const profile = cds.env.profiles.find((profile) => profile === "development");
-  if (!profile || process.env.NODE_ENV === "production") {
+  if (!config.developmentMode) {
     return;
   }
-
-  const tenantIds = await getAllTenantIds();
+  const tenantIds = config.isMultiTenancy ? await getAllTenantIds() : [null];
   for (const tenantId of tenantIds) {
     await cds.tx({ tenant: tenantId }, async (tx) => {
       await tx.run(DELETE.from(config.tableNameEventLock));
@@ -252,6 +258,18 @@ const _registerUnsubscribe = () => {
   });
 };
 
+const _disableAdminService = () => {
+  if (config.enableAdminService) {
+    return;
+  }
+  cds.on("loaded", (model) => {
+    const srvDefinition = model.definitions["EventQueueAdminService"];
+    if (srvDefinition) {
+      srvDefinition["@protocol"] = "none";
+    }
+  });
+};
+
 module.exports = {
   initialize,
 };

package/src/processEventQueue.js

@@ -339,7 +339,7 @@ const _processEvent = async (eventTypeInstance, processContext, key, queueEntrie
         return eventTypeInstance.handleErrorDuringProcessing(err, queueEntries);
       }
     },
-    { traceContext }
+    { traceContext, attributes: { eventIds: queueEntries.map(({ ID }) => ID) } }
   );
 };
 

package/src/runner/runner.js

@@ -22,7 +22,8 @@ const EVENT_QUEUE_RUN_ID = "RUN_ID";
 const EVENT_QUEUE_RUN_TS = "RUN_TS";
 const EVENT_QUEUE_RUN_REDIS_CHECK = "RUN_REDIS_CHECK";
 const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "UPDATE_PERIODIC_EVENTS";
-let OFFSET_FIRST_RUN = 10 * 1000;
+
+let OFFSET_FIRST_RUN;
 
 let tenantIdHash;
 let singleRunDone;
@@ -399,7 +400,7 @@ const _acquireRunId = async (context) => {
 };
 
 const _calculateOffsetForFirstRun = async () => {
-  let offsetDependingOnLastRun = OFFSET_FIRST_RUN;
+  let offsetDependingOnLastRun = OFFSET_FIRST_RUN ?? (config.developmentMode ? 500 : 10 * 1000);
   const now = Date.now();
   // NOTE: this is only supported with Redis, because this is a tenant agnostic information
   //       currently there is no proper place to store this information beside t0 schema

package/src/shared/cdsHelper.js

@@ -116,21 +116,29 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
   }
 }
 
-const getAllTenantIds = async () => {
+const _getAllTenantBase = async () => {
   if (!config.isMultiTenancy) {
     return null;
   }
 
   // NOTE: tmp workaround until cds-mtxs fixes the connect.to service
   for (let i = 0; i < 10; i++) {
-    if (cds.services["saas-registry"]) {
+    if (cds.services["cds.xt.SaasProvisioningService"] || cds.services["saas-registry"]) {
       break;
     }
     await new Promise((resolve) => setTimeout(resolve, 1000));
   }
 
   const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");
-  const response = await ssp.get("/tenant");
+  return await ssp.get("/tenant");
+};
+
+const getAllTenantIds = async () => {
+  const response = await _getAllTenantBase();
+  if (!response) {
+    return null;
+  }
+
   return response
     .map((tenant) => tenant.subscribedTenantId ?? tenant.tenant)
     .reduce(async (result, tenantId) => {
@@ -142,7 +150,27 @@ const getAllTenantIds = async () => {
     }, []);
 };
 
+const getAllTenantWithSubdomain = async () => {
+  const response = await _getAllTenantBase();
+  if (!response) {
+    return null;
+  }
+
+  return response.reduce(async (result, row) => {
+    const tenantId = row.subscribedTenantId ?? row.tenant;
+    result = await result;
+    if (await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId)) {
+      result.push({
+        ID: tenantId,
+        subdomain: row.subscribedSubdomain,
+      });
+    }
+    return result;
+  }, []);
+};
+
 module.exports = {
   executeInNewTransaction,
   getAllTenantIds,
+  getAllTenantWithSubdomain,
 };

package/src/shared/distributedLock.js

@@ -17,7 +17,7 @@ const acquireLock = async (
   if (config.redisEnabled) {
     return await _acquireLockRedis(context, fullKey, expiryTime, { keepTrackOfLock });
   } else {
-    return await _acquireLockDB(context, fullKey, expiryTime);
+    return await _acquireLockDB(context, fullKey, expiryTime, { keepTrackOfLock });
   }
 };
 
@@ -73,7 +73,7 @@ const _acquireLockRedis = async (
   context,
   fullKey,
   expiryTime,
-  { value = "true", overrideValue = false, keepTrackOfLock } = {}
+  { value = Date.now(), overrideValue = false, keepTrackOfLock } = {}
 ) => {
   const client = await redis.createMainClientAndConnect(config.redisOptions);
   const result = await client.set(fullKey, value, {
@@ -82,7 +82,7 @@ const _acquireLockRedis = async (
   });
   const isOk = result === REDIS_COMMAND_OK;
   if (isOk && keepTrackOfLock) {
-    existingLocks[fullKey] = 1;
+    existingLocks[fullKey] = context.tenant;
   }
   return isOk;
 };
@@ -129,9 +129,15 @@ const _releaseLockDb = async (context, fullKey) => {
   await cdsHelper.executeInNewTransaction(context, "distributedLock-release", async (tx) => {
     await tx.run(DELETE.from(config.tableNameEventLock).where("code =", fullKey));
   });
+  delete existingLocks[fullKey];
 };
 
-const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", overrideValue = false } = {}) => {
+const _acquireLockDB = async (
+  context,
+  fullKey,
+  expiryTime,
+  { value = "true", overrideValue = false, keepTrackOfLock } = {}
+) => {
   let result;
   await cdsHelper.executeInNewTransaction(context, "distributedLock-acquire", async (tx) => {
     try {
@@ -171,6 +177,9 @@ const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", ov
       }
     }
   });
+  if (result && keepTrackOfLock) {
+    existingLocks[fullKey] = context.tenant;
+  }
   return result;
 };
 
@@ -181,14 +190,58 @@ const _generateKey = (context, tenantScoped, key) => {
   return `${keyParts.join("##")}`;
 };
 
+const getAllLocksRedis = async () => {
+  const client = await redis.createMainClientAndConnect(config.redisOptions);
+  const keys = await client.keys(`${config.redisOptions.redisNamespace}*`);
+  const relevantKeys = {};
+  for (const key of keys) {
+    const [, tenant, guidOrType, subType] = key.split("##");
+    if (subType) {
+      relevantKeys[key] = { tenant, guidOrType, subType };
+    }
+  }
+
+  if (!Object.keys(relevantKeys)) {
+    return [];
+  }
+
+  const pipeline = client.multi();
+  for (const key in relevantKeys) {
+    pipeline.ttl(key);
+    pipeline.get(key);
+  }
+
+  const replies = await pipeline.exec();
+
+  let counter = 0;
+  const result = [];
+  for (const value of Object.values(relevantKeys)) {
+    const ttl = replies[counter];
+    const createdAt = replies[counter + 1];
+    result.push({
+      tenant: value.tenant,
+      type: value.guidOrType,
+      subType: value.subType,
+      ttl,
+      createdAt,
+    });
+    counter = counter + 2;
+  }
+  return result;
+};
+
 const shutdownHandler = async () => {
   const logger = cds.log(COMPONENT_NAME);
   logger.info("received shutdown event, trying to release all locks", {
     numberOfLocks: Object.keys(existingLocks).length,
   });
   const result = await Promise.allSettled(
-    Object.keys(existingLocks).map(async (key) => {
-      await _releaseLockRedis(null, key);
+    Object.entries(existingLocks).map(async ([key, tenant]) => {
+      if (config.redisEnabled) {
+        await _releaseLockRedis({ tenant }, key);
+      } else {
+        await _releaseLockDb({ tenant }, key);
+      }
       logger.info("lock released", { key });
     })
   );
@@ -206,4 +259,5 @@ module.exports = {
   setValueWithExpire,
   shutdownHandler,
   renewLock,
+  getAllLocksRedis,
 };

package/src/shared/openTelemetry.js

@@ -85,11 +85,22 @@ const _startOtelTrace = async (ctxWithSpan, traceContext, span, fn) => {
 const _setAttributes = (context, span, attributes) => {
   span.setAttribute("sap.tenancy.tenant_id", context.tenant);
   span.setAttribute("sap.correlation_id", context.id);
+  _sanitizeAttributes(attributes);
   for (const attributeKey in attributes) {
     span.setAttribute(attributeKey, attributes[attributeKey]);
   }
 };
 
+const _sanitizeAttributes = (attributes = {}) => {
+  for (const attributeKey in attributes) {
+    attributes[attributeKey] =
+      typeof attributes[attributeKey] !== "string"
+        ? JSON.stringify(attributes[attributeKey])
+        : attributes[attributeKey];
+  }
+  return attributes;
+};
+
 const getCurrentTraceContext = () => {
   if (!otel) {
     return null;

package/srv/index.cds

@@ -0,0 +1 @@
+using from './service';

package/srv/service/admin-service.cds

@@ -0,0 +1,42 @@
+using sap.eventqueue as db from '../../db';
+
+@path: 'event-queue/admin'
+@impl: './admin-service.js'
+@requires: 'internal-user'
+service EventQueueAdminService {
+
+  @readonly
+  @cds.persistence.skip
+  entity Event as projection on db.Event {
+        null as tenant: String,
+        null as landscape: String,
+        null as space: String,
+        *
+  } actions {
+    action setStatusAndAttempts(
+      // TODO: remove tenant as soon as CAP issue is fixed https://github.tools.sap/cap/issues/issues/18445
+      @mandatory
+      tenant: String,
+      status: db.Status,
+      @assert.range: [0,100]
+      attempts: Integer) returns Event;
+  }
+
+  @cds.persistence.exists
+  entity Lock {
+    key tenant: String;
+    key type: String;
+    key subType: String;
+    landscape: String;
+    space: String;
+    ttl: Integer;
+    createdAt: Integer;
+  }
+
+   @readonly
+   @cds.persistence.skip
+   entity Tenant {
+      Key ID: String;
+      subdomain: String;
+   }
+}

package/srv/service/admin-service.js

@@ -0,0 +1,111 @@
+"use strict";
+
+const cds = require("@sap/cds");
+const cdsHelper = require("../../src/shared/cdsHelper");
+const { EventProcessingStatus } = require("../../src");
+const config = require("../../src/config");
+const distributedLock = require("../../src/shared/distributedLock");
+
+module.exports = class AdminService extends cds.ApplicationService {
+  async init() {
+    const { Event: EventService, Tenant, Lock: LockService } = this.entities();
+    const { Event: EventDb } = cds.db.entities("sap.eventqueue");
+    const { landscape, space } = this.getLandscapeAndSpace();
+
+    this.before("*", (req) => {
+      if (!config.enableAdminService) {
+        req.reject(403, "Admin service is disabled by configuration");
+      }
+
+      if (req.target.name === Tenant.name) {
+        return;
+      }
+      const headers = Object.assign({}, req.headers, req.req?.headers);
+      const tenant = headers["z-id"] ?? req.data.tenant;
+
+      if (config.isMultiTenancy && tenant == null) {
+        req.reject(400, "Missing tenant ID in request header (z-id)");
+      }
+      req.headers ??= {};
+      req.headers["z-id"] = tenant;
+    });
+
+    this.on("READ", EventService, async (req) => {
+      const tenant = req.headers["z-id"];
+      return await cds.tx({ tenant: tenant }, async (tx) => {
+        if (req.query.SELECT.from.ref[0].id) {
+          req.query.SELECT.from.ref[0].id = EventDb.name;
+        } else {
+          req.query.SELECT.from.ref[0] = EventDb.name;
+        }
+        const events = await tx.run(req.query);
+        return events?.map((event) => {
+          event.landscape = landscape;
+          event.space = space;
+          event.tenant = tenant;
+          return event;
+        });
+      });
+    });
+
+    this.on("READ", LockService, async () => {
+      if (!config.redisEnabled) {
+        return [];
+      }
+      const locks = await distributedLock.getAllLocksRedis();
+      return locks.map((lock) => ({
+        ...lock,
+        landscape: landscape,
+        space: space,
+      }));
+    });
+
+    this.on("READ", Tenant, async () => {
+      const tenants = await cdsHelper.getAllTenantWithSubdomain();
+      return tenants ?? [];
+    });
+
+    this.on("setStatusAndAttempts", async (req) => {
+      const tenant = req.headers["z-id"];
+      cds.log("eventQueue").info("Restarting processing for event queue");
+      const updateData = {};
+
+      if (Number.isInteger(req.data.attempts)) {
+        updateData.attempts = req.data.attempts;
+      }
+
+      if (Object.values(EventProcessingStatus).includes(req.data.status)) {
+        updateData.status = req.data.status;
+      }
+
+      if (!Object.keys(updateData).length) {
+        return req.reject(400, "No status or attempts provided");
+      }
+
+      await cds.tx({ tenant, headers: { "z-id": tenant } }, async () => {
+        await UPDATE.entity(EventDb)
+          .set(updateData)
+          .where({ ID: req.params[0].ID ?? req.params[0] });
+      });
+      return await this.send(new cds.Request({ query: req.query, headers: req.headers }));
+    });
+
+    await super.init();
+  }
+
+  getLandscapeAndSpace() {
+    const url = cds.requires.db.credentials.url;
+    if (!url) {
+      return { landscape: "eu10-canary", space: "local-dev" };
+    }
+    const match = url.match(/https?:\/\/[^.]+\.authentication\.([^.]+)\.hana\.ondemand\.com/);
+    const landscape = (match?.[1] ?? "sap") === "sap" ? "eu10-canary" : match?.[1];
+    let space = "local-dev";
+    try {
+      space = JSON.parse(process.env.VCAP_APPLICATION)?.space_name;
+    } catch {
+      /* empty */
+    }
+    return { landscape, space };
+  }
+};

package/srv/service/index.cds

@@ -0,0 +1 @@
+using from './admin-service';