@cap-js-community/event-queue 1.10.3 → 1.11.0-beta.0

package/index.cds

@@ -1 +1,2 @@
 using from './db';
+using from './srv';

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "1.10.3",
+  "version": "1.11.0-beta.0",
   "description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
   "main": "src/index.js",
   "types": "src/index.d.ts",
   "files": [
     "src",
+    "srv",
     "db",
     "cds-plugin.js",
     "index.cds"
@@ -21,6 +22,8 @@
     "multi-tenancy"
   ],
   "scripts": {
+    "start": "PORT=4005 cds-serve",
+    "watch": "PORT=4005 cds watch",
     "test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
     "test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
     "voter:test:integration": "jest --testPathIgnorePatterns=\"/test/\" --forceExit",
@@ -45,7 +48,7 @@
   },
   "dependencies": {
     "@sap/xssec": "^4.6.0",
-    "cron-parser": "^5.1.0",
+    "cron-parser": "^5.2.0",
     "redis": "^4.7.0",
     "verror": "^1.10.1",
     "yaml": "^2.7.1"

package/src/EventQueueProcessorBase.js

@@ -174,7 +174,12 @@ class EventQueueProcessorBase {
       eventSubType: this.#eventSubType,
       iterationCounter,
     });
-    this.#eventSchedulerInstance.scheduleEvent(this.__context.tenant, this.#eventType, this.#eventSubType, new Date());
+    this.#eventSchedulerInstance.scheduleEvent(
+      this.__context.tenant,
+      this.#eventType,
+      this.#eventSubType,
+      new Date(Date.now() + 5 * 1000) // add some offset to make sure all locks are released
+    );
   }
 
   logStartMessage() {

package/src/config.js

@@ -117,6 +117,7 @@ class Config {
   #tenantIdFilterEventProcessingCb;
   #configEvents;
   #configPeriodicEvents;
+  #enableAdminService;
   static #instance;
   constructor() {
     this.#logger = cds.log(COMPONENT_NAME);
@@ -638,18 +639,17 @@ class Config {
       let cron;
 
       // NOTE: logic is as follows:
-      // - if event.utc is true --> always use UTC
+      // - if event.utc is true --> always use UTC (default is false)
       // - if event.useCronTimezone is false OR event.cronTimezone is not defined --> use UTC as well
       // - if event.utc is not true AND event.cronTimezone is set AND event.useCronTimezone is NOT set to false use event.cronTimezone
       event.utc = event.utc ?? UTC_DEFAULT;
-
-      if (!event.cronTimezone) {
+      if (!this.cronTimezone) {
         event.useCronTimezone = false;
       } else {
         event.useCronTimezone = event.useCronTimezone ?? USE_CRON_TZ_DEFAULT;
       }
 
-      event.tz = event.utc || !event.useCronTimezone ? "UTC" : event.cronTimezone;
+      event.tz = event.utc || !event.useCronTimezone ? "UTC" : this.cronTimezone;
 
       try {
         cron = CronExpressionParser.parse(event.cron);
@@ -972,6 +972,18 @@ class Config {
     return this.#eventMap;
   }
 
+  get developmentMode() {
+    return cds.env.profiles.find((profile) => profile === "development");
+  }
+
+  get enableAdminService() {
+    return this.#enableAdminService;
+  }
+
+  set enableAdminService(value) {
+    this.#enableAdminService = value;
+  }
+
   /**
     @return { Config }
   **/

package/src/index.d.ts

@@ -257,6 +257,8 @@ declare class Config {
   get insertEventsBeforeCommit(): any;
   set enableTelemetry(value: any);
   get enableTelemetry(): any;
+  set enableAdminService(value: any);
+  get enableAdminService(): any;
   get isMultiTenancy(): boolean;
 }
 

package/src/initialize.js

@@ -46,6 +46,7 @@ const CONFIG_VARS = [
   ["redisNamespace", null],
   ["publishEventBlockList", true],
   ["crashOnRedisUnavailable", false],
+  ["enableAdminService", false],
 ];
 
 /**
@@ -87,7 +88,7 @@ const initialize = async (options = {}) => {
       "Event queue initialization skipped: no configFilePath provided, and event queue is not configured as a CAP outbox."
     );
   }
-
+  _disableAdminService();
   const redisEnabled = config.checkRedisEnabled();
   let resolveFn;
   let initFinished = new Promise((resolve) => (resolveFn = resolve));
@@ -182,6 +183,14 @@ const monkeyPatchCAPOutbox = () => {
       get: () => eventQueueAsOutbox.unboxed,
       configurable: true,
     });
+    Object.defineProperty(cds, "queued", {
+      get: () => eventQueueAsOutbox.outboxed,
+      configurable: true,
+    });
+    Object.defineProperty(cds, "unqueued", {
+      get: () => eventQueueAsOutbox.unboxed,
+      configurable: true,
+    });
   }
 };
 
@@ -193,34 +202,34 @@ const mixConfigVarsWithEnv = (options) => {
 };
 
 const registerCdsShutdown = () => {
-  const isTestProfile = cds.env.profiles.find((profile) => profile.includes("test"));
-  if (isTestProfile || !config.redisEnabled) {
+  if (!config.developmentMode) {
     return;
   }
   cds.on("shutdown", async () => {
     return await new Promise((resolve) => {
-      const timeoutRef = setTimeout(() => {
+      let timeoutRef;
+      timeoutRef = setTimeout(() => {
         clearTimeout(timeoutRef);
         cds.log(COMPONENT).info("shutdown timeout reached - some locks might not have been released!");
         resolve();
       }, TIMEOUT_SHUTDOWN);
-      distributedLock.shutdownHandler().then(() =>
-        Promise.allSettled([redis.closeMainClient(), redis.closeSubscribeClient()]).then((result) => {
+      distributedLock.shutdownHandler().then(() => {
+        Promise.allSettled(
+          config.redisEnabled ? [redis.closeMainClient(), redis.closeSubscribeClient()] : [Promise.resolve()]
+        ).then((result) => {
           clearTimeout(timeoutRef);
           resolve(result);
-        })
-      );
+        });
+      });
     });
   });
 };
 
 const registerCleanupForDevDb = async () => {
-  const profile = cds.env.profiles.find((profile) => profile === "development");
-  if (!profile || process.env.NODE_ENV === "production") {
+  if (!config.developmentMode) {
     return;
   }
-
-  const tenantIds = await getAllTenantIds();
+  const tenantIds = config.isMultiTenancy ? await getAllTenantIds() : [null];
   for (const tenantId of tenantIds) {
     await cds.tx({ tenant: tenantId }, async (tx) => {
       await tx.run(DELETE.from(config.tableNameEventLock));
@@ -252,6 +261,18 @@ const _registerUnsubscribe = () => {
   });
 };
 
+const _disableAdminService = () => {
+  if (config.enableAdminService) {
+    return;
+  }
+  cds.on("loaded", (model) => {
+    const srvDefinition = model.definitions["EventQueueAdminService"];
+    if (srvDefinition) {
+      srvDefinition["@protocol"] = "none";
+    }
+  });
+};
+
 module.exports = {
   initialize,
 };

package/src/processEventQueue.js

@@ -339,7 +339,7 @@ const _processEvent = async (eventTypeInstance, processContext, key, queueEntrie
         return eventTypeInstance.handleErrorDuringProcessing(err, queueEntries);
       }
     },
-    { traceContext }
+    { traceContext, attributes: { eventIds: queueEntries.map(({ ID }) => ID) } }
   );
 };
 

package/src/runner/runner.js

@@ -22,7 +22,8 @@ const EVENT_QUEUE_RUN_ID = "RUN_ID";
 const EVENT_QUEUE_RUN_TS = "RUN_TS";
 const EVENT_QUEUE_RUN_REDIS_CHECK = "RUN_REDIS_CHECK";
 const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "UPDATE_PERIODIC_EVENTS";
-let OFFSET_FIRST_RUN = 10 * 1000;
+
+let OFFSET_FIRST_RUN;
 
 let tenantIdHash;
 let singleRunDone;
@@ -399,7 +400,7 @@ const _acquireRunId = async (context) => {
 };
 
 const _calculateOffsetForFirstRun = async () => {
-  let offsetDependingOnLastRun = OFFSET_FIRST_RUN;
+  let offsetDependingOnLastRun = OFFSET_FIRST_RUN ?? (config.developmentMode ? 500 : 10 * 1000);
   const now = Date.now();
   // NOTE: this is only supported with Redis, because this is a tenant agnostic information
   //       currently there is no proper place to store this information beside t0 schema

package/src/shared/cdsHelper.js

@@ -116,7 +116,7 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
   }
 }
 
-const getAllTenantIds = async () => {
+const _getAllTenantBase = async () => {
   if (!config.isMultiTenancy) {
     return null;
   }
@@ -130,7 +130,15 @@ const getAllTenantIds = async () => {
   }
 
   const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");
-  const response = await ssp.get("/tenant");
+  return await ssp.get("/tenant");
+};
+
+const getAllTenantIds = async () => {
+  const response = await _getAllTenantBase();
+  if (!response) {
+    return null;
+  }
+
   return response
     .map((tenant) => tenant.subscribedTenantId ?? tenant.tenant)
     .reduce(async (result, tenantId) => {
@@ -142,7 +150,27 @@ const getAllTenantIds = async () => {
     }, []);
 };
 
+const getAllTenantWithSubdomain = async () => {
+  const response = await _getAllTenantBase();
+  if (!response) {
+    return null;
+  }
+
+  return response.reduce(async (result, row) => {
+    const tenantId = row.subscribedTenantId ?? row.tenant;
+    result = await result;
+    if (await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId)) {
+      result.push({
+        ID: tenantId,
+        subdomain: row.subscribedSubdomain,
+      });
+    }
+    return result;
+  }, []);
+};
+
 module.exports = {
   executeInNewTransaction,
   getAllTenantIds,
+  getAllTenantWithSubdomain,
 };

package/src/shared/openTelemetry.js

@@ -85,11 +85,22 @@ const _startOtelTrace = async (ctxWithSpan, traceContext, span, fn) => {
 const _setAttributes = (context, span, attributes) => {
   span.setAttribute("sap.tenancy.tenant_id", context.tenant);
   span.setAttribute("sap.correlation_id", context.id);
+  _sanitizeAttributes(attributes);
   for (const attributeKey in attributes) {
     span.setAttribute(attributeKey, attributes[attributeKey]);
   }
 };
 
+const _sanitizeAttributes = (attributes = {}) => {
+  for (const attributeKey in attributes) {
+    attributes[attributeKey] =
+      typeof attributes[attributeKey] !== "string"
+        ? JSON.stringify(attributes[attributeKey])
+        : attributes[attributeKey];
+  }
+  return attributes;
+};
+
 const getCurrentTraceContext = () => {
   if (!otel) {
     return null;

package/srv/index.cds

@@ -0,0 +1 @@
+using from './service';

package/srv/service/admin-service.cds

@@ -0,0 +1,31 @@
+using sap.eventqueue as db from '../../db';
+
+@path: 'event-queue/admin'
+@impl: './admin-service.js'
+@requires: 'internal-user'
+service EventQueueAdminService {
+
+  @readonly
+  @cds.persistence.skip
+  entity Event as projection on db.Event {
+        null as tenant: String,
+        null as landscape: String,
+        null as space: String,
+        *
+  } actions {
+    action setStatusAndAttempts(
+      // TODO: remove tenant as soon as CAP issue is fixed https://github.tools.sap/cap/issues/issues/18445
+      @mandatory
+      tenant: String,
+      status: db.Status,
+      @assert.range: [0,100]
+      attempts: Integer) returns Event;
+  }
+
+   @readonly
+   @cds.persistence.skip
+   entity Tenant {
+      Key ID: String;
+      subdomain: String;
+   }
+}

package/srv/service/admin-service.js

@@ -0,0 +1,98 @@
+"use strict";
+
+const cds = require("@sap/cds");
+const cdsHelper = require("../../src/shared/cdsHelper");
+const { EventProcessingStatus } = require("../../src");
+const config = require("../../src/config");
+
+module.exports = class AdminService extends cds.ApplicationService {
+  async init() {
+    const { Event: EventService, Tenant } = this.entities();
+    const { Event: EventDb } = cds.db.entities("sap.eventqueue");
+    const { landscape, space } = this.getLandscapeAndSpace();
+
+    this.before("*", async (req) => {
+      if (!config.enableAdminService) {
+        req.reject(403, "Admin service is disabled by configuration");
+      }
+
+      if (req.target.name === Tenant.name) {
+        return;
+      }
+      const headers = Object.assign({}, req.headers, req.req?.headers);
+      const tenant = headers["z-id"] ?? req.data.tenant;
+
+      if (config.isMultiTenancy && tenant == null) {
+        req.reject(400, "Missing tenant ID in request header (z-id)");
+      }
+      req.headers ??= {};
+      req.headers["z-id"] = tenant;
+    });
+
+    this.on("READ", EventService, async (req) => {
+      const tenant = req.headers["z-id"];
+      return await cds.tx({ tenant: tenant }, async (tx) => {
+        if (req.query.SELECT.from.ref[0].id) {
+          req.query.SELECT.from.ref[0].id = EventDb.name;
+        } else {
+          req.query.SELECT.from.ref[0] = EventDb.name;
+        }
+        const events = await tx.run(req.query);
+        return events?.map((event) => {
+          event.landscape = landscape;
+          event.space = space;
+          event.tenant = tenant;
+          return event;
+        });
+      });
+    });
+
+    this.on("READ", Tenant, async () => {
+      const tenants = await cdsHelper.getAllTenantWithSubdomain();
+      return tenants ?? [];
+    });
+
+    this.on("setStatusAndAttempts", async (req) => {
+      const tenant = req.headers["z-id"];
+      cds.log("eventQueue").info("Restarting processing for event queue");
+      const updateData = {};
+
+      if (Number.isInteger(req.data.attempts)) {
+        updateData.attempts = req.data.attempts;
+      }
+
+      if (Object.values(EventProcessingStatus).includes(req.data.status)) {
+        updateData.status = req.data.status;
+      }
+
+      if (!Object.keys(updateData).length) {
+        return req.reject(400, "No status or attempts provided");
+      }
+
+      await cds.tx({ tenant, headers: { "z-id": tenant } }, async () => {
+        await UPDATE.entity(EventDb)
+          .set(updateData)
+          .where({ ID: req.params[0].ID ?? req.params[0] });
+      });
+      return await this.send(new cds.Request({ query: req.query, headers: req.headers }));
+    });
+
+    await super.init();
+  }
+
+  getLandscapeAndSpace() {
+    const url = cds.requires.db.credentials.url;
+    if (!url) {
+      return { landscape: "eu10-canary", space: "local-dev" };
+    }
+    const match = url.match(/https?:\/\/[^.]+\.authentication\.([^.]+)\.hana\.ondemand\.com/);
+    const landscape = (match?.[1] ?? "sap") === "sap" ? "eu10-canary" : match?.[1];
+    let space = "local-dev";
+    try {
+      space = JSON.parse(process.env.VCAP_APPLICATION)?.space_name;
+    } catch {
+      /* empty */
+    }
+    return { landscape, space };
+  }
+};

package/srv/service/index.cds

@@ -0,0 +1 @@
+using from './admin-service';