@cap-js-community/event-queue 1.11.0-beta.5 → 2.0.0-beta.0

package/src/shared/cdsHelper.js

@@ -165,38 +165,7 @@ const getAllTenantIds = async () => {
   }, []);
 };
 
-const TENANT_COLUMNS = ["subscribedSubdomain", "createdAt", "modifiedAt"];
-
-const getAllTenantWithMetadata = async () => {
-  const response = await _getAllTenantBase();
-  if (!response) {
-    return null;
-  }
-
-  return response.reduce(async (result, row) => {
-    const tenantId = row.subscribedTenantId ?? row.tenant;
-    result = await result;
-    if (await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId)) {
-      const data = Object.entries(row).reduce(
-        (result, [key, value]) => {
-          if (TENANT_COLUMNS.includes(key)) {
-            result[key] = value;
-          } else {
-            result.metadata[key] = value;
-          }
-          return result;
-        },
-        { metadata: {} }
-      );
-      data.metadata = JSON.stringify(data.metadata);
-      result.push(data);
-    }
-    return result;
-  }, []);
-};
-
 module.exports = {
   executeInNewTransaction,
   getAllTenantIds,
-  getAllTenantWithMetadata,
 };

package/src/shared/common.js

@@ -1,5 +1,6 @@
 "use strict";
 
+const { AsyncResource } = require("async_hooks");
 const crypto = require("crypto");
 
 const cds = require("@sap/cds");
@@ -92,7 +93,7 @@ const hashStringTo32Bit = (value) => crypto.createHash("sha256").update(String(v
 const _getNewAuthContext = async (tenantId) => {
   try {
     if (!_getNewAuthContext._xsuaaService) {
-      _getNewAuthContext._xsuaaService = new xssec.XsuaaService(cds.requires.auth.credentials);
+      _getNewAuthContext._xsuaaService = new xssec.XsuaaService(cds.requires["xsuaa-eventQueue"]?.credentials);
     }
     const authService = _getNewAuthContext._xsuaaService;
     const token = await authService.fetchClientCredentialsToken({ zid: tenantId });
@@ -104,6 +105,7 @@ const _getNewAuthContext = async (tenantId) => {
       err: err.message,
       responseCode: err.responseCode,
       responseText: err.responseText,
+      tenantId,
     });
 
     if (err.responseCode === 404) {
@@ -131,7 +133,10 @@ const getAuthContext = async (tenantId, { returnError = false } = {}) => {
   }
 
   getAuthContext._cache = getAuthContext._cache ?? new ExpiringLazyCache();
-  const result = await getAuthContext._cache.getSetCb(tenantId, async () => _getNewAuthContext(tenantId));
+  const result = await getAuthContext._cache.getSetCb(
+    tenantId,
+    AsyncResource.bind(async () => _getNewAuthContext(tenantId))
+  );
   if (returnError) {
     return result;
   } else {

package/src/shared/distributedLock.js

@@ -60,12 +60,21 @@ const releaseLock = async (context, key, { tenantScoped = true } = {}) => {
   }
 };
 
-const checkLockExistsAndReturnValue = async (context, key, { tenantScoped = true } = {}) => {
+const checkLockExists = async (context, key, { tenantScoped = true } = {}) => {
   const fullKey = _generateKey(context, tenantScoped, key);
   if (config.redisEnabled) {
-    return await _checkLockExistsRedis(context, fullKey);
+    return !!(await _getLockValueRedis(context, fullKey));
   } else {
-    return await _checkLockExistsDb(context, fullKey);
+    return !!(await _getLockValueDb(context, fullKey));
+  }
+};
+
+const getValue = async (context, key, { tenantScoped = true } = {}) => {
+  const fullKey = _generateKey(context, tenantScoped, key);
+  if (config.redisEnabled) {
+    return await _getLockValueRedis(context, fullKey);
+  } else {
+    return await _getLockValueDb(context, fullKey);
   }
 };
 
@@ -75,7 +84,7 @@ const _acquireLockRedis = async (
   expiryTime,
   { value = Date.now(), overrideValue = false, keepTrackOfLock } = {}
 ) => {
-  const client = await redis.createMainClientAndConnect(config.redisOptions);
+  const client = await redis.createMainClientAndConnect();
   const result = await client.set(fullKey, value, {
     PX: Math.round(expiryTime),
     ...(overrideValue ? null : { NX: true }),
@@ -88,7 +97,7 @@ const _acquireLockRedis = async (
 };
 
 const _renewLockRedis = async (context, fullKey, expiryTime, { value = "true" } = {}) => {
-  const client = await redis.createMainClientAndConnect(config.redisOptions);
+  const client = await redis.createMainClientAndConnect();
   let result = await client.set(fullKey, value, {
     PX: Math.round(expiryTime),
     XX: true,
@@ -106,12 +115,12 @@ const _renewLockRedis = async (context, fullKey, expiryTime, { value = "true" }
   return result === REDIS_COMMAND_OK;
 };
 
-const _checkLockExistsRedis = async (context, fullKey) => {
-  const client = await redis.createMainClientAndConnect(config.redisOptions);
-  return await client.exists(fullKey);
+const _getLockValueRedis = async (context, fullKey) => {
+  const client = await redis.createMainClientAndConnect();
+  return await client.get(fullKey);
 };
 
-const _checkLockExistsDb = async (context, fullKey) => {
+const _getLockValueDb = async (context, fullKey) => {
   let result;
   await cdsHelper.executeInNewTransaction(context, "distributedLock-checkExists", async (tx) => {
     result = await tx.run(SELECT.one.from(config.tableNameEventLock).where("code =", fullKey));
@@ -120,7 +129,7 @@ const _checkLockExistsDb = async (context, fullKey) => {
 };
 
 const _releaseLockRedis = async (context, fullKey) => {
-  const client = await redis.createMainClientAndConnect(config.redisOptions);
+  const client = await redis.createMainClientAndConnect();
   const result = await client.del(fullKey);
   delete existingLocks[fullKey];
   return result === 1;
@@ -186,14 +195,14 @@ const _acquireLockDB = async (
 };
 
 const _generateKey = (context, tenantScoped, key) => {
-  const keyParts = [config.redisOptions.redisNamespace];
+  const keyParts = [config.redisNamespace];
   tenantScoped && keyParts.push(context.tenant);
   keyParts.push(key);
   return `${keyParts.join("##")}`;
 };
 
 const getAllLocksRedis = async () => {
-  const clientOrCluster = await redis.createMainClientAndConnect(config.redisOptions);
+  const clientOrCluster = await redis.createMainClientAndConnect();
   const output = [];
   const results = [];
 
@@ -259,7 +268,8 @@ const shutdownHandler = async () => {
 module.exports = {
   acquireLock,
   releaseLock,
-  checkLockExistsAndReturnValue,
+  checkLockExists,
+  getValue,
   setValueWithExpire,
   shutdownHandler,
   renewLock,

package/src/shared/eventScheduler.js

@@ -15,9 +15,9 @@ class EventScheduler {
     config.attachUnsubscribeHandler(this.clearForTenant.bind(this));
   }
 
-  scheduleEvent(tenantId, type, subType, startAfter) {
+  scheduleEvent(tenantId, type, subType, namespace, startAfter) {
     const { date, relative } = this.calculateOffset(type, subType, startAfter);
-    const key = [tenantId, type, subType, date.toISOString()].join("##");
+    const key = [tenantId, type, subType, namespace, date.toISOString()].join("##");
     if (this.#scheduledEvents[key]) {
       return; // event combination already scheduled
     }
@@ -34,7 +34,7 @@ class EventScheduler {
       clearTimeout(timeout);
       delete this.#eventsByTenants[tenantId][timeout];
       delete this.#scheduledEvents[key];
-      redisPub.broadcastEvent(tenantId, { type, subType }).catch((err) => {
+      redisPub.broadcastEvent(tenantId, { type, subType, namespace }).catch((err) => {
         cds.log(COMPONENT_NAME).error("could not execute scheduled event", err, {
           tenantId,
           type,

package/src/shared/redis/client.js

@@ -0,0 +1,46 @@
+"use strict";
+
+const { RedisClient } = require("@cap-js-community/common");
+
+const config = require("../../config");
+
+const REDIS_CLIENT_NAME = "eventQueue";
+
+const createMainClientAndConnect = async () => {
+  const redisClient = RedisClient.create(REDIS_CLIENT_NAME);
+  return await redisClient.createMainClientAndConnect(config.redisOptions);
+};
+
+const subscribeRedisChannel = async (channel, subscribeHandler) => {
+  const redisClient = RedisClient.create(REDIS_CLIENT_NAME);
+  const channelWithNamespace = [config.redisNamespace, channel].join("##");
+  return await redisClient.subscribeChannel(config.redisOptions, channelWithNamespace, subscribeHandler);
+};
+
+const publishMessage = async (channel, message) => {
+  const redisClient = RedisClient.create(REDIS_CLIENT_NAME);
+  const channelWithNamespace = [config.redisNamespace, channel].join("##");
+  return await redisClient.publishMessage(config.redisOptions, channelWithNamespace, message);
+};
+
+const connectionCheck = async () => {
+  const redisClient = RedisClient.create(REDIS_CLIENT_NAME);
+  return await redisClient.connectionCheck(config.redisOptions);
+};
+
+const isClusterMode = () => {
+  return RedisClient.create(REDIS_CLIENT_NAME).isCluster;
+};
+
+const registerShutdownHandler = (cb) => {
+  RedisClient.create(REDIS_CLIENT_NAME).beforeCloseHandler = cb;
+};
+
+module.exports = {
+  createMainClientAndConnect,
+  subscribeRedisChannel,
+  publishMessage,
+  connectionCheck,
+  isClusterMode,
+  registerShutdownHandler,
+};

package/src/shared/redis/index.js

@@ -0,0 +1,45 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const client = require("./client");
+const config = require("../../config");
+
+const COMPONENT_NAME = "/eventQueue/redis";
+const REDIS_OFFBOARD_TENANT_CHANNEL = "REDIS_OFFBOARD_TENANT_CHANNEL";
+
+const attachRedisUnsubscribeHandler = () => {
+  cds.log(COMPONENT_NAME).info("attached redis handle for unsubscribe events");
+  client
+    .subscribeRedisChannel(REDIS_OFFBOARD_TENANT_CHANNEL, (messageData) => {
+      try {
+        const { tenantId } = JSON.parse(messageData);
+        cds.log(COMPONENT_NAME).info("received unsubscribe broadcast event", { tenantId });
+        this.executeUnsubscribeHandlers(tenantId);
+      } catch (err) {
+        cds.log(COMPONENT_NAME).error("could not parse unsubscribe broadcast event", err, {
+          messageData,
+        });
+      }
+    })
+    .catch((err) => _errorHandlerSubscribeChannel(REDIS_OFFBOARD_TENANT_CHANNEL, err));
+};
+
+const handleUnsubscribe = (tenantId) => {
+  if (config.redisEnabled) {
+    client.publishMessage(REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId })).catch((error) => {
+      cds.log(COMPONENT_NAME).error(`publishing tenant unsubscribe failed. tenantId: ${tenantId}`, error);
+    });
+  } else {
+    config.executeUnsubscribeHandlers(tenantId);
+  }
+};
+
+const _errorHandlerSubscribeChannel = (channelName, err) =>
+  cds.log(COMPONENT_NAME).error("error subscribing to channel", err, { channelName });
+
+module.exports = {
+  ...client,
+  attachRedisUnsubscribeHandler,
+  handleUnsubscribe,
+};

package/srv/service/admin-service.cds

@@ -42,12 +42,4 @@ service EventQueueAdminService {
         @mandatory
         subType: String) returns Boolean;
     }
-
-   @readonly
-   @cds.persistence.skip
-   entity Tenant {
-      Key ID: String;
-      subdomain: String;
-      metadata: String;
-   }
 }

package/srv/service/admin-service.js

@@ -1,7 +1,6 @@
 "use strict";
 
 const cds = require("@sap/cds");
-const cdsHelper = require("../../src/shared/cdsHelper");
 const { EventProcessingStatus } = require("../../src");
 const config = require("../../src/config");
 const distributedLock = require("../../src/shared/distributedLock");
@@ -9,7 +8,7 @@ const redisPub = require("../../src/redis/redisPub");
 
 module.exports = class AdminService extends cds.ApplicationService {
   async init() {
-    const { Event: EventService, Tenant, Lock: LockService } = this.entities();
+    const { Event: EventService, Lock: LockService } = this.entities;
     const { Event: EventDb } = cds.db.entities("sap.eventqueue");
     const { landscape, space } = this.getLandscapeAndSpace();
 
@@ -18,9 +17,6 @@ module.exports = class AdminService extends cds.ApplicationService {
         req.reject(403, "Admin service is disabled by configuration");
       }
 
-      if (req.target.name === Tenant.name) {
-        return;
-      }
       const headers = Object.assign({}, req.headers, req.req?.headers);
       const tenant = headers["z-id"] ?? req.data.tenant;
 
@@ -61,11 +57,6 @@ module.exports = class AdminService extends cds.ApplicationService {
       }));
     });
 
-    this.on("READ", Tenant, async () => {
-      const tenants = await cdsHelper.getAllTenantWithMetadata();
-      return tenants ?? [];
-    });
-
     this.on("setStatusAndAttempts", async (req) => {
       const tenant = req.headers["z-id"];
       cds.log("eventQueue").info("Restarting processing for event queue");

package/src/shared/redis.js

@@ -1,199 +0,0 @@
-"use strict";
-
-const redis = require("redis");
-
-const { getEnvInstance } = require("./env");
-const EventQueueError = require("../EventQueueError");
-
-const COMPONENT_NAME = "/eventQueue/shared/redis";
-const LOG_AFTER_SEC = 5;
-
-let mainClientPromise;
-let subscriberClientPromise;
-const subscribedChannels = {};
-let lastErrorLog = Date.now();
-
-const createMainClientAndConnect = (options) => {
-  if (mainClientPromise) {
-    return mainClientPromise;
-  }
-
-  const errorHandlerCreateClient = (err) => {
-    mainClientPromise?.then?.(_resilientClientClose);
-    cds.log(COMPONENT_NAME).error("error from redis main client:", err);
-    mainClientPromise = null;
-    setTimeout(() => createMainClientAndConnect(options), LOG_AFTER_SEC * 1000).unref();
-  };
-
-  mainClientPromise = createClientAndConnect(options, errorHandlerCreateClient);
-  return mainClientPromise;
-};
-
-const _createClientBase = (redisOptions = {}) => {
-  const env = getEnvInstance();
-  try {
-    const { credentials, options } = env.redisRequires;
-    const socket = Object.assign(
-      {
-        host: credentials.hostname,
-        tls: !!credentials.tls,
-        port: credentials.port,
-      },
-      options?.socket,
-      redisOptions.socket
-    );
-    const socketOptions = Object.assign({}, options, redisOptions, {
-      password: redisOptions.password ?? options.password ?? credentials.password,
-      socket,
-    });
-    delete socketOptions.redisNamespace;
-    if (credentials.cluster_mode) {
-      return redis.createCluster({
-        rootNodes: [socketOptions],
-        defaults: socketOptions,
-      });
-    }
-    return redis.createClient(socketOptions);
-  } catch (err) {
-    throw EventQueueError.redisConnectionFailure(err);
-  }
-};
-
-const createClientAndConnect = async (options, errorHandlerCreateClient, isConnectionCheck) => {
-  try {
-    const client = _createClientBase(options);
-    if (!isConnectionCheck) {
-      client.on("error", (err) => {
-        const dateNow = Date.now();
-        if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
-          cds.log(COMPONENT_NAME).error("error redis client:", err);
-          lastErrorLog = dateNow;
-        }
-      });
-
-      client.on("reconnecting", () => {
-        const dateNow = Date.now();
-        if (dateNow - lastErrorLog > LOG_AFTER_SEC * 1000) {
-          cds.log(COMPONENT_NAME).info("redis client trying reconnect...");
-          lastErrorLog = dateNow;
-        }
-      });
-    }
-    await client.connect();
-    return client;
-  } catch (err) {
-    errorHandlerCreateClient(err);
-  }
-};
-
-const subscribeRedisChannel = (options, channel, subscribeHandler) => {
-  subscribedChannels[channel] = subscribeHandler;
-  const errorHandlerCreateClient = (err) => {
-    cds.log(COMPONENT_NAME).error(`error from redis client for pub/sub failed for channel ${channel}`, err);
-    subscriberClientPromise?.then?.(_resilientClientClose);
-    subscriberClientPromise = null;
-    setTimeout(() => _subscribeChannels(options, subscribedChannels, subscribeHandler), LOG_AFTER_SEC * 1000).unref();
-  };
-
-  _subscribeChannels(options, { [channel]: subscribeHandler }, errorHandlerCreateClient);
-};
-
-const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClient) => {
-  subscriberClientPromise = createClientAndConnect(options, errorHandlerCreateClient)
-    .then((client) => {
-      for (const channel in subscribedChannels) {
-        const fn = subscribedChannels[channel];
-        client._subscribedChannels ??= {};
-        if (client._subscribedChannels[channel]) {
-          continue;
-        }
-        const prefixedChannelName = [options.redisNamespace, channel].join("_");
-        cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel: prefixedChannelName });
-        client
-          .subscribe(prefixedChannelName, fn)
-          .then(() => {
-            client._subscribedChannels ??= {};
-            client._subscribedChannels[channel] = 1;
-          })
-          .catch(() => {
-            cds.log(COMPONENT_NAME).error("error subscribe to channel - retrying...");
-            setTimeout(() => _subscribeChannels(options, [channel], fn), LOG_AFTER_SEC * 1000).unref();
-          });
-      }
-    })
-    .catch((err) => {
-      cds
-        .log(COMPONENT_NAME)
-        .error(
-          `error from redis client for pub/sub failed during startup - trying to reconnect - ${Object.keys(
-            subscribedChannels
-          ).join(", ")}`,
-          err
-        );
-    });
-};
-
-const publishMessage = async (options, channel, message) => {
-  const client = await createMainClientAndConnect(options);
-  return await client.publish([options.redisNamespace, channel].join("_"), message);
-};
-
-const closeMainClient = async () => {
-  await _resilientClientClose(await mainClientPromise);
-  cds.log(COMPONENT_NAME).info("main redis client closed!");
-};
-
-const closeSubscribeClient = async () => {
-  await _resilientClientClose(await subscriberClientPromise);
-  cds.log(COMPONENT_NAME).info("subscribe redis client closed!");
-};
-
-const _resilientClientClose = async (client) => {
-  try {
-    if (client?.quit) {
-      await client.quit();
-    }
-  } catch (err) {
-    cds.log(COMPONENT_NAME).info("error during redis close - continuing...", err);
-  }
-};
-
-const connectionCheck = async (options) => {
-  return new Promise((resolve, reject) => {
-    createClientAndConnect(options, reject, true)
-      .then((client) => {
-        if (client) {
-          _resilientClientClose(client);
-          resolve();
-        } else {
-          reject(new Error());
-        }
-      })
-      .catch(reject);
-  })
-    .then(() => true)
-    .catch((err) => {
-      cds.log(COMPONENT_NAME).error("Redis connection check failed! Falling back to NO_REDIS mode", err);
-      return false;
-    });
-};
-
-const isClusterMode = () => {
-  if (!("__clusterMode" in isClusterMode)) {
-    const env = getEnvInstance();
-    const { credentials } = env.redisRequires;
-    isClusterMode.__clusterMode = credentials.cluster_mode;
-  }
-  return isClusterMode.__clusterMode;
-};
-
-module.exports = {
-  createClientAndConnect,
-  createMainClientAndConnect,
-  subscribeRedisChannel,
-  publishMessage,
-  closeMainClient,
-  closeSubscribeClient,
-  connectionCheck,
-  isClusterMode,
-};