@cap-js-community/event-queue 0.1.49

package/src/shared/PerformanceTracer.js

@@ -0,0 +1,74 @@
+"use strict";
+
+const CUSTOM_FIELD_EXECUTION_TIME = "response_time_ms";
+const CUSTOM_FIELD_QUANTITY = "quantity";
+
+class PerformanceTracer {
+  /**
+   * The constructor of the performance tracer
+   * @param {Object} logger logger
+   * @param {string} name name of the performance trace
+   * @param {Object} options An options object with additional properties (optional)
+   * @param {string} options.level The level to be used for logging
+   * @param {string} options.summary The summary to be used for logging
+   * @param {Object} options.startMessage Writes a log record with the provided details when starting the action
+   */
+  constructor(logger, name, options = {}) {
+    this.__start = new Date();
+    this.__logger = logger;
+    this.__name = name;
+    options.startMessage &&
+      logger.info("Performance measurement started", {
+        name: name,
+        ...options.startMessage,
+      });
+  }
+
+  /**
+   * Ends the performance trace
+   * @param {Object} options An options object with additional properties (optional)
+   * @param {Number} options.quantity A case-specific quantity such as a node count that influences the execution time (optional)
+   * @param {Number} options.threshold Only write the log above verbose level, if threshold in ms. is met (optional)
+   * @param {Number} options.additionalQuantityThreshold Value multiplied with quantity and added to the threshold (optional)
+   */
+  endPerformanceTrace(...args) {
+    let options = {};
+    //determine, if an options object was provided as first argument
+    if (
+      typeof args?.[0] === "object" &&
+      (args[0].quantity >= 0 ||
+        args[0].threshold > 0 ||
+        args[0].additionalQuantityThreshold > 0)
+    ) {
+      options = args.shift();
+    }
+    const currentTime = new Date();
+    const executionTime = currentTime - this.__start;
+    this.__start = currentTime;
+    const isBelowThreshold =
+      options &&
+      options.threshold &&
+      executionTime <
+        options.threshold +
+          (options.additionalQuantityThreshold > 0 && options.quantity > 0
+            ? options.additionalQuantityThreshold * options.quantity
+            : 0);
+
+    if (isBelowThreshold) {
+      return;
+    }
+
+    const customFields = {
+      [CUSTOM_FIELD_EXECUTION_TIME]: executionTime,
+      [CUSTOM_FIELD_QUANTITY]: options.quantity,
+    };
+
+    this.__logger.info("Performance measurement executed", {
+      name: this.__name,
+      milliseconds: executionTime,
+      customFields,
+    });
+  }
+}
+
+module.exports = PerformanceTracer;

package/src/shared/WorkerQueue.js

@@ -0,0 +1,78 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const { getConfigInstance } = require("../config");
+
+const COMPONENT_NAME = "eventQueue/WorkerQueue";
+
+let instance = null;
+
+class WorkerQueue {
+  constructor(concurrency) {
+    if (Number.isNaN(concurrency) || concurrency <= 0) {
+      this.__concurrencyLimit = 1;
+    } else {
+      this.__concurrencyLimit = concurrency;
+    }
+    this.__runningPromises = [];
+    this.__queue = [];
+  }
+
+  addToQueue(cb) {
+    const p = new Promise((resolve, reject) => {
+      this.__queue.push([cb, resolve, reject]);
+    });
+    this._checkForNext();
+    return p;
+  }
+
+  _executeFunction(cb, resolve, reject) {
+    const promise = Promise.resolve().then(() => cb());
+    this.__runningPromises.push(promise);
+    promise
+      .finally(() => {
+        this.__runningPromises.splice(
+          this.__runningPromises.indexOf(promise),
+          1
+        );
+        this._checkForNext();
+      })
+      .then((...results) => {
+        resolve(...results);
+      })
+      .catch((err) => {
+        cds
+          .log(COMPONENT_NAME)
+          .error(
+            "Error happened in WorkQueue. Errors should be caught before! Error:",
+            err
+          );
+        reject(err);
+      });
+  }
+
+  _checkForNext() {
+    if (
+      !this.__queue.length ||
+      this.__runningPromises.length >= this.__concurrencyLimit
+    ) {
+      return;
+    }
+    const [cb, resolve, reject] = this.__queue.shift();
+    this._executeFunction(cb, resolve, reject);
+  }
+}
+
+module.exports = {
+  getWorkerPoolInstance: () => {
+    if (!instance) {
+      const configInstance = getConfigInstance();
+      instance = new WorkerQueue(configInstance.parallelTenantProcessing);
+    }
+    return instance;
+  },
+  _: {
+    WorkerQueue,
+  },
+};

package/src/shared/cdsHelper.js

@@ -0,0 +1,136 @@
+"use strict";
+
+const VError = require("verror");
+const cds = require("@sap/cds");
+
+const config = require("../config");
+
+const subdomainCache = {};
+
+const VERROR_CLUSTER_NAME = "ExecuteInNewTransactionError";
+const COMPONENT_NAME = "eventQueue/cdsHelper";
+
+/**
+ * Execute logic in a new managed CDS transaction context, auto-handling commit, rollback and error/exception situations.
+ * Includes logging of start, end and error situation with additional info object and unique transaction id (txId)
+ * @param context {object} Current CDS request context
+ * @param transactionTag A tag identifying the transaction
+ * @param fn {function} Callback function (logic) to be executed in context of new managed CDS transaction
+ * @param args {array|object|any} Array of function arguments passed to callback function as spread arguments. Object or primitive types are auto-normalized to array.
+ * @param info {object} Additional information object attached to logging
+ * @returns {Promise<boolean>} Promise resolving to true if everything worked fine / false if an error occurred
+ */
+async function executeInNewTransaction(
+  context = {},
+  transactionTag,
+  fn,
+  args,
+  { info = {} } = {}
+) {
+  const parameters = Array.isArray(args) ? args : [args];
+  const logger = cds.log(COMPONENT_NAME);
+  try {
+    if (cds.db.kind === "hana") {
+      await cds.tx(
+        {
+          id: context.id,
+          tenant: context.tenant,
+          locale: context.locale,
+          user: context.user,
+          headers: context.headers,
+          http: context.http,
+        },
+        async (tx) => {
+          tx.context._ = context._ ?? {};
+          return await fn(tx, ...parameters);
+        }
+      );
+    } else {
+      const contextTx = cds.tx(context);
+      const contextTxState = contextTx.ready;
+      if (
+        !contextTxState ||
+        ["committed", "rolled back"].includes(contextTxState)
+      ) {
+        await cds.tx(
+          {
+            id: context.id,
+            tenant: context.tenant,
+            locale: context.locale,
+            user: context.user,
+            headers: context.headers,
+            http: context.http,
+          },
+          async (tx) => fn(tx, ...parameters)
+        );
+      } else {
+        await fn(contextTx, ...parameters);
+      }
+    }
+  } catch (err) {
+    if (!(err instanceof TriggerRollback)) {
+      if (err instanceof VError) {
+        Object.assign(err.jse_info, {
+          newTx: info,
+        });
+        throw err;
+      } else {
+        throw new VError(
+          {
+            name: VERROR_CLUSTER_NAME,
+            cause: err,
+            info,
+          },
+          "Execution in new transaction failed"
+        );
+      }
+    }
+    return false;
+  } finally {
+    logger.debug("Execution in new transaction finished", info);
+  }
+  return true;
+}
+
+/**
+ * Error class to be used to force rollback in executionInNewTransaction
+ * Error will not be logged, as it assumes that error handling has been done before...
+ */
+class TriggerRollback extends VError {
+  constructor() {
+    super("Rollback triggered");
+  }
+}
+
+const getSubdomainForTenantId = async (tenantId) => {
+  if (!config.getConfigInstance().isMultiTenancy) {
+    return null;
+  }
+  if (subdomainCache[tenantId]) {
+    return subdomainCache[tenantId];
+  }
+  try {
+    const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");
+    const response = await ssp.get("/tenant", { subscribedTenantId: tenantId });
+    subdomainCache[tenantId] = response.subscribedSubdomain;
+    return response.subscribedSubdomain;
+  } catch (err) {
+    return null;
+  }
+};
+
+const getAllTenantIds = async () => {
+  if (!config.getConfigInstance().isMultiTenancy) {
+    return null;
+  }
+  const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");
+  const response = await ssp.get("/tenant");
+  return response.map((tenant) => tenant.subscribedTenantId ?? tenant.tenant);
+};
+
+module.exports = {
+  executeInNewTransaction,
+  TriggerRollback,
+  getSubdomainForTenantId,
+  getAllTenantIds,
+};

package/src/shared/common.js

@@ -0,0 +1,115 @@
+"use strict";
+
+const { floor, abs, min } = Math;
+
+const arrayToFlatMap = (array, key = "ID") => {
+  return array.reduce((result, element) => {
+    result[element[key]] = element;
+    return result;
+  }, {});
+};
+
+/**
+ * Establish a "Funnel" instance to limit how much
+ * load can be processed in parallel. This is somewhat
+ * similar to the limiter function however it has some
+ * distinctintly different features. The Funnel will
+ * not know in advance which functions and how many
+ * loads it will have to process.
+ */
+class Funnel {
+  /**
+   * Create a funnel with specified capacity
+   * @param capacity - the capacity of the funnel (integer, sign will be ignored)
+   */
+  constructor(capacity = 100) {
+    this.runningPromises = [];
+    this.capacity = floor(abs(capacity));
+  }
+
+  /**
+   * Asynchronously run a function that will put a specified load to the funnel.
+   * The total amount of load of all running functions shall not
+   * exceed the capacity of the funnel. If the desired load exceeds the capacity
+   * the funnel will wait until sufficient capacity is available.
+   * If a function requires a load >= capacity, then it will run
+   * exclusively.
+   * @param load - the load (integer, sign will be ignored)
+   * @param f
+   * @param args
+   * @return {Promise<unknown>}
+   */
+  async run(load, f, ...args) {
+    load = min(floor(abs(load)), Number.MAX_SAFE_INTEGER);
+
+    // wait for sufficient capacity
+    while (this.capacity < load && this.runningPromises.length > 0) {
+      try {
+        await Promise.race(this.runningPromises);
+      } catch {
+        // Yes, we must ignore exceptions here. The
+        // caller expects exceptions from f and no
+        // exceptions from other workloads.
+        // Other exceptions must be handled by the
+        // other callers. See (*) below.
+      }
+    }
+
+    // map function call to promise
+    const p =
+      f.constructor.name === "AsyncFunction"
+        ? f(...args)
+        : Promise.resolve().then(() => f(...args));
+
+    // create promise for book keeping
+    const workload = p.finally(() => {
+      // remove workload
+      this.runningPromises.splice(this.runningPromises.indexOf(workload), 1);
+      // and reclaim its capacity
+      this.capacity += load;
+    });
+
+    // claim the capacity and schedule workload
+    this.capacity -= load;
+    this.runningPromises.push(workload);
+
+    // make the caller wait for the workload
+    // this also establish the seemingly missing
+    // exception handling. See (*) above.
+    return workload;
+  }
+}
+
+/**
+ * Defines a promise that resolves when all payloads are processed by the iterator, but limits
+ * the number concurrent executions.
+ *
+ * @param limit     number of concurrent executions
+ * @param payloads  array where each element is an array of arguments passed to the iterator
+ * @param iterator  (async) function to process a payload
+ * @returns {Promise<[]>} promise for an array of iterator results
+ */
+const limiter = async (limit, payloads, iterator) => {
+  const returnPromises = [];
+  const runningPromises = [];
+  for (const payload of payloads) {
+    const p =
+      iterator.constructor.name === "AsyncFunction"
+        ? iterator(payload)
+        : Promise.resolve().then(() => iterator(payload));
+    returnPromises.push(p);
+
+    if (limit <= payloads.length) {
+      const e = p
+        .catch(() => {})
+        .finally(() => runningPromises.splice(runningPromises.indexOf(e), 1));
+      runningPromises.push(e);
+      if (limit <= runningPromises.length) {
+        await Promise.race(runningPromises);
+      }
+    }
+  }
+  return Promise.allSettled(returnPromises);
+};
+
+module.exports = { arrayToFlatMap, Funnel, limiter };

package/src/shared/distributedLock.js

@@ -0,0 +1,191 @@
+"use strict";
+
+const redis = require("./redis");
+const config = require("../config");
+const cdsHelper = require("./cdsHelper");
+const { getConfigInstance } = require("../config");
+
+const acquireLock = async (
+  context,
+  key,
+  {
+    tenantScoped = true,
+    expiryTime = config.getConfigInstance().globalTxTimeout,
+  } = {}
+) => {
+  const fullKey = _generateKey(context, tenantScoped, key);
+  if (config.getConfigInstance().redisEnabled) {
+    return await _acquireLockRedis(context, fullKey, expiryTime);
+  } else {
+    return await _acquireLockDB(context, fullKey, expiryTime);
+  }
+};
+
+const setValueWithExpire = async (
+  context,
+  key,
+  value,
+  {
+    tenantScoped = true,
+    expiryTime = config.getConfigInstance().globalTxTimeout,
+    overrideValue = false,
+  } = {}
+) => {
+  const fullKey = _generateKey(context, tenantScoped, key);
+  if (config.getConfigInstance().redisEnabled) {
+    return await _acquireLockRedis(context, fullKey, expiryTime, {
+      value,
+      overrideValue,
+    });
+  } else {
+    return await _acquireLockDB(context, fullKey, expiryTime, {
+      value,
+      overrideValue,
+    });
+  }
+};
+
+const releaseLock = async (context, key, { tenantScoped = true } = {}) => {
+  const fullKey = _generateKey(context, tenantScoped, key);
+  if (config.getConfigInstance().redisEnabled) {
+    return await _releaseLockRedis(context, fullKey);
+  } else {
+    return await _releaseLockDb(context, fullKey);
+  }
+};
+
+const checkLockExistsAndReturnValue = async (
+  context,
+  key,
+  { tenantScoped = true } = {}
+) => {
+  const fullKey = _generateKey(context, tenantScoped, key);
+  if (config.getConfigInstance().redisEnabled) {
+    return await _checkLockExistsRedis(context, fullKey);
+  } else {
+    return await _checkLockExistsDb(context, fullKey);
+  }
+};
+
+const _acquireLockRedis = async (
+  context,
+  fullKey,
+  expiryTime,
+  { value = "true", overrideValue = false } = {}
+) => {
+  const client = await redis.createMainClientAndConnect();
+  const result = await client.set(fullKey, value, {
+    PX: expiryTime,
+    ...(overrideValue ? null : { NX: true }),
+  });
+  return result === "OK";
+};
+
+const _checkLockExistsRedis = async (context, fullKey) => {
+  const client = await redis.createMainClientAndConnect();
+  return await client.get(fullKey);
+};
+
+const _checkLockExistsDb = async (context, fullKey) => {
+  let result;
+  const configInstance = getConfigInstance();
+  await cdsHelper.executeInNewTransaction(
+    context,
+    "distributedLock-checkExists",
+    async (tx) => {
+      result = await tx.run(
+        SELECT.one
+          .from(configInstance.tableNameEventLock)
+          .where("code =", fullKey)
+      );
+    }
+  );
+  return result?.value;
+};
+
+const _releaseLockRedis = async (context, fullKey) => {
+  const client = await redis.createMainClientAndConnect();
+  await client.del(fullKey);
+};
+
+const _releaseLockDb = async (context, fullKey) => {
+  const configInstance = getConfigInstance();
+  await cdsHelper.executeInNewTransaction(
+    context,
+    "distributedLock-release",
+    async (tx) => {
+      await tx.run(
+        DELETE.from(configInstance.tableNameEventLock).where("code =", fullKey)
+      );
+    }
+  );
+};
+
+const _acquireLockDB = async (
+  context,
+  fullKey,
+  expiryTime,
+  { value = "true", overrideValue = false } = {}
+) => {
+  let result;
+  const configInstance = getConfigInstance();
+  await cdsHelper.executeInNewTransaction(
+    context,
+    "distributedLock-acquire",
+    async (tx) => {
+      try {
+        await tx.run(
+          INSERT.into(configInstance.tableNameEventLock).entries({
+            code: fullKey,
+            value,
+          })
+        );
+        result = true;
+      } catch (err) {
+        let currentEntry;
+
+        if (!overrideValue) {
+          currentEntry = await tx.run(
+            SELECT.one
+              .from(configInstance.tableNameEventLock)
+              .forUpdate({ wait: config.getConfigInstance().forUpdateTimeout })
+              .where("code =", fullKey)
+          );
+        }
+        if (
+          overrideValue ||
+          (currentEntry &&
+            new Date(currentEntry.createdAt).getTime() + expiryTime <=
+              Date.now())
+        ) {
+          await tx.run(
+            UPDATE.entity(configInstance.tableNameEventLock)
+              .set({
+                createdAt: new Date().toISOString(),
+                value,
+              })
+              .where("code =", currentEntry.code)
+          );
+          result = true;
+        } else {
+          result = false;
+        }
+      }
+    }
+  );
+  return result;
+};
+
+const _generateKey = (context, tenantScoped, key) => {
+  const keyParts = [];
+  tenantScoped && keyParts.push(context.tenant);
+  keyParts.push(key);
+  return keyParts.join("##");
+};
+
+module.exports = {
+  acquireLock,
+  releaseLock,
+  checkLockExistsAndReturnValue,
+  setValueWithExpire,
+};

package/src/shared/env.js

@@ -0,0 +1,9 @@
+"use strict";
+
+const isLocal = process.env.USER !== "vcap";
+const isOnCF = !isLocal;
+
+module.exports = {
+  isOnCF,
+  isLocal,
+};

package/src/shared/redis.js

@@ -0,0 +1,70 @@
+"use strict";
+
+const redis = require("redis");
+
+const config = require("../config");
+const EventQueueError = require("../EventQueueError");
+
+const COMPONENT_NAME = "eventQueue/shared/redis";
+
+let subscriberClientPromise;
+
+const createMainClientAndConnect = () => {
+  if (subscriberClientPromise) {
+    return subscriberClientPromise;
+  }
+
+  const errorHandlerCreateClient = (err) => {
+    cds
+      .log(COMPONENT_NAME)
+      .error("error from redis client for pub/sub failed", err);
+    subscriberClientPromise = null;
+    setTimeout(createMainClientAndConnect, 5 * 1000).unref();
+  };
+  subscriberClientPromise = createClientAndConnect(errorHandlerCreateClient);
+  return subscriberClientPromise;
+};
+
+const _createClientBase = () => {
+  const configInstance = config.getConfigInstance();
+  if (configInstance.isOnCF) {
+    try {
+      const credentials = configInstance.getRedisCredentialsFromEnv();
+      // NOTE: settings the user explicitly to empty resolves auth problems, see
+      // https://github.com/go-redis/redis/issues/1343
+      const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
+      return redis.createClient({ url });
+    } catch (err) {
+      throw EventQueueError.redisConnectionFailure(err);
+    }
+  } else {
+    return redis.createClient({
+      socket: { reconnectStrategy: _localReconnectStrategy },
+    });
+  }
+};
+
+const createClientAndConnect = async (errorHandlerCreateClient) => {
+  let client = null;
+  try {
+    client = _createClientBase();
+  } catch (err) {
+    throw EventQueueError.redisConnectionFailure(err);
+  }
+
+  client.on("error", errorHandlerCreateClient);
+
+  try {
+    await client.connect();
+  } catch (err) {
+    errorHandlerCreateClient(err);
+  }
+  return client;
+};
+
+const _localReconnectStrategy = () => EventQueueError.redisNoReconnect();
+
+module.exports = {
+  createClientAndConnect,
+  createMainClientAndConnect,
+};