@cap-js-community/event-queue 0.1.53 → 0.1.54

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/event-queue",
-  "version": "0.1.53",
+  "version": "0.1.54",
   "description": "event queue for cds",
   "main": "src/index.js",
   "files": [

package/src/config.js

@@ -2,30 +2,33 @@
 
 const cds = require("@sap/cds");
 
-const env = require("./shared/env");
+const { getEnvInstance: getEnvInstance } = require("./shared/env");
+const redis = require("./shared/redis");
 
 let instance;
 
 const FOR_UPDATE_TIMEOUT = 10;
 const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
+const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
+const COMPONENT_NAME = "eventQueue/config";
 
 class Config {
   constructor() {
+    this.__logger = cds.log(COMPONENT_NAME);
     this.__config = null;
     this.__forUpdateTimeout = FOR_UPDATE_TIMEOUT;
     this.__globalTxTimeout = GLOBAL_TX_TIMEOUT;
     this.__runInterval = null;
     this.__redisEnabled = null;
-    this.__isOnCF = env.isOnCF;
     this.__initialized = false;
     this.__parallelTenantProcessing = null;
     this.__tableNameEventQueue = null;
     this.__tableNameEventLock = null;
-    this.__vcapServices = this._parseVcapServices();
     this.__isRunnerDeactivated = false;
     this.__configFilePath = null;
     this.__processEventsAfterPublish = null;
     this.__skipCsnCheck = null;
+    this.__env = getEnvInstance();
   }
 
   getEventConfig(type, subType) {
@@ -37,23 +40,37 @@ class Config {
   }
 
   _checkRedisIsBound() {
-    return !!this.getRedisCredentialsFromEnv();
+    return !!this.__env.getRedisCredentialsFromEnv();
   }
 
-  getRedisCredentialsFromEnv() {
-    return this.__vcapServices["redis-cache"]?.[0]?.credentials;
+  checkRedisEnabled() {
+    this.__redisEnabled = this._checkRedisIsBound() && this.__env.isOnCF;
   }
 
-  _parseVcapServices() {
-    try {
-      return JSON.parse(process.env.VCAP_SERVICES);
-    } catch {
-      return {};
-    }
+  attachConfigChangeHandler() {
+    redis.subscribeRedisChannel(REDIS_CONFIG_CHANNEL, (messageData) => {
+      try {
+        const { key, value } = JSON.parse(messageData);
+        if (this[key] !== value) {
+          this.__logger.info("received config change", { key, value });
+          this[key] = value;
+        }
+      } catch (err) {
+        this.__logger.error("could not parse event config change", {
+          messageData,
+        });
+      }
+    });
   }
 
-  calculateIsRedisEnabled() {
-    this.__redisEnabled = this._checkRedisIsBound() && this.__isOnCF;
+  publishConfigChange(key, value) {
+    if (!this.redisEnabled) {
+      this.__logger.info("redis not connected, config change won't be published", { key, value });
+      return;
+    }
+    redis.publishMessage(REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
+      this.__logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
+    });
   }
 
   get isRunnerDeactivated() {
@@ -112,14 +129,6 @@ class Config {
     this.__redisEnabled = value;
   }
 
-  get isOnCF() {
-    return this.__isOnCF;
-  }
-
-  set isOnCF(value) {
-    this.__isOnCF = value;
-  }
-
   get initialized() {
     return this.__initialized;
   }

package/src/initialize.js

@@ -28,6 +28,7 @@ const initialize = async ({
   configFilePath,
   registerAsEventProcessor,
   processEventsAfterPublish,
+  isRunnerDeactivated,
   runInterval,
   parallelTenantProcessing,
   tableNameEventQueue,
@@ -48,6 +49,7 @@ const initialize = async ({
     configFilePath,
     registerAsEventProcessor,
     processEventsAfterPublish,
+    isRunnerDeactivated,
     runInterval,
     parallelTenantProcessing,
     tableNameEventQueue,
@@ -57,7 +59,7 @@ const initialize = async ({
 
   const logger = cds.log(COMPONENT);
   configInstance.fileContent = await readConfigFromFile(configInstance.configFilePath);
-  configInstance.calculateIsRedisEnabled();
+  configInstance.checkRedisEnabled();
 
   const dbService = await cds.connect.to("db");
   await (cds.model ? Promise.resolve() : new Promise((resolve) => cds.on("serving", resolve)));
@@ -107,6 +109,7 @@ const registerEventProcessors = () => {
 
   if (configInstance.redisEnabled) {
     initEventQueueRedisSubscribe();
+    configInstance.attachConfigChangeHandler();
     runner.multiTenancyRedis();
   } else {
     runner.multiTenancyDb();
@@ -161,6 +164,7 @@ const mixConfigVarsWithEnv = (
   configFilePath,
   registerAsEventProcessor,
   processEventsAfterPublish,
+  isRunnerDeactivated,
   runInterval,
   parallelTenantProcessing,
   tableNameEventQueue,
@@ -172,6 +176,7 @@ const mixConfigVarsWithEnv = (
   configInstance.configFilePath = configFilePath ?? cds.env.eventQueue?.configFilePath;
   configInstance.registerAsEventProcessor =
     registerAsEventProcessor ?? cds.env.eventQueue?.registerAsEventProcessor ?? true;
+  configInstance.isRunnerDeactivated = isRunnerDeactivated ?? cds.env.eventQueue?.isRunnerDeactivated ?? false;
   configInstance.processEventsAfterPublish =
     processEventsAfterPublish ?? cds.env.eventQueue?.processEventsAfterPublish ?? true;
   configInstance.runInterval = runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;

package/src/redisPubSub.js

@@ -7,83 +7,50 @@ const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
 const config = require("./config");
 const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
 
-const MESSAGE_CHANNEL = "cdsEventQueue";
+const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
 const COMPONENT_NAME = "eventQueue/redisPubSub";
 
-let publishClient;
 let subscriberClientPromise;
 
 const initEventQueueRedisSubscribe = () => {
   if (subscriberClientPromise || !config.getConfigInstance().redisEnabled) {
     return;
   }
-  subscribeRedisClient();
-};
-
-const subscribeRedisClient = () => {
-  const errorHandlerCreateClient = (err) => {
-    cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
-    subscriberClientPromise = null;
-    setTimeout(subscribeRedisClient, 5 * 1000).unref();
-  };
-  subscriberClientPromise = redis.createClientAndConnect(errorHandlerCreateClient);
-  subscriberClientPromise
-    .then((client) => {
-      cds.log(COMPONENT_NAME).info("subscribe redis client connected");
-      client.subscribe(MESSAGE_CHANNEL, messageHandlerProcessEvents);
-    })
-    .catch((err) => {
-      cds
-        .log(COMPONENT_NAME)
-        .error("error from redis client for pub/sub failed during startup - trying to reconnect", err);
-    });
+  redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL, messageHandlerProcessEvents);
 };
 
 const messageHandlerProcessEvents = async (messageData) => {
-  let tenantId, type, subType;
+  const logger = cds.log(COMPONENT_NAME);
   try {
-    ({ tenantId, type, subType } = JSON.parse(messageData));
+    const { tenantId, type, subType } = JSON.parse(messageData);
+    const subdomain = await getSubdomainForTenantId(tenantId);
+    const context = new cds.EventContext({
+      tenant: tenantId,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    });
+    cds.context = context;
+    logger.debug("received redis event", {
+      tenantId,
+      type,
+      subType,
+    });
+    getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
   } catch (err) {
-    cds.log(COMPONENT_NAME).error("could not parse event information", {
+    logger.error("could not parse event information", {
       messageData,
     });
-    return;
   }
-  const subdomain = await getSubdomainForTenantId(tenantId);
-  const context = new cds.EventContext({
-    tenant: tenantId,
-    // NOTE: we need this because of logging otherwise logs would not contain the subdomain
-    http: { req: { authInfo: { getSubdomain: () => subdomain } } },
-  });
-  cds.context = context;
-  cds.log(COMPONENT_NAME).debug("received redis event", {
-    tenantId,
-    type,
-    subType,
-  });
-  getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
 };
 
 const publishEvent = async (tenantId, type, subType) => {
+  const logger = cds.log(COMPONENT_NAME);
   const configInstance = config.getConfigInstance();
   if (!configInstance.redisEnabled) {
     await _handleEventInternally(tenantId, type, subType);
     return;
   }
-
-  const logger = cds.log(COMPONENT_NAME);
-  const errorHandlerCreateClient = (err) => {
-    logger.error("error from redis client for pub/sub failed", {
-      err,
-    });
-    publishClient = null;
-  };
   try {
-    if (!publishClient) {
-      publishClient = await redis.createClientAndConnect(errorHandlerCreateClient);
-      logger.info("publish redis client connected");
-    }
-
     const result = await checkLockExistsAndReturnValue(
       new cds.EventContext({ tenant: tenantId }),
       [type, subType].join("##")
@@ -97,7 +64,7 @@ const publishEvent = async (tenantId, type, subType) => {
       type,
       subType,
     });
-    await publishClient.publish(MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
+    await redis.publishMessage(EVENT_MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
   } catch (err) {
     logger.error(`publish event failed with error: ${err.toString()}`, {
       tenantId,
@@ -108,8 +75,7 @@ const publishEvent = async (tenantId, type, subType) => {
 };
 
 const _handleEventInternally = async (tenantId, type, subType) => {
-  const logger = cds.log(COMPONENT_NAME);
-  logger.info("processEventQueue internally", {
+  cds.log(COMPONENT_NAME).info("processEventQueue internally", {
     tenantId,
     type,
     subType,

package/src/runner.js

@@ -14,8 +14,6 @@ const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
 const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
 const OFFSET_FIRST_RUN = 10 * 1000;
 
-const LOGGER = cds.log(COMPONENT_NAME);
-
 const singleTenant = () => _scheduleFunction(_executeRunForTenant);
 
 const multiTenancyDb = () => _scheduleFunction(_multiTenancyDb);
@@ -23,16 +21,18 @@ const multiTenancyDb = () => _scheduleFunction(_multiTenancyDb);
 const multiTenancyRedis = () => _scheduleFunction(_multiTenancyRedis);
 
 const _scheduleFunction = async (fn) => {
+  const logger = cds.log(COMPONENT_NAME);
   const configInstance = eventQueueConfig.getConfigInstance();
   const eventsForAutomaticRun = configInstance.events;
   if (!eventsForAutomaticRun.length) {
-    LOGGER.warn("no events for automatic run are configured - skipping runner registration");
+    logger.warn("no events for automatic run are configured - skipping runner registration");
     return;
   }
 
   const fnWithRunningCheck = () => {
+    const logger = cds.log(COMPONENT_NAME);
     if (configInstance.isRunnerDeactivated) {
-      LOGGER.info("runner is deactivated via config variable. Skipping this run.");
+      logger.info("runner is deactivated via config variable. Skipping this run.");
       return;
     }
     return fn();
@@ -40,7 +40,7 @@ const _scheduleFunction = async (fn) => {
 
   const offsetDependingOnLastRun = await _calculateOffsetForFirstRun();
 
-  LOGGER.info("first event-queue run scheduled", {
+  logger.info("first event-queue run scheduled", {
     firstRunScheduledFor: new Date(Date.now() + offsetDependingOnLastRun).toISOString(),
   });
 
@@ -52,13 +52,14 @@ const _scheduleFunction = async (fn) => {
 };
 
 const _multiTenancyRedis = async () => {
+  const logger = cds.log(COMPONENT_NAME);
   const emptyContext = new cds.EventContext({});
-  LOGGER.info("executing event queue run for multi instance and tenant");
+  logger.info("executing event queue run for multi instance and tenant");
   const tenantIds = await cdsHelper.getAllTenantIds();
   const runId = await _acquireRunId(emptyContext);
 
   if (!runId) {
-    LOGGER.error("could not acquire runId, skip processing events!");
+    logger.error("could not acquire runId, skip processing events!");
     return;
   }
 
@@ -66,12 +67,13 @@ const _multiTenancyRedis = async () => {
 };
 
 const _multiTenancyDb = async () => {
+  const logger = cds.log(COMPONENT_NAME);
   try {
-    LOGGER.info("executing event queue run for single instance and multi tenant");
+    logger.info("executing event queue run for single instance and multi tenant");
     const tenantIds = await cdsHelper.getAllTenantIds();
     _executeAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
   } catch (err) {
-    LOGGER.error(
+    logger.error(
       `Couldn't fetch tenant ids for event queue processing! Next try after defined interval. Error: ${err}`
     );
   }
@@ -92,7 +94,7 @@ const _executeAllTenants = (tenantIds, runId) => {
         }
         await _executeRunForTenant(tenantId, runId);
       } catch (err) {
-        LOGGER.error("executing event-queue run for tenant failed", {
+        cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
           tenantId,
         });
       }
@@ -101,6 +103,7 @@ const _executeAllTenants = (tenantIds, runId) => {
 };
 
 const _executeRunForTenant = async (tenantId, runId) => {
+  const logger = cds.log(COMPONENT_NAME);
   const configInstance = eventQueueConfig.getConfigInstance();
   try {
     const eventsForAutomaticRun = configInstance.events;
@@ -111,14 +114,14 @@ const _executeRunForTenant = async (tenantId, runId) => {
       http: { req: { authInfo: { getSubdomain: () => subdomain } } },
     });
     cds.context = context;
-    LOGGER.info("executing eventQueue run", {
+    logger.info("executing eventQueue run", {
       tenantId,
       subdomain,
       ...(runId ? { runId } : null),
     });
     await eventQueueRunner(context, eventsForAutomaticRun);
   } catch (err) {
-    LOGGER.error(`Couldn't process eventQueue for tenant! Next try after defined interval. Error: ${err}`, {
+    logger.error(`Couldn't process eventQueue for tenant! Next try after defined interval. Error: ${err}`, {
       tenantId,
       redisEnabled: configInstance.redisEnabled,
     });
@@ -177,10 +180,12 @@ const _calculateOffsetForFirstRun = async () => {
       offsetDependingOnLastRun = new Date(lastRunTs).getTime() + configInstance.runInterval - now;
     }
   } catch (err) {
-    LOGGER.error(
-      "calculating offset for first run failed, falling back to default. Runs might be out-of-sync. Error:",
-      err
-    );
+    cds
+      .log(COMPONENT_NAME)
+      .error(
+        "calculating offset for first run failed, falling back to default. Runs might be out-of-sync. Error:",
+        err
+      );
   }
   return offsetDependingOnLastRun;
 };

package/src/shared/SetIntervalDriftSafe.js

@@ -35,7 +35,7 @@ class SetIntervalDriftSafe {
     setTimeout(() => {
       this.run(fn);
       fn();
-    }, this.#adjustedInterval);
+    }, this.#adjustedInterval).unref();
   }
 }
 

package/src/shared/env.js

@@ -1,9 +1,53 @@
 "use strict";
 
-const isLocal = process.env.USER !== "vcap";
-const isOnCF = !isLocal;
+let instance;
+
+class Env {
+  #isLocal;
+  #isOnCF;
+  #vcapServices;
+
+  constructor() {
+    this.#isLocal = process.env.USER !== "vcap";
+    this.#isOnCF = !this.#isLocal;
+    try {
+      this.#vcapServices = JSON.parse(process.env.VCAP_SERVICES);
+    } catch {
+      this.#vcapServices = {};
+    }
+  }
+
+  getRedisCredentialsFromEnv() {
+    return this.#vcapServices["redis-cache"]?.[0]?.credentials;
+  }
+
+  set isLocal(value) {
+    this.#isLocal = value;
+  }
+  get isLocal() {
+    return this.#isLocal;
+  }
+
+  set isOnCF(value) {
+    this.#isOnCF = value;
+  }
+  get isOnCF() {
+    return this.#isOnCF;
+  }
+
+  set vcapServices(value) {
+    this.#vcapServices = value;
+  }
+  get vcapServices() {
+    return this.#vcapServices;
+  }
+}
 
 module.exports = {
-  isOnCF,
-  isLocal,
+  getEnvInstance: () => {
+    if (!instance) {
+      instance = new Env();
+    }
+    return instance;
+  },
 };

package/src/shared/redis.js

@@ -2,32 +2,33 @@
 
 const redis = require("redis");
 
-const config = require("../config");
+const { getInstance: getEnvInstance } = require("./env");
 const EventQueueError = require("../EventQueueError");
 
 const COMPONENT_NAME = "eventQueue/shared/redis";
 
-let subscriberClientPromise;
+let mainClientPromise;
+const subscriberChannelClientPromise = {};
 
 const createMainClientAndConnect = () => {
-  if (subscriberClientPromise) {
-    return subscriberClientPromise;
+  if (mainClientPromise) {
+    return mainClientPromise;
   }
 
   const errorHandlerCreateClient = (err) => {
     cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
-    subscriberClientPromise = null;
+    mainClientPromise = null;
     setTimeout(createMainClientAndConnect, 5 * 1000).unref();
   };
-  subscriberClientPromise = createClientAndConnect(errorHandlerCreateClient);
-  return subscriberClientPromise;
+  mainClientPromise = createClientAndConnect(errorHandlerCreateClient);
+  return mainClientPromise;
 };
 
 const _createClientBase = () => {
-  const configInstance = config.getConfigInstance();
-  if (configInstance.isOnCF) {
+  const env = getEnvInstance();
+  if (env.isOnCF) {
     try {
-      const credentials = configInstance.getRedisCredentialsFromEnv();
+      const credentials = env.getRedisCredentialsFromEnv();
       // NOTE: settings the user explicitly to empty resolves auth problems, see
       // https://github.com/go-redis/redis/issues/1343
       const url = credentials.uri.replace(/(?<=rediss:\/\/)[\w-]+?(?=:)/, "");
@@ -60,9 +61,35 @@ const createClientAndConnect = async (errorHandlerCreateClient) => {
   return client;
 };
 
+const subscribeRedisChannel = (channel, subscribeCb) => {
+  const errorHandlerCreateClient = (err) => {
+    cds.log(COMPONENT_NAME).error(`error from redis client for pub/sub failed for channel ${channel}`, err);
+    subscriberChannelClientPromise[channel] = null;
+    setTimeout(() => subscribeRedisChannel(channel, subscribeCb), 5 * 1000).unref();
+  };
+  subscriberChannelClientPromise[channel] = createClientAndConnect(errorHandlerCreateClient);
+  subscriberChannelClientPromise[channel]
+    .then((client) => {
+      cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel });
+      client.subscribe(channel, subscribeCb);
+    })
+    .catch((err) => {
+      cds
+        .log(COMPONENT_NAME)
+        .error(`error from redis client for pub/sub failed during startup - trying to reconnect - ${channel}`, err);
+    });
+};
+
+const publishMessage = async (channel, message) => {
+  const client = await createMainClientAndConnect();
+  return await client.publish(channel, message);
+};
+
 const _localReconnectStrategy = () => EventQueueError.redisNoReconnect();
 
 module.exports = {
   createClientAndConnect,
   createMainClientAndConnect,
+  subscribeRedisChannel,
+  publishMessage,
 };