@cap-js-community/event-queue 0.1.49

package/src/processEventQueue.js

@@ -0,0 +1,297 @@
+"use strict";
+
+const pathLib = require("path");
+
+const cds = require("@sap/cds");
+
+const { getConfigInstance } = require("./config");
+const { limiter, Funnel } = require("./shared/common");
+
+const EventQueueBase = require("./EventQueueProcessorBase");
+const {
+  executeInNewTransaction,
+  TriggerRollback,
+} = require("./shared/cdsHelper");
+
+const COMPONENT_NAME = "eventQueue/processEventQueue";
+const MAX_EXECUTION_TIME = 5 * 60 * 1000;
+
+const eventQueueRunner = async (context, events) => {
+  const startTime = new Date();
+  const funnel = new Funnel();
+  await Promise.allSettled(
+    events.map((event) =>
+      funnel.run(event.load, async () =>
+        processEventQueue(context, event.type, event.subType, startTime)
+      )
+    )
+  );
+};
+
+const processEventQueue = async (
+  context,
+  eventType,
+  eventSubType,
+  startTime = new Date()
+) => {
+  let iterationCounter = 0;
+  let shouldContinue = true;
+  let baseInstance;
+  try {
+    let eventTypeInstance;
+    const eventConfig = getConfigInstance().getEventConfig(
+      eventType,
+      eventSubType
+    );
+    const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
+    if (
+      !eventConfig ||
+      err ||
+      !(typeof EventTypeClass.constructor === "function")
+    ) {
+      await EventQueueBase.handleMissingTypeImplementation(
+        context,
+        eventType,
+        eventSubType
+      );
+      return;
+    }
+    baseInstance = new EventTypeClass(
+      context,
+      eventType,
+      eventSubType,
+      eventConfig
+    );
+    const continueProcessing = await baseInstance.handleDistributedLock();
+    if (!continueProcessing) {
+      return;
+    }
+    eventConfig.startTime = startTime;
+    while (shouldContinue) {
+      iterationCounter++;
+      await executeInNewTransaction(
+        context,
+        `eventQueue-pre-processing-${eventType}##${eventSubType}`,
+        async (tx) => {
+          eventTypeInstance = new EventTypeClass(
+            tx.context,
+            eventType,
+            eventSubType,
+            eventConfig
+          );
+          const queueEntries =
+            await eventTypeInstance.getQueueEntriesAndSetToInProgress();
+          eventTypeInstance.startPerformanceTracerPreprocessing();
+          for (const queueEntry of queueEntries) {
+            try {
+              eventTypeInstance.modifyQueueEntry(queueEntry);
+              const payload =
+                await eventTypeInstance.checkEventAndGeneratePayload(
+                  queueEntry
+                );
+              if (payload === null) {
+                eventTypeInstance.setStatusToDone(queueEntry);
+                continue;
+              }
+              if (payload === undefined) {
+                eventTypeInstance.handleInvalidPayloadReturned(queueEntry);
+                continue;
+              }
+              eventTypeInstance.addEventWithPayloadForProcessing(
+                queueEntry,
+                payload
+              );
+            } catch (err) {
+              eventTypeInstance.handleErrorDuringProcessing(err, queueEntry);
+            }
+          }
+          throw new TriggerRollback();
+        }
+      );
+      eventTypeInstance.exceededEvents.length &&
+        (await executeInNewTransaction(
+          context,
+          `eventQueue-handleExceededEvents-${eventType}##${eventSubType}`,
+          async (tx) => {
+            eventTypeInstance.processEventContext = tx.context;
+            await eventTypeInstance.handleExceededEvents(
+              eventTypeInstance.exceededEvents
+            );
+          }
+        ));
+      if (!eventTypeInstance) {
+        return;
+      }
+      eventTypeInstance.endPerformanceTracerPreprocessing();
+      if (Object.keys(eventTypeInstance.queueEntriesWithPayloadMap).length) {
+        await executeInNewTransaction(
+          context,
+          `eventQueue-processing-${eventType}##${eventSubType}`,
+          async (tx) => {
+            eventTypeInstance.processEventContext = tx.context;
+            try {
+              eventTypeInstance.clusterQueueEntries();
+              await processEventMap(eventTypeInstance);
+            } catch (err) {
+              eventTypeInstance.handleErrorDuringClustering(err);
+            }
+            if (
+              eventTypeInstance.shouldTriggerRollback ||
+              Object.entries(eventTypeInstance.eventProcessingMap).some(
+                ([key]) => eventTypeInstance.shouldRollbackTransaction(key)
+              )
+            ) {
+              throw new TriggerRollback();
+            }
+          }
+        );
+      }
+      await executeInNewTransaction(
+        context,
+        `eventQueue-persistStatus-${eventType}##${eventSubType}`,
+        async (tx) => {
+          await eventTypeInstance.persistEventStatus(tx);
+        }
+      );
+      shouldContinue = reevaluateShouldContinue(
+        eventTypeInstance,
+        iterationCounter,
+        startTime
+      );
+    }
+  } catch (err) {
+    cds
+      .log(COMPONENT_NAME)
+      .error(
+        "Processing event queue failed with unexpected error. Error:",
+        err,
+        {
+          eventType,
+          eventSubType,
+        }
+      );
+  } finally {
+    await baseInstance?.handleReleaseLock();
+  }
+};
+
+const reevaluateShouldContinue = (
+  eventTypeInstance,
+  iterationCounter,
+  startTime
+) => {
+  if (!eventTypeInstance.getSelectNextChunk()) {
+    return false; // no select next chunk configured for this event
+  }
+  if (eventTypeInstance.emptyChunkSelected) {
+    return false; // the last selected chunk was empty - no more data for processing
+  }
+  if (new Date(startTime.getTime() + MAX_EXECUTION_TIME) > new Date()) {
+    return true;
+  }
+  eventTypeInstance.logTimeExceeded(iterationCounter);
+  return false;
+};
+
+const processEventMap = async (eventTypeInstance) => {
+  eventTypeInstance.startPerformanceTracerEvents();
+  await eventTypeInstance.beforeProcessingEvents();
+  if (eventTypeInstance.commitOnEventLevel) {
+    eventTypeInstance.txUsageAllowed = false;
+  }
+  await limiter(
+    eventTypeInstance.parallelEventProcessing,
+    Object.entries(eventTypeInstance.eventProcessingMap),
+    async ([key, { queueEntries, payload }]) => {
+      if (eventTypeInstance.commitOnEventLevel) {
+        let statusMap;
+        await executeInNewTransaction(
+          eventTypeInstance.baseContext,
+          `eventQueue-processEvent-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
+          async (tx) => {
+            statusMap = await _processEvent(
+              eventTypeInstance,
+              tx.context,
+              key,
+              queueEntries,
+              payload
+            );
+            if (
+              eventTypeInstance.statusMapContainsError(statusMap) ||
+              eventTypeInstance.shouldRollbackTransaction(key)
+            ) {
+              throw new TriggerRollback();
+            }
+          }
+        );
+        await executeInNewTransaction(
+          eventTypeInstance.baseContext,
+          `eventQueue-persistStatus-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
+          async (tx) => {
+            eventTypeInstance.processEventContext = tx.context;
+            await eventTypeInstance.persistEventStatus(tx, {
+              skipChecks: true,
+              statusMap,
+            });
+          }
+        );
+      } else {
+        await _processEvent(
+          eventTypeInstance,
+          eventTypeInstance.context,
+          key,
+          queueEntries,
+          payload
+        );
+      }
+    }
+  ).finally(() => {
+    eventTypeInstance.clearEventProcessingContext();
+    if (eventTypeInstance.commitOnEventLevel) {
+      eventTypeInstance.txUsageAllowed = true;
+    }
+  });
+  eventTypeInstance.endPerformanceTracerEvents();
+};
+
+const _processEvent = async (
+  eventTypeInstance,
+  processContext,
+  key,
+  queueEntries,
+  payload
+) => {
+  try {
+    eventTypeInstance.logStartMessage(queueEntries);
+    const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(
+      queueEntries
+    );
+    if (eventOutdated) {
+      return;
+    }
+    eventTypeInstance.setTxForEventProcessing(key, cds.tx(processContext));
+    const statusTuple = await eventTypeInstance.processEvent(
+      processContext,
+      key,
+      queueEntries,
+      payload
+    );
+    return eventTypeInstance.setEventStatus(queueEntries, statusTuple);
+  } catch (err) {
+    return eventTypeInstance.handleErrorDuringProcessing(err, queueEntries);
+  }
+};
+
+const resilientRequire = (path) => {
+  try {
+    const module = require(pathLib.join(process.cwd(), path));
+    return [null, module];
+  } catch (err) {
+    return [err, null];
+  }
+};
+
+module.exports = {
+  processEventQueue,
+  eventQueueRunner,
+};

package/src/publishEvent.js

@@ -0,0 +1,25 @@
+"use strict";
+
+const config = require("./config");
+const EventQueueError = require("./EventQueueError");
+
+const publishEvent = async (tx, events) => {
+  const configInstance = config.getConfigInstance();
+  if (!configInstance.initialized) {
+    throw EventQueueError.notInitialized();
+  }
+  const eventsForProcessing = Array.isArray(events) ? events : [events];
+  for (const { type, subType } of eventsForProcessing) {
+    const eventConfig = configInstance.getEventConfig(type, subType);
+    if (!eventConfig) {
+      throw EventQueueError.unknownEventType(type, subType);
+    }
+  }
+  await tx.run(
+    INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing)
+  );
+};
+
+module.exports = {
+  publishEvent,
+};

package/src/redisPubSub.js

@@ -0,0 +1,143 @@
+"use strict";
+
+const redis = require("./shared/redis");
+const { processEventQueue } = require("./processEventQueue");
+const { getSubdomainForTenantId } = require("./shared/cdsHelper");
+const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
+const config = require("./config");
+const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
+
+const MESSAGE_CHANNEL = "cdsEventQueue";
+const COMPONENT_NAME = "eventQueue/redisPubSub";
+
+let publishClient;
+let subscriberClientPromise;
+
+const initEventQueueRedisSubscribe = () => {
+  if (subscriberClientPromise || !config.getConfigInstance().redisEnabled) {
+    return;
+  }
+  subscribeRedisClient();
+};
+
+const subscribeRedisClient = () => {
+  const errorHandlerCreateClient = (err) => {
+    cds
+      .log(COMPONENT_NAME)
+      .error("error from redis client for pub/sub failed", err);
+    subscriberClientPromise = null;
+    setTimeout(subscribeRedisClient, 5 * 1000).unref();
+  };
+  subscriberClientPromise = redis.createClientAndConnect(
+    errorHandlerCreateClient
+  );
+  subscriberClientPromise
+    .then((client) => {
+      cds.log(COMPONENT_NAME).info("subscribe redis client connected");
+      client.subscribe(MESSAGE_CHANNEL, messageHandlerProcessEvents);
+    })
+    .catch((err) => {
+      cds
+        .log(COMPONENT_NAME)
+        .error(
+          "error from redis client for pub/sub failed during startup - trying to reconnect",
+          err
+        );
+    });
+};
+
+const messageHandlerProcessEvents = async (messageData) => {
+  let tenantId, type, subType;
+  try {
+    ({ tenantId, type, subType } = JSON.parse(messageData));
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("could not parse event information", {
+      messageData,
+    });
+    return;
+  }
+  const subdomain = await getSubdomainForTenantId(tenantId);
+  const context = new cds.EventContext({
+    tenant: tenantId,
+    // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+    http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+  });
+  cds.context = context;
+  cds.log(COMPONENT_NAME).debug("received redis event", {
+    tenantId,
+    type,
+    subType,
+  });
+  getWorkerPoolInstance().addToQueue(async () =>
+    processEventQueue(context, type, subType)
+  );
+};
+
+const publishEvent = async (tenantId, type, subType) => {
+  const configInstance = config.getConfigInstance();
+  if (!configInstance.redisEnabled) {
+    await _handleEventInternally(tenantId, type, subType);
+    return;
+  }
+
+  const logger = cds.log(COMPONENT_NAME);
+  const errorHandlerCreateClient = (err) => {
+    logger.error("error from redis client for pub/sub failed", {
+      err,
+    });
+    publishClient = null;
+  };
+  try {
+    if (!publishClient) {
+      publishClient = await redis.createClientAndConnect(
+        errorHandlerCreateClient
+      );
+      logger.info("publish redis client connected");
+    }
+
+    const result = await checkLockExistsAndReturnValue(
+      new cds.EventContext({ tenant: tenantId }),
+      [type, subType].join("##")
+    );
+    if (result) {
+      logger.info("skip publish redis event as no lock is available");
+      return;
+    }
+    logger.debug("publishing redis event", {
+      tenantId,
+      type,
+      subType,
+    });
+    await publishClient.publish(
+      MESSAGE_CHANNEL,
+      JSON.stringify({ tenantId, type, subType })
+    );
+  } catch (err) {
+    logger.error(`publish event failed with error: ${err.toString()}`, {
+      tenantId,
+      type,
+      subType,
+    });
+  }
+};
+
+const _handleEventInternally = async (tenantId, type, subType) => {
+  const logger = cds.log(COMPONENT_NAME);
+  logger.info("processEventQueue internally", {
+    tenantId,
+    type,
+    subType,
+  });
+  const subdomain = await getSubdomainForTenantId(tenantId);
+  const context = new cds.EventContext({
+    tenant: tenantId,
+    // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+    http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+  });
+  processEventQueue(context, type, subType);
+};
+
+module.exports = {
+  initEventQueueRedisSubscribe,
+  publishEvent,
+};

package/src/runner.js

@@ -0,0 +1,237 @@
+"use strict";
+
+const uuid = require("uuid");
+
+const eventQueueConfig = require("./config");
+const cdsHelper = require("./shared/cdsHelper");
+const { eventQueueRunner } = require("./processEventQueue");
+const distributedLock = require("./shared/distributedLock");
+const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
+const { getConfigInstance } = require("./config");
+
+const COMPONENT_NAME = "eventQueue/runner";
+const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
+const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
+const OFFSET_FIRST_RUN = 10 * 1000;
+
+const LOGGER = cds.log(COMPONENT_NAME);
+
+const singleTenant = () => _scheduleFunction(_executeRunForTenant);
+
+const multiTenancyDb = () => _scheduleFunction(_multiTenancyDb);
+
+const multiTenancyRedis = () => _scheduleFunction(_multiTenancyRedis);
+
+const _scheduleFunction = async (fn) => {
+  const configInstance = eventQueueConfig.getConfigInstance();
+  const eventsForAutomaticRun = configInstance.events;
+  if (!eventsForAutomaticRun.length) {
+    LOGGER.warn(
+      "no events for automatic run are configured - skipping runner registration"
+    );
+    return;
+  }
+
+  const fnWithRunningCheck = () => {
+    if (configInstance.isRunnerDeactivated) {
+      LOGGER.info(
+        "runner is deactivated via config variable. Skipping this run."
+      );
+      return;
+    }
+    return fn();
+  };
+
+  const offsetDependingOnLastRun = await _calculateOffsetForFirstRun();
+
+  LOGGER.info("first event-queue run scheduled", {
+    firstRunScheduledFor: new Date(
+      Date.now() + offsetDependingOnLastRun
+    ).toISOString(),
+  });
+
+  setTimeout(() => {
+    fnWithRunningCheck();
+    setInterval(fnWithRunningCheck, configInstance.runInterval).unref();
+  }, offsetDependingOnLastRun).unref();
+};
+
+const _multiTenancyRedis = async () => {
+  const emptyContext = new cds.EventContext({});
+  LOGGER.info("executing event queue run for multi instance and tenant");
+  const tenantIds = await cdsHelper.getAllTenantIds();
+  const runId = await _acquireRunId(emptyContext);
+
+  if (!runId) {
+    LOGGER.error("could not acquire runId, skip processing events!");
+    return;
+  }
+
+  _executeAllTenants(tenantIds, runId);
+};
+
+const _multiTenancyDb = async () => {
+  try {
+    LOGGER.info(
+      "executing event queue run for single instance and multi tenant"
+    );
+    const tenantIds = await cdsHelper.getAllTenantIds();
+    _executeAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
+  } catch (err) {
+    LOGGER.error(
+      `Couldn't fetch tenant ids for event queue processing! Next try after defined interval. Error: ${err}`
+    );
+  }
+};
+
+const _executeAllTenants = (tenantIds, runId) => {
+  const configInstance = eventQueueConfig.getConfigInstance();
+  const workerQueueInstance = getWorkerPoolInstance();
+  tenantIds.forEach((tenantId) => {
+    workerQueueInstance.addToQueue(async () => {
+      try {
+        const tenantContext = new cds.EventContext({ tenant: tenantId });
+        const couldAcquireLock = await distributedLock.acquireLock(
+          tenantContext,
+          runId,
+          {
+            expiryTime: configInstance.runInterval * 0.95,
+          }
+        );
+        if (!couldAcquireLock) {
+          return;
+        }
+        await _executeRunForTenant(tenantId, runId);
+      } catch (err) {
+        LOGGER.error("executing event-queue run for tenant failed", {
+          tenantId,
+        });
+      }
+    });
+  });
+};
+
+const _executeRunForTenant = async (tenantId, runId) => {
+  const configInstance = eventQueueConfig.getConfigInstance();
+  try {
+    const eventsForAutomaticRun = configInstance.events;
+    const subdomain = await cdsHelper.getSubdomainForTenantId(tenantId);
+    const context = new cds.EventContext({
+      tenant: tenantId,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    });
+    cds.context = context;
+    LOGGER.info("executing eventQueue run", {
+      tenantId,
+      subdomain,
+      ...(runId ? { runId } : null),
+    });
+    await eventQueueRunner(context, eventsForAutomaticRun);
+  } catch (err) {
+    LOGGER.error(
+      `Couldn't process eventQueue for tenant! Next try after defined interval. Error: ${err}`,
+      {
+        tenantId,
+        redisEnabled: configInstance.redisEnabled,
+      }
+    );
+  }
+};
+
+const _acquireRunId = async (context) => {
+  const configInstance = eventQueueConfig.getConfigInstance();
+  let runId = uuid.v4();
+  const couldSetValue = await distributedLock.setValueWithExpire(
+    context,
+    EVENT_QUEUE_RUN_ID,
+    runId,
+    {
+      tenantScoped: false,
+      expiryTime: configInstance.runInterval * 0.95,
+    }
+  );
+
+  if (couldSetValue) {
+    await distributedLock.setValueWithExpire(
+      context,
+      EVENT_QUEUE_RUN_TS,
+      new Date().toISOString(),
+      {
+        tenantScoped: false,
+        expiryTime: configInstance.runInterval,
+        overrideValue: true,
+      }
+    );
+  } else {
+    runId = await distributedLock.checkLockExistsAndReturnValue(
+      context,
+      EVENT_QUEUE_RUN_ID,
+      {
+        tenantScoped: false,
+      }
+    );
+  }
+
+  return runId;
+};
+
+const _calculateOffsetForFirstRun = async () => {
+  const configInstance = getConfigInstance();
+  let offsetDependingOnLastRun = OFFSET_FIRST_RUN;
+  const now = Date.now();
+  // NOTE: this is only supported with Redis, because this is a tenant agnostic information
+  //       currently there is no proper place to store this information beside t0 schema
+  try {
+    if (configInstance.redisEnabled) {
+      const dummyContext = new cds.EventContext({});
+      let lastRunTs = await distributedLock.checkLockExistsAndReturnValue(
+        dummyContext,
+        EVENT_QUEUE_RUN_TS,
+        { tenantScoped: false }
+      );
+      if (!lastRunTs) {
+        const ts = new Date(now).toISOString();
+        const couldSetValue = await distributedLock.setValueWithExpire(
+          dummyContext,
+          EVENT_QUEUE_RUN_TS,
+          ts,
+          {
+            tenantScoped: false,
+            expiryTime: configInstance.runInterval,
+          }
+        );
+        if (couldSetValue) {
+          lastRunTs = ts;
+        } else {
+          lastRunTs = await distributedLock.checkLockExistsAndReturnValue(
+            dummyContext,
+            EVENT_QUEUE_RUN_TS,
+            { tenantScoped: false }
+          );
+        }
+      }
+      offsetDependingOnLastRun =
+        new Date(lastRunTs).getTime() + configInstance.runInterval - now;
+    }
+  } catch (err) {
+    LOGGER.error(
+      "calculating offset for first run failed, falling back to default. Runs might be out-of-sync. Error:",
+      err
+    );
+  }
+  return offsetDependingOnLastRun;
+};
+
+module.exports = {
+  singleTenant,
+  multiTenancyDb,
+  multiTenancyRedis,
+  _: {
+    _multiTenancyRedis,
+    _multiTenancyDb,
+    _calculateOffsetForFirstRun,
+    _acquireRunId,
+    EVENT_QUEUE_RUN_TS,
+  },
+};