@cap-js-community/event-queue 0.1.49

package/src/config.js

@@ -0,0 +1,196 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const env = require("./shared/env");
+
+let instance;
+
+const FOR_UPDATE_TIMEOUT = 10;
+const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
+
+class Config {
+  constructor() {
+    this.__config = null;
+    this.__forUpdateTimeout = FOR_UPDATE_TIMEOUT;
+    this.__globalTxTimeout = GLOBAL_TX_TIMEOUT;
+    this.__runInterval = null;
+    this.__redisEnabled = null;
+    this.__isOnCF = env.isOnCF;
+    this.__initialized = false;
+    this.__parallelTenantProcessing = null;
+    this.__tableNameEventQueue = null;
+    this.__tableNameEventLock = null;
+    this.__vcapServices = this._parseVcapServices();
+    this.__isRunnerDeactivated = false;
+    this.__eventsForAutomaticRun = null;
+    this.__configFilePath = null;
+    this.__processEventsAfterPublish = null;
+    this.__skipCsnCheck = null;
+  }
+
+  getEventConfig(type, subType) {
+    return this.__eventMap[[type, subType].join("##")];
+  }
+
+  hasEventAfterCommitFlag(type, subType) {
+    return (
+      this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true
+    );
+  }
+
+  _checkRedisIsBound() {
+    return !!this.getRedisCredentialsFromEnv();
+  }
+
+  getRedisCredentialsFromEnv() {
+    return this.__vcapServices["redis-cache"]?.[0]?.credentials;
+  }
+
+  _parseVcapServices() {
+    try {
+      return JSON.parse(process.env.VCAP_SERVICES);
+    } catch {
+      return {};
+    }
+  }
+
+  calculateIsRedisEnabled() {
+    this.__redisEnabled = this._checkRedisIsBound() && this.__isOnCF;
+  }
+
+  get isRunnerDeactivated() {
+    return this.__isRunnerDeactivated;
+  }
+
+  set isRunnerDeactivated(value) {
+    this.__isRunnerDeactivated = value;
+  }
+
+  set fileContent(config) {
+    this.__config = config;
+    this.__eventMap = config.events.reduce((result, event) => {
+      result[[event.type, event.subType].join("##")] = event;
+      return result;
+    }, {});
+  }
+
+  get fileContent() {
+    return this.__config;
+  }
+
+  get events() {
+    return this.__config.events;
+  }
+
+  get forUpdateTimeout() {
+    return this.__forUpdateTimeout;
+  }
+
+  get globalTxTimeout() {
+    return this.__globalTxTimeout;
+  }
+
+  set forUpdateTimeout(value) {
+    this.__forUpdateTimeout = value;
+  }
+
+  set globalTxTimeout(value) {
+    this.__globalTxTimeout = value;
+  }
+
+  get runInterval() {
+    return this.__runInterval;
+  }
+
+  set runInterval(value) {
+    this.__runInterval = value;
+  }
+
+  get redisEnabled() {
+    return this.__redisEnabled;
+  }
+
+  set redisEnabled(value) {
+    this.__redisEnabled = value;
+  }
+
+  get isOnCF() {
+    return this.__isOnCF;
+  }
+
+  set isOnCF(value) {
+    this.__isOnCF = value;
+  }
+
+  get initialized() {
+    return this.__initialized;
+  }
+
+  set initialized(value) {
+    this.__initialized = value;
+  }
+
+  get parallelTenantProcessing() {
+    return this.__parallelTenantProcessing;
+  }
+
+  set parallelTenantProcessing(value) {
+    this.__parallelTenantProcessing = value;
+  }
+
+  get tableNameEventQueue() {
+    return this.__tableNameEventQueue;
+  }
+
+  set tableNameEventQueue(value) {
+    this.__tableNameEventQueue = value;
+  }
+
+  get tableNameEventLock() {
+    return this.__tableNameEventLock;
+  }
+
+  set tableNameEventLock(value) {
+    this.__tableNameEventLock = value;
+  }
+
+  set configFilePath(value) {
+    this.__configFilePath = value;
+  }
+
+  get configFilePath() {
+    return this.__configFilePath;
+  }
+
+  set processEventsAfterPublish(value) {
+    this.__processEventsAfterPublish = value;
+  }
+
+  get processEventsAfterPublish() {
+    return this.__processEventsAfterPublish;
+  }
+
+  set skipCsnCheck(value) {
+    this.__skipCsnCheck = value;
+  }
+
+  get skipCsnCheck() {
+    return this.__skipCsnCheck;
+  }
+
+  get isMultiTenancy() {
+    return !!cds.requires.multitenancy;
+  }
+}
+
+const getConfigInstance = () => {
+  if (!instance) {
+    instance = new Config();
+  }
+  return instance;
+};
+
+module.exports = {
+  getConfigInstance,
+};

package/src/constants.js

@@ -0,0 +1,11 @@
+"use strict";
+
+module.exports = {
+  EventProcessingStatus: {
+    Open: 0,
+    InProgress: 1,
+    Done: 2,
+    Error: 3,
+    Exceeded: 4,
+  },
+};

package/src/dbHandler.js

@@ -0,0 +1,40 @@
+"use strict";
+
+const { publishEvent } = require("./redisPubSub");
+const config = require("./config");
+
+const registerEventQueueDbHandler = (dbService) => {
+  const configInstance = config.getConfigInstance();
+  const def = dbService.model.definitions[configInstance.tableNameEventQueue];
+  dbService.after("CREATE", def, (_, req) => {
+    req.tx._ = req.tx._ ?? {};
+    req.tx._.eventQueuePublishEvents = req.tx._.eventQueuePublishEvents ?? {};
+    const eventQueuePublishEvents = req.tx._.eventQueuePublishEvents;
+    const data = Array.isArray(req.data) ? req.data : [req.data];
+    const eventCombinations = Object.keys(
+      data.reduce((result, event) => {
+        const key = [event.type, event.subType].join("##");
+        if (
+          !configInstance.hasEventAfterCommitFlag(event.type, event.subType) ||
+          eventQueuePublishEvents[key]
+        ) {
+          return result;
+        }
+        eventQueuePublishEvents[key] = true;
+        result[key] = true;
+        return result;
+      }, {})
+    );
+
+    eventCombinations.length &&
+      req.on("succeeded", () => {
+        for (const eventCombination of eventCombinations) {
+          publishEvent(req.tenant, ...eventCombination.split("##"));
+        }
+      });
+  });
+};
+
+module.exports = {
+  registerEventQueueDbHandler,
+};

package/src/index.js

@@ -0,0 +1,20 @@
+"use strict";
+
+// TODO: how to deal with fatal logs
+// TODO: think about situations where isInitialized need to be checked - publishEvent access config which is not initialized
+// TODO: add tests for config --> similar to csn check
+// TODO: redis client check reconnect strategy
+
+// TODO: for test
+// --> deeper look into the functions e.g. getQueueEntriesAndSetToInProgress
+// TODO: add test for commit on event level and stuff like that
+
+module.exports = {
+  ...require("./initialize"),
+  ...require("./config"),
+  ...require("./processEventQueue"),
+  ...require("./dbHandler"),
+  ...require("./constants"),
+  ...require("./publishEvent"),
+  EventQueueProcessorBase: require("./EventQueueProcessorBase"),
+};

package/src/initialize.js

@@ -0,0 +1,216 @@
+"use strict";
+
+const { promisify } = require("util");
+const fs = require("fs");
+const path = require("path");
+
+const cds = require("@sap/cds");
+
+const yaml = require("yaml");
+const VError = require("verror");
+
+const EventQueueError = require("./EventQueueError");
+const runner = require("./runner");
+const dbHandler = require("./dbHandler");
+const { getConfigInstance } = require("./config");
+const { initEventQueueRedisSubscribe } = require("./redisPubSub");
+
+const readFileAsync = promisify(fs.readFile);
+
+const VERROR_CLUSTER_NAME = "EventQueueInitialization";
+const COMPONENT = "eventQueue/initialize";
+const BASE_TABLES = {
+  EVENT: "sap.eventqueue.Event",
+  LOCK: "sap.eventqueue.Lock",
+};
+
+const initialize = async ({
+  configFilePath,
+  registerAsEventProcessor,
+  processEventsAfterPublish,
+  runInterval,
+  parallelTenantProcessing,
+  tableNameEventQueue,
+  tableNameEventLock,
+  skipCsnCheck,
+} = {}) => {
+  // TODO: initialize check:
+  // - content of yaml check
+  // - betweenRuns and parallelTenantProcessing
+
+  const configInstance = getConfigInstance();
+  if (configInstance.initialized) {
+    return;
+  }
+  configInstance.initialized = true;
+
+  mixConfigVarsWithEnv(
+    configFilePath,
+    registerAsEventProcessor,
+    processEventsAfterPublish,
+    runInterval,
+    parallelTenantProcessing,
+    tableNameEventQueue,
+    tableNameEventLock,
+    skipCsnCheck
+  );
+
+  const logger = cds.log(COMPONENT);
+  configInstance.fileContent = await readConfigFromFile(
+    configInstance.configFilePath
+  );
+  configInstance.calculateIsRedisEnabled();
+
+  const dbService = await cds.connect.to("db");
+  await (cds.model
+    ? Promise.resolve()
+    : new Promise((resolve) => cds.on("serving", resolve)));
+  !configInstance.skipCsnCheck && (await csnCheck());
+  if (configInstance.processEventsAfterPublish) {
+    // TODO: remove this as soon as CDS fixes the current plugin model issues --> cds 7
+    if (BASE_TABLES.EVENT === configInstance.tableNameEventQueue) {
+      cds.db.model.definitions[BASE_TABLES.EVENT] =
+        cds.model.definitions[BASE_TABLES.EVENT];
+    }
+    if (BASE_TABLES.LOCK === configInstance.tableNameEventLock) {
+      cds.db.model.definitions[BASE_TABLES.LOCK] =
+        cds.model.definitions[BASE_TABLES.LOCK];
+    }
+    dbHandler.registerEventQueueDbHandler(dbService);
+  }
+
+  registerEventProcessors();
+  logger.info("event queue initialized", {
+    registerAsEventProcessor: configInstance.registerAsEventProcessor,
+    multiTenancyEnabled: configInstance.isMultiTenancy,
+    redisEnabled: configInstance.redisEnabled,
+    runInterval: configInstance.runInterval,
+    parallelTenantProcessing: configInstance.parallelTenantProcessing,
+  });
+};
+
+const readConfigFromFile = async (configFilepath) => {
+  const fileData = await readFileAsync(configFilepath);
+  if (/\.ya?ml$/i.test(configFilepath)) {
+    return yaml.parse(fileData.toString());
+  }
+  if (/\.json$/i.test(configFilepath)) {
+    return JSON.parse(fileData.toString());
+  }
+  throw new VError(
+    {
+      name: VERROR_CLUSTER_NAME,
+      info: { configFilepath },
+    },
+    "configFilepath with unsupported extension, allowed extensions are .yaml and .json"
+  );
+};
+
+const registerEventProcessors = () => {
+  const configInstance = getConfigInstance();
+
+  if (!configInstance.registerAsEventProcessor) {
+    return;
+  }
+
+  if (!configInstance.isMultiTenancy) {
+    runner.singleTenant();
+    return;
+  }
+
+  if (configInstance.redisEnabled) {
+    initEventQueueRedisSubscribe();
+    runner.multiTenancyRedis();
+  } else {
+    runner.multiTenancyDb();
+  }
+};
+
+const csnCheck = async () => {
+  const configInstance = getConfigInstance();
+  const eventCsn = cds.model.definitions[configInstance.tableNameEventQueue];
+  if (!eventCsn) {
+    throw EventQueueError.missingTableInCsn(configInstance.tableNameEventQueue);
+  }
+
+  const lockCsn = cds.model.definitions[configInstance.tableNameEventLock];
+  if (!lockCsn) {
+    throw EventQueueError.missingTableInCsn(configInstance.tableNameEventLock);
+  }
+
+  if (
+    configInstance.tableNameEventQueue === BASE_TABLES.EVENT &&
+    configInstance.tableNameEventLock === BASE_TABLES.LOCK
+  ) {
+    return; // no need to check base tables
+  }
+
+  const csn = await cds.load(path.join(__dirname, "..", "db"));
+  const baseEvent = csn.definitions["sap.eventqueue.Event"];
+  const baseLock = csn.definitions["sap.eventqueue.Lock"];
+
+  checkCustomTable(baseEvent, eventCsn);
+  checkCustomTable(baseLock, lockCsn);
+};
+
+const checkCustomTable = (baseCsn, customCsn) => {
+  for (const columnName in baseCsn.elements) {
+    if (!customCsn.elements[columnName]) {
+      throw EventQueueError.missingElementInTable(customCsn.name, columnName);
+    }
+
+    if (
+      customCsn.elements[columnName].type !== "cds.Association" &&
+      customCsn.elements[columnName].type !==
+        baseCsn.elements[columnName].type &&
+      columnName === "status" &&
+      customCsn.elements[columnName].type !== "cds.Integer"
+    ) {
+      throw EventQueueError.typeMismatchInTable(customCsn.name, columnName);
+    }
+  }
+};
+
+const mixConfigVarsWithEnv = (
+  configFilePath,
+  registerAsEventProcessor,
+  processEventsAfterPublish,
+  runInterval,
+  parallelTenantProcessing,
+  tableNameEventQueue,
+  tableNameEventLock,
+  skipCsnCheck
+) => {
+  const configInstance = getConfigInstance();
+
+  configInstance.configFilePath =
+    configFilePath ?? cds.env.eventQueue?.configFilePath;
+  configInstance.registerAsEventProcessor =
+    registerAsEventProcessor ??
+    cds.env.eventQueue?.registerAsEventProcessor ??
+    true;
+  configInstance.processEventsAfterPublish =
+    processEventsAfterPublish ??
+    cds.env.eventQueue?.processEventsAfterPublish ??
+    true;
+  configInstance.runInterval =
+    runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
+  configInstance.parallelTenantProcessing =
+    parallelTenantProcessing ??
+    cds.env.eventQueue?.parallelTenantProcessing ??
+    5;
+  configInstance.tableNameEventQueue =
+    tableNameEventQueue ??
+    cds.env.eventQueue?.tableNameEventQueue ??
+    BASE_TABLES.EVENT;
+  configInstance.tableNameEventLock =
+    tableNameEventLock ??
+    cds.env.eventQueue?.tableNameEventLock ??
+    BASE_TABLES.LOCK;
+  configInstance.skipCsnCheck =
+    skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
+};
+
+module.exports = {
+  initialize,
+};