@cap-js-community/event-queue 0.2.1 → 0.2.3

package/src/periodicEvents.js

@@ -0,0 +1,107 @@
+"use strict";
+
+const cds = require("@sap/cds");
+
+const { EventProcessingStatus } = require("./constants");
+const { processChunkedSync } = require("./shared/common");
+const eventConfig = require("./config");
+
+const COMPONENT_NAME = "eventQueue/periodicEvents";
+
+const checkAndInsertPeriodicEvents = async (context) => {
+  const tx = cds.tx(context);
+  const baseCqn = SELECT.from(eventConfig.tableNameEventQueue)
+    .where([
+      { list: [{ ref: ["type"] }, { ref: ["subType"] }] },
+      "IN",
+      {
+        list: eventConfig.periodicEvents.map((periodicEvent) => ({
+          list: [{ val: periodicEvent.type }, { val: periodicEvent.subType }],
+        })),
+      },
+      "AND",
+      { ref: ["status"] },
+      "=",
+      { val: EventProcessingStatus.Open },
+    ])
+    .columns(["ID", "type", "subType", "startAfter"]);
+  const currentPeriodEvents = await tx.run(baseCqn);
+
+  if (!currentPeriodEvents.length) {
+    // fresh insert all
+    return await insertPeriodEvents(tx, eventConfig.periodicEvents);
+  }
+
+  const exitingEventMap = currentPeriodEvents.reduce((result, current) => {
+    const key = _generateKey(current);
+    result[key] = current;
+    return result;
+  }, {});
+
+  const { newEvents, existingEvents } = eventConfig.periodicEvents.reduce(
+    (result, event) => {
+      if (exitingEventMap[_generateKey(event)]) {
+        result.existingEvents.push(exitingEventMap[_generateKey(event)]);
+      } else {
+        result.newEvents.push(event);
+      }
+      return result;
+    },
+    { newEvents: [], existingEvents: [] }
+  );
+
+  const currentDate = new Date();
+  const exitingWithNotMatchingInterval = existingEvents.filter((existingEvent) => {
+    const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType);
+    const eventStartAfter = new Date(existingEvent.startAfter);
+    // check if to far in future
+    const dueInWithNewInterval = new Date(currentDate.getTime() + config.interval * 1000);
+    return eventStartAfter >= dueInWithNewInterval;
+  });
+
+  exitingWithNotMatchingInterval.length &&
+    cds.log(COMPONENT_NAME).info("deleting periodic events because they have changed", {
+      changedEvents: exitingWithNotMatchingInterval.map(({ type, subType }) => ({ type, subType })),
+    });
+  await tx.run(
+    DELETE.from(eventConfig.tableNameEventQueue).where(
+      "ID IN",
+      exitingWithNotMatchingInterval.map(({ ID }) => ID)
+    )
+  );
+
+  const newOrChangedEvents = newEvents.concat(exitingWithNotMatchingInterval);
+
+  if (!newOrChangedEvents.length) {
+    return;
+  }
+
+  return await insertPeriodEvents(tx, newOrChangedEvents);
+};
+
+const insertPeriodEvents = async (tx, events) => {
+  const startAfter = new Date();
+  processChunkedSync(events, 4, (chunk) => {
+    cds.log(COMPONENT_NAME).info("inserting changed or new periodic events", {
+      events: chunk.map(({ type, subType }) => {
+        const { interval } = eventConfig.getEventConfig(type, subType);
+        return { type, subType, interval };
+      }),
+    });
+  });
+  const periodEventsInsert = events.map((periodicEvent) => ({
+    type: periodicEvent.type,
+    subType: periodicEvent.subType,
+    startAfter: startAfter,
+  }));
+
+  tx._skipEventQueueBroadcase = true;
+  await tx.run(INSERT.into(eventConfig.tableNameEventQueue).entries(periodEventsInsert));
+  tx._skipEventQueueBroadcase = false;
+};
+
+const _generateKey = ({ type, subType }) => [type, subType].join("##");
+
+module.exports = {
+  checkAndInsertPeriodicEvents,
+};

package/src/processEventQueue.js

@@ -4,7 +4,7 @@ const pathLib = require("path");
 
 const cds = require("@sap/cds");
 
-const { getConfigInstance } = require("./config");
+const config = require("./config");
 const { TransactionMode } = require("./constants");
 const { limiter, Funnel } = require("./shared/common");
 
@@ -29,7 +29,7 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
   let baseInstance;
   try {
     let eventTypeInstance;
-    const eventConfig = getConfigInstance().getEventConfig(eventType, eventSubType);
+    const eventConfig = config.getEventConfig(eventType, eventSubType);
     const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
     if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
       cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
@@ -43,6 +43,9 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
     if (!continueProcessing) {
       return;
     }
+    if (baseInstance.isPeriodicEvent) {
+      return await processPeriodicEvent(baseInstance);
+    }
     eventConfig.startTime = startTime;
     while (shouldContinue) {
       iterationCounter++;
@@ -131,6 +134,75 @@ const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime
   return false;
 };
 
+// TODO: don't forget to release lock
+const processPeriodicEvent = async (eventTypeInstance) => {
+  let queueEntry;
+  let processNext = true;
+
+  try {
+    while (processNext) {
+      await executeInNewTransaction(
+        eventTypeInstance.context,
+        `eventQueue-periodic-scheduleNext-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
+        async (tx) => {
+          eventTypeInstance.processEventContext = tx.context;
+          const queueEntries = await eventTypeInstance.getQueueEntriesAndSetToInProgress();
+          if (!queueEntries.length) {
+            return;
+          }
+          if (queueEntries.length > 1) {
+            queueEntry = await eventTypeInstance.handleDuplicatedPeriodicEventEntry(queueEntries);
+          } else {
+            queueEntry = queueEntries[0];
+          }
+          processNext = await eventTypeInstance.scheduleNextPeriodEvent(queueEntry);
+        }
+      );
+
+      if (!queueEntry) {
+        return;
+      }
+
+      await executeInNewTransaction(
+        eventTypeInstance.context,
+        `eventQueue-periodic-process-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
+        async (tx) => {
+          eventTypeInstance.processEventContext = tx.context;
+          eventTypeInstance.setTxForEventProcessing(queueEntry.ID, cds.tx(tx.context));
+          try {
+            await eventTypeInstance.processEvent(tx.context, queueEntry.ID, [queueEntry]);
+          } catch (err) {
+            eventTypeInstance.handleErrorDuringPeriodicEventProcessing(err, queueEntry);
+            throw new TriggerRollback();
+          }
+          if (
+            eventTypeInstance.transactionMode !== TransactionMode.alwaysCommit ||
+            eventTypeInstance.shouldRollbackTransaction(queueEntry.ID)
+          ) {
+            throw new TriggerRollback();
+          }
+        }
+      );
+
+      await executeInNewTransaction(
+        eventTypeInstance.context,
+        `eventQueue-periodic-setStatus-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
+        async (tx) => {
+          eventTypeInstance.processEventContext = tx.context;
+          await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID);
+        }
+      );
+    }
+  } catch (err) {
+    cds.log(COMPONENT_NAME).error("Processing periodic events failed with unexpected error. Error:", err, {
+      eventType: eventTypeInstance?.eventType,
+      eventSubType: eventTypeInstance?.eventSubType,
+    });
+  } finally {
+    await eventTypeInstance?.handleReleaseLock();
+  }
+};
+
 const processEventMap = async (eventTypeInstance) => {
   eventTypeInstance.startPerformanceTracerEvents();
   await eventTypeInstance.beforeProcessingEvents();

package/src/publishEvent.js

@@ -21,27 +21,34 @@ const EventQueueError = require("./EventQueueError");
  *     createdAt: Timestamp, // Timestamp of event creation. This field is automatically set on insert.
  *     startAfter: Timestamp, // Timestamp indicating when the event should start after.
  *   }
+ * @param {Boolean} skipBroadcast - (Optional) If set to true, event broadcasting will be skipped. Defaults to false.
  * @throws {EventQueueError} Throws an error if the configuration is not initialized.
  * @throws {EventQueueError} Throws an error if the event type is unknown.
  * @throws {EventQueueError} Throws an error if the startAfter field is not a valid date.
  * @returns {Promise} Returns a promise which resolves to the result of the database insert operation.
  */
-const publishEvent = async (tx, events) => {
-  const configInstance = config.getConfigInstance();
-  if (!configInstance.initialized) {
+const publishEvent = async (tx, events, skipBroadcast = false) => {
+  if (!config.initialized) {
     throw EventQueueError.notInitialized();
   }
   const eventsForProcessing = Array.isArray(events) ? events : [events];
   for (const { type, subType, startAfter } of eventsForProcessing) {
-    const eventConfig = configInstance.getEventConfig(type, subType);
+    const eventConfig = config.getEventConfig(type, subType);
     if (!eventConfig) {
       throw EventQueueError.unknownEventType(type, subType);
     }
     if (startAfter && !common.isValidDate(startAfter)) {
       throw EventQueueError.malformedDate(startAfter);
     }
+
+    if (eventConfig.isPeriodic) {
+      throw EventQueueError.manuelPeriodicEventInsert(type, subType);
+    }
   }
-  return await tx.run(INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing));
+  tx._skipEventQueueBroadcase = skipBroadcast;
+  const result = await tx.run(INSERT.into(config.tableNameEventQueue).entries(eventsForProcessing));
+  tx._skipEventQueueBroadcase = false;
+  return result;
 };
 
 module.exports = {

package/src/redisPubSub.js

@@ -11,7 +11,7 @@ const COMPONENT_NAME = "eventQueue/redisPubSub";
 let subscriberClientPromise;
 
 const initEventQueueRedisSubscribe = () => {
-  if (subscriberClientPromise || !config.getConfigInstance().redisEnabled) {
+  if (subscriberClientPromise || !config.redisEnabled) {
     return;
   }
   redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL, messageHandlerProcessEvents);
@@ -36,9 +36,8 @@ const messageHandlerProcessEvents = async (messageData) => {
 
 const broadcastEvent = async (tenantId, type, subType) => {
   const logger = cds.log(COMPONENT_NAME);
-  const configInstance = config.getConfigInstance();
-  if (!configInstance.redisEnabled) {
-    if (configInstance.registerAsEventProcessor) {
+  if (!config.redisEnabled) {
+    if (config.registerAsEventProcessor) {
       await runEventCombinationForTenant(tenantId, type, subType);
     }
     return;

package/src/runner.js

@@ -9,22 +9,27 @@ const cdsHelper = require("./shared/cdsHelper");
 const distributedLock = require("./shared/distributedLock");
 const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
 const { getSubdomainForTenantId } = require("./shared/cdsHelper");
+const periodicEvents = require("./periodicEvents");
+const { hashStringTo32Bit } = require("./shared/common");
 
 const COMPONENT_NAME = "eventQueue/runner";
 const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
 const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
+const EVENT_QUEUE_RUN_PERIODIC_EVENT = "EVENT_QUEUE_RUN_PERIODIC_EVENT";
 const OFFSET_FIRST_RUN = 10 * 1000;
 
-const singleTenant = () => _scheduleFunction(_executeRunForTenant);
+let tenantIdHash;
+let singleRunDone;
 
-const multiTenancyDb = () => _scheduleFunction(_multiTenancyDb);
+const singleTenant = () => _scheduleFunction(_checkPeriodicEventsSingleTenant, _executeRunForTenant);
 
-const multiTenancyRedis = () => _scheduleFunction(_multiTenancyRedis);
+const multiTenancyDb = () => _scheduleFunction(_multiTenancyPeriodicEvents, _multiTenancyDb);
 
-const _scheduleFunction = async (fn) => {
+const multiTenancyRedis = () => _scheduleFunction(_multiTenancyPeriodicEvents, _multiTenancyRedis);
+
+const _scheduleFunction = async (singleRunFn, periodicFn) => {
   const logger = cds.log(COMPONENT_NAME);
-  const configInstance = eventQueueConfig.getConfigInstance();
-  const eventsForAutomaticRun = configInstance.events;
+  const eventsForAutomaticRun = eventQueueConfig.allEvents;
   if (!eventsForAutomaticRun.length) {
     logger.warn("no events for automatic run are configured - skipping runner registration");
     return;
@@ -32,11 +37,15 @@ const _scheduleFunction = async (fn) => {
 
   const fnWithRunningCheck = () => {
     const logger = cds.log(COMPONENT_NAME);
-    if (configInstance.isRunnerDeactivated) {
+    if (eventQueueConfig.isRunnerDeactivated) {
       logger.info("runner is deactivated via config variable. Skipping this run.");
       return;
     }
-    return fn();
+    if (!singleRunDone) {
+      singleRunDone = true;
+      singleRunFn().catch(() => (singleRunDone = false));
+    }
+    return periodicFn();
   };
 
   const offsetDependingOnLastRun = await _calculateOffsetForFirstRun();
@@ -47,7 +56,7 @@ const _scheduleFunction = async (fn) => {
 
   setTimeout(() => {
     fnWithRunningCheck();
-    const intervalRunner = new SetIntervalDriftSafe(configInstance.runInterval);
+    const intervalRunner = new SetIntervalDriftSafe(eventQueueConfig.runInterval);
     intervalRunner.run(fnWithRunningCheck);
   }, offsetDependingOnLastRun).unref();
 };
@@ -57,6 +66,8 @@ const _multiTenancyRedis = async () => {
   const emptyContext = new cds.EventContext({});
   logger.info("executing event queue run for multi instance and tenant");
   const tenantIds = await cdsHelper.getAllTenantIds();
+  _checkAndTriggerPeriodicEventUpdate(tenantIds);
+
   const runId = await _acquireRunId(emptyContext);
 
   if (!runId) {
@@ -67,33 +78,33 @@ const _multiTenancyRedis = async () => {
   _executeAllTenants(tenantIds, runId);
 };
 
-const _multiTenancyDb = async () => {
-  const logger = cds.log(COMPONENT_NAME);
-  try {
-    logger.info("executing event queue run for single instance and multi tenant");
-    const tenantIds = await cdsHelper.getAllTenantIds();
-    _executeAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
-  } catch (err) {
-    logger.error(
-      `Couldn't fetch tenant ids for event queue processing! Next try after defined interval. Error: ${err}`
-    );
+const _checkAndTriggerPeriodicEventUpdate = (tenantIds) => {
+  const hash = hashStringTo32Bit(JSON.stringify(tenantIds));
+  if (!tenantIdHash) {
+    tenantIdHash = hash;
+    return;
+  }
+  if (tenantIdHash && tenantIdHash !== hash) {
+    cds.log(COMPONENT_NAME).info("tenant id hash changed, triggering updating periodic events!");
+    _multiTenancyPeriodicEvents().catch((err) => {
+      cds.log(COMPONENT_NAME).error("Error during triggering updating periodic events! Error:", err);
+    });
   }
 };
 
-const _executeAllTenants = (tenantIds, runId) => {
-  const configInstance = eventQueueConfig.getConfigInstance();
+const _executeAllTenantsGeneric = (tenantIds, runId, fn) => {
   const workerQueueInstance = getWorkerPoolInstance();
   tenantIds.forEach((tenantId) => {
     workerQueueInstance.addToQueue(async () => {
       try {
         const tenantContext = new cds.EventContext({ tenant: tenantId });
         const couldAcquireLock = await distributedLock.acquireLock(tenantContext, runId, {
-          expiryTime: configInstance.runInterval * 0.95,
+          expiryTime: eventQueueConfig.runInterval * 0.95,
         });
         if (!couldAcquireLock) {
           return;
         }
-        await _executeRunForTenant(tenantId, runId);
+        await fn(tenantId, runId);
       } catch (err) {
         cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
           tenantId,
@@ -103,11 +114,15 @@ const _executeAllTenants = (tenantIds, runId) => {
   });
 };
 
+const _executeAllTenants = (tenantIds, runId) => _executeAllTenantsGeneric(tenantIds, runId, _executeRunForTenant);
+
+const _executePeriodicEventsAllTenants = (tenantIds, runId) =>
+  _executeAllTenantsGeneric(tenantIds, runId, _checkPeriodicEventsSingleTenant);
+
 const _executeRunForTenant = async (tenantId, runId) => {
   const logger = cds.log(COMPONENT_NAME);
-  const configInstance = eventQueueConfig.getConfigInstance();
   try {
-    const eventsForAutomaticRun = configInstance.events;
+    const eventsForAutomaticRun = eventQueueConfig.allEvents;
     const subdomain = await cdsHelper.getSubdomainForTenantId(tenantId);
     const context = new cds.EventContext({
       tenant: tenantId,
@@ -124,23 +139,22 @@ const _executeRunForTenant = async (tenantId, runId) => {
   } catch (err) {
     logger.error(`Couldn't process eventQueue for tenant! Next try after defined interval. Error: ${err}`, {
       tenantId,
-      redisEnabled: configInstance.redisEnabled,
+      redisEnabled: eventQueueConfig.redisEnabled,
     });
   }
 };
 
 const _acquireRunId = async (context) => {
-  const configInstance = eventQueueConfig.getConfigInstance();
   let runId = randomUUID();
   const couldSetValue = await distributedLock.setValueWithExpire(context, EVENT_QUEUE_RUN_ID, runId, {
     tenantScoped: false,
-    expiryTime: configInstance.runInterval * 0.95,
+    expiryTime: eventQueueConfig.runInterval * 0.95,
   });
 
   if (couldSetValue) {
     await distributedLock.setValueWithExpire(context, EVENT_QUEUE_RUN_TS, new Date().toISOString(), {
       tenantScoped: false,
-      expiryTime: configInstance.runInterval,
+      expiryTime: eventQueueConfig.runInterval,
       overrideValue: true,
     });
   } else {
@@ -153,13 +167,12 @@ const _acquireRunId = async (context) => {
 };
 
 const _calculateOffsetForFirstRun = async () => {
-  const configInstance = eventQueueConfig.getConfigInstance();
   let offsetDependingOnLastRun = OFFSET_FIRST_RUN;
   const now = Date.now();
   // NOTE: this is only supported with Redis, because this is a tenant agnostic information
   //       currently there is no proper place to store this information beside t0 schema
   try {
-    if (configInstance.redisEnabled) {
+    if (eventQueueConfig.redisEnabled) {
       const dummyContext = new cds.EventContext({});
       let lastRunTs = await distributedLock.checkLockExistsAndReturnValue(dummyContext, EVENT_QUEUE_RUN_TS, {
         tenantScoped: false,
@@ -168,7 +181,7 @@ const _calculateOffsetForFirstRun = async () => {
         const ts = new Date(now).toISOString();
         const couldSetValue = await distributedLock.setValueWithExpire(dummyContext, EVENT_QUEUE_RUN_TS, ts, {
           tenantScoped: false,
-          expiryTime: configInstance.runInterval,
+          expiryTime: eventQueueConfig.runInterval,
         });
         if (couldSetValue) {
           lastRunTs = ts;
@@ -178,7 +191,7 @@ const _calculateOffsetForFirstRun = async () => {
           });
         }
       }
-      offsetDependingOnLastRun = new Date(lastRunTs).getTime() + configInstance.runInterval - now;
+      offsetDependingOnLastRun = new Date(lastRunTs).getTime() + eventQueueConfig.runInterval - now;
     }
   } catch (err) {
     cds
@@ -211,6 +224,59 @@ const runEventCombinationForTenant = async (tenantId, type, subType) => {
   }
 };
 
+const _multiTenancyDb = async () => {
+  const logger = cds.log(COMPONENT_NAME);
+  try {
+    logger.info("executing event queue run for single instance and multi tenant");
+    const tenantIds = await cdsHelper.getAllTenantIds();
+    _checkAndTriggerPeriodicEventUpdate(tenantIds);
+    _executeAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
+  } catch (err) {
+    logger.error(
+      `Couldn't fetch tenant ids for event queue processing! Next try after defined interval. Error: ${err}`
+    );
+  }
+};
+
+const _multiTenancyPeriodicEvents = async () => {
+  const logger = cds.log(COMPONENT_NAME);
+  try {
+    logger.info("executing event queue update periodic events");
+    const tenantIds = await cdsHelper.getAllTenantIds();
+    _executePeriodicEventsAllTenants(tenantIds, EVENT_QUEUE_RUN_PERIODIC_EVENT);
+  } catch (err) {
+    logger.error(`Couldn't fetch tenant ids for updating periodic event processing! Error: ${err}`);
+  }
+};
+
+const _checkPeriodicEventsSingleTenant = async (tenantId) => {
+  const logger = cds.log(COMPONENT_NAME);
+  if (!eventQueueConfig.updatePeriodicEvents) {
+    logger.info("updating of periodic events is disabled");
+  }
+  try {
+    const subdomain = await cdsHelper.getSubdomainForTenantId(tenantId);
+    const context = new cds.EventContext({
+      tenant: tenantId,
+      // NOTE: we need this because of logging otherwise logs would not contain the subdomain
+      http: { req: { authInfo: { getSubdomain: () => subdomain } } },
+    });
+    cds.context = context;
+    logger.info("executing updating periotic events", {
+      tenantId,
+      subdomain,
+    });
+    await cdsHelper.executeInNewTransaction(context, "update-periodic-events", async (tx) => {
+      await periodicEvents.checkAndInsertPeriodicEvents(tx.context);
+    });
+  } catch (err) {
+    logger.error(`Couldn't process eventQueue for tenant! Next try after defined interval. Error: ${err}`, {
+      tenantId,
+      redisEnabled: eventQueueConfig.redisEnabled,
+    });
+  }
+};
+
 module.exports = {
   singleTenant,
   multiTenancyDb,

package/src/shared/WorkerQueue.js

@@ -2,7 +2,7 @@
 
 const cds = require("@sap/cds");
 
-const { getConfigInstance } = require("../config");
+const config = require("../config");
 
 const COMPONENT_NAME = "eventQueue/WorkerQueue";
 
@@ -56,8 +56,7 @@ class WorkerQueue {
 module.exports = {
   getWorkerPoolInstance: () => {
     if (!instance) {
-      const configInstance = getConfigInstance();
-      instance = new WorkerQueue(configInstance.parallelTenantProcessing);
+      instance = new WorkerQueue(config.parallelTenantProcessing);
     }
     return instance;
   },

package/src/shared/cdsHelper.js

@@ -94,7 +94,7 @@ class TriggerRollback extends VError {
 }
 
 const getSubdomainForTenantId = async (tenantId) => {
-  if (!config.getConfigInstance().isMultiTenancy) {
+  if (!config.isMultiTenancy) {
     return null;
   }
   if (subdomainCache[tenantId]) {
@@ -111,7 +111,7 @@ const getSubdomainForTenantId = async (tenantId) => {
 };
 
 const getAllTenantIds = async () => {
-  if (!config.getConfigInstance().isMultiTenancy) {
+  if (!config.isMultiTenancy) {
     return null;
   }
   const ssp = await cds.connect.to("cds.xt.SaasProvisioningService");

package/src/shared/common.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const crypto = require("crypto");
+
 const { floor, abs, min } = Math;
 
 const arrayToFlatMap = (array, key = "ID") => {
@@ -118,4 +120,16 @@ const isValidDate = (value) => {
   }
 };
 
-module.exports = { arrayToFlatMap, Funnel, limiter, isValidDate };
+const processChunkedSync = (inputs, chunkSize, chunkHandler) => {
+  let start = 0;
+  while (start < inputs.length) {
+    let end = start + chunkSize > inputs.length ? inputs.length : start + chunkSize;
+    const chunk = inputs.slice(start, end);
+    chunkHandler(chunk);
+    start = end;
+  }
+};
+
+const hashStringTo32Bit = (value) => crypto.createHash("sha256").update(String(value)).digest("base64").slice(0, 32);
+
+module.exports = { arrayToFlatMap, Funnel, limiter, isValidDate, processChunkedSync, hashStringTo32Bit };