@hotmeshio/hotmesh 0.0.49 → 0.0.50

package/build/services/task/index.js

@@ -5,18 +5,20 @@ const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const pipe_1 = require("../pipe");
 const hotmesh_1 = require("../../types/hotmesh");
+const key_1 = require("../../modules/key");
 class TaskService {
     constructor(store, logger) {
         this.cleanupTimeout = null;
         this.isScout = false;
+        this.errorCount = 0;
         this.logger = logger;
         this.store = store;
     }
     async processWebHooks(hookEventCallback) {
         const workItemKey = await this.store.getActiveTaskQueue();
         if (workItemKey) {
-            const [topic, sourceKey, scrub, ...sdata] = workItemKey.split('::');
-            const data = JSON.parse(sdata.join('::'));
+            const [topic, sourceKey, scrub, ...sdata] = workItemKey.split(key_1.WEBSEP);
+            const data = JSON.parse(sdata.join(key_1.WEBSEP));
             const destinationKey = `${sourceKey}:processed`;
             const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
             if (jobId) {
@@ -41,11 +43,11 @@ class TaskService {
             await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
         }
     }
-    async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, multi) {
+    async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, multi) {
         const fromNow = Date.now() + (inSeconds * 1000);
         const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
         const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
-        await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, multi);
+        await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
     }
     /**
      * Should this engine instance play the role of 'scout' on behalf
@@ -89,11 +91,13 @@ class TaskService {
                         await timeEventCallback(target, gId, activityId, type);
                     }
                     await (0, utils_1.sleepFor)(0);
+                    this.errorCount = 0;
                     this.processTimeHooks(timeEventCallback, listKey);
                 }
                 else if (workListTask) {
                     //a worklist was just emptied; try again immediately
                     await (0, utils_1.sleepFor)(0);
+                    this.errorCount = 0;
                     this.processTimeHooks(timeEventCallback);
                 }
                 else {
@@ -101,12 +105,18 @@ class TaskService {
                     let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
                     this.cleanupTimeout = sleep.timerId;
                     await sleep.promise;
+                    this.errorCount = 0;
                     this.processTimeHooks(timeEventCallback);
                 }
             }
             catch (err) {
-                //todo: retry connect to redis
-                this.logger.error('task-process-timehooks-error', err);
+                //most common reasons: deleted job not found; container stopping; test stopping
+                //less common: redis/cluster down; retry with fallback (5s max main reassignment)
+                this.logger.warn('task-process-timehooks-error', err);
+                await (0, utils_1.sleepFor)(1000 * this.errorCount++);
+                if (this.errorCount < 5) {
+                    this.processTimeHooks(timeEventCallback);
+                }
             }
         }
         else {
@@ -135,10 +145,17 @@ class TaskService {
             const jobId = context.metadata.jid;
             const gId = context.metadata.gid;
             const activityId = hookRule.to;
+            //composite keys are used to fully describe the task target
+            const compositeJobKey = [
+                activityId,
+                dad,
+                gId,
+                jobId
+            ].join(key_1.WEBSEP);
             const hook = {
                 topic,
                 resolved,
-                jobId: `${activityId}::${dad}::${gId}::${jobId}`,
+                jobId: compositeJobKey,
             };
             await this.store.setHookSignal(hook, multi);
             return jobId;
@@ -157,17 +174,17 @@ class TaskService {
             const resolved = pipe_1.Pipe.resolve(mapExpression, context);
             const hookSignalId = await this.store.getHookSignal(topic, resolved);
             if (!hookSignalId) {
-                //messages can be double-processed; not an issue; return undefined
-                //users can also provide a bogus topic; not an issue; return undefined
+                //messages can be double-processed; not an issue; return `undefined`
+                //users can also provide a bogus topic; not an issue; return `undefined`
                 return undefined;
             }
-            //`aid` is part of composit key, but the hook `topic` is its public interface;
+            //`aid` is part of composite key, but the hook `topic` is its public interface;
             // this means that a new version of the graph can be deployed and the
             // topic can be re-mapped to a different activity id. Outside callers
             // can adhere to the unchanged contract (calling the same topic),
-            // while the internal system can be updated in real time as necessary.
-            const [_aid, dad, gid, ...jid] = hookSignalId.split('::');
-            return [jid.join('::'), hookRule.to, dad, gid];
+            // while the internal system can be updated in real-time as necessary.
+            const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
+            return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
         }
         else {
             throw new Error('signal-not-found');

package/build/services/worker/index.d.ts

@@ -1,17 +1,20 @@
+/// <reference types="node" />
 import { ILogger } from "../logger";
 import { Router } from "../router";
 import { StoreService } from '../store';
 import { StreamService } from '../stream';
 import { SubService } from '../sub';
 import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
-import { SubscriptionCallback } from "../../types/quorum";
+import { RollCallMessage, SubscriptionCallback } from "../../types/quorum";
 import { RedisClient, RedisMulti } from "../../types/redis";
+import { StreamData, StreamDataResponse } from "../../types/stream";
 declare class WorkerService {
     namespace: string;
     appId: string;
     guid: string;
     topic: string;
     config: HotMeshConfig;
+    callback: (streamData: StreamData) => Promise<StreamDataResponse | void>;
     store: StoreService<RedisClient, RedisMulti> | null;
     stream: StreamService<RedisClient, RedisMulti> | null;
     subscribe: SubService<RedisClient, RedisMulti> | null;
@@ -19,6 +22,7 @@ declare class WorkerService {
     logger: ILogger;
     reporting: boolean;
     inited: string;
+    rollCallInterval: NodeJS.Timeout;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
     verifyWorkerFields(worker: HotMeshWorker): void;
     initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;
@@ -26,7 +30,14 @@ declare class WorkerService {
     initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
     initRouter(worker: HotMeshWorker, logger: ILogger): Router;
     subscriptionHandler(): SubscriptionCallback;
-    sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    doRollCall(message: RollCallMessage): Promise<void>;
+    cancelRollCall(): void;
+    stop(): void;
+    sayPong(appId: string, guid: string, originator?: string, details?: boolean, signature?: boolean): Promise<void>;
     throttle(delayInMillis: number): Promise<void>;
 }
 export { WorkerService };

package/build/services/worker/index.js

@@ -12,6 +12,7 @@ const redis_2 = require("../stream/clients/redis");
 const ioredis_3 = require("../sub/clients/ioredis");
 const redis_3 = require("../sub/clients/redis");
 const stream_1 = require("../../types/stream");
+const enums_1 = require("../../modules/enums");
 class WorkerService {
     constructor() {
         this.reporting = false;
@@ -26,6 +27,7 @@ class WorkerService {
                 service.namespace = namespace;
                 service.appId = appId;
                 service.guid = guid;
+                service.callback = worker.callback;
                 service.topic = worker.topic;
                 service.config = config;
                 service.logger = logger;
@@ -95,14 +97,51 @@ class WorkerService {
         return async (topic, message) => {
             self.logger.debug('worker-event-received', { topic, type: message.type });
             if (message.type === 'throttle') {
-                self.throttle(message.throttle);
+                if (message.topic !== null) { //undefined allows passthrough
+                    self.throttle(message.throttle);
+                }
             }
             else if (message.type === 'ping') {
                 self.sayPong(self.appId, self.guid, message.originator, message.details);
             }
+            else if (message.type === 'rollcall') {
+                if (message.topic !== null) { //undefined allows passthrough
+                    self.doRollCall(message);
+                }
+            }
         };
     }
-    async sayPong(appId, guid, originator, details = false) {
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    async doRollCall(message) {
+        let iteration = 0;
+        let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
+        if (this.rollCallInterval)
+            clearTimeout(this.rollCallInterval);
+        const base = (message.interval / 2);
+        const amount = base + Math.ceil(Math.random() * base);
+        do {
+            await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
+            await this.sayPong(this.appId, this.guid, null, true, message.signature);
+            if (!message.interval)
+                return;
+            const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
+            this.rollCallInterval = timerId;
+            await promise;
+        } while (this.rollCallInterval && iteration++ < max - 1);
+    }
+    cancelRollCall() {
+        if (this.rollCallInterval) {
+            clearTimeout(this.rollCallInterval);
+            delete this.rollCallInterval;
+        }
+    }
+    stop() {
+        this.cancelRollCall();
+    }
+    async sayPong(appId, guid, originator, details = false, signature = false) {
         let profile;
         if (details) {
             const params = {
@@ -122,11 +161,13 @@ class WorkerService {
                 reclaimDelay: this.router.reclaimDelay,
                 reclaimCount: this.router.reclaimCount,
                 system: await (0, utils_1.getSystemHealth)(),
+                signature: signature ? this.callback.toString() : undefined,
             };
         }
         this.store.publish(key_1.KeyType.QUORUM, {
             type: 'pong',
-            guid, originator,
+            guid,
+            originator,
             profile,
         }, appId);
     }

package/build/types/activity.d.ts

@@ -74,6 +74,7 @@ interface AwaitActivity extends BaseActivity {
     type: 'await';
     eventName: string;
     timeout: number;
+    await?: boolean;
 }
 interface WorkerActivity extends BaseActivity {
     type: 'worker';

package/build/types/exporter.d.ts

@@ -21,6 +21,8 @@ export interface JobTimeline {
     dimension: string;
     duplex: 'entry' | 'exit';
     timestamp: string;
+    created?: string;
+    updated?: string;
     actions?: ActivityAction[];
 }
 export interface DependencyExport {

package/build/types/job.d.ts

@@ -15,6 +15,7 @@ type JobMetadata = {
     pg?: string;
     pd?: string;
     pa?: string;
+    px?: boolean;
     ngn?: string;
     app: string;
     vrs: string;

package/build/types/quorum.d.ts

@@ -45,42 +45,56 @@ export interface QuorumProfile {
     reclaimDelay?: number;
     reclaimCount?: number;
     system?: SystemHealth;
+    signature?: string;
 }
-export interface PingMessage {
+interface QuorumMessageBase {
+    guid?: string;
+    topic?: string;
+    type?: string;
+}
+export interface PingMessage extends QuorumMessageBase {
     type: 'ping';
     originator: string;
     details?: boolean;
 }
-export interface WorkMessage {
+export interface WorkMessage extends QuorumMessageBase {
     type: 'work';
     originator: string;
 }
-export interface CronMessage {
+export interface CronMessage extends QuorumMessageBase {
     type: 'cron';
     originator: string;
 }
-export interface PongMessage {
+export interface PongMessage extends QuorumMessageBase {
     type: 'pong';
     guid: string;
     originator: string;
     profile?: QuorumProfile;
 }
-export interface ActivateMessage {
+export interface ActivateMessage extends QuorumMessageBase {
     type: 'activate';
     cache_mode: 'nocache' | 'cache';
     until_version: string;
 }
-export interface JobMessage {
+export interface JobMessage extends QuorumMessageBase {
     type: 'job';
     topic: string;
     job: JobOutput;
 }
-export interface ThrottleMessage {
+export interface ThrottleMessage extends QuorumMessageBase {
     type: 'throttle';
     guid?: string;
     topic?: string;
     throttle: number;
 }
+export interface RollCallMessage extends QuorumMessageBase {
+    type: 'rollcall';
+    guid?: string;
+    topic?: string | null;
+    interval: number;
+    max?: number;
+    signature?: boolean;
+}
 export interface JobMessageCallback {
     (topic: string, message: JobOutput): void;
 }
@@ -96,5 +110,5 @@ export interface QuorumMessageCallback {
  * These messages serve to coordinate the cache invalidation and switch-over
  * to the new version without any downtime and a coordinating parent server.
  */
-export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | RollCallMessage | CronMessage;
 export {};

package/build/types/stream.d.ts

@@ -37,6 +37,7 @@ export interface StreamData {
         trc?: string;
         spn?: string;
         try?: number;
+        await?: boolean;
     };
     type?: StreamDataType;
     data: Record<string, unknown>;

package/modules/enums.ts

@@ -23,6 +23,7 @@ export const HMSH_CODE_DURABLE_RETRYABLE = 599;
 export const HMSH_STATUS_UNKNOWN = 'unknown';
 
 // QUORUM
+export const HMSH_QUORUM_ROLLCALL_CYCLES = 12; //max iterations
 export const HMSH_QUORUM_DELAY_MS = 250;
 export const HMSH_ACTIVATION_MAX_RETRY = 3;
 

package/modules/key.ts

@@ -28,7 +28,12 @@ import { KeyStoreParams, KeyType } from '../types/hotmesh';
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
 
-const HMNS = "hmsh"; //default
+const HMNS = "hmsh";
+
+const KEYSEP = ':';  //default delimiter for keys 
+const VALSEP = '::'; //default delimiter for vals
+const WEBSEP = '::'; //default delimiter for webhook vals
+const TYPSEP = '::'; //delimiter for ZSET task typing (how should a list be used?)
 
 class KeyService {
 
@@ -93,4 +98,4 @@ class KeyService {
   }
 }
 
-export { KeyService, KeyType, KeyStoreParams, HMNS };
+export { KeyService, KeyType, KeyStoreParams, HMNS, KEYSEP, TYPSEP, WEBSEP, VALSEP };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.49",
+  "version": "0.0.50",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -28,6 +28,7 @@
     "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:emit": "NODE_ENV=test jest ./tests/functional/emit/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:await": "NODE_ENV=test jest ./tests/functional/awaiter/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:hook": "NODE_ENV=test jest ./tests/functional/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:signal": "NODE_ENV=test jest ./tests/functional/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:interrupt": "NODE_ENV=test jest ./tests/functional/interrupt/index.test.ts --detectOpenHandles --forceExit --verbose",

package/services/activities/await.ts

@@ -95,6 +95,12 @@ class Await extends Activity {
       type: StreamDataType.AWAIT,
       data: this.context.data
     };
+    if (this.config.await !== true) {
+      const doAwait = Pipe.resolve(this.config.await, this.context);
+      if (doAwait === false) {
+        streamData.metadata.await = false;
+      }
+    }
     if (this.config.retry) {
       streamData.policies = {
         retry: this.config.retry

package/services/activities/hook.ts

@@ -148,6 +148,7 @@ class Hook extends Activity {
         `${this.metadata.aid}${this.metadata.dad || ''}`,
         'sleep',
         duration,
+        this.metadata.dad || '',
       );
       return this.context.metadata.jid;
     }

package/services/activities/trigger.ts

@@ -15,6 +15,7 @@ import {
 import { JobState } from '../../types/job';
 import { RedisMulti } from '../../types/redis';
 import { StringScalarType } from '../../types/serializer';
+import { WorkListTaskType } from '../../types/task';
 
 class Trigger extends Activity {
   config: TriggerActivity;
@@ -50,6 +51,10 @@ class Trigger extends Activity {
       await this.registerJobDependency(multi);
       await multi.exec();
 
+      //if the parent (spawner) chose not to await,
+      // emit the job_id as the data payload { job_id }
+      this.execAdjacentParent();
+
       telemetry.mapActivityAttributes();
       const jobStatus = Number(this.context.metadata.js);
       telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
@@ -79,6 +84,12 @@ class Trigger extends Activity {
     this.context.metadata.js = amount;
   }
 
+  async execAdjacentParent() {
+    if (this.context.metadata.px) {
+      await this.engine.execAdjacentParent(this.context, {metadata: this.context.metadata, data: { job_id: this.context.metadata.jid }});
+    }
+  }
+
   createInputContext(): Partial<JobState> {
     const input = { 
       [this.metadata.aid]: {
@@ -115,6 +126,7 @@ class Trigger extends Activity {
         pg: this.context.metadata.pg,
         pd: this.context.metadata.pd,
         pa: this.context.metadata.pa,
+        px: this.context.metadata.px,
         app: id,
         vrs: version,
         tpc: this.config.subscribes,
@@ -193,12 +205,23 @@ class Trigger extends Activity {
     }
     if (resolvedDepKey) {
       const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
+      let type: WorkListTaskType;
+      if (isParentOrigin) {
+        if (this.context.metadata.px) {
+          type = 'child'
+        } else {
+          type = 'expire-child'
+        }
+      } else {
+        type = 'expire';
+      }
       await this.store.registerJobDependency(
-        isParentOrigin ? 'expire-child' : 'expire',
+        type,
         resolvedDepKey,
         this.context.metadata.tpc,
         this.context.metadata.jid,
         this.context.metadata.gid,
+        this.context.metadata.pd,
         multi,
       );
     }
@@ -209,6 +232,7 @@ class Trigger extends Activity {
         this.context.metadata.tpc,
         this.context.metadata.jid,
         this.context.metadata.gid,
+        this.context.metadata.pd,
         multi,
       );
     }

package/services/durable/exporter.ts

@@ -13,6 +13,7 @@ import {
   JobTimeline } from '../../types/exporter';
 import { SerializerService } from '../serializer';
 import { restoreHierarchy } from '../../modules/utils';
+import { VALSEP } from '../../modules/key';
 
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
@@ -116,19 +117,31 @@ class ExporterService {
       const activityName = item[1].split('/')[0];
       const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
       const timestamp = item[2];
-      const event: JobTimeline = {
+      let event: JobTimeline = {
         activity: activityName,
         duplex: duplex as 'entry' | 'exit',
         dimension: dimensions,
         timestamp,
+        created: timestamp,
+        updated: timestamp,
       };
-      timeline.push(event);
+      const prior = timeline[timeline.length - 1];
+      if (prior && prior.activity === event.activity && prior.duplex !== event.duplex && prior.dimension === event.dimension) {
+        if (event.duplex === 'exit') {
+          prior.updated = event.timestamp;
+        } else {
+          prior.created = event.timestamp;
+        }
+        event = prior;
+      } else {
+        timeline.push(event);
+      }
 
       if (this.isMainEntry(item[1])) {
         event.actions = [] as ActivityAction[];
         this.interleaveActions(actions.main, event.actions);
       } else if (this.isHookEntry(item[1])) {
-        const hookDimension =  `/${parts[1]}/${parts[2]}`;
+        const hookDimension = `/${parts[1]}/${parts[2]}`;
         const hookActions = actions.hooks[hookDimension];
         event.actions = [] as ActivityAction[];
         this.interleaveActions(hookActions, event.actions);
@@ -191,17 +204,15 @@ class ExporterService {
    * @returns - the organized dependency data
    */
   inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
-    //console.log('dependency data>', data);
     const hookReg = /([0-9,]+)-(\d+)$/;
     const flowReg = /-(\d+)$/;
     return data.map((dependency, index: number): DependencyExport => {
-      const [action, topic, gid, ...jid] = dependency.split('::');
-      const jobId = jid.join('::');
+      const [action, topic, gid, _pd, ...jid] = dependency.split(VALSEP);
+      const jobId = jid.join(VALSEP);
       const match = jobId.match(hookReg);
       let prefix: string;
       let type: 'hook' | 'flow' | 'other';
       let dimensionKey: string = '';
-
       if (match) {
         //hook-originating dependency
         const [_, dimension, counter] = match;

package/services/engine/index.ts

@@ -1,4 +1,4 @@
-import { KeyType } from '../../modules/key';
+import { KeyType, VALSEP } from '../../modules/key';
 import {
   HMSH_OTT_WAIT_TIME,
   HMSH_CODE_SUCCESS,
@@ -386,9 +386,10 @@ class EngineService {
         pg: streamData.metadata.gid,
         pd: streamData.metadata.dad,
         pa: streamData.metadata.aid,
+        px: streamData.metadata.await === false, //sever the parent connection (px)
         trc: streamData.metadata.trc,
         spn: streamData.metadata.spn,
-       };
+      };
       const activityHandler = await this.initActivity(
         streamData.metadata.topic,
         streamData.data,
@@ -459,7 +460,10 @@ class EngineService {
       return (await this.router?.publishMessage(null, streamData)) as string;
     }
   }
-  hasParentJob(context: JobState): boolean {
+  hasParentJob(context: JobState, checkSevered = false): boolean {
+    if (checkSevered) {
+      return Boolean(context.metadata.pj && context.metadata.pa && !context.metadata.px);
+    }
     return Boolean(context.metadata.pj && context.metadata.pa);
   }
   resolveError(metadata: JobMetadata): StreamError | undefined {
@@ -537,7 +541,11 @@ class EngineService {
         const taskService = new TaskService(this.store, this.logger);
         await taskService.enqueueWorkItems(
           workItems.map(
-            workItem => `${hookTopic}::${workItem}::${keyResolver.scrub || false}::${JSON.stringify(data)}`
+            workItem => [
+              hookTopic,
+              workItem,
+              keyResolver.scrub || false,
+              JSON.stringify(data)].join(VALSEP)
         ));
         this.store.publish(
           KeyType.QUORUM,
@@ -663,7 +671,7 @@ class EngineService {
   // ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
   async runJobCompletionTasks(context: JobState, options: JobCompletionOptions = {}): Promise<string | void> {
     //'emit' indicates the job is still active
-    const isAwait = this.hasParentJob(context);
+    const isAwait = this.hasParentJob(context, true);
     const isOneTimeSub = this.hasOneTimeSubscription(context);
     const topic = await this.getPublishesTopic(context);
     let msgId: string;

package/services/exporter/index.ts

@@ -12,6 +12,7 @@ import {
   JobExport } from '../../types/exporter';
 import { SerializerService } from '../serializer';
 import { restoreHierarchy } from '../../modules/utils';
+import { VALSEP } from '../../modules/key';
 
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
@@ -108,8 +109,8 @@ class ExporterService {
     const hookReg = /([0-9,]+)-(\d+)$/;
     const flowReg = /-(\d+)$/;
     return data.map((dependency, index: number): DependencyExport => {
-      const [action, topic, gid, ...jid] = dependency.split('::');
-      const jobId = jid.join('::');
+      const [action, topic, gid, _pd, ...jid] = dependency.split(VALSEP);
+      const jobId = jid.join(VALSEP);
       const match = jobId.match(hookReg);
       let prefix: string;
       let type: 'hook' | 'flow' | 'other';

package/services/hotmesh/index.ts

@@ -222,6 +222,10 @@ class HotMeshService {
 
   stop() {
     this.engine?.taskService.cancelCleanup();
+    this.quorum?.stop();
+    this.workers?.forEach((worker: WorkerService) => {
+      worker.stop();
+    });
   }
 
   async compress(terms: string[]): Promise<boolean> {