@hotmeshio/hotmesh 0.0.41 → 0.0.43

package/build/services/engine/index.js

@@ -226,6 +226,7 @@ class EngineService {
         });
         const context = {
             metadata: {
+                guid: streamData.metadata.guid,
                 jid: streamData.metadata.jid,
                 gid: streamData.metadata.gid,
                 dad: streamData.metadata.dad,
@@ -353,15 +354,11 @@ class EngineService {
         };
         return await this.router.publishMessage(null, streamData);
     }
-    async hookTime(jobId, gId, activityId, type) {
-        if (type === 'interrupt') {
-            return await this.interrupt(activityId, //note: 'activityId' is the actually job topic
-            jobId, { suppress: true, expire: 1 });
+    async hookTime(jobId, gId, topicOrActivity, type) {
+        if (type === 'interrupt' || type === 'expire') {
+            return await this.interrupt(topicOrActivity, jobId, { suppress: true, expire: 1 });
         }
-        else if (type === 'expire') {
-            return await this.store.expireJob(jobId, 1);
-        }
-        const [aid, ...dimensions] = activityId.split(',');
+        const [aid, ...dimensions] = topicOrActivity.split(',');
         const dad = `,${dimensions.join(',')}`;
         const streamData = {
             type: stream_1.StreamDataType.TIMEHOOK,

package/build/services/quorum/index.d.ts

@@ -3,8 +3,8 @@ import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
-import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
 import { HotMeshConfig } from '../../types/hotmesh';
+import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
@@ -30,6 +30,9 @@ declare class QuorumService {
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
     rollCall(delay?: number): Promise<QuorumProfile[]>;
-    activate(version: string, delay?: number): Promise<boolean>;
+    /**
+     * request a quorum; if successful activate the app version
+     */
+    activate(version: string, delay?: number, count?: number): Promise<boolean>;
 }
 export { QuorumService };

package/build/services/quorum/index.js

@@ -1,15 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.QuorumService = void 0;
-const key_1 = require("../../modules/key");
+const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const compiler_1 = require("../compiler");
 const redis_1 = require("../store/clients/redis");
 const ioredis_1 = require("../store/clients/ioredis");
 const ioredis_2 = require("../sub/clients/ioredis");
 const redis_2 = require("../sub/clients/redis");
-//wait time to see if quorum is reached
-const QUORUM_DELAY = 250;
+const hotmesh_1 = require("../../types/hotmesh");
 class QuorumService {
     constructor() {
         this.profiles = [];
@@ -30,8 +29,10 @@ class QuorumService {
             //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
             await instance.initStoreChannel(config.engine.store);
             await instance.initSubChannel(config.engine.sub);
-            await instance.subscribe.subscribe(key_1.KeyType.QUORUM, instance.subscriptionHandler(), appId); //general quorum subscription
-            await instance.subscribe.subscribe(key_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid); //app-specific quorum subscription (used for pubsub one-time request/response)
+            //general quorum subscription
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
+            //app-specific quorum subscription (used for pubsub one-time request/response)
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
             instance.engine.processWebHooks();
             instance.engine.processTimeHooks();
             return instance;
@@ -102,20 +103,20 @@ class QuorumService {
                 engine_id: this.guid,
                 namespace: this.namespace,
                 app_id: this.appId,
-                stream: this.engine.stream.mintKey(key_1.KeyType.STREAMS, { appId: this.appId })
+                stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId })
             };
         }
-        this.store.publish(key_1.KeyType.QUORUM, {
+        this.store.publish(hotmesh_1.KeyType.QUORUM, {
             type: 'pong',
             guid, originator,
             profile,
         }, appId);
     }
-    async requestQuorum(delay = QUORUM_DELAY, details = false) {
+    async requestQuorum(delay = enums_1.HMSH_QUORUM_DELAY_MS, details = false) {
         const quorum = this.quorum;
         this.quorum = 0;
         this.profiles.length = 0;
-        await this.store.publish(key_1.KeyType.QUORUM, {
+        await this.store.publish(hotmesh_1.KeyType.QUORUM, {
             type: 'ping',
             originator: this.guid,
             details,
@@ -126,7 +127,7 @@ class QuorumService {
     // ************* PUB/SUB METHODS *************
     //publish a message to the quorum
     async pub(quorumMessage) {
-        return await this.store.publish(key_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
+        return await this.store.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
     }
     //subscribe user to quorum messages
     async sub(callback) {
@@ -139,7 +140,7 @@ class QuorumService {
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
     // ************* COMPILER METHODS *************
-    async rollCall(delay = QUORUM_DELAY) {
+    async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
         await this.requestQuorum(delay, true);
         const targetStreams = [];
         const multi = this.store.getMulti();
@@ -160,18 +161,29 @@ class QuorumService {
         });
         return this.profiles;
     }
-    async activate(version, delay = QUORUM_DELAY) {
+    /**
+     * request a quorum; if successful activate the app version
+     */
+    async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
         version = version.toString();
+        const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
+        if (!canActivate) {
+            //another engine is already activating the app version
+            this.logger.debug('quorum-activation-awaiting', { version });
+            await (0, utils_1.sleepFor)(delay * 6);
+            const app = await this.store.getApp(this.appId, true);
+            return app?.active == true && app?.version === version;
+        }
         const config = await this.engine.getVID();
-        //request a quorum to activate the version
         await this.requestQuorum(delay);
         const q1 = await this.requestQuorum(delay);
         const q2 = await this.requestQuorum(delay);
         const q3 = await this.requestQuorum(delay);
         if (q1 && q1 === q2 && q2 === q3) {
             this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
-            this.store.publish(key_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
+            this.store.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
             await new Promise(resolve => setTimeout(resolve, delay));
+            await this.store.releaseScoutRole('activate');
             //confirm we received the activation message
             if (this.engine.untilVersion === version) {
                 this.logger.info('quorum-activation-succeeded', { version });
@@ -185,7 +197,12 @@ class QuorumService {
             }
         }
         else {
-            this.logger.info('quorum-rollcall-error', { q1, q2, q3 });
+            this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
+            this.store.releaseScoutRole('activate');
+            if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
+                //increase the delay (give the quorum time to respond) and try again
+                return await this.activate(version, delay * 2, count + 1);
+            }
             throw new Error(`Quorum not reached. Version ${version} not activated.`);
         }
     }

package/build/services/store/clients/redis.js

@@ -6,6 +6,7 @@ class RedisStoreService extends index_1.StoreService {
     constructor(redisClient) {
         super(redisClient);
         this.commands = {
+            set: 'SET',
             setnx: 'SETNX',
             del: 'DEL',
             expire: 'EXPIRE',

package/build/services/store/index.d.ts

@@ -46,7 +46,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * check for and process work items in the
      * time and signal task queues.
      */
-    reserveScoutRole(scoutType: 'time' | 'signal', delay?: number): Promise<boolean>;
+    reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay?: number): Promise<boolean>;
+    releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean>;
     getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
     setSettings(manifest: HotMeshSettings): Promise<any>;
     reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;
@@ -86,7 +87,18 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      */
     getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
+    /**
+     * collate is a generic method for incrementing a value in a hash
+     * in order to track their progress during processing.
+     */
     collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
+    /**
+     * synthentic collation affects those activities in the graph
+     * that represent the synthetic DAG that was materialized during compilation;
+     * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+     * purposeful re-entry.
+     */
+    collateSynthetic(jobId: string, guid: string, amount: number, multi?: U): Promise<number>;
     setStateNX(jobId: string, appId: string): Promise<boolean>;
     getSchema(activityId: string, appVersion: AppVID): Promise<ActivityType>;
     getSchemas(appVersion: AppVID): Promise<Record<string, ActivityType>>;

package/build/services/store/index.js

@@ -33,6 +33,7 @@ const errors_1 = require("../../modules/errors");
 class StoreService {
     constructor(redisClient) {
         this.commands = {
+            set: 'set',
             setnx: 'setnx',
             del: 'del',
             expire: 'expire',
@@ -106,12 +107,13 @@ class StoreService {
      */
     async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
         const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
-        const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`);
-        if (this.isSuccessful(success)) {
-            await this.redisClient[this.commands.expire](key, delay - 1);
-            return true;
-        }
-        return false;
+        const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
+        return this.isSuccessful(success);
+    }
+    async releaseScoutRole(scoutType) {
+        const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
+        const success = await this.exec('DEL', key);
+        return this.isSuccessful(success);
     }
     async getSettings(bCreate = false) {
         let settings = this.cache?.getSettings();
@@ -485,6 +487,10 @@ class StoreService {
             throw new errors_1.GetStateError(jobId);
         }
     }
+    /**
+     * collate is a generic method for incrementing a value in a hash
+     * in order to track their progress during processing.
+     */
     async collate(jobId, activityId, amount, dIds, multi) {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         const collationKey = `${activityId}/output/metadata/as`; //activity state
@@ -497,6 +503,16 @@ class StoreService {
         const targetId = Object.keys(hashData)[0];
         return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
     }
+    /**
+     * synthentic collation affects those activities in the graph
+     * that represent the synthetic DAG that was materialized during compilation;
+     * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+     * purposeful re-entry.
+     */
+    async collateSynthetic(jobId, guid, amount, multi) {
+        const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+        return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
+    }
     async setStateNX(jobId, appId) {
         const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
         const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', '1');

package/build/types/hotmesh.d.ts

@@ -42,7 +42,7 @@ type KeyStoreParams = {
     facet?: string;
     topic?: string;
     timeValue?: number;
-    scoutType?: 'signal' | 'time';
+    scoutType?: 'signal' | 'time' | 'activate';
 };
 type HotMesh = typeof HotMeshService;
 type RedisConfig = {

package/build/types/job.d.ts

@@ -6,6 +6,7 @@ type ActivityData = {
 };
 type JobMetadata = {
     key?: string;
+    guid?: string;
     gid: string;
     jid: string;
     dad: string;

package/modules/enums.ts

@@ -22,6 +22,10 @@ export const HMSH_CODE_DURABLE_RETRYABLE = 599;
 
 export const HMSH_STATUS_UNKNOWN = 'unknown';
 
+// QUORUM
+export const HMSH_QUORUM_DELAY_MS = 250;
+export const HMSH_ACTIVATION_MAX_RETRY = 3;
+
 // ENGINE
 export const HMSH_OTT_WAIT_TIME = parseInt(process.env.HMSH_OTT_WAIT_TIME, 10) || 1000;
 export const HMSH_EXPIRE_JOB_SECONDS = parseInt(process.env.HMSH_EXPIRE_JOB_SECONDS, 10) || 1;

package/modules/errors.ts

@@ -45,20 +45,6 @@ class DurableWaitForSignalError extends Error {
   }
 }
 
-/* @deprecated */
-class DurableSleepError extends Error {
-  code: number;
-  duration: number; //seconds
-  index: number;    //execution order in the workflow
-  dimension: string; //hook dimension (e.g., ',0,1,0') (uses empty string for `null`)
-  constructor(message: string, duration: number, index: number, dimension: string) {
-    super(message);
-    this.duration = duration;
-    this.index = index;
-    this.dimension = dimension;
-    this.code = 595;
-  }
-}
 class DurableSleepForError extends Error {
   code: number;
   duration: number; //seconds
@@ -175,7 +161,6 @@ export {
   DurableIncompleteSignalError,
   DurableMaxedError,
   DurableRetryError,
-  DurableSleepError,
   DurableSleepForError,
   DurableTimeoutError,
   DurableWaitForSignalError,

package/modules/utils.ts

@@ -10,7 +10,7 @@ export async function sleepFor(ms: number) {
 }
 
 export function guid(): string {
-  return nanoid();
+  return nanoid().replace(/[_-]/g, '0');
 }
 
 export function deterministicRandom(seed: number): number {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.41",
+  "version": "0.0.43",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/services/activities/activity.ts

@@ -90,6 +90,22 @@ class Activity {
     await CollatorService.notarizeEntry(this);
   }
 
+  /**
+   * Upon entering leg 2 of a duplexed activty, verify
+   * all aspects of the re-entry including job and activty state
+   */
+  async verifyReentry(): Promise<number> {
+    const guid = this.context.metadata.guid;
+    this.setLeg(2);
+    await this.getState();
+    CollatorService.assertJobActive(
+      this.context.metadata.js,
+      this.context.metadata.jid,
+      this.metadata.aid
+    );
+    return await CollatorService.notarizeReentry(this, guid);
+  }
+
   //********  DUPLEX RE-ENTRY POINT  ********//
   async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output'): Promise<void> {
     this.setLeg(2);
@@ -103,23 +119,20 @@ class Activity {
     this.code = code;
     this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
     let telemetry: TelemetryService;
-    try {
-      await this.getState();
-      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
-      const aState = await CollatorService.notarizeReentry(this);
-      this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
 
-      if (isComplete) {
-        this.logger.warn('activity-process-event-duplicate', { jid, aid });
-        this.logger.debug('activity-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
-        return;
-      }
+    try {
+      const collationKey = await this.verifyReentry();
 
+      this.adjacentIndex = CollatorService.getDimensionalIndex(collationKey);
+      telemetry = new TelemetryService(
+        this.engine.appId,
+        this.config,
+        this.metadata,
+        this.context,
+      );
       telemetry.startActivitySpan(this.leg);
       let multiResponse: MultiResponseFlags;
+
       if (status === StreamStatus.PENDING) {
         multiResponse = await this.processPending(telemetry, type);
       } else if (status === StreamStatus.SUCCESS) {
@@ -384,7 +397,7 @@ class Activity {
     TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
     let { dad, jid } = this.context.metadata;
     const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
-    //`state` is a flat hash; context is a tree
+    //`state` is a unidimensional hash; context is a tree
     const [state, status] = await this.store.getState(jid, consumes, dIds);
     this.context = restoreHierarchy(state) as JobState;
     this.assertGenerationalId(this.context.metadata.gid, gid);
@@ -395,7 +408,9 @@ class Activity {
 
   /**
    * if the job is created/deleted/created with the same key,
-   * the 'gid' ensures no stale messages enter the stream
+   * the 'gid' ensures no stale messages (such as sleep delays)
+   * enter the workstream. Any message with a mismatched gid
+   * belongs to a prior job and can safely be ignored/dropped.
    */
   assertGenerationalId(jobGID: string, msgGID?: string) {
     if (msgGID !== jobGID) {

package/services/collator/index.ts

@@ -78,11 +78,28 @@ class CollatorService {
     return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_001 - decrement, this.getDimensionalAddress(activity), multi);
   };
 
-  static async notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number> {
+  /**
+   * verifies both the concrete and synthetic keys for the activity; concrete keys
+   * exist in the original model and are effectively the 'real' keys. In reality,
+   * hook activities are atomized during compilation to create a synthetic DAG that
+   * is used to track the status of the graph in a distributed environment. The
+   * synthetic key represents different dimensional realities and is used to
+   * track re-entry overages (it distinguishes between the original and re-entry).
+   * The essential challenge is: is this a re-entry that is purposeful in
+   * order to induce cycles, or is the re-entry due to a failure in the system?
+   */
+  static async notarizeReentry(activity: Activity, guid: string, multi?: RedisMulti): Promise<number> {
+    const jid = activity.context.metadata.jid;
+    const localMulti = multi || activity.store.getMulti();
     //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
-    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_000, this.getDimensionalAddress(activity, true), multi);
-    this.verifyInteger(amount, 2, 'enter');
-    return amount;
+    await activity.store.collate(jid, activity.metadata.aid, 1_000_000, this.getDimensionalAddress(activity, true), localMulti);
+    await activity.store.collateSynthetic(jid, guid, 1_000_000, localMulti);
+    const [_amountConcrete, _amountSynthetic] = await localMulti.exec();
+    const amountConcrete = Array.isArray(_amountConcrete) ? _amountConcrete[1] : _amountConcrete;
+    const amountSynthetic = Array.isArray(_amountSynthetic) ? _amountSynthetic[1] : _amountSynthetic;
+    this.verifyInteger(amountConcrete as number, 2, 'enter');
+    this.verifySyntheticInteger(amountSynthetic as number);
+    return amountConcrete as number;
   };
 
   static async notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number> {
@@ -134,6 +151,26 @@ class CollatorService {
     }
   }
 
+  /**
+   * During compilation, the graphs are compiled into structures necessary
+   * for distributed processing; these are referred to as 'synthetic DAGs',
+   * because they are not part of the original graph, but are used to track
+   * the status of the graph in a distributed environment. This check ensures
+   * that the 'synthetic key' is not a duplicate. (which is different than
+   * saying the 'key' is not a duplicate)
+   */
+  static verifySyntheticInteger(amount: number): void {
+    const samount = amount.toString();
+    const isCompletedValue = parseInt(samount[samount.length - 1], 10);
+    if (isCompletedValue > 0) {
+      //already done error (ack/delete clearly failed; this is a duplicate)
+      throw new CollationError(amount, 2, 'enter', CollationFaultType.INACTIVE);
+    } else if (amount >= 2_000_000) {
+      //duplicate synthetic key (todo: need to resolve/fix this!!)
+      throw new CollationError(amount, 2, 'enter', CollationFaultType.DUPLICATE);
+    }
+  }
+
   static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void {
     let faultType: CollationFaultType | undefined;
     if (leg === 1 && stage === 'enter') {
@@ -239,10 +276,6 @@ class CollatorService {
     });
   }
 
-  static isActivityComplete(status: number): boolean {
-    return (status - 0) <= 0;
-  }
-
   /**
    * All activities exist on a dimensional plane. Zero
    * is the default. A value of 

package/services/durable/client.ts

@@ -11,13 +11,14 @@ import { JobState } from '../../types/job';
 import { KeyService, KeyType } from '../../modules/key';
 import { Search } from './search';
 import { StreamStatus } from '../../types';
-import { HMSH_LOGLEVEL, HMSH_EXPIRE_JOB_SECONDS } from '../../modules/enums';
+import { HMSH_LOGLEVEL, HMSH_EXPIRE_JOB_SECONDS, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
+import { sleepFor } from '../../modules/utils';
 
 export class ClientService {
 
   connection: Connection;
-  topics: string[] = [];
   options: WorkflowOptions;
+  static topics: string[] = [];
   static instances = new Map<string, HotMesh | Promise<HotMesh>>();
 
   constructor(config: ClientConfig) {
@@ -29,8 +30,9 @@ export class ClientService {
     const instanceId = 'SINGLETON';
     if (ClientService.instances.has(instanceId)) {
       const hotMeshClient = await ClientService.instances.get(instanceId);
-      if (!this.topics.includes(workflowTopic)) {
-        this.topics.push(workflowTopic);
+      await this.verifyWorkflowActive(hotMeshClient, namespace ?? APP_ID);
+      if (!ClientService.topics.includes(workflowTopic)) {
+        ClientService.topics.push(workflowTopic);
         await this.createStream(hotMeshClient, workflowTopic, namespace);
       }
       return hotMeshClient;
@@ -196,6 +198,19 @@ export class ClientService {
     }
   }
 
+  async verifyWorkflowActive(hotMesh: HotMesh, appId = APP_ID, count = 0): Promise<boolean> {
+    const app = await hotMesh.engine.store.getApp(appId);
+    const appVersion = app?.version as unknown as number;
+    if(isNaN(appVersion)) {
+      if (count > 10) {
+        throw new Error('Workflow failed to activate');
+      }
+      await sleepFor(HMSH_QUORUM_DELAY_MS * 2);
+      return await this.verifyWorkflowActive(hotMesh, appId, count + 1);
+    }
+    return true;
+  }
+
   async activateWorkflow(hotMesh: HotMesh, appId = APP_ID, version = APP_VERSION): Promise<void> {
     const app = await hotMesh.engine.store.getApp(appId);
     const appVersion = app?.version as unknown as number;