@hotmeshio/hotmesh 0.0.48 → 0.0.50

package/README.md

@@ -1,7 +1,7 @@
 # HotMesh
 ![alpha release](https://img.shields.io/badge/release-alpha-yellow)
 
-HotMesh elevates Redis from an in-memory data cache to a distributed orchestration engine.
+HotMesh transforms Redis into a distributed orchestration engine.
 
 *Write functions in your own preferred style, and let Redis govern their execution, reliably and durably.*
 

package/build/modules/enums.d.ts

@@ -15,6 +15,7 @@ export declare const HMSH_CODE_DURABLE_MAXED = 597;
 export declare const HMSH_CODE_DURABLE_FATAL = 598;
 export declare const HMSH_CODE_DURABLE_RETRYABLE = 599;
 export declare const HMSH_STATUS_UNKNOWN = "unknown";
+export declare const HMSH_QUORUM_ROLLCALL_CYCLES = 12;
 export declare const HMSH_QUORUM_DELAY_MS = 250;
 export declare const HMSH_ACTIVATION_MAX_RETRY = 3;
 export declare const HMSH_OTT_WAIT_TIME: number;

package/build/modules/enums.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
+exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
 // HOTMESH SYSTEM
 exports.HMSH_LOGLEVEL = process.env.HMSH_LOGLEVEL || 'info';
 // STATUS CODES AND MESSAGES
@@ -20,6 +20,7 @@ exports.HMSH_CODE_DURABLE_FATAL = 598;
 exports.HMSH_CODE_DURABLE_RETRYABLE = 599;
 exports.HMSH_STATUS_UNKNOWN = 'unknown';
 // QUORUM
+exports.HMSH_QUORUM_ROLLCALL_CYCLES = 12; //max iterations
 exports.HMSH_QUORUM_DELAY_MS = 250;
 exports.HMSH_ACTIVATION_MAX_RETRY = 3;
 // ENGINE

package/build/modules/key.d.ts

@@ -27,6 +27,10 @@ import { KeyStoreParams, KeyType } from '../types/hotmesh';
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
 declare const HMNS = "hmsh";
+declare const KEYSEP = ":";
+declare const VALSEP = "::";
+declare const WEBSEP = "::";
+declare const TYPSEP = "::";
 declare class KeyService {
     /**
      * returns a key that can be used to access a value in the key/value store
@@ -41,4 +45,4 @@ declare class KeyService {
      */
     static mintKey(namespace: string, keyType: KeyType, params: KeyStoreParams): string;
 }
-export { KeyService, KeyType, KeyStoreParams, HMNS };
+export { KeyService, KeyType, KeyStoreParams, HMNS, KEYSEP, TYPSEP, WEBSEP, VALSEP };

package/build/modules/key.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HMNS = exports.KeyType = exports.KeyService = void 0;
+exports.VALSEP = exports.WEBSEP = exports.TYPSEP = exports.KEYSEP = exports.HMNS = exports.KeyType = exports.KeyService = void 0;
 const hotmesh_1 = require("../types/hotmesh");
 Object.defineProperty(exports, "KeyType", { enumerable: true, get: function () { return hotmesh_1.KeyType; } });
 /**
@@ -30,8 +30,16 @@ Object.defineProperty(exports, "KeyType", { enumerable: true, get: function () {
  * hmsh:<appid>:sym:keys:<activityid|$subscribes> ->  {hash}    list of symbols based upon schema enums (initially) and adaptively optimized (later) during runtime; if '$subscribes' is used as the activityid, it is a top-level `job` symbol set (for job keys)
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
-const HMNS = "hmsh"; //default
+const HMNS = "hmsh";
 exports.HMNS = HMNS;
+const KEYSEP = ':'; //default delimiter for keys 
+exports.KEYSEP = KEYSEP;
+const VALSEP = '::'; //default delimiter for vals
+exports.VALSEP = VALSEP;
+const WEBSEP = '::'; //default delimiter for webhook vals
+exports.WEBSEP = WEBSEP;
+const TYPSEP = '::'; //delimiter for ZSET task typing (how should a list be used?)
+exports.TYPSEP = TYPSEP;
 class KeyService {
     /**
      * returns a key that can be used to access a value in the key/value store

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.48",
+    "version": "0.0.50",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",
@@ -28,6 +28,7 @@
         "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
         "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
         "test:emit": "NODE_ENV=test jest ./tests/functional/emit/index.test.ts --detectOpenHandles --forceExit --verbose",
+        "test:await": "NODE_ENV=test jest ./tests/functional/awaiter/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:hook": "NODE_ENV=test jest ./tests/functional/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:signal": "NODE_ENV=test jest ./tests/functional/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:interrupt": "NODE_ENV=test jest ./tests/functional/interrupt/index.test.ts --detectOpenHandles --forceExit --verbose",

package/build/services/activities/await.js

@@ -78,6 +78,12 @@ class Await extends activity_1.Activity {
             type: stream_1.StreamDataType.AWAIT,
             data: this.context.data
         };
+        if (this.config.await !== true) {
+            const doAwait = pipe_1.Pipe.resolve(this.config.await, this.context);
+            if (doAwait === false) {
+                streamData.metadata.await = false;
+            }
+        }
         if (this.config.retry) {
             streamData.policies = {
                 retry: this.config.retry

package/build/services/activities/hook.js

@@ -107,7 +107,7 @@ class Hook extends activity_1.Activity {
         }
         else if (this.config.sleep) {
             const duration = pipe_1.Pipe.resolve(this.config.sleep, this.context);
-            await this.engine.taskService.registerTimeHook(this.context.metadata.jid, this.context.metadata.gid, `${this.metadata.aid}${this.metadata.dad || ''}`, 'sleep', duration);
+            await this.engine.taskService.registerTimeHook(this.context.metadata.jid, this.context.metadata.gid, `${this.metadata.aid}${this.metadata.dad || ''}`, 'sleep', duration, this.metadata.dad || '');
             return this.context.metadata.jid;
         }
     }

package/build/services/activities/trigger.d.ts

@@ -8,6 +8,7 @@ declare class Trigger extends Activity {
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
     process(): Promise<string>;
     setStatus(amount: number): Promise<void>;
+    execAdjacentParent(): Promise<void>;
     createInputContext(): Partial<JobState>;
     getState(): Promise<void>;
     bindJobMetadataPaths(): string[];

package/build/services/activities/trigger.js

@@ -31,6 +31,9 @@ class Trigger extends activity_1.Activity {
             await this.setStats(multi);
             await this.registerJobDependency(multi);
             await multi.exec();
+            //if the parent (spawner) chose not to await,
+            // emit the job_id as the data payload { job_id }
+            this.execAdjacentParent();
             telemetry.mapActivityAttributes();
             const jobStatus = Number(this.context.metadata.js);
             telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
@@ -61,6 +64,11 @@ class Trigger extends activity_1.Activity {
     async setStatus(amount) {
         this.context.metadata.js = amount;
     }
+    async execAdjacentParent() {
+        if (this.context.metadata.px) {
+            await this.engine.execAdjacentParent(this.context, { metadata: this.context.metadata, data: { job_id: this.context.metadata.jid } });
+        }
+    }
     createInputContext() {
         const input = {
             [this.metadata.aid]: {
@@ -95,6 +103,7 @@ class Trigger extends activity_1.Activity {
                 pg: this.context.metadata.pg,
                 pd: this.context.metadata.pd,
                 pa: this.context.metadata.pa,
+                px: this.context.metadata.px,
                 app: id,
                 vrs: version,
                 tpc: this.config.subscribes,
@@ -165,10 +174,22 @@ class Trigger extends activity_1.Activity {
         }
         if (resolvedDepKey) {
             const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
-            await this.store.registerJobDependency(isParentOrigin ? 'expire-child' : 'expire', resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
+            let type;
+            if (isParentOrigin) {
+                if (this.context.metadata.px) {
+                    type = 'child';
+                }
+                else {
+                    type = 'expire-child';
+                }
+            }
+            else {
+                type = 'expire';
+            }
+            await this.store.registerJobDependency(type, resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, this.context.metadata.pd, multi);
         }
         if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
-            await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
+            await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, this.context.metadata.pd, multi);
         }
     }
     async setStats(multi) {

package/build/services/durable/exporter.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExporterService = void 0;
 const serializer_1 = require("../serializer");
 const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
  * Splits, Inflates, and Sorts the job data for use in durable contexts
@@ -94,13 +95,27 @@ class ExporterService {
             const activityName = item[1].split('/')[0];
             const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
             const timestamp = item[2];
-            const event = {
+            let event = {
                 activity: activityName,
                 duplex: duplex,
                 dimension: dimensions,
                 timestamp,
+                created: timestamp,
+                updated: timestamp,
             };
-            timeline.push(event);
+            const prior = timeline[timeline.length - 1];
+            if (prior && prior.activity === event.activity && prior.duplex !== event.duplex && prior.dimension === event.dimension) {
+                if (event.duplex === 'exit') {
+                    prior.updated = event.timestamp;
+                }
+                else {
+                    prior.created = event.timestamp;
+                }
+                event = prior;
+            }
+            else {
+                timeline.push(event);
+            }
             if (this.isMainEntry(item[1])) {
                 event.actions = [];
                 this.interleaveActions(actions.main, event.actions);
@@ -163,12 +178,11 @@ class ExporterService {
      * @returns - the organized dependency data
      */
     inflateDependencyData(data, actions) {
-        //console.log('dependency data>', data);
         const hookReg = /([0-9,]+)-(\d+)$/;
         const flowReg = /-(\d+)$/;
         return data.map((dependency, index) => {
-            const [action, topic, gid, ...jid] = dependency.split('::');
-            const jobId = jid.join('::');
+            const [action, topic, gid, _pd, ...jid] = dependency.split(key_1.VALSEP);
+            const jobId = jid.join(key_1.VALSEP);
             const match = jobId.match(hookReg);
             let prefix;
             let type;

package/build/services/durable/meshos.js

@@ -187,13 +187,18 @@ class MeshOSService {
         }
         else {
             //limit which hash fields to return
-            if (options.return?.length) {
-                args.push('RETURN');
-                args.push(options.return.length.toString());
-                options.return.forEach(returnField => {
+            args.push('RETURN');
+            args.push(((options.return?.length ?? 0) + 1).toString());
+            args.push('$');
+            options.return?.forEach(returnField => {
+                if (returnField.startsWith('"')) {
+                    //allow literal values to be requested
+                    args.push(returnField.slice(1, -1));
+                }
+                else {
                     args.push(`_${returnField}`);
-                });
-            }
+                }
+            });
             //paginate
             if (options.limit) {
                 args.push('LIMIT', options.limit.start.toString(), options.limit.size.toString());

package/build/services/durable/search.d.ts

@@ -30,10 +30,29 @@ export declare class Search {
      * calling any method that produces side effects (changes the value)
      */
     getSearchSessionGuid(): string;
-    set(...args: string[]): Promise<void>;
+    /**
+     * Sets the fields listed in args. Returns the
+     * count of new fields that were set (does not
+     * count fields that were updated)
+     */
+    set(...args: string[]): Promise<number>;
     get(key: string): Promise<string>;
     mget(...args: string[]): Promise<string[]>;
+    /**
+     * Deletes the fields listed in args. Returns the
+     * count of fields that were deleted.
+     */
     del(...args: string[]): Promise<number | void>;
+    /**
+     * Increments the value of a field by the given amount. Returns the
+     * new value of the field after the increment. Can be
+     * used to decrement the value of a field by specifying a negative.
+     */
     incr(key: string, val: number): Promise<number>;
+    /**
+     * Multiplies the value of a field by the given amount. Returns the
+     * new value of the field after the multiplication. NOTE:
+     * this is exponential multiplication.
+     */
     mult(key: string, val: number): Promise<number>;
 }

package/build/services/durable/search.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Search = void 0;
 const key_1 = require("../../modules/key");
+const storage_1 = require("../../modules/storage");
 class Search {
     constructor(workflowId, hotMeshClient, searchSessionId) {
         this.searchSessionIndex = 0;
@@ -57,9 +58,15 @@ class Search {
      * @returns {Promise<string[]>} - the list of search indexes
      */
     static async listSearchIndexes(hotMeshClient) {
-        const store = hotMeshClient.engine.store;
-        const searchIndexes = await store.exec('FT._LIST');
-        return searchIndexes;
+        try {
+            const store = hotMeshClient.engine.store;
+            const searchIndexes = await store.exec('FT._LIST');
+            return searchIndexes;
+        }
+        catch (err) {
+            hotMeshClient.engine.logger.info('durable-client-search-list-err', { err });
+            return [];
+        }
     }
     /**
      * increments the index to return a unique search session guid when
@@ -69,18 +76,28 @@ class Search {
         //return the search session as it would exist in the search session index
         return `${this.searchSessionId}-${this.searchSessionIndex++}-`;
     }
+    /**
+     * Sets the fields listed in args. Returns the
+     * count of new fields that were set (does not
+     * count fields that were updated)
+     */
     async set(...args) {
         const ssGuid = this.getSearchSessionGuid();
-        const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
-        if (ssGuidValue === 1) {
-            const safeArgs = [];
-            for (let i = 0; i < args.length; i += 2) {
-                const key = this.safeKey(args[i]);
-                const value = args[i + 1].toString();
-                safeArgs.push(key, value);
-            }
-            await this.store.exec('HSET', this.jobId, ...safeArgs);
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Number(replay[ssGuid]);
         }
+        const safeArgs = [];
+        for (let i = 0; i < args.length; i += 2) {
+            const key = this.safeKey(args[i]);
+            const value = args[i + 1].toString();
+            safeArgs.push(key, value);
+        }
+        const fieldCount = await this.store.exec('HSET', this.jobId, ...safeArgs);
+        //no need to wait; set this interim value in the replay
+        this.store.exec('HSET', this.jobId, ssGuid, fieldCount.toString());
+        return Number(fieldCount);
     }
     async get(key) {
         try {
@@ -104,32 +121,63 @@ class Search {
             return [];
         }
     }
+    /**
+     * Deletes the fields listed in args. Returns the
+     * count of fields that were deleted.
+     */
     async del(...args) {
         const ssGuid = this.getSearchSessionGuid();
-        const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
-        if (ssGuidValue === 1) {
-            const safeArgs = [];
-            for (let i = 0; i < args.length; i++) {
-                safeArgs.push(this.safeKey(args[i]));
-            }
-            const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
-            return isNaN(response) ? undefined : Number(response);
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Number(replay[ssGuid]);
+        }
+        const safeArgs = [];
+        for (let i = 0; i < args.length; i++) {
+            safeArgs.push(this.safeKey(args[i]));
         }
+        const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
+        const formattedResponse = isNaN(response) ? 0 : Number(response);
+        //no need to wait; set this interim value in the replay
+        this.store.exec('HSET', this.jobId, ssGuid, formattedResponse.toString());
+        return formattedResponse;
     }
+    /**
+     * Increments the value of a field by the given amount. Returns the
+     * new value of the field after the increment. Can be
+     * used to decrement the value of a field by specifying a negative.
+     */
     async incr(key, val) {
         const ssGuid = this.getSearchSessionGuid();
-        const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
-        if (ssGuidValue === 1) {
-            return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()));
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Number(replay[ssGuid]);
         }
+        const num = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString());
+        //no need to wait; set this interim value in the replay
+        this.store.exec('HSET', this.jobId, ssGuid, num.toString());
+        return Number(num);
     }
+    /**
+     * Multiplies the value of a field by the given amount. Returns the
+     * new value of the field after the multiplication. NOTE:
+     * this is exponential multiplication.
+     */
     async mult(key, val) {
         const ssGuid = this.getSearchSessionGuid();
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Math.exp(Number(replay[ssGuid]));
+        }
         const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
         if (ssGuidValue === 1) {
             const log = Math.log(val);
-            const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()));
-            return Math.exp(logTotal);
+            const logTotal = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString());
+            //no need to wait; set this interim value in the replay
+            this.store.exec('HSET', this.jobId, ssGuid, logTotal.toString());
+            return Math.exp(Number(logTotal));
         }
     }
 }

package/build/services/durable/worker.js

@@ -164,16 +164,26 @@ class WorkerService {
                     // garbage collect (expire) this job when originJobId is expired
                     context.set('originJobId', workflowInput.originJobId);
                 }
+                let replayQuery = '';
                 if (workflowInput.workflowDimension) {
                     //every hook function runs in an isolated dimension controlled
                     //by the index assigned when the signal was received; even if the
                     //hook function re-runs, its scope will always remain constant
                     context.set('workflowDimension', workflowInput.workflowDimension);
+                    replayQuery = `-*${workflowInput.workflowDimension}-*`;
+                }
+                else {
+                    //last letter of words like 'hook', 'sleep', 'wait', 'signal', 'search', 'start'
+                    replayQuery = '-*[ehklpt]-*';
                 }
                 context.set('workflowTopic', workflowTopic);
                 context.set('workflowName', workflowTopic.split('-').pop());
                 context.set('workflowTrace', data.metadata.trc);
                 context.set('workflowSpan', data.metadata.spn);
+                const store = this.workflowRunner.engine.store;
+                const [cursor, replay] = await store.findJobFields(workflowInput.workflowId, replayQuery, 50000, 5000);
+                context.set('replay', replay);
+                context.set('cursor', cursor); // if != 0, more remain
                 const workflowResponse = await storage_1.asyncLocalStorage.run(context, async () => {
                     return await workflowFunction.apply(this, workflowInput.arguments);
                 });

package/build/services/durable/workflow.d.ts

@@ -87,6 +87,7 @@ export declare class WorkflowService {
         workflowTopic: any;
         workflowDimension: any;
         counter: any;
+        replay: any;
     };
     /**
      * Executes a function once and caches the result. If the function is called

package/build/services/durable/workflow.js

@@ -76,6 +76,10 @@ class WorkflowService {
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
         const sessionId = `-start${workflowDimension}-${execIndex}-`;
+        const replay = store.get('replay');
+        if (sessionId in replay) {
+            return replay[sessionId];
+        }
         //NOTE: this is the hash prefix; necessary for the search index to locate the entity
         const entityOrEmptyString = options.entity ?? '';
         //If the workflowId is not provided, it is generated from the entity and the workflow name
@@ -172,6 +176,8 @@ class WorkflowService {
     static getContext() {
         const store = storage_1.asyncLocalStorage.getStore();
         const workflowId = store.get('workflowId');
+        const replay = store.get('replay');
+        const cursor = store.get('cursor');
         const workflowDimension = store.get('workflowDimension') ?? '';
         const workflowTopic = store.get('workflowTopic');
         const namespace = store.get('namespace');
@@ -180,7 +186,9 @@ class WorkflowService {
         const COUNTER = store.get('counter');
         return {
             counter: COUNTER.counter,
+            cursor,
             namespace,
+            replay,
             workflowId,
             workflowDimension,
             workflowTopic,
@@ -201,6 +209,10 @@ class WorkflowService {
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
         const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
+        const replay = store.get('replay');
+        if (sessionId in replay) {
+            return false;
+        }
         const keyParams = {
             appId: hotMeshClient.appId,
             jobId: workflowId
@@ -274,6 +286,7 @@ class WorkflowService {
             workflowTopic: store.get('workflowTopic'),
             workflowDimension: store.get('workflowDimension') ?? '',
             counter: store.get('counter'),
+            replay: store.get('replay'),
         };
     }
     /**
@@ -284,9 +297,12 @@ class WorkflowService {
      * @template T - the result type
      */
     static async once(fn, ...args) {
-        const { workflowId, namespace, workflowTopic, workflowDimension, counter: COUNTER, } = WorkflowService.getLocalState();
+        const { workflowId, namespace, workflowTopic, workflowDimension, counter: COUNTER, replay, } = WorkflowService.getLocalState();
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
         const sessionId = `-once${workflowDimension}-${execIndex}-`;
+        if (sessionId in replay) {
+            return JSON.parse(replay[sessionId]);
+        }
         const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
         const keyParams = {
             appId: hotMeshClient.appId,

package/build/services/engine/index.d.ts

@@ -65,7 +65,7 @@ declare class EngineService {
     resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
     processStreamMessage(streamData: StreamDataResponse): Promise<void>;
     execAdjacentParent(context: JobState, jobOutput: JobOutput, emit?: boolean): Promise<string>;
-    hasParentJob(context: JobState): boolean;
+    hasParentJob(context: JobState, checkSevered?: boolean): boolean;
     resolveError(metadata: JobMetadata): StreamError | undefined;
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<string>;
     scrub(jobId: string): Promise<void>;

package/build/services/engine/index.js

@@ -259,6 +259,7 @@ class EngineService {
                 pg: streamData.metadata.gid,
                 pd: streamData.metadata.dad,
                 pa: streamData.metadata.aid,
+                px: streamData.metadata.await === false,
                 trc: streamData.metadata.trc,
                 spn: streamData.metadata.spn,
             };
@@ -316,7 +317,10 @@ class EngineService {
             return (await this.router?.publishMessage(null, streamData));
         }
     }
-    hasParentJob(context) {
+    hasParentJob(context, checkSevered = false) {
+        if (checkSevered) {
+            return Boolean(context.metadata.pj && context.metadata.pa && !context.metadata.px);
+        }
         return Boolean(context.metadata.pj && context.metadata.pa);
     }
     resolveError(metadata) {
@@ -385,7 +389,12 @@ class EngineService {
             const workItems = await reporter.getWorkItems(resolvedQuery, queryFacets);
             if (workItems.length) {
                 const taskService = new task_1.TaskService(this.store, this.logger);
-                await taskService.enqueueWorkItems(workItems.map(workItem => `${hookTopic}::${workItem}::${keyResolver.scrub || false}::${JSON.stringify(data)}`));
+                await taskService.enqueueWorkItems(workItems.map(workItem => [
+                    hookTopic,
+                    workItem,
+                    keyResolver.scrub || false,
+                    JSON.stringify(data)
+                ].join(key_1.VALSEP)));
                 this.store.publish(key_1.KeyType.QUORUM, { type: 'work', originator: this.guid }, this.appId);
             }
             return workItems;
@@ -504,7 +513,7 @@ class EngineService {
     // ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
     async runJobCompletionTasks(context, options = {}) {
         //'emit' indicates the job is still active
-        const isAwait = this.hasParentJob(context);
+        const isAwait = this.hasParentJob(context, true);
         const isOneTimeSub = this.hasOneTimeSubscription(context);
         const topic = await this.getPublishesTopic(context);
         let msgId;

package/build/services/exporter/index.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExporterService = void 0;
 const serializer_1 = require("../serializer");
 const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
  * Expands process data and includes dependency list
@@ -87,8 +88,8 @@ class ExporterService {
         const hookReg = /([0-9,]+)-(\d+)$/;
         const flowReg = /-(\d+)$/;
         return data.map((dependency, index) => {
-            const [action, topic, gid, ...jid] = dependency.split('::');
-            const jobId = jid.join('::');
+            const [action, topic, gid, _pd, ...jid] = dependency.split(key_1.VALSEP);
+            const jobId = jid.join(key_1.VALSEP);
             const match = jobId.match(hookReg);
             let prefix;
             let type;

package/build/services/hotmesh/index.js

@@ -167,6 +167,10 @@ class HotMeshService {
     }
     stop() {
         this.engine?.taskService.cancelCleanup();
+        this.quorum?.stop();
+        this.workers?.forEach((worker) => {
+            worker.stop();
+        });
     }
     async compress(terms) {
         return await this.engine?.compress(terms);