@hotmeshio/hotmesh 0.0.41 → 0.0.42

package/build/modules/errors.d.ts

@@ -23,13 +23,6 @@ declare class DurableWaitForSignalError extends Error {
         index: number;
     }[]);
 }
-declare class DurableSleepError extends Error {
-    code: number;
-    duration: number;
-    index: number;
-    dimension: string;
-    constructor(message: string, duration: number, index: number, dimension: string);
-}
 declare class DurableSleepForError extends Error {
     code: number;
     duration: number;
@@ -86,4 +79,4 @@ declare class CollationError extends Error {
     fault: CollationFaultType;
     constructor(status: number, leg: ActivityDuplex, stage: CollationStage, fault?: CollationFaultType);
 }
-export { CollationError, DurableFatalError, DurableIncompleteSignalError, DurableMaxedError, DurableRetryError, DurableSleepError, DurableSleepForError, DurableTimeoutError, DurableWaitForSignalError, DuplicateJobError, ExecActivityError, GenerationalError, GetStateError, InactiveJobError, MapDataError, RegisterTimeoutError, SetStateError, };
+export { CollationError, DurableFatalError, DurableIncompleteSignalError, DurableMaxedError, DurableRetryError, DurableSleepForError, DurableTimeoutError, DurableWaitForSignalError, DuplicateJobError, ExecActivityError, GenerationalError, GetStateError, InactiveJobError, MapDataError, RegisterTimeoutError, SetStateError, };

package/build/modules/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SetStateError = exports.RegisterTimeoutError = exports.MapDataError = exports.InactiveJobError = exports.GetStateError = exports.GenerationalError = exports.ExecActivityError = exports.DuplicateJobError = exports.DurableWaitForSignalError = exports.DurableTimeoutError = exports.DurableSleepForError = exports.DurableSleepError = exports.DurableRetryError = exports.DurableMaxedError = exports.DurableIncompleteSignalError = exports.DurableFatalError = exports.CollationError = void 0;
+exports.SetStateError = exports.RegisterTimeoutError = exports.MapDataError = exports.InactiveJobError = exports.GetStateError = exports.GenerationalError = exports.ExecActivityError = exports.DuplicateJobError = exports.DurableWaitForSignalError = exports.DurableTimeoutError = exports.DurableSleepForError = exports.DurableRetryError = exports.DurableMaxedError = exports.DurableIncompleteSignalError = exports.DurableFatalError = exports.CollationError = void 0;
 const enums_1 = require("./enums");
 class GetStateError extends Error {
     constructor(jobId) {
@@ -34,17 +34,6 @@ class DurableWaitForSignalError extends Error {
     }
 }
 exports.DurableWaitForSignalError = DurableWaitForSignalError;
-/* @deprecated */
-class DurableSleepError extends Error {
-    constructor(message, duration, index, dimension) {
-        super(message);
-        this.duration = duration;
-        this.index = index;
-        this.dimension = dimension;
-        this.code = 595;
-    }
-}
-exports.DurableSleepError = DurableSleepError;
 class DurableSleepForError extends Error {
     constructor(message, duration, index, dimension) {
         super(message);

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.41",
+    "version": "0.0.42",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/activity.d.ts

@@ -31,6 +31,11 @@ declare class Activity {
      * all aspects of the entry including job and activty state
      */
     verifyEntry(): Promise<void>;
+    /**
+     * Upon entering leg 2 of a duplexed activty, verify
+     * all aspects of the re-entry including job and activty state
+     */
+    verifyReentry(): Promise<number>;
     processEvent(status?: StreamStatus, code?: StreamCode, type?: 'hook' | 'output'): Promise<void>;
     processPending(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags>;
     processSuccess(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags>;
@@ -57,7 +62,9 @@ declare class Activity {
     getState(): Promise<void>;
     /**
      * if the job is created/deleted/created with the same key,
-     * the 'gid' ensures no stale messages enter the stream
+     * the 'gid' ensures no stale messages (such as sleep delays)
+     * enter the workstream. Any message with a mismatched gid
+     * belongs to a prior job and can safely be ignored/dropped.
      */
     assertGenerationalId(jobGID: string, msgGID?: string): void;
     initDimensionalAddress(dad: string): void;

package/build/services/activities/activity.js

@@ -40,6 +40,17 @@ class Activity {
         collator_1.CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
         await collator_1.CollatorService.notarizeEntry(this);
     }
+    /**
+     * Upon entering leg 2 of a duplexed activty, verify
+     * all aspects of the re-entry including job and activty state
+     */
+    async verifyReentry() {
+        const guid = this.context.metadata.guid;
+        this.setLeg(2);
+        await this.getState();
+        collator_1.CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
+        return await collator_1.CollatorService.notarizeReentry(this, guid);
+    }
     //********  DUPLEX RE-ENTRY POINT  ********//
     async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200, type = 'output') {
         this.setLeg(2);
@@ -54,17 +65,9 @@ class Activity {
         this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
         let telemetry;
         try {
-            await this.getState();
-            collator_1.CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
-            const aState = await collator_1.CollatorService.notarizeReentry(this);
-            this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(aState);
+            const collationKey = await this.verifyReentry();
+            this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(collationKey);
             telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-            let isComplete = collator_1.CollatorService.isActivityComplete(this.context.metadata.js);
-            if (isComplete) {
-                this.logger.warn('activity-process-event-duplicate', { jid, aid });
-                this.logger.debug('activity-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
-                return;
-            }
             telemetry.startActivitySpan(this.leg);
             let multiResponse;
             if (status === stream_1.StreamStatus.PENDING) {
@@ -306,7 +309,7 @@ class Activity {
         telemetry_1.TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
         let { dad, jid } = this.context.metadata;
         const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
-        //`state` is a flat hash; context is a tree
+        //`state` is a unidimensional hash; context is a tree
         const [state, status] = await this.store.getState(jid, consumes, dIds);
         this.context = (0, utils_1.restoreHierarchy)(state);
         this.assertGenerationalId(this.context.metadata.gid, gid);
@@ -316,7 +319,9 @@ class Activity {
     }
     /**
      * if the job is created/deleted/created with the same key,
-     * the 'gid' ensures no stale messages enter the stream
+     * the 'gid' ensures no stale messages (such as sleep delays)
+     * enter the workstream. Any message with a mismatched gid
+     * belongs to a prior job and can safely be ignored/dropped.
      */
     assertGenerationalId(jobGID, msgGID) {
         if (msgGID !== jobGID) {

package/build/services/collator/index.d.ts

@@ -30,7 +30,17 @@ declare class CollatorService {
     static authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
     static notarizeEarlyExit(activity: Activity, multi?: RedisMulti): Promise<number>;
     static notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
-    static notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
+    /**
+     * verifies both the concrete and synthetic keys for the activity; concrete keys
+     * exist in the original model and are effectively the 'real' keys. In reality,
+     * hook activities are atomized during compilation to create a synthetic DAG that
+     * is used to track the status of the graph in a distributed environment. The
+     * synthetic key represents different dimensional realities and is used to
+     * track re-entry overages (it distinguishes between the original and re-entry).
+     * The essential challenge is: is this a re-entry that is purposeful in
+     * order to induce cycles, or is the re-entry due to a failure in the system?
+     */
+    static notarizeReentry(activity: Activity, guid: string, multi?: RedisMulti): Promise<number>;
     static notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number>;
     static notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
     static getDigitAtIndex(num: number, targetDigitIndex: number): number | null;
@@ -38,6 +48,15 @@ declare class CollatorService {
     static isDuplicate(num: number, targetDigitIndex: number): boolean;
     static isInactive(num: number): boolean;
     static isPrimed(amount: number, leg: ActivityDuplex): boolean;
+    /**
+     * During compilation, the graphs are compiled into structures necessary
+     * for distributed processing; these are referred to as 'synthetic DAGs',
+     * because they are not part of the original graph, but are used to track
+     * the status of the graph in a distributed environment. This check ensures
+     * that the 'synthetic key' is not a duplicate. (which is different than
+     * saying the 'key' is not a duplicate)
+     */
+    static verifySyntheticInteger(amount: number): void;
     static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
     static getDimensionsById(ancestors: string[], dad: string): Record<string, string>;
     /**
@@ -72,7 +91,6 @@ declare class CollatorService {
      *
      */
     static bindAncestorArray(graphs: HotMeshGraph[]): void;
-    static isActivityComplete(status: number): boolean;
     /**
      * All activities exist on a dimensional plane. Zero
      * is the default. A value of

package/build/services/collator/index.js

@@ -67,11 +67,28 @@ class CollatorService {
         return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000001 - decrement, this.getDimensionalAddress(activity), multi);
     }
     ;
-    static async notarizeReentry(activity, multi) {
+    /**
+     * verifies both the concrete and synthetic keys for the activity; concrete keys
+     * exist in the original model and are effectively the 'real' keys. In reality,
+     * hook activities are atomized during compilation to create a synthetic DAG that
+     * is used to track the status of the graph in a distributed environment. The
+     * synthetic key represents different dimensional realities and is used to
+     * track re-entry overages (it distinguishes between the original and re-entry).
+     * The essential challenge is: is this a re-entry that is purposeful in
+     * order to induce cycles, or is the re-entry due to a failure in the system?
+     */
+    static async notarizeReentry(activity, guid, multi) {
+        const jid = activity.context.metadata.jid;
+        const localMulti = multi || activity.store.getMulti();
         //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
-        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000000, this.getDimensionalAddress(activity, true), multi);
-        this.verifyInteger(amount, 2, 'enter');
-        return amount;
+        await activity.store.collate(jid, activity.metadata.aid, 1000000, this.getDimensionalAddress(activity, true), localMulti);
+        await activity.store.collateSynthetic(jid, guid, 1000000, localMulti);
+        const [_amountConcrete, _amountSynthetic] = await localMulti.exec();
+        const amountConcrete = Array.isArray(_amountConcrete) ? _amountConcrete[1] : _amountConcrete;
+        const amountSynthetic = Array.isArray(_amountSynthetic) ? _amountSynthetic[1] : _amountSynthetic;
+        this.verifyInteger(amountConcrete, 2, 'enter');
+        this.verifySyntheticInteger(amountSynthetic);
+        return amountConcrete;
     }
     ;
     static async notarizeContinuation(activity, multi) {
@@ -119,6 +136,26 @@ class CollatorService {
                 this.getDigitAtIndex(amount, 1) < 9;
         }
     }
+    /**
+     * During compilation, the graphs are compiled into structures necessary
+     * for distributed processing; these are referred to as 'synthetic DAGs',
+     * because they are not part of the original graph, but are used to track
+     * the status of the graph in a distributed environment. This check ensures
+     * that the 'synthetic key' is not a duplicate. (which is different than
+     * saying the 'key' is not a duplicate)
+     */
+    static verifySyntheticInteger(amount) {
+        const samount = amount.toString();
+        const isCompletedValue = parseInt(samount[samount.length - 1], 10);
+        if (isCompletedValue > 0) {
+            //already done error (ack/delete clearly failed; this is a duplicate)
+            throw new errors_1.CollationError(amount, 2, 'enter', collator_1.CollationFaultType.INACTIVE);
+        }
+        else if (amount >= 2000000) {
+            //duplicate synthetic key (todo: need to resolve/fix this!!)
+            throw new errors_1.CollationError(amount, 2, 'enter', collator_1.CollationFaultType.DUPLICATE);
+        }
+    }
     static verifyInteger(amount, leg, stage) {
         let faultType;
         if (leg === 1 && stage === 'enter') {
@@ -222,9 +259,6 @@ class CollatorService {
             dfs(startingNode, []);
         });
     }
-    static isActivityComplete(status) {
-        return (status - 0) <= 0;
-    }
     /**
      * All activities exist on a dimensional plane. Zero
      * is the default. A value of

package/build/services/durable/factory.d.ts

@@ -7,7 +7,6 @@
  * ERROR CODES:
  *      594: waitforsignal
  *      592: sleepFor
- *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */

package/build/services/durable/factory.js

@@ -10,7 +10,6 @@ exports.DEFAULT_COEFFICIENT = exports.APP_ID = exports.APP_VERSION = exports.get
  * ERROR CODES:
  *      594: waitforsignal
  *      592: sleepFor
- *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */
@@ -125,19 +124,6 @@ const getWorkflowYAML = (app, version) => {
               maps:
                 index: '{$self.output.data.index}'
                 signals: '{$self.output.data.signals}'
-            595:
-              schema:
-                type: object
-                properties:
-                  duration:
-                    type: number
-                    description: sleep duration in seconds
-                  index:
-                    type: number
-                    description: the current index
-              maps:
-                duration: '{$self.output.data.duration}'
-                index: '{$self.output.data.index}'
             592:
               schema:
                 type: object
@@ -232,19 +218,6 @@ const getWorkflowYAML = (app, version) => {
               maps:
                 index: '{$self.output.data.index}'
                 signals: '{$self.output.data.signals}'
-            595:
-              schema:
-                type: object
-                properties:
-                  duration:
-                    type: number
-                    description: sleep duration in seconds
-                  index:
-                    type: number
-                    description: the current index
-              maps:
-                duration: '{$self.output.data.duration}'
-                index: '{$self.output.data.index}'
             592:
               schema:
                 type: object
@@ -323,57 +296,6 @@ const getWorkflowYAML = (app, version) => {
             maps:
               duration: '{siga1.output.data.duration}'
 
-        siga595:
-          title: Signal In - Sleep before trying again
-          type: await
-          topic: ${app}.sleep.execute
-          input:
-            schema:
-              type: object
-              properties:
-                duration:
-                  type: number
-                index:
-                  type: number
-                workflowId:
-                  type: string
-                parentWorkflowId:
-                  type: string
-                originJobId:
-                  type: string
-            maps:
-              duration: '{sigw1.output.data.duration}'
-              index: '{sigw1.output.data.index}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-s']
-                  - ['{@string.concat}']
-              originJobId:
-                '@pipe':
-                  - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
-                  - ['{@conditional.ternary}']
-
-              workflowId:
-                '@pipe':
-                  - ['-', '{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
-                  - ['{@string.concat}']
-          output:
-            schema:
-              type: object
-              properties:
-                done:
-                  type: boolean
-            maps:
-              done: '{sigw1.output.data.done}'
-
-        sigc595:
-          title: Signal In - Goto Activity siga1
-          type: cycle
-          ancestor: siga1
-          input:
-            maps:
-              duration: '{siga1.output.data.duration}'
-
         siga592:
           title: Signal In - Sleep For a duration and then cycle/goto
           type: hook
@@ -467,56 +389,6 @@ const getWorkflowYAML = (app, version) => {
             maps:
               duration: '{a1.output.data.duration}'
 
-        a595:
-          title: Sleep before trying again
-          type: await
-          topic: ${app}.sleep.execute
-          input:
-            schema:
-              type: object
-              properties:
-                duration:
-                  type: number
-                index:
-                  type: number
-                workflowId:
-                  type: string
-                parentWorkflowId:
-                  type: string
-                originJobId:
-                  type: string
-            maps:
-              duration: '{w1.output.data.duration}'
-              index: '{w1.output.data.index}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-s']
-                  - ['{@string.concat}']
-              originJobId:
-                '@pipe':
-                  - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
-                  - ['{@conditional.ternary}']
-              workflowId:
-                '@pipe':
-                  - ['-', '{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
-                  - ['{@string.concat}']
-          output:
-            schema:
-              type: object
-              properties:
-                done:
-                  type: boolean
-            maps:
-              done: '{w1.output.data.done}'
-
-        c595:
-          title: Goto Activity a1
-          type: cycle
-          ancestor: a1
-          input:
-            maps:
-              duration: '{a1.output.data.duration}'
-
         a592:
           title: Sleep For a duration and then cycle/goto
           type: hook
@@ -574,9 +446,6 @@ const getWorkflowYAML = (app, version) => {
           - to: siga594
             conditions:
               code: 594
-          - to: siga595
-            conditions:
-              code: 595
           - to: siga592
             conditions:
               code: 592
@@ -585,8 +454,6 @@ const getWorkflowYAML = (app, version) => {
               code: 599
         siga594:
           - to: sigc594
-        siga595:
-          - to: sigc595
         siga592:
           - to: sigc592
         siga599:
@@ -597,9 +464,6 @@ const getWorkflowYAML = (app, version) => {
           - to: a594
             conditions:
               code: 594
-          - to: a595
-            conditions:
-              code: 595
           - to: a592
             conditions:
               code: 592
@@ -611,8 +475,6 @@ const getWorkflowYAML = (app, version) => {
               code: [200, 598, 597, 596]
         a594:
           - to: c594
-        a595:
-          - to: c595
         a592:
           - to: c592
         a599:

package/build/services/durable/worker.js

@@ -187,21 +187,6 @@ class WorkerService {
             catch (err) {
                 //not an error...just a trigger to sleep
                 if (err instanceof errors_1.DurableSleepForError) {
-                    return {
-                        status: stream_1.StreamStatus.SUCCESS,
-                        code: err.code,
-                        metadata: { ...data.metadata },
-                        data: {
-                            code: err.code,
-                            message: JSON.stringify({ duration: err.duration, index: err.index, dimension: err.dimension }),
-                            duration: err.duration,
-                            index: err.index,
-                            dimension: err.dimension
-                        }
-                    };
-                    //deprecated format; not an error...just a trigger to sleep
-                }
-                else if (err instanceof errors_1.DurableSleepError) {
                     return {
                         status: stream_1.StreamStatus.SUCCESS,
                         code: err.code,

package/build/services/durable/workflow.d.ts

@@ -112,15 +112,6 @@ export declare class WorkflowService {
      * @returns {Promise<number>}
      */
     static sleepFor(duration: string): Promise<number>;
-    /**
-     * Sleeps the workflow for a duration. As the function is reentrant,
-     * upon reentry, the function will traverse prior execution paths up
-     * until the sleep command and then resume execution from that point.
-     * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
-     * @returns {Promise<number>}
-     * @deprecated - use `sleepFor` instead
-     */
-    static sleep(duration: string): Promise<number>;
     /**
      * Waits for a signal to awaken
      * @param {string[]} signals - the signals to wait for

package/build/services/durable/workflow.js

@@ -338,35 +338,6 @@ class WorkflowService {
         }
         return seconds;
     }
-    /**
-     * Sleeps the workflow for a duration. As the function is reentrant,
-     * upon reentry, the function will traverse prior execution paths up
-     * until the sleep command and then resume execution from that point.
-     * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
-     * @returns {Promise<number>}
-     * @deprecated - use `sleepFor` instead
-     */
-    static async sleep(duration) {
-        const seconds = (0, ms_1.default)(duration) / 1000;
-        const store = storage_1.asyncLocalStorage.getStore();
-        const COUNTER = store.get('counter');
-        const execIndex = COUNTER.counter = COUNTER.counter + 1;
-        const workflowId = store.get('workflowId');
-        const workflowTopic = store.get('workflowTopic');
-        const workflowDimension = store.get('workflowDimension') ?? '';
-        const namespace = store.get('namespace');
-        const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
-        try {
-            const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
-            await hotMeshClient.getState(`${hotMeshClient.appId}.sleep.execute`, sleepJobId);
-            //if no error is thrown, we've already slept, return the delay
-            return seconds;
-        }
-        catch (e) {
-            // spawn a new sleep job if error code 595 is thrown by the worker)
-            throw new errors_1.DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
-        }
-    }
     /**
      * Waits for a signal to awaken
      * @param {string[]} signals - the signals to wait for

package/build/services/engine/index.d.ts

@@ -66,7 +66,7 @@ declare class EngineService {
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<string>;
     scrub(jobId: string): Promise<void>;
     hook(topic: string, data: JobData, status?: StreamStatus, code?: StreamCode): Promise<string>;
-    hookTime(jobId: string, gId: string, activityId: string, type?: WorkListTaskType): Promise<string | void>;
+    hookTime(jobId: string, gId: string, topicOrActivity: string, type?: WorkListTaskType): Promise<string | void>;
     hookAll(hookTopic: string, data: JobData, keyResolver: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
     pub(topic: string, data: JobData, context?: JobState): Promise<string>;
     sub(topic: string, callback: JobMessageCallback): Promise<void>;

package/build/services/engine/index.js

@@ -226,6 +226,7 @@ class EngineService {
         });
         const context = {
             metadata: {
+                guid: streamData.metadata.guid,
                 jid: streamData.metadata.jid,
                 gid: streamData.metadata.gid,
                 dad: streamData.metadata.dad,
@@ -353,15 +354,11 @@ class EngineService {
         };
         return await this.router.publishMessage(null, streamData);
     }
-    async hookTime(jobId, gId, activityId, type) {
-        if (type === 'interrupt') {
-            return await this.interrupt(activityId, //note: 'activityId' is the actually job topic
-            jobId, { suppress: true, expire: 1 });
+    async hookTime(jobId, gId, topicOrActivity, type) {
+        if (type === 'interrupt' || type === 'expire') {
+            return await this.interrupt(topicOrActivity, jobId, { suppress: true, expire: 1 });
         }
-        else if (type === 'expire') {
-            return await this.store.expireJob(jobId, 1);
-        }
-        const [aid, ...dimensions] = activityId.split(',');
+        const [aid, ...dimensions] = topicOrActivity.split(',');
         const dad = `,${dimensions.join(',')}`;
         const streamData = {
             type: stream_1.StreamDataType.TIMEHOOK,

package/build/services/store/index.d.ts

@@ -86,7 +86,18 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      */
     getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
+    /**
+     * collate is a generic method for incrementing a value in a hash
+     * in order to track their progress during processing.
+     */
     collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
+    /**
+     * synthentic collation affects those activities in the graph
+     * that represent the synthetic DAG that was materialized during compilation;
+     * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+     * purposeful re-entry.
+     */
+    collateSynthetic(jobId: string, guid: string, amount: number, multi?: U): Promise<number>;
     setStateNX(jobId: string, appId: string): Promise<boolean>;
     getSchema(activityId: string, appVersion: AppVID): Promise<ActivityType>;
     getSchemas(appVersion: AppVID): Promise<Record<string, ActivityType>>;

package/build/services/store/index.js

@@ -485,6 +485,10 @@ class StoreService {
             throw new errors_1.GetStateError(jobId);
         }
     }
+    /**
+     * collate is a generic method for incrementing a value in a hash
+     * in order to track their progress during processing.
+     */
     async collate(jobId, activityId, amount, dIds, multi) {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         const collationKey = `${activityId}/output/metadata/as`; //activity state
@@ -497,6 +501,16 @@ class StoreService {
         const targetId = Object.keys(hashData)[0];
         return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
     }
+    /**
+     * synthentic collation affects those activities in the graph
+     * that represent the synthetic DAG that was materialized during compilation;
+     * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+     * purposeful re-entry.
+     */
+    async collateSynthetic(jobId, guid, amount, multi) {
+        const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+        return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
+    }
     async setStateNX(jobId, appId) {
         const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
         const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', '1');

package/build/types/job.d.ts

@@ -6,6 +6,7 @@ type ActivityData = {
 };
 type JobMetadata = {
     key?: string;
+    guid?: string;
     gid: string;
     jid: string;
     dad: string;

package/modules/errors.ts

@@ -45,20 +45,6 @@ class DurableWaitForSignalError extends Error {
   }
 }
 
-/* @deprecated */
-class DurableSleepError extends Error {
-  code: number;
-  duration: number; //seconds
-  index: number;    //execution order in the workflow
-  dimension: string; //hook dimension (e.g., ',0,1,0') (uses empty string for `null`)
-  constructor(message: string, duration: number, index: number, dimension: string) {
-    super(message);
-    this.duration = duration;
-    this.index = index;
-    this.dimension = dimension;
-    this.code = 595;
-  }
-}
 class DurableSleepForError extends Error {
   code: number;
   duration: number; //seconds
@@ -175,7 +161,6 @@ export {
   DurableIncompleteSignalError,
   DurableMaxedError,
   DurableRetryError,
-  DurableSleepError,
   DurableSleepForError,
   DurableTimeoutError,
   DurableWaitForSignalError,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.41",
+  "version": "0.0.42",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/services/activities/activity.ts

@@ -90,6 +90,22 @@ class Activity {
     await CollatorService.notarizeEntry(this);
   }
 
+  /**
+   * Upon entering leg 2 of a duplexed activty, verify
+   * all aspects of the re-entry including job and activty state
+   */
+  async verifyReentry(): Promise<number> {
+    const guid = this.context.metadata.guid;
+    this.setLeg(2);
+    await this.getState();
+    CollatorService.assertJobActive(
+      this.context.metadata.js,
+      this.context.metadata.jid,
+      this.metadata.aid
+    );
+    return await CollatorService.notarizeReentry(this, guid);
+  }
+
   //********  DUPLEX RE-ENTRY POINT  ********//
   async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output'): Promise<void> {
     this.setLeg(2);
@@ -103,23 +119,20 @@ class Activity {
     this.code = code;
     this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
     let telemetry: TelemetryService;
-    try {
-      await this.getState();
-      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
-      const aState = await CollatorService.notarizeReentry(this);
-      this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
 
-      if (isComplete) {
-        this.logger.warn('activity-process-event-duplicate', { jid, aid });
-        this.logger.debug('activity-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
-        return;
-      }
+    try {
+      const collationKey = await this.verifyReentry();
 
+      this.adjacentIndex = CollatorService.getDimensionalIndex(collationKey);
+      telemetry = new TelemetryService(
+        this.engine.appId,
+        this.config,
+        this.metadata,
+        this.context,
+      );
       telemetry.startActivitySpan(this.leg);
       let multiResponse: MultiResponseFlags;
+
       if (status === StreamStatus.PENDING) {
         multiResponse = await this.processPending(telemetry, type);
       } else if (status === StreamStatus.SUCCESS) {
@@ -384,7 +397,7 @@ class Activity {
     TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
     let { dad, jid } = this.context.metadata;
     const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
-    //`state` is a flat hash; context is a tree
+    //`state` is a unidimensional hash; context is a tree
     const [state, status] = await this.store.getState(jid, consumes, dIds);
     this.context = restoreHierarchy(state) as JobState;
     this.assertGenerationalId(this.context.metadata.gid, gid);
@@ -395,7 +408,9 @@ class Activity {
 
   /**
    * if the job is created/deleted/created with the same key,
-   * the 'gid' ensures no stale messages enter the stream
+   * the 'gid' ensures no stale messages (such as sleep delays)
+   * enter the workstream. Any message with a mismatched gid
+   * belongs to a prior job and can safely be ignored/dropped.
    */
   assertGenerationalId(jobGID: string, msgGID?: string) {
     if (msgGID !== jobGID) {

package/services/collator/index.ts

@@ -78,11 +78,28 @@ class CollatorService {
     return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_001 - decrement, this.getDimensionalAddress(activity), multi);
   };
 
-  static async notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number> {
+  /**
+   * verifies both the concrete and synthetic keys for the activity; concrete keys
+   * exist in the original model and are effectively the 'real' keys. In reality,
+   * hook activities are atomized during compilation to create a synthetic DAG that
+   * is used to track the status of the graph in a distributed environment. The
+   * synthetic key represents different dimensional realities and is used to
+   * track re-entry overages (it distinguishes between the original and re-entry).
+   * The essential challenge is: is this a re-entry that is purposeful in
+   * order to induce cycles, or is the re-entry due to a failure in the system?
+   */
+  static async notarizeReentry(activity: Activity, guid: string, multi?: RedisMulti): Promise<number> {
+    const jid = activity.context.metadata.jid;
+    const localMulti = multi || activity.store.getMulti();
     //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
-    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_000, this.getDimensionalAddress(activity, true), multi);
-    this.verifyInteger(amount, 2, 'enter');
-    return amount;
+    await activity.store.collate(jid, activity.metadata.aid, 1_000_000, this.getDimensionalAddress(activity, true), localMulti);
+    await activity.store.collateSynthetic(jid, guid, 1_000_000, localMulti);
+    const [_amountConcrete, _amountSynthetic] = await localMulti.exec();
+    const amountConcrete = Array.isArray(_amountConcrete) ? _amountConcrete[1] : _amountConcrete;
+    const amountSynthetic = Array.isArray(_amountSynthetic) ? _amountSynthetic[1] : _amountSynthetic;
+    this.verifyInteger(amountConcrete as number, 2, 'enter');
+    this.verifySyntheticInteger(amountSynthetic as number);
+    return amountConcrete as number;
   };
 
   static async notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number> {
@@ -134,6 +151,26 @@ class CollatorService {
     }
   }
 
+  /**
+   * During compilation, the graphs are compiled into structures necessary
+   * for distributed processing; these are referred to as 'synthetic DAGs',
+   * because they are not part of the original graph, but are used to track
+   * the status of the graph in a distributed environment. This check ensures
+   * that the 'synthetic key' is not a duplicate. (which is different than
+   * saying the 'key' is not a duplicate)
+   */
+  static verifySyntheticInteger(amount: number): void {
+    const samount = amount.toString();
+    const isCompletedValue = parseInt(samount[samount.length - 1], 10);
+    if (isCompletedValue > 0) {
+      //already done error (ack/delete clearly failed; this is a duplicate)
+      throw new CollationError(amount, 2, 'enter', CollationFaultType.INACTIVE);
+    } else if (amount >= 2_000_000) {
+      //duplicate synthetic key (todo: need to resolve/fix this!!)
+      throw new CollationError(amount, 2, 'enter', CollationFaultType.DUPLICATE);
+    }
+  }
+
   static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void {
     let faultType: CollationFaultType | undefined;
     if (leg === 1 && stage === 'enter') {
@@ -239,10 +276,6 @@ class CollatorService {
     });
   }
 
-  static isActivityComplete(status: number): boolean {
-    return (status - 0) <= 0;
-  }
-
   /**
    * All activities exist on a dimensional plane. Zero
    * is the default. A value of 

package/services/durable/factory.ts

@@ -7,7 +7,6 @@
  * ERROR CODES:
  *      594: waitforsignal
  *      592: sleepFor
- *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */
@@ -122,19 +121,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               maps:
                 index: '{$self.output.data.index}'
                 signals: '{$self.output.data.signals}'
-            595:
-              schema:
-                type: object
-                properties:
-                  duration:
-                    type: number
-                    description: sleep duration in seconds
-                  index:
-                    type: number
-                    description: the current index
-              maps:
-                duration: '{$self.output.data.duration}'
-                index: '{$self.output.data.index}'
             592:
               schema:
                 type: object
@@ -229,19 +215,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               maps:
                 index: '{$self.output.data.index}'
                 signals: '{$self.output.data.signals}'
-            595:
-              schema:
-                type: object
-                properties:
-                  duration:
-                    type: number
-                    description: sleep duration in seconds
-                  index:
-                    type: number
-                    description: the current index
-              maps:
-                duration: '{$self.output.data.duration}'
-                index: '{$self.output.data.index}'
             592:
               schema:
                 type: object
@@ -320,57 +293,6 @@ const getWorkflowYAML = (app: string, version: string) => {
             maps:
               duration: '{siga1.output.data.duration}'
 
-        siga595:
-          title: Signal In - Sleep before trying again
-          type: await
-          topic: ${app}.sleep.execute
-          input:
-            schema:
-              type: object
-              properties:
-                duration:
-                  type: number
-                index:
-                  type: number
-                workflowId:
-                  type: string
-                parentWorkflowId:
-                  type: string
-                originJobId:
-                  type: string
-            maps:
-              duration: '{sigw1.output.data.duration}'
-              index: '{sigw1.output.data.index}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-s']
-                  - ['{@string.concat}']
-              originJobId:
-                '@pipe':
-                  - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
-                  - ['{@conditional.ternary}']
-
-              workflowId:
-                '@pipe':
-                  - ['-', '{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
-                  - ['{@string.concat}']
-          output:
-            schema:
-              type: object
-              properties:
-                done:
-                  type: boolean
-            maps:
-              done: '{sigw1.output.data.done}'
-
-        sigc595:
-          title: Signal In - Goto Activity siga1
-          type: cycle
-          ancestor: siga1
-          input:
-            maps:
-              duration: '{siga1.output.data.duration}'
-
         siga592:
           title: Signal In - Sleep For a duration and then cycle/goto
           type: hook
@@ -464,56 +386,6 @@ const getWorkflowYAML = (app: string, version: string) => {
             maps:
               duration: '{a1.output.data.duration}'
 
-        a595:
-          title: Sleep before trying again
-          type: await
-          topic: ${app}.sleep.execute
-          input:
-            schema:
-              type: object
-              properties:
-                duration:
-                  type: number
-                index:
-                  type: number
-                workflowId:
-                  type: string
-                parentWorkflowId:
-                  type: string
-                originJobId:
-                  type: string
-            maps:
-              duration: '{w1.output.data.duration}'
-              index: '{w1.output.data.index}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-s']
-                  - ['{@string.concat}']
-              originJobId:
-                '@pipe':
-                  - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
-                  - ['{@conditional.ternary}']
-              workflowId:
-                '@pipe':
-                  - ['-', '{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
-                  - ['{@string.concat}']
-          output:
-            schema:
-              type: object
-              properties:
-                done:
-                  type: boolean
-            maps:
-              done: '{w1.output.data.done}'
-
-        c595:
-          title: Goto Activity a1
-          type: cycle
-          ancestor: a1
-          input:
-            maps:
-              duration: '{a1.output.data.duration}'
-
         a592:
           title: Sleep For a duration and then cycle/goto
           type: hook
@@ -571,9 +443,6 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: siga594
             conditions:
               code: 594
-          - to: siga595
-            conditions:
-              code: 595
           - to: siga592
             conditions:
               code: 592
@@ -582,8 +451,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               code: 599
         siga594:
           - to: sigc594
-        siga595:
-          - to: sigc595
         siga592:
           - to: sigc592
         siga599:
@@ -594,9 +461,6 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: a594
             conditions:
               code: 594
-          - to: a595
-            conditions:
-              code: 595
           - to: a592
             conditions:
               code: 592
@@ -608,8 +472,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               code: [200, 598, 597, 596]
         a594:
           - to: c594
-        a595:
-          - to: c595
         a592:
           - to: c592
         a599:

package/services/durable/worker.ts

@@ -3,7 +3,6 @@ import {
   DurableIncompleteSignalError,
   DurableMaxedError,
   DurableRetryError,
-  DurableSleepError,
   DurableSleepForError,
   DurableTimeoutError, 
   DurableWaitForSignalError} from '../../modules/errors';
@@ -249,21 +248,6 @@ export class WorkerService {
             }
           } as StreamDataResponse;
 
-        //deprecated format; not an error...just a trigger to sleep
-        } else if (err instanceof DurableSleepError) {
-          return {
-            status: StreamStatus.SUCCESS,
-            code: err.code,
-            metadata: { ...data.metadata },
-            data: {
-              code: err.code,
-              message: JSON.stringify({ duration: err.duration, index: err.index, dimension: err.dimension }),
-              duration: err.duration,
-              index: err.index,
-              dimension: err.dimension
-            }
-          } as StreamDataResponse;
-
         //not an error...just a trigger to wait for a signal
         } else if (err instanceof DurableWaitForSignalError) {
           return {

package/services/durable/workflow.ts

@@ -2,7 +2,6 @@ import ms from 'ms';
 
 import {
   DurableIncompleteSignalError,
-  DurableSleepError,
   DurableSleepForError,
   DurableWaitForSignalError } from '../../modules/errors';
 import { KeyService, KeyType } from '../../modules/key';
@@ -376,37 +375,6 @@ export class WorkflowService {
     return seconds;
   }
 
-  /**
-   * Sleeps the workflow for a duration. As the function is reentrant, 
-   * upon reentry, the function will traverse prior execution paths up
-   * until the sleep command and then resume execution from that point.
-   * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
-   * @returns {Promise<number>}
-   * @deprecated - use `sleepFor` instead
-   */
-  static async sleep(duration: string): Promise<number> {
-    const seconds = ms(duration) / 1000;
-
-    const store = asyncLocalStorage.getStore();
-    const COUNTER = store.get('counter');
-    const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    const workflowId = store.get('workflowId');
-    const workflowTopic = store.get('workflowTopic');
-    const workflowDimension = store.get('workflowDimension') ?? '';
-    const namespace = store.get('namespace');
-    const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
-
-    try {
-      const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
-      await hotMeshClient.getState(`${hotMeshClient.appId}.sleep.execute`, sleepJobId);
-      //if no error is thrown, we've already slept, return the delay
-      return seconds;
-    } catch (e) {
-      // spawn a new sleep job if error code 595 is thrown by the worker)
-      throw new DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
-    }
-  }
-
   /**
    * Waits for a signal to awaken
    * @param {string[]} signals - the signals to wait for

package/services/engine/index.ts

@@ -332,6 +332,7 @@ class EngineService {
     });
     const context: PartialJobState = {
       metadata: {
+        guid: streamData.metadata.guid,
         jid: streamData.metadata.jid,
         gid: streamData.metadata.gid,
         dad: streamData.metadata.dad,
@@ -490,17 +491,15 @@ class EngineService {
     };
     return await this.router.publishMessage(null, streamData) as string;
   }
-  async hookTime(jobId: string, gId: string, activityId: string, type?: WorkListTaskType): Promise<string | void> {
-    if (type === 'interrupt') {
+  async hookTime(jobId: string, gId: string, topicOrActivity: string, type?: WorkListTaskType): Promise<string | void> {
+    if (type === 'interrupt' || type === 'expire') {
       return await this.interrupt(
-        activityId, //note: 'activityId' is the actually job topic
+        topicOrActivity,
         jobId,
         { suppress: true, expire: 1 },
       );
-    } else if (type === 'expire') {
-      return await this.store.expireJob(jobId, 1);
     }
-    const [aid, ...dimensions] = activityId.split(',');
+    const [aid, ...dimensions] = topicOrActivity.split(',');
     const dad = `,${dimensions.join(',')}`;
     const streamData: StreamData = {
       type: StreamDataType.TIMEHOOK,

package/services/store/index.ts

@@ -586,6 +586,10 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
+  /**
+   * collate is a generic method for incrementing a value in a hash
+   * in order to track their progress during processing.
+   */
   async collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi? : U): Promise<number> {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     const collationKey = `${activityId}/output/metadata/as`; //activity state
@@ -600,6 +604,17 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
   }
 
+  /**
+   * synthentic collation affects those activities in the graph
+   * that represent the synthetic DAG that was materialized during compilation;
+   * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+   * purposeful re-entry.
+   */
+  async collateSynthetic(jobId: string, guid: string, amount: number, multi? : U): Promise<number> {
+    const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+    return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
+  }
+
   async setStateNX(jobId: string, appId: string): Promise<boolean> {
     const hashKey = this.mintKey(KeyType.JOB_STATE, { appId, jobId });
     const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', '1');

package/types/job.ts

@@ -8,6 +8,7 @@ type ActivityData = {
 
 type JobMetadata = {
   key?: string; //job_key
+  guid?: string; //system assigned guid that corresponds to the transition message guid that spawned reentry
   gid: string;  //system assigned guid; ensured created/deleted/created jobs are unique
   jid: string;  //job_id (jid+dad+aid) is composite key for activity
   dad: string;  //dimensional address for the activity (,0,0,1)