@hotmeshio/hotmesh 0.0.43 → 0.0.44

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.43",
+    "version": "0.0.44",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/trigger.js

@@ -158,11 +158,17 @@ class Trigger extends activity_1.Activity {
     async registerJobDependency(multi) {
         const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
         let resolvedDepKey = depKey ? pipe_1.Pipe.resolve(depKey, this.context) : '';
+        const adjKey = this.config.stats?.adjacent;
+        let resolvedAdjKey = depKey ? pipe_1.Pipe.resolve(adjKey, this.context) : '';
         if (!resolvedDepKey) {
             resolvedDepKey = this.context.metadata.pj;
         }
         if (resolvedDepKey) {
-            await this.store.registerJobDependency(resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
+            const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
+            await this.store.registerJobDependency(isParentOrigin ? 'expire-child' : 'expire', resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
+        }
+        if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
+            await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
         }
     }
     async setStats(multi) {

package/build/services/durable/exporter.d.ts

@@ -0,0 +1,105 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { StringStringType, Symbols } from "../../types";
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { ActivityAction, DependencyExport, ExportItem, ExportOptions, JobAction, JobActionExport, DurableJobExport, JobTimeline } from '../../types/exporter';
+import { SerializerService } from '../serializer';
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Splits, Inflates, and Sorts the job data for use in durable contexts
+ */
+declare class ExporterService {
+    appId: string;
+    logger: ILogger;
+    serializer: SerializerService;
+    store: StoreService<RedisClient, RedisMulti>;
+    symbols: Promise<Symbols> | Symbols;
+    /**
+     * Friendly names for the activity ids
+     */
+    activitySymbols: Symbols;
+    transitions: {
+        trigger: string[];
+        pivot: string[];
+        worker: string[];
+        sleeper: string[];
+        awaiter: string[];
+        retryer: string[];
+        hook: string[];
+        hook_pivot: string[];
+        hook_worker: string[];
+        hook_sleeper: string[];
+        hook_awaiter: string[];
+        hook_retryer: string[];
+    };
+    cycles: {
+        sleep_cycler: string[];
+        await_cycler: string[];
+        retry_cycler: string[];
+        hook_sleep_cycler: string[];
+        hook_await_cycler: string[];
+        hook_retry_cycler: string[];
+    };
+    constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    /**
+     * Convert the job hash and dependency list into a DurableJobExport object.
+     * This object contains various facets that describe the interaction
+     * in terms relevant to narrative storytelling.
+     */
+    export(jobId: string, options?: ExportOptions): Promise<DurableJobExport>;
+    /**
+     * Interleave actions into the replay timeline to create
+     * a time-ordered timeline of the entire interaction, beginning
+     * with the entry trigger and concluding with the scrubber
+     * activity. Using the returned timeline, it is possible to
+     * create an animated narrative of the job, highlighting
+     * activities in the graph according to the timeline's
+     * activity-created (/ac) and activity-updated (/au) entries.
+     */
+    createTimeline(replay: ExportItem[], actions: JobActionExport): JobTimeline[];
+    /**
+     * Interleave actions into the 'worker' and 'hook_worker'
+     * activities (between their /ac and /au entries)
+     */
+    interleaveActions(target: JobAction, actions: ActivityAction[]): void;
+    isPausingAction(actionType: string): boolean;
+    isMainEntry(key: string): boolean;
+    isHookEntry(key: string): boolean;
+    /**
+     * Inflates the key from Redis, 3-character symbol
+     * into a human-readable JSON path, reflecting the
+     * tree-like structure of the unidimensional Hash
+     */
+    inflateKey(key: string): string;
+    /**
+     * Inflates the dependency data from Redis into a DurableJobExport object by
+     * organizing the dimensional isolate in sch a way asto interleave
+     * into a story
+     * @param data - the dependency data from Redis
+     * @returns - the organized dependency data
+     */
+    inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
+    /**
+     * Adds historical actions (proxyActivity, executeChild)
+     * using the `dependency list` to determine
+     * after-the-fact what happened within the 'black-box'
+     * worker function. This is necessary to interleave the
+     * actions into the replay timeline, given that it isn't
+     * really possible to know the inner-workings of the user's
+     * function
+     *
+     */
+    seedActions(type: 'flow' | 'hook' | 'other', action: string, topic: string, dep: string, prefix: string, dimensionKey: string, actions: JobActionExport, jobId: string): void;
+    /**
+     * Inflates the job data from Redis into a DurableJobExport object
+     * @param jobHash - the job data from Redis
+     * @param dependencyList - the list of dependencies for the job
+     * @returns - the inflated job data
+     */
+    inflate(jobHash: StringStringType, dependencyList: string[]): DurableJobExport;
+    inflateProcess(match: RegExpMatchArray, value: string, replay: ExportItem[]): void;
+    inflateActions(key: string, value: string, actions: JobActionExport): void;
+    reverseSort(aKey: ExportItem, bKey: ExportItem): 1 | -1 | 0;
+    dateSort(aKey: ExportItem, bKey: ExportItem): 1 | -1 | 0;
+}
+export { ExporterService };

package/build/services/durable/exporter.js

@@ -0,0 +1,374 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExporterService = void 0;
+const serializer_1 = require("../serializer");
+const utils_1 = require("../../modules/utils");
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Splits, Inflates, and Sorts the job data for use in durable contexts
+ */
+class ExporterService {
+    constructor(appId, store, logger) {
+        /**
+         * Friendly names for the activity ids
+         */
+        this.activitySymbols = {
+            t1: 'trigger',
+            a1: 'pivot',
+            w1: 'worker',
+            a592: 'sleeper',
+            a594: 'awaiter',
+            a599: 'retryer',
+            c592: 'sleep_cycler',
+            c594: 'await_cycler',
+            c599: 'retry_cycler',
+            s5: 'scrubber',
+            sig: 'hook',
+            siga1: 'hook_pivot',
+            sigw1: 'hook_worker',
+            siga592: 'hook_sleeper',
+            siga594: 'hook_awaiter',
+            siga599: 'hook_retryer',
+            sigc592: 'hook_sleep_cycler',
+            sigc594: 'hook_await_cycler',
+            sigc599: 'hook_retry_cycler',
+        };
+        //adjacent transitions
+        this.transitions = {
+            trigger: ['pivot', 'hook'],
+            pivot: ['worker'],
+            worker: ['sleeper', 'awaiter', 'retryer', 'scrubber'],
+            sleeper: ['sleep_cycler'],
+            awaiter: ['await_cycler'],
+            retryer: ['retry_cycler'],
+            hook: ['hook_pivot'],
+            hook_pivot: ['hook_worker'],
+            hook_worker: ['hook_sleeper', 'hook_awaiter', 'hook_retryer'],
+            hook_sleeper: ['hook_sleep_cycler'],
+            hook_awaiter: ['hook_await_cycler'],
+            hook_retryer: ['hook_retry_cycler'],
+        };
+        //goto transitions
+        this.cycles = {
+            sleep_cycler: ['pivot'],
+            await_cycler: ['pivot'],
+            retry_cycler: ['pivot'],
+            hook_sleep_cycler: ['hook_pivot'],
+            hook_await_cycler: ['hook_pivot'],
+            hook_retry_cycler: ['hook_pivot'],
+        };
+        this.appId = appId;
+        this.logger = logger;
+        this.store = store;
+        this.serializer = new serializer_1.SerializerService();
+    }
+    /**
+     * Convert the job hash and dependency list into a DurableJobExport object.
+     * This object contains various facets that describe the interaction
+     * in terms relevant to narrative storytelling.
+     */
+    async export(jobId, options = {}) {
+        if (!this.symbols) {
+            this.symbols = this.store.getAllSymbols();
+            this.symbols = await this.symbols;
+        }
+        const depData = await this.store.getDependencies(jobId);
+        const jobData = await this.store.getRaw(jobId);
+        const jobExport = this.inflate(jobData, depData);
+        return jobExport;
+    }
+    /**
+     * Interleave actions into the replay timeline to create
+     * a time-ordered timeline of the entire interaction, beginning
+     * with the entry trigger and concluding with the scrubber
+     * activity. Using the returned timeline, it is possible to
+     * create an animated narrative of the job, highlighting
+     * activities in the graph according to the timeline's
+     * activity-created (/ac) and activity-updated (/au) entries.
+     */
+    createTimeline(replay, actions) {
+        const timeline = [];
+        replay.forEach((item) => {
+            const dimensions = item[0];
+            const parts = dimensions.split('/');
+            const activityName = item[1].split('/')[0];
+            const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
+            const timestamp = item[2];
+            const event = {
+                activity: activityName,
+                duplex: duplex,
+                dimension: dimensions,
+                timestamp,
+            };
+            timeline.push(event);
+            if (this.isMainEntry(item[1])) {
+                event.actions = [];
+                this.interleaveActions(actions.main, event.actions);
+            }
+            else if (this.isHookEntry(item[1])) {
+                const hookDimension = `/${parts[1]}/${parts[2]}`;
+                const hookActions = actions.hooks[hookDimension];
+                event.actions = [];
+                this.interleaveActions(hookActions, event.actions);
+            }
+        });
+        return timeline;
+    }
+    /**
+     * Interleave actions into the 'worker' and 'hook_worker'
+     * activities (between their /ac and /au entries)
+     */
+    interleaveActions(target, actions) {
+        if (target) {
+            for (let i = target.cursor + 1; i < target.items.length; i++) {
+                const [_, actionType, jobOrIndex] = target.items[i];
+                actions.push({ action: actionType, target: jobOrIndex });
+                target.cursor = i;
+                if (this.isPausingAction(actionType)) {
+                    break;
+                }
+            }
+        }
+    }
+    isPausingAction(actionType) {
+        return actionType === 'sleep' || actionType === 'waitForSignal';
+    }
+    isMainEntry(key) {
+        return key.startsWith('worker/') && key.endsWith('/ac');
+    }
+    isHookEntry(key) {
+        return key.startsWith('hook_worker/') && key.endsWith('/ac');
+    }
+    /**
+     * Inflates the key from Redis, 3-character symbol
+     * into a human-readable JSON path, reflecting the
+     * tree-like structure of the unidimensional Hash
+     */
+    inflateKey(key) {
+        if (key in this.symbols) {
+            const path = this.symbols[key];
+            const parts = path.split('/');
+            if (parts[0] in this.activitySymbols) {
+                parts[0] = this.activitySymbols[parts[0]];
+            }
+            return parts.join('/');
+        }
+        return key;
+    }
+    /**
+     * Inflates the dependency data from Redis into a DurableJobExport object by
+     * organizing the dimensional isolate in sch a way asto interleave
+     * into a story
+     * @param data - the dependency data from Redis
+     * @returns - the organized dependency data
+     */
+    inflateDependencyData(data, actions) {
+        //console.log('dependency data>', data);
+        const hookReg = /([0-9,]+)-(\d+)$/;
+        const flowReg = /-(\d+)$/;
+        return data.map((dependency, index) => {
+            const [action, topic, gid, ...jid] = dependency.split('::');
+            const jobId = jid.join('::');
+            const match = jobId.match(hookReg);
+            let prefix;
+            let type;
+            let dimensionKey = '';
+            if (match) {
+                //hook-originating dependency
+                const [_, dimension, counter] = match;
+                dimensionKey = dimension.split(',').join('/');
+                prefix = `${dimensionKey}[${counter}]`;
+                type = 'hook';
+            }
+            else {
+                const match = jobId.match(flowReg);
+                if (match) {
+                    //main workflow-originating dependency
+                    const [_, counter] = match;
+                    prefix = `[${counter}]`;
+                    type = 'flow';
+                }
+                else {
+                    //'other' types like signal cleanup
+                    prefix = '/';
+                    type = 'other';
+                }
+            }
+            this.seedActions(type, action, topic, dependency, prefix, dimensionKey, actions, jobId);
+            return {
+                type: action,
+                topic,
+                gid,
+                jid: jobId,
+            };
+        });
+    }
+    /**
+     * Adds historical actions (proxyActivity, executeChild)
+     * using the `dependency list` to determine
+     * after-the-fact what happened within the 'black-box'
+     * worker function. This is necessary to interleave the
+     * actions into the replay timeline, given that it isn't
+     * really possible to know the inner-workings of the user's
+     * function
+     *
+     */
+    seedActions(type, action, topic, dep, prefix, dimensionKey, actions, jobId) {
+        if (type !== 'other' && action === 'expire-child') {
+            let depType;
+            if (topic == `${this.appId}.activity.execute`) {
+                depType = 'proxyActivity';
+            }
+            else if (topic == `${this.appId}.execute`) {
+                depType = 'executeChild';
+            }
+            else if (topic == `${this.appId}.wfsc.execute`) {
+                depType = 'waitForSignal';
+            }
+            if (depType) {
+                if (type === 'flow') {
+                    actions.main.items.push([prefix, depType, jobId]);
+                }
+                else if (type === 'hook') {
+                    if (!actions.hooks[dimensionKey]) {
+                        actions.hooks[dimensionKey] = {
+                            cursor: -1,
+                            items: [],
+                        };
+                    }
+                    actions.hooks[dimensionKey].items.push([prefix, depType, jobId]);
+                }
+            }
+        }
+    }
+    /**
+     * Inflates the job data from Redis into a DurableJobExport object
+     * @param jobHash - the job data from Redis
+     * @param dependencyList - the list of dependencies for the job
+     * @returns - the inflated job data
+     */
+    inflate(jobHash, dependencyList) {
+        //the list of actions taken in the workflow and hook functions
+        const actions = {
+            hooks: {},
+            main: { cursor: -1, items: [] },
+        };
+        const dependencies = this.inflateDependencyData(dependencyList, actions);
+        const state = {};
+        const data = {};
+        const other = [];
+        const replay = [];
+        const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
+        Object.entries(jobHash).forEach(([key, value]) => {
+            const match = key.match(regex);
+            if (match) {
+                //activity process state
+                this.inflateProcess(match, value, replay);
+            }
+            else if (key.length === 3) {
+                //job state
+                state[this.inflateKey(key)] = this.serializer.fromString(value);
+            }
+            else if (key.startsWith('_')) {
+                //job data
+                data[key.substring(1)] = value;
+            }
+            else if (key.startsWith('-')) {
+                //actions with side effect (replayable)
+                this.inflateActions(key, value, actions);
+            }
+            else {
+                //collator guids, etc
+                other.push([null, key, value]);
+            }
+        });
+        replay.sort(this.dateSort);
+        actions.main.items.sort(this.reverseSort);
+        Object.entries(actions.hooks).forEach(([key, value]) => {
+            value.items.sort(this.reverseSort);
+        });
+        return {
+            data: (0, utils_1.restoreHierarchy)(data),
+            dependencies,
+            state: Object.entries((0, utils_1.restoreHierarchy)(state))[0][1],
+            status: jobHash[':'],
+            timeline: this.createTimeline(replay, actions),
+            transitions: { ...this.transitions },
+            cycles: { ...this.cycles },
+        };
+    }
+    inflateProcess(match, value, replay) {
+        const [_, letters, numbers] = match;
+        const path = this.inflateKey(letters);
+        if (path.endsWith('/output/metadata/ac') ||
+            path.endsWith('/output/metadata/au')) {
+            const dimensions = `/${numbers.replace(/,/g, '/')}`;
+            const resolved = this.serializer.fromString(value);
+            replay.push([
+                dimensions,
+                path,
+                resolved,
+            ]);
+        }
+    }
+    inflateActions(key, value, actions) {
+        let [_, dimensionalType, counter, subcounter] = key.split('-');
+        if (subcounter) {
+            counter = `${counter}.${subcounter}`;
+        }
+        const [type, ...dimensions] = dimensionalType.split(',');
+        let dimensionKey = '';
+        let isHook = false;
+        if (dimensions.length > 0) {
+            dimensionKey = `/${dimensions.join('/')}`;
+            isHook = true;
+        }
+        let targetList;
+        if (isHook) {
+            if (!actions.hooks[dimensionKey]) {
+                actions.hooks[dimensionKey] = {
+                    cursor: -1,
+                    items: [],
+                };
+            }
+            targetList = actions.hooks[dimensionKey].items;
+        }
+        else {
+            targetList = actions.main.items;
+        }
+        targetList.push([
+            `${dimensionKey}[${counter}]`,
+            type,
+            value,
+        ]);
+    }
+    reverseSort(aKey, bKey) {
+        if (aKey[0] > bKey[0]) {
+            return 1;
+        }
+        else if (aKey[0] < bKey[0]) {
+            return -1;
+        }
+        else {
+            if (aKey[1] > bKey[1]) {
+                return 1;
+            }
+            else if (aKey[1] < bKey[1]) {
+                return -1;
+            }
+            return 0;
+        }
+    }
+    dateSort(aKey, bKey) {
+        if (aKey[2] > bKey[2]) {
+            return 1;
+        }
+        else if (aKey[2] < bKey[2]) {
+            return -1;
+        }
+        else {
+            return 0;
+        }
+    }
+}
+exports.ExporterService = ExporterService;

package/build/services/durable/factory.js

@@ -60,6 +60,7 @@ const getWorkflowYAML = (app, version) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
           job:
             maps:
               done: false
@@ -263,10 +264,7 @@ const getWorkflowYAML = (app, version) => {
                   description: index will be appended later
             maps:
               signals: '{sigw1.output.data.signals}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-w']
-                  - ['{@string.concat}']
+              parentWorkflowId: '{$job.metadata.jid}'
               originJobId:
                 '@pipe':
                   - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
@@ -356,10 +354,7 @@ const getWorkflowYAML = (app, version) => {
                   description: index will be appended later
             maps:
               signals: '{w1.output.data.signals}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-w']
-                  - ['{@string.concat}']
+              parentWorkflowId: '{$job.metadata.jid}'
               originJobId:
                 '@pipe':
                   - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
@@ -526,6 +521,7 @@ const getWorkflowYAML = (app, version) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         w1a:
           title: Activity Worker - Calls Activity Functions
@@ -566,61 +562,6 @@ const getWorkflowYAML = (app, version) => {
         t1a:
           - to: w1a
 
-    - subscribes: ${app}.sleep.execute
-      publishes: ${app}.sleep.executed
-
-      expire: 0
-
-      input:
-        schema:
-          type: object
-          properties:
-            parentWorkflowId:
-              type: string
-            originJobId:
-              type: string
-            workflowId:
-              type: string
-            duration:
-              type: number
-              description: in seconds
-            index:
-              type: number
-      output:
-        schema:
-          type: object
-          properties:
-            done:
-              type: boolean
-            duration:
-              type: number
-            index:
-              type: number
-
-      activities:
-        t1s:
-          title: Sleep Flow Trigger
-          type: trigger
-          stats:
-            id: '{$self.input.data.workflowId}'
-            key: '{$self.input.data.parentWorkflowId}'
-            parent: '{$self.input.data.originJobId}'
-
-        a1s:
-          title: Sleep for a duration
-          type: hook
-          sleep: '{t1s.output.data.duration}'
-          job:
-            maps:
-              done: true
-              duration: '{t1s.output.data.duration}'
-              index: '{t1s.output.data.index}'
-              workflowId: '{t1s.output.data.workflowId}'
-
-      transitions:
-        t1s:
-          - to: a1s
-
     - subscribes: ${app}.wfsc.execute
       publishes: ${app}.wfsc.executed
 
@@ -665,6 +606,7 @@ const getWorkflowYAML = (app, version) => {
           stats:
             id: '{$self.input.data.cycleWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         a1wc:
           title: Pivot - All Cycling Descendants Point Here
@@ -836,6 +778,7 @@ const getWorkflowYAML = (app, version) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         a1ww:
           title: WFS - signal entry point

package/build/services/durable/handle.d.ts

@@ -1,10 +1,14 @@
+import { ExporterService } from './exporter';
 import { HotMeshService as HotMesh } from '../hotmesh';
+import { DurableJobExport } from '../../types/exporter';
 import { JobInterruptOptions } from '../../types/job';
 export declare class WorkflowHandleService {
+    exporter: ExporterService;
     hotMesh: HotMesh;
     workflowTopic: string;
     workflowId: string;
     constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string);
+    export(): Promise<DurableJobExport>;
     /**
      * Sends a signal to the workflow. This is a way to send
      * a message to a workflow that is paused due to having

package/build/services/durable/handle.js

@@ -2,11 +2,16 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkflowHandleService = void 0;
 const enums_1 = require("../../modules/enums");
+const exporter_1 = require("./exporter");
 class WorkflowHandleService {
     constructor(hotMesh, workflowTopic, workflowId) {
         this.workflowTopic = workflowTopic;
         this.workflowId = workflowId;
         this.hotMesh = hotMesh;
+        this.exporter = new exporter_1.ExporterService(this.hotMesh.appId, this.hotMesh.engine.store, this.hotMesh.engine.logger);
+    }
+    async export() {
+        return this.exporter.export(this.workflowId);
     }
     /**
      * Sends a signal to the workflow. This is a way to send