@hotmeshio/hotmesh 0.0.52 → 0.0.53

package/services/durable/exporter.ts

@@ -1,184 +1,43 @@
+import { VALSEP } from '../../modules/key';
+import { restoreHierarchy } from '../../modules/utils';
 import { ILogger } from '../logger';
+import { SerializerService } from '../serializer';
 import { StoreService } from '../store';
-import { StringStringType, Symbols } from "../../types";
-import { RedisClient, RedisMulti } from '../../types/redis';
 import {
-  ActivityAction,
-  DependencyExport,
   ExportItem,
   ExportOptions,
-  JobAction,
-  JobActionExport,
-  DurableJobExport,
-  JobTimeline } from '../../types/exporter';
-import { SerializerService } from '../serializer';
-import { restoreHierarchy } from '../../modules/utils';
-import { VALSEP } from '../../modules/key';
+  DurableJobExport, 
+  IdemType,
+  TimelineEntry } from '../../types/exporter';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { StringStringType, Symbols } from "../../types/serializer";
 
-/**
- * Downloads job data from Redis (hscan, hmget, hgetall)
- * Splits, Inflates, and Sorts the job data for use in durable contexts
- */
 class ExporterService {
   appId: string;
   logger: ILogger;
-  serializer: SerializerService
   store: StoreService<RedisClient, RedisMulti>;
   symbols: Promise<Symbols> | Symbols;
 
-  /**
-   * Friendly names for the activity ids
-   */
-  activitySymbols: Symbols = {
-    t1: 'trigger',
-    a1: 'pivot',
-    w1: 'worker',
-    a592: 'sleeper',
-    a594: 'awaiter',
-    a599: 'retryer',
-    c592: 'sleep_cycler',
-    c594: 'await_cycler',
-    c599: 'retry_cycler',
-    s5: 'scrubber',
-    sig: 'hook',
-    siga1: 'hook_pivot',
-    sigw1: 'hook_worker',
-    siga592: 'hook_sleeper',
-    siga594: 'hook_awaiter',
-    siga599: 'hook_retryer',
-    sigc592: 'hook_sleep_cycler',
-    sigc594: 'hook_await_cycler',
-    sigc599: 'hook_retry_cycler',
-  }
-
-  //adjacent transitions
-  transitions = {
-    trigger: ['pivot', 'hook'],
-    pivot: ['worker'],
-    worker: ['sleeper', 'awaiter', 'retryer', 'scrubber'],
-    sleeper: ['sleep_cycler'],
-    awaiter: ['await_cycler'],
-    retryer: ['retry_cycler'],
-    hook: ['hook_pivot'],
-    hook_pivot: ['hook_worker'],
-    hook_worker: ['hook_sleeper', 'hook_awaiter', 'hook_retryer'],
-    hook_sleeper: ['hook_sleep_cycler'],
-    hook_awaiter: ['hook_await_cycler'],
-    hook_retryer: ['hook_retry_cycler'],
-  };
-
-  //goto transitions
-  cycles = {
-    sleep_cycler: ['pivot'],
-    await_cycler: ['pivot'],
-    retry_cycler: ['pivot'],
-    hook_sleep_cycler: ['hook_pivot'],
-    hook_await_cycler: ['hook_pivot'],
-    hook_retry_cycler: ['hook_pivot'],
-  }
-
   constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
     this.appId = appId;
     this.logger = logger;
     this.store = store;
-    this.serializer = new SerializerService();
   }
 
   /**
-   * Convert the job hash and dependency list into a DurableJobExport object.
-   * This object contains various facets that describe the interaction
-   * in terms relevant to narrative storytelling.
+   * Convert the job hash from its compiles format into a DurableJobExport object with
+   * facets that describe the workflow in terms relevant to narrative storytelling.
    */
   async export(jobId: string, options: ExportOptions = {}): Promise<DurableJobExport> {
     if (!this.symbols) {
       this.symbols = this.store.getAllSymbols();
       this.symbols = await this.symbols;
     }
-    const depData = await this.store.getDependencies(jobId);
     const jobData = await this.store.getRaw(jobId);
-    const jobExport = this.inflate(jobData, depData);
+    const jobExport = this.inflate(jobData/*, depData*/);
     return jobExport;
   }
 
-  /**
-   * Interleave actions into the replay timeline to create
-   * a time-ordered timeline of the entire interaction, beginning
-   * with the entry trigger and concluding with the scrubber
-   * activity. Using the returned timeline, it is possible to
-   * create an animated narrative of the job, highlighting
-   * activities in the graph according to the timeline's 
-   * activity-created (/ac) and activity-updated (/au) entries.
-   */
-  createTimeline(replay: ExportItem[], actions: JobActionExport): JobTimeline[] {
-    const timeline: JobTimeline[] = [];
-    replay.forEach((item) => {
-      const dimensions = item[0];
-      const parts = dimensions.split('/');
-      const activityName = item[1].split('/')[0];
-      const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
-      const timestamp = item[2];
-      let event: JobTimeline = {
-        activity: activityName,
-        duplex: duplex as 'entry' | 'exit',
-        dimension: dimensions,
-        timestamp,
-        created: timestamp,
-        updated: timestamp,
-      };
-      const prior = timeline[timeline.length - 1];
-      if (prior && prior.activity === event.activity && prior.duplex !== event.duplex && prior.dimension === event.dimension) {
-        if (event.duplex === 'exit') {
-          prior.updated = event.timestamp;
-        } else {
-          prior.created = event.timestamp;
-        }
-        event = prior;
-      } else {
-        timeline.push(event);
-      }
-
-      if (this.isMainEntry(item[1])) {
-        event.actions = [] as ActivityAction[];
-        this.interleaveActions(actions.main, event.actions);
-      } else if (this.isHookEntry(item[1])) {
-        const hookDimension = `/${parts[1]}/${parts[2]}`;
-        const hookActions = actions.hooks[hookDimension];
-        event.actions = [] as ActivityAction[];
-        this.interleaveActions(hookActions, event.actions);
-      }
-    });
-    return timeline;
-  }
-
-  /**
-   * Interleave actions into the 'worker' and 'hook_worker'
-   * activities (between their /ac and /au entries)
-   */
-  interleaveActions(target: JobAction, actions: ActivityAction[]) {
-    if (target) {
-      for (let i = target.cursor + 1; i < target.items.length; i++) {
-        const [_, actionType, jobOrIndex] = target.items[i];
-        actions.push({ action: actionType, target: jobOrIndex });
-        target.cursor = i;
-        if (this.isPausingAction(actionType)) {
-          break;
-        }
-      }
-    }
-  }
-
-  isPausingAction(actionType: string): boolean {
-    return actionType === 'sleep' || actionType === 'waitForSignal';
-  }
-
-  isMainEntry(key: string): boolean {
-    return key.startsWith('worker/') && key.endsWith('/ac');
-  }
-
-  isHookEntry(key: string): boolean {
-    return key.startsWith('hook_worker/') && key.endsWith('/ac');
-  }
-
   /**
    * Inflates the key from Redis, 3-character symbol
    * into a human-readable JSON path, reflecting the
@@ -188,9 +47,6 @@ class ExporterService {
     if (key in this.symbols) {
       const path = this.symbols[key];
       const parts = path.split('/');
-      if (parts[0] in this.activitySymbols) {
-        parts[0] = this.activitySymbols[parts[0]];
-      }
       return parts.join('/');
     }
     return key;
@@ -203,89 +59,19 @@ class ExporterService {
    * @param data - the dependency data from Redis
    * @returns - the organized dependency data
    */
-  inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
-    const hookReg = /([0-9,]+)-(\d+)$/;
-    const flowReg = /-(\d+)$/;
-    return data.map((dependency, index: number): DependencyExport => {
-      const [action, topic, gid, _pd, ...jid] = dependency.split(VALSEP);
-      const jobId = jid.join(VALSEP);
-      const match = jobId.match(hookReg);
-      let prefix: string;
-      let type: 'hook' | 'flow' | 'other';
-      let dimensionKey: string = '';
-      if (match) {
-        //hook-originating dependency
-        const [_, dimension, counter] = match;
-        dimensionKey = dimension.split(',').join('/');
-        prefix = `${dimensionKey}[${counter}]`;
-        type = 'hook';
-      } else {
-        const match = jobId.match(flowReg);
-        if (match) {
-          //main workflow-originating dependency
-          const [_, counter] = match;
-          prefix = `[${counter}]`;
-          type = 'flow';
-        } else {
-          //'other' types like signal cleanup
-          prefix = '/';
-          type = 'other';
-        }
-      }
-      this.seedActions(
-        type,
-        action,
-        topic,
-        dependency,
-        prefix,
-        dimensionKey,
-        actions,
-        jobId,
-      );
+  inflateDependencyData(data: string[]): Record<string, any>[] {
+    return data.map((dependency, index: number): Record<string, any> => {
+      const [action, topic, gid, dimension, ...jid] = dependency.split(VALSEP);
+      const job_id = jid.join(VALSEP);
       return {
-        type: action,
+        index,
+        action,
         topic,
         gid,
-        jid: jobId,
-      } as unknown as DependencyExport;
-    });
-  }
-
-  /**
-   * Adds historical actions (proxyActivity, executeChild)
-   * using the `dependency list` to determine
-   * after-the-fact what happened within the 'black-box'
-   * worker function. This is necessary to interleave the
-   * actions into the replay timeline, given that it isn't
-   * really possible to know the inner-workings of the user's
-   * function
-   *  
-   */
-  seedActions(type: 'flow'|'hook'|'other', action: string, topic: string, dep: string, prefix: string, dimensionKey: string, actions: JobActionExport, jobId: string) {
-    if (type !== 'other' && action === 'expire-child') {
-      let depType: string;
-      if (topic == `${this.appId}.activity.execute`) {
-        depType = 'proxyActivity';
-      } else if (topic == `${this.appId}.execute`) {
-        depType = 'executeChild';
-      } else if (topic == `${this.appId}.wfsc.execute`) {
-        depType = 'waitForSignal';
-      }
-      
-      if (depType) {
-        if (type === 'flow') {
-          actions.main.items.push([prefix, depType, jobId]);
-        } else if (type === 'hook') {
-          if (!actions.hooks[dimensionKey]) {
-            actions.hooks[dimensionKey] = {
-              cursor: -1,
-              items: [],
-            };
-          }
-          actions.hooks[dimensionKey].items.push([prefix, depType, jobId]);
-        }
+        dimension,
+        job_id,
       }
-    }
+    });
   }
 
   /**
@@ -294,17 +80,12 @@ class ExporterService {
    * @param dependencyList - the list of dependencies for the job
    * @returns - the inflated job data
    */
-  inflate(jobHash: StringStringType, dependencyList: string[]): DurableJobExport {
-    //the list of actions taken in the workflow and hook functions
-    const actions: JobActionExport = {
-      hooks: {},
-      main: { cursor: -1, items: [] },
-    };
-    const dependencies = this.inflateDependencyData(dependencyList, actions);
+  inflate(jobHash: StringStringType): DurableJobExport {
+    const idempotents: IdemType[] = [];
     const state: StringStringType = {};
     const data: StringStringType = {};
     const other: ExportItem[] = [];
-    const replay: ExportItem[] = [];
+    const replay: Record<string, TimelineEntry> = {};
     const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
 
     Object.entries(jobHash).forEach(([key, value]) => {
@@ -314,104 +95,115 @@ class ExporterService {
         this.inflateProcess(match, value, replay);
       } else if (key.length === 3) {
         //job state
-        state[this.inflateKey(key)] = this.serializer.fromString(value);
+        state[this.inflateKey(key)] = SerializerService.fromString(value);
       } else if (key.startsWith('_')) {
         //job data
         data[key.substring(1)] = value;
       } else if (key.startsWith('-')) {
         //actions with side effect (replayable)
-        this.inflateActions(key, value, actions);
+        idempotents.push({
+          key,
+          value: SerializerService.fromString(value),
+          parts: extractParts(key),
+        });
       } else {
         //collator guids, etc
         other.push([null, key, value]);
       }
     });
 
-    replay.sort(this.dateSort)
-    actions.main.items.sort(this.reverseSort);
-    Object.entries(actions.hooks).forEach(([key, value]) => {
-      value.items.sort(this.reverseSort);
-    });
+    const sortEntriesByCreated = (obj: { [key: string]: TimelineEntry }): TimelineEntry[]  => {
+      const entriesArray: TimelineEntry[] = Object.values(obj);
+      entriesArray.sort((a, b) => {
+        return (a.created || a.updated).localeCompare(b.created || b.updated);
+      });
+      return entriesArray;
+    }
+
+    /**
+     * idem list has a complicated sort order based on indexes and dimensions
+     */
+    const sortParts = (parts: IdemType[]): IdemType[]=> {
+      return parts.sort((a, b) => {
+        const { dimension: aDim, index: aIdx, secondary: aSec } = a.parts;
+        const { dimension: bDim, index: bIdx, secondary: bSec } = b.parts;
+    
+        if (aDim === undefined && bDim !== undefined) return -1;
+        if (aDim !== undefined && bDim === undefined) return 1;
+        if (aDim !== undefined && bDim !== undefined) {
+          if (aDim < bDim) return -1;
+          if (aDim > bDim) return 1;
+        }
+    
+        if (aIdx < bIdx) return -1;
+        if (aIdx > bIdx) return 1;
+    
+        if (aSec === undefined && bSec !== undefined) return -1;
+        if (aSec !== undefined && bSec === undefined) return 1;
+        if (aSec !== undefined && bSec !== undefined) {
+          if (aSec < bSec) return -1;
+          if (aSec > bSec) return 1;
+        }
+    
+        return 0;
+      });
+    };
     
+    function extractParts(key: string): {index: number, dimension?: string, secondary?: number} {
+      function extractDimension(label: string): string {
+        const parts = label.split(',');
+        if (parts.length > 1) {
+          parts.shift();
+          return parts.join(',');
+        }
+      }
+
+      const parts = key.split('-');
+      if (parts.length === 4) {
+        //-proxy-5-     -search-1-1-
+        return {
+          index: parseInt(parts[2], 10),
+          dimension: extractDimension(parts[1]),
+        }
+      } else {
+        //-search,0,0-1-1-    -proxy,0,0-1-
+        return {
+          index: parseInt(parts[2], 10),
+          secondary: parseInt(parts[3], 10),
+          dimension: extractDimension(parts[1]),
+        }
+      }
+    }
+
     return {
       data: restoreHierarchy(data),
-      dependencies,
+      idempotents: sortParts(idempotents),
       state: Object.entries(restoreHierarchy(state))[0][1],
       status: jobHash[':'],
-      timeline: this.createTimeline(replay, actions),
-      transitions: { ...this.transitions },
-      cycles: { ...this.cycles },
+      replay: sortEntriesByCreated(replay),
     };
   }
 
-  inflateProcess(match: RegExpMatchArray, value: string, replay: ExportItem[]) {
-    const [_, letters, numbers] = match;
+  inflateProcess(match: RegExpMatchArray, value: string, replay: Record<string, Record<string, any>>) {
+    const [_, letters, dimensions] = match;
     const path = this.inflateKey(letters);
-    if (path.endsWith('/output/metadata/ac') ||
-        path.endsWith('/output/metadata/au')) {
-      const dimensions = `/${numbers.replace(/,/g, '/')}`;
-      const resolved = this.serializer.fromString(value);
-      replay.push([
-        dimensions,
-        path,
-        resolved,
-      ]);
-    }
-  }
-
-  inflateActions(key: string, value: string, actions: JobActionExport) {
-    let [_, dimensionalType, counter, subcounter] = key.split('-');
-    if (subcounter) {
-      counter = `${counter}.${subcounter}`;
-    }
-    const [type, ...dimensions] = dimensionalType.split(',');
-    let dimensionKey = '';
-    let isHook = false;
-    if (dimensions.length > 0) {
-      dimensionKey = `/${dimensions.join('/')}`;
-      isHook = true;
-    }
-    let targetList: ExportItem[];
-    if (isHook) {
-      if (!actions.hooks[dimensionKey]) {
-        actions.hooks[dimensionKey] = {
-          cursor: -1,
-          items: [],
+    const parts = path.split('/');
+    const activity = parts[0];
+    const isCreate = path.endsWith('/output/metadata/ac');
+    const isUpdate = path.endsWith('/output/metadata/au');
+    if (isCreate || isUpdate) {
+      const targetName = `${activity},${dimensions}`;
+      let target = replay[targetName];
+      if (!target) {
+        replay[targetName] = {
+          activity,
+          dimensions,
+          created: isCreate ? value : null,
+          updated: isUpdate ? value : null,
         };
+      } else {
+        target[isCreate ? 'created' : 'updated'] = value;
       }
-      targetList = actions.hooks[dimensionKey].items;
-    } else {
-      targetList = actions.main.items;
-    }
-    targetList.push([
-      `${dimensionKey}[${counter}]`,
-      type,
-      value,
-    ]);
-  }
-
-  reverseSort(aKey: ExportItem, bKey: ExportItem) {
-    if (aKey[0] > bKey[0]) {
-      return 1;
-    } else if (aKey[0] < bKey[0]) {
-      return -1;
-    } else {
-      if (aKey[1] > bKey[1]) {
-        return 1;
-      } else if (aKey[1] < bKey[1]) {
-        return -1;
-      }
-      return 0;
-    }
-  }
-
-  dateSort(aKey: ExportItem, bKey: ExportItem) {
-    if (aKey[2] > bKey[2]) {
-      return 1;
-    } else if (aKey[2] < bKey[2]) {
-      return -1;
-    } else {
-      return 0;
     }
   }
 }

package/services/durable/handle.ts

@@ -1,4 +1,3 @@
-import { HMSH_CODE_INTERRUPT } from '../../modules/enums';
 import { ExporterService } from './exporter';
 import { HotMeshService as HotMesh } from '../hotmesh';
 import { DurableJobExport } from '../../types/exporter';
@@ -29,8 +28,8 @@ export class WorkflowHandleService {
   /**
    * Sends a signal to the workflow. This is a way to send
    * a message to a workflow that is paused due to having
-   * executed a `waitForSignal` workflow extension. Awakens
-   * the workflow if no other signals are pending.
+   * executed `Durable.workflow.waitFor`. The workflow
+   * will awaken if no other signals are pending.
    */
   async signal(signalId: string, data: Record<any, any>): Promise<void> {
     await this.hotMesh.hook(`${this.hotMesh.appId}.wfs.signal`, { id: signalId, data });
@@ -81,67 +80,81 @@ export class WorkflowHandleService {
   }
 
   /**
-   * Awaits for the workflow to complete and returns the result. If
-   * the workflow thows and error, this method will likewise throw
-   * an error.
+   * Waits for the workflow to complete and returns the result. If
+   * the workflow response includes an error, this method will rethrow
+   * the error, including the stack trace if available. 
+   * Wrap calls in a try/catch as necessary to avoid unhandled exceptions.
    */
-  async result(loadState?: boolean): Promise<any> {
-    if (loadState) {
-      const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
-      if (!state.data && state.metadata.err) {
-        throw new Error(JSON.parse(state.metadata.err));
-      }
-      if (state?.data?.done) {
-        //child flows are never 'done'; they use a hook
-        //that only closes upon parent flow completion.
-        return state.data.response;
-      }
-    }
-    let status = await this.hotMesh.getStatus(this.workflowId);
+  async result<T>(config?: {state?: boolean, throwOnError?: boolean}): Promise<T | StreamError> {
     const topic = `${this.hotMesh.appId}.executed.${this.workflowId}`;
-  
-    return new Promise((resolve, reject) => {
-      let isResolved = false;
-      //common fulfill/unsubscribe
-      const complete = async (response?: any, err?: string) => {
+    let isResolved = false;
+
+    return new Promise(async (resolve, reject) => {
+
+      /**
+       * rejects/resolves the promise based on the `throwOnError`
+       * default behavior is to throw if error
+       */
+      const safeReject = (err: StreamError) => {
+        if (config?.throwOnError === false) {
+          return resolve(err);
+        }
+        reject(err);
+      };
+
+      /**
+       * Common completion function that unsubscribes from the topic/returns
+       */
+      const complete = async (response?: T, err?: StreamError) => {
         if (isResolved) return;
         isResolved = true;
-        this.hotMesh.unsub(topic);
+
         if (err) {
-          return reject(JSON.parse(err));
+          return safeReject(err as StreamError);
         } else if (!response) {
           const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
-          if (state.metadata.err) {
-            const error = JSON.parse(state.metadata.err) as StreamError;
-            if (error.code === HMSH_CODE_INTERRUPT || !state.data) {
-              return reject({ ...error, job_id: this.workflowId });
-            }
+          if (state.data?.done && !state.data?.$error) {
+            return resolve(state.data.response as T);
+          } else if (state.data?.$error) {
+            return safeReject(state.data.$error as StreamError)
+          } else if (state.metadata.err) {
+            return safeReject(JSON.parse(state.metadata.err) as StreamError);
           }
-          response = state.data?.response;
+          response = state.data?.response as T;
         }
-        resolve(response);
+        resolve(response as T);
       };
-      //check for done
-      if (status <= 0) {
-        return complete();
+
+      //more expensive; fetches the entire job, not just the `status`
+      if (config?.state) {
+        const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
+        if (state?.data?.done && !state.data?.$error) {
+          return complete(state.data.response as T);
+        } else if (state.data?.$error) {
+          return complete(null, state.data.$error as StreamError);
+        } else if (state.metadata.err) {
+          return complete(null, JSON.parse(state.metadata.err) as StreamError);
+        }
       }
-      //subscribe to topic
-      this.hotMesh.sub(topic, async (topic: string, state: JobOutput) => {
-        if (state.metadata.err) {
+
+      //subscribe to 'done' topic
+      this.hotMesh.sub(topic, async (_topic: string, state: JobOutput) => {
+        this.hotMesh.unsub(topic);
+        if (state.data.done && !state.data?.$error) {
+          await complete(state.data?.response as T);
+        } else if (state.data?.$error) {
+          return complete(null, state.data.$error as StreamError)
+        } else if (state.metadata.err) {
           const error = JSON.parse(state.metadata.err) as StreamError;
-          if (error.code === HMSH_CODE_INTERRUPT || !state.data) {
-            return await complete(null, state.metadata.err);
-          }
+          return await complete(null, error);
         }
-        await complete(state.data?.response);
       });
-      //resolve for race condition
-      setTimeout(async () => {
-        status = await this.hotMesh.getStatus(this.workflowId);
-        if (status <= 0) {
-          await complete();
-        }
-      }, 0);
+
+      //check state in case completed during wiring
+      const status = await this.hotMesh.getStatus(this.workflowId);
+      if (status <= 0) {
+        await complete();
+      }
     });
   }
 }

package/services/durable/index.ts

@@ -1,6 +1,5 @@
 import { ClientService } from './client';
 import { ConnectionService } from './connection';
-import { MeshOSService } from './meshos';
 import { Search } from './search';
 import { WorkerService } from './worker';
 import { WorkflowService } from './workflow';
@@ -11,7 +10,6 @@ export const Durable = {
   Client: ClientService,
   Connection: ConnectionService,
   Search,
-  MeshOS: MeshOSService,
   Worker: WorkerService,
   workflow: WorkflowService,