@hotmeshio/hotmesh 0.0.51 → 0.0.53

package/services/durable/client.ts

@@ -1,4 +1,17 @@
-import { APP_ID, APP_VERSION, DEFAULT_COEFFICIENT, getWorkflowYAML } from './factory';
+import ms from 'ms';
+
+import {
+  APP_ID,
+  APP_VERSION,
+  getWorkflowYAML } from './schemas/factory';
+import {
+  HMSH_LOGLEVEL,
+  HMSH_EXPIRE_JOB_SECONDS,
+  HMSH_QUORUM_DELAY_MS,
+  HMSH_DURABLE_EXP_BACKOFF,
+  HMSH_DURABLE_MAX_ATTEMPTS,
+  HMSH_DURABLE_MAX_INTERVAL } from '../../modules/enums';
+import { sleepFor } from '../../modules/utils';
 import { WorkflowHandleService } from './handle';
 import { HotMeshService as HotMesh } from '../hotmesh';
 import {
@@ -11,8 +24,6 @@ import { JobState } from '../../types/job';
 import { KeyService, KeyType } from '../../modules/key';
 import { Search } from './search';
 import { StreamStatus } from '../../types';
-import { HMSH_LOGLEVEL, HMSH_EXPIRE_JOB_SECONDS, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
-import { sleepFor } from '../../modules/utils';
 
 export class ClientService {
 
@@ -32,7 +43,7 @@ export class ClientService {
       await this.verifyWorkflowActive(hotMeshClient, targetNS);
       if (!ClientService.topics.includes(workflowTopic)) {
         ClientService.topics.push(workflowTopic);
-        await this.createStream(hotMeshClient, workflowTopic, namespace);
+        await ClientService.createStream(hotMeshClient, workflowTopic, namespace);
       }
       return hotMeshClient;
     }
@@ -49,7 +60,7 @@ export class ClientService {
       }
     });
     ClientService.instances.set(targetNS, hotMeshClient);
-    await this.createStream(await hotMeshClient, workflowTopic, namespace);
+    await ClientService.createStream(await hotMeshClient, workflowTopic, namespace);
     await this.activateWorkflow(await hotMeshClient, targetNS);
     return hotMeshClient;
   }
@@ -60,7 +71,7 @@ export class ClientService {
    * has not yet been initialized, so this call ensures that the channel
    * exists and is ready to serve as a container for events.
    */
-  createStream = async(hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => {
+  static createStream = async(hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => {
     const store = hotMeshClient.engine.store;
     const params = { appId: namespace ?? APP_ID, topic: workflowTopic };
     const streamKey = store.mintKey(KeyType.STREAMS, params);
@@ -71,6 +82,23 @@ export class ClientService {
     }
   }
 
+  /**
+   * It is possible for a client to invoke a workflow without first
+   * creating the stream. This method will verify that the stream
+   * exists and if not, create it.
+   */
+  static verifyStream = async(workflowTopic: string, namespace?: string) => {
+    const targetNS = namespace ?? APP_ID;
+    if (ClientService.instances.has(targetNS)) {
+      const hotMeshClient = await ClientService.instances.get(targetNS);
+      if (!ClientService.topics.includes(workflowTopic)) {
+        ClientService.topics.push(workflowTopic);
+        await ClientService.createStream(hotMeshClient, workflowTopic, namespace);
+      }
+      return hotMeshClient;
+    }
+  }
+
   /**
    * For those deployments with a redis stack backend (with the FT module),
    * this method will configure the search index for the workflow.
@@ -95,8 +123,8 @@ export class ClientService {
         const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
         const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
         await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
-      } catch (err) {
-        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
+      } catch (error) {
+        hotMeshClient.engine.logger.info('durable-client-search-err', { ...error });
       }
     }
   }
@@ -127,8 +155,11 @@ export class ClientService {
         parentWorkflowId: options.parentWorkflowId,
         workflowId: options.workflowId || HotMesh.guid(),
         workflowTopic: workflowTopic,
-        backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
+        backoffCoefficient: options.config?.backoffCoefficient || HMSH_DURABLE_EXP_BACKOFF,
+        maximumAttempts: options.config?.maximumAttempts || HMSH_DURABLE_MAX_ATTEMPTS,
+        maximumInterval: ms(options.config?.maximumInterval || HMSH_DURABLE_MAX_INTERVAL) / 1000,
       }
+
       const context = { metadata: { trc, spn }, data: {}};
       const jobId = await hotMeshClient.pub(
         `${options.namespace ?? APP_ID}.execute`,
@@ -165,7 +196,9 @@ export class ClientService {
         arguments: [...options.args],
         id: options.workflowId,
         workflowTopic,
-        backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
+        backoffCoefficient: options.config?.backoffCoefficient || HMSH_DURABLE_EXP_BACKOFF,
+        maximumAttempts: options.config?.maximumAttempts || HMSH_DURABLE_MAX_ATTEMPTS,
+        maximumInterval: ms(options.config?.maximumInterval || HMSH_DURABLE_MAX_INTERVAL) / 1000,
       }
       //seed search data if presentthe hook before entering
       const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
@@ -190,9 +223,9 @@ export class ClientService {
       const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
       try {
         return await this.search(hotMeshClient, index, query);
-      } catch (err) {
-        hotMeshClient.engine.logger.error('durable-client-search-err', { err });
-        throw err;
+      } catch (error) {
+        hotMeshClient.engine.logger.error('durable-client-search-err', { ...error });
+        throw error;
       }
     }
   }
@@ -218,7 +251,7 @@ export class ClientService {
         await hotMesh.deploy(getWorkflowYAML(appId, version));
         await hotMesh.activate(version);
       } catch (error) {
-        hotMesh.engine.logger.error('durable-client-deploy-activate-err', { error });
+        hotMesh.engine.logger.error('durable-client-deploy-activate-err', { ...error });
         throw error;
       }
     } else if(app && !app.active) {

package/services/durable/exporter.ts

@@ -1,184 +1,43 @@
+import { VALSEP } from '../../modules/key';
+import { restoreHierarchy } from '../../modules/utils';
 import { ILogger } from '../logger';
+import { SerializerService } from '../serializer';
 import { StoreService } from '../store';
-import { StringStringType, Symbols } from "../../types";
-import { RedisClient, RedisMulti } from '../../types/redis';
 import {
-  ActivityAction,
-  DependencyExport,
   ExportItem,
   ExportOptions,
-  JobAction,
-  JobActionExport,
-  DurableJobExport,
-  JobTimeline } from '../../types/exporter';
-import { SerializerService } from '../serializer';
-import { restoreHierarchy } from '../../modules/utils';
-import { VALSEP } from '../../modules/key';
+  DurableJobExport, 
+  IdemType,
+  TimelineEntry } from '../../types/exporter';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { StringStringType, Symbols } from "../../types/serializer";
 
-/**
- * Downloads job data from Redis (hscan, hmget, hgetall)
- * Splits, Inflates, and Sorts the job data for use in durable contexts
- */
 class ExporterService {
   appId: string;
   logger: ILogger;
-  serializer: SerializerService
   store: StoreService<RedisClient, RedisMulti>;
   symbols: Promise<Symbols> | Symbols;
 
-  /**
-   * Friendly names for the activity ids
-   */
-  activitySymbols: Symbols = {
-    t1: 'trigger',
-    a1: 'pivot',
-    w1: 'worker',
-    a592: 'sleeper',
-    a594: 'awaiter',
-    a599: 'retryer',
-    c592: 'sleep_cycler',
-    c594: 'await_cycler',
-    c599: 'retry_cycler',
-    s5: 'scrubber',
-    sig: 'hook',
-    siga1: 'hook_pivot',
-    sigw1: 'hook_worker',
-    siga592: 'hook_sleeper',
-    siga594: 'hook_awaiter',
-    siga599: 'hook_retryer',
-    sigc592: 'hook_sleep_cycler',
-    sigc594: 'hook_await_cycler',
-    sigc599: 'hook_retry_cycler',
-  }
-
-  //adjacent transitions
-  transitions = {
-    trigger: ['pivot', 'hook'],
-    pivot: ['worker'],
-    worker: ['sleeper', 'awaiter', 'retryer', 'scrubber'],
-    sleeper: ['sleep_cycler'],
-    awaiter: ['await_cycler'],
-    retryer: ['retry_cycler'],
-    hook: ['hook_pivot'],
-    hook_pivot: ['hook_worker'],
-    hook_worker: ['hook_sleeper', 'hook_awaiter', 'hook_retryer'],
-    hook_sleeper: ['hook_sleep_cycler'],
-    hook_awaiter: ['hook_await_cycler'],
-    hook_retryer: ['hook_retry_cycler'],
-  };
-
-  //goto transitions
-  cycles = {
-    sleep_cycler: ['pivot'],
-    await_cycler: ['pivot'],
-    retry_cycler: ['pivot'],
-    hook_sleep_cycler: ['hook_pivot'],
-    hook_await_cycler: ['hook_pivot'],
-    hook_retry_cycler: ['hook_pivot'],
-  }
-
   constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
     this.appId = appId;
     this.logger = logger;
     this.store = store;
-    this.serializer = new SerializerService();
   }
 
   /**
-   * Convert the job hash and dependency list into a DurableJobExport object.
-   * This object contains various facets that describe the interaction
-   * in terms relevant to narrative storytelling.
+   * Convert the job hash from its compiles format into a DurableJobExport object with
+   * facets that describe the workflow in terms relevant to narrative storytelling.
    */
   async export(jobId: string, options: ExportOptions = {}): Promise<DurableJobExport> {
     if (!this.symbols) {
       this.symbols = this.store.getAllSymbols();
       this.symbols = await this.symbols;
     }
-    const depData = await this.store.getDependencies(jobId);
     const jobData = await this.store.getRaw(jobId);
-    const jobExport = this.inflate(jobData, depData);
+    const jobExport = this.inflate(jobData/*, depData*/);
     return jobExport;
   }
 
-  /**
-   * Interleave actions into the replay timeline to create
-   * a time-ordered timeline of the entire interaction, beginning
-   * with the entry trigger and concluding with the scrubber
-   * activity. Using the returned timeline, it is possible to
-   * create an animated narrative of the job, highlighting
-   * activities in the graph according to the timeline's 
-   * activity-created (/ac) and activity-updated (/au) entries.
-   */
-  createTimeline(replay: ExportItem[], actions: JobActionExport): JobTimeline[] {
-    const timeline: JobTimeline[] = [];
-    replay.forEach((item) => {
-      const dimensions = item[0];
-      const parts = dimensions.split('/');
-      const activityName = item[1].split('/')[0];
-      const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
-      const timestamp = item[2];
-      let event: JobTimeline = {
-        activity: activityName,
-        duplex: duplex as 'entry' | 'exit',
-        dimension: dimensions,
-        timestamp,
-        created: timestamp,
-        updated: timestamp,
-      };
-      const prior = timeline[timeline.length - 1];
-      if (prior && prior.activity === event.activity && prior.duplex !== event.duplex && prior.dimension === event.dimension) {
-        if (event.duplex === 'exit') {
-          prior.updated = event.timestamp;
-        } else {
-          prior.created = event.timestamp;
-        }
-        event = prior;
-      } else {
-        timeline.push(event);
-      }
-
-      if (this.isMainEntry(item[1])) {
-        event.actions = [] as ActivityAction[];
-        this.interleaveActions(actions.main, event.actions);
-      } else if (this.isHookEntry(item[1])) {
-        const hookDimension = `/${parts[1]}/${parts[2]}`;
-        const hookActions = actions.hooks[hookDimension];
-        event.actions = [] as ActivityAction[];
-        this.interleaveActions(hookActions, event.actions);
-      }
-    });
-    return timeline;
-  }
-
-  /**
-   * Interleave actions into the 'worker' and 'hook_worker'
-   * activities (between their /ac and /au entries)
-   */
-  interleaveActions(target: JobAction, actions: ActivityAction[]) {
-    if (target) {
-      for (let i = target.cursor + 1; i < target.items.length; i++) {
-        const [_, actionType, jobOrIndex] = target.items[i];
-        actions.push({ action: actionType, target: jobOrIndex });
-        target.cursor = i;
-        if (this.isPausingAction(actionType)) {
-          break;
-        }
-      }
-    }
-  }
-
-  isPausingAction(actionType: string): boolean {
-    return actionType === 'sleep' || actionType === 'waitForSignal';
-  }
-
-  isMainEntry(key: string): boolean {
-    return key.startsWith('worker/') && key.endsWith('/ac');
-  }
-
-  isHookEntry(key: string): boolean {
-    return key.startsWith('hook_worker/') && key.endsWith('/ac');
-  }
-
   /**
    * Inflates the key from Redis, 3-character symbol
    * into a human-readable JSON path, reflecting the
@@ -188,9 +47,6 @@ class ExporterService {
     if (key in this.symbols) {
       const path = this.symbols[key];
       const parts = path.split('/');
-      if (parts[0] in this.activitySymbols) {
-        parts[0] = this.activitySymbols[parts[0]];
-      }
       return parts.join('/');
     }
     return key;
@@ -203,89 +59,19 @@ class ExporterService {
    * @param data - the dependency data from Redis
    * @returns - the organized dependency data
    */
-  inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
-    const hookReg = /([0-9,]+)-(\d+)$/;
-    const flowReg = /-(\d+)$/;
-    return data.map((dependency, index: number): DependencyExport => {
-      const [action, topic, gid, _pd, ...jid] = dependency.split(VALSEP);
-      const jobId = jid.join(VALSEP);
-      const match = jobId.match(hookReg);
-      let prefix: string;
-      let type: 'hook' | 'flow' | 'other';
-      let dimensionKey: string = '';
-      if (match) {
-        //hook-originating dependency
-        const [_, dimension, counter] = match;
-        dimensionKey = dimension.split(',').join('/');
-        prefix = `${dimensionKey}[${counter}]`;
-        type = 'hook';
-      } else {
-        const match = jobId.match(flowReg);
-        if (match) {
-          //main workflow-originating dependency
-          const [_, counter] = match;
-          prefix = `[${counter}]`;
-          type = 'flow';
-        } else {
-          //'other' types like signal cleanup
-          prefix = '/';
-          type = 'other';
-        }
-      }
-      this.seedActions(
-        type,
-        action,
-        topic,
-        dependency,
-        prefix,
-        dimensionKey,
-        actions,
-        jobId,
-      );
+  inflateDependencyData(data: string[]): Record<string, any>[] {
+    return data.map((dependency, index: number): Record<string, any> => {
+      const [action, topic, gid, dimension, ...jid] = dependency.split(VALSEP);
+      const job_id = jid.join(VALSEP);
       return {
-        type: action,
+        index,
+        action,
         topic,
         gid,
-        jid: jobId,
-      } as unknown as DependencyExport;
-    });
-  }
-
-  /**
-   * Adds historical actions (proxyActivity, executeChild)
-   * using the `dependency list` to determine
-   * after-the-fact what happened within the 'black-box'
-   * worker function. This is necessary to interleave the
-   * actions into the replay timeline, given that it isn't
-   * really possible to know the inner-workings of the user's
-   * function
-   *  
-   */
-  seedActions(type: 'flow'|'hook'|'other', action: string, topic: string, dep: string, prefix: string, dimensionKey: string, actions: JobActionExport, jobId: string) {
-    if (type !== 'other' && action === 'expire-child') {
-      let depType: string;
-      if (topic == `${this.appId}.activity.execute`) {
-        depType = 'proxyActivity';
-      } else if (topic == `${this.appId}.execute`) {
-        depType = 'executeChild';
-      } else if (topic == `${this.appId}.wfsc.execute`) {
-        depType = 'waitForSignal';
-      }
-      
-      if (depType) {
-        if (type === 'flow') {
-          actions.main.items.push([prefix, depType, jobId]);
-        } else if (type === 'hook') {
-          if (!actions.hooks[dimensionKey]) {
-            actions.hooks[dimensionKey] = {
-              cursor: -1,
-              items: [],
-            };
-          }
-          actions.hooks[dimensionKey].items.push([prefix, depType, jobId]);
-        }
+        dimension,
+        job_id,
       }
-    }
+    });
   }
 
   /**
@@ -294,17 +80,12 @@ class ExporterService {
    * @param dependencyList - the list of dependencies for the job
    * @returns - the inflated job data
    */
-  inflate(jobHash: StringStringType, dependencyList: string[]): DurableJobExport {
-    //the list of actions taken in the workflow and hook functions
-    const actions: JobActionExport = {
-      hooks: {},
-      main: { cursor: -1, items: [] },
-    };
-    const dependencies = this.inflateDependencyData(dependencyList, actions);
+  inflate(jobHash: StringStringType): DurableJobExport {
+    const idempotents: IdemType[] = [];
     const state: StringStringType = {};
     const data: StringStringType = {};
     const other: ExportItem[] = [];
-    const replay: ExportItem[] = [];
+    const replay: Record<string, TimelineEntry> = {};
     const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
 
     Object.entries(jobHash).forEach(([key, value]) => {
@@ -314,104 +95,115 @@ class ExporterService {
         this.inflateProcess(match, value, replay);
       } else if (key.length === 3) {
         //job state
-        state[this.inflateKey(key)] = this.serializer.fromString(value);
+        state[this.inflateKey(key)] = SerializerService.fromString(value);
       } else if (key.startsWith('_')) {
         //job data
         data[key.substring(1)] = value;
       } else if (key.startsWith('-')) {
         //actions with side effect (replayable)
-        this.inflateActions(key, value, actions);
+        idempotents.push({
+          key,
+          value: SerializerService.fromString(value),
+          parts: extractParts(key),
+        });
       } else {
         //collator guids, etc
         other.push([null, key, value]);
       }
     });
 
-    replay.sort(this.dateSort)
-    actions.main.items.sort(this.reverseSort);
-    Object.entries(actions.hooks).forEach(([key, value]) => {
-      value.items.sort(this.reverseSort);
-    });
+    const sortEntriesByCreated = (obj: { [key: string]: TimelineEntry }): TimelineEntry[]  => {
+      const entriesArray: TimelineEntry[] = Object.values(obj);
+      entriesArray.sort((a, b) => {
+        return (a.created || a.updated).localeCompare(b.created || b.updated);
+      });
+      return entriesArray;
+    }
+
+    /**
+     * idem list has a complicated sort order based on indexes and dimensions
+     */
+    const sortParts = (parts: IdemType[]): IdemType[]=> {
+      return parts.sort((a, b) => {
+        const { dimension: aDim, index: aIdx, secondary: aSec } = a.parts;
+        const { dimension: bDim, index: bIdx, secondary: bSec } = b.parts;
+    
+        if (aDim === undefined && bDim !== undefined) return -1;
+        if (aDim !== undefined && bDim === undefined) return 1;
+        if (aDim !== undefined && bDim !== undefined) {
+          if (aDim < bDim) return -1;
+          if (aDim > bDim) return 1;
+        }
+    
+        if (aIdx < bIdx) return -1;
+        if (aIdx > bIdx) return 1;
+    
+        if (aSec === undefined && bSec !== undefined) return -1;
+        if (aSec !== undefined && bSec === undefined) return 1;
+        if (aSec !== undefined && bSec !== undefined) {
+          if (aSec < bSec) return -1;
+          if (aSec > bSec) return 1;
+        }
+    
+        return 0;
+      });
+    };
     
+    function extractParts(key: string): {index: number, dimension?: string, secondary?: number} {
+      function extractDimension(label: string): string {
+        const parts = label.split(',');
+        if (parts.length > 1) {
+          parts.shift();
+          return parts.join(',');
+        }
+      }
+
+      const parts = key.split('-');
+      if (parts.length === 4) {
+        //-proxy-5-     -search-1-1-
+        return {
+          index: parseInt(parts[2], 10),
+          dimension: extractDimension(parts[1]),
+        }
+      } else {
+        //-search,0,0-1-1-    -proxy,0,0-1-
+        return {
+          index: parseInt(parts[2], 10),
+          secondary: parseInt(parts[3], 10),
+          dimension: extractDimension(parts[1]),
+        }
+      }
+    }
+
     return {
       data: restoreHierarchy(data),
-      dependencies,
+      idempotents: sortParts(idempotents),
       state: Object.entries(restoreHierarchy(state))[0][1],
       status: jobHash[':'],
-      timeline: this.createTimeline(replay, actions),
-      transitions: { ...this.transitions },
-      cycles: { ...this.cycles },
+      replay: sortEntriesByCreated(replay),
     };
   }
 
-  inflateProcess(match: RegExpMatchArray, value: string, replay: ExportItem[]) {
-    const [_, letters, numbers] = match;
+  inflateProcess(match: RegExpMatchArray, value: string, replay: Record<string, Record<string, any>>) {
+    const [_, letters, dimensions] = match;
     const path = this.inflateKey(letters);
-    if (path.endsWith('/output/metadata/ac') ||
-        path.endsWith('/output/metadata/au')) {
-      const dimensions = `/${numbers.replace(/,/g, '/')}`;
-      const resolved = this.serializer.fromString(value);
-      replay.push([
-        dimensions,
-        path,
-        resolved,
-      ]);
-    }
-  }
-
-  inflateActions(key: string, value: string, actions: JobActionExport) {
-    let [_, dimensionalType, counter, subcounter] = key.split('-');
-    if (subcounter) {
-      counter = `${counter}.${subcounter}`;
-    }
-    const [type, ...dimensions] = dimensionalType.split(',');
-    let dimensionKey = '';
-    let isHook = false;
-    if (dimensions.length > 0) {
-      dimensionKey = `/${dimensions.join('/')}`;
-      isHook = true;
-    }
-    let targetList: ExportItem[];
-    if (isHook) {
-      if (!actions.hooks[dimensionKey]) {
-        actions.hooks[dimensionKey] = {
-          cursor: -1,
-          items: [],
+    const parts = path.split('/');
+    const activity = parts[0];
+    const isCreate = path.endsWith('/output/metadata/ac');
+    const isUpdate = path.endsWith('/output/metadata/au');
+    if (isCreate || isUpdate) {
+      const targetName = `${activity},${dimensions}`;
+      let target = replay[targetName];
+      if (!target) {
+        replay[targetName] = {
+          activity,
+          dimensions,
+          created: isCreate ? value : null,
+          updated: isUpdate ? value : null,
         };
+      } else {
+        target[isCreate ? 'created' : 'updated'] = value;
       }
-      targetList = actions.hooks[dimensionKey].items;
-    } else {
-      targetList = actions.main.items;
-    }
-    targetList.push([
-      `${dimensionKey}[${counter}]`,
-      type,
-      value,
-    ]);
-  }
-
-  reverseSort(aKey: ExportItem, bKey: ExportItem) {
-    if (aKey[0] > bKey[0]) {
-      return 1;
-    } else if (aKey[0] < bKey[0]) {
-      return -1;
-    } else {
-      if (aKey[1] > bKey[1]) {
-        return 1;
-      } else if (aKey[1] < bKey[1]) {
-        return -1;
-      }
-      return 0;
-    }
-  }
-
-  dateSort(aKey: ExportItem, bKey: ExportItem) {
-    if (aKey[2] > bKey[2]) {
-      return 1;
-    } else if (aKey[2] < bKey[2]) {
-      return -1;
-    } else {
-      return 0;
     }
   }
 }