@hotmeshio/hotmesh 0.0.43 → 0.0.45

package/build/types/exporter.d.ts

@@ -0,0 +1,51 @@
+import { StringAnyType } from "./serializer";
+export type ExportItem = [(string | null), string, any];
+export interface ExportOptions {
+}
+export type JobAction = {
+    cursor: number;
+    items: ExportItem[];
+};
+export interface JobActionExport {
+    hooks: {
+        [key: string]: JobAction;
+    };
+    main: JobAction;
+}
+export interface ActivityAction {
+    action: string;
+    target: string;
+}
+export interface JobTimeline {
+    activity: string;
+    dimension: string;
+    duplex: 'entry' | 'exit';
+    timestamp: string;
+    actions?: ActivityAction[];
+}
+export interface DependencyExport {
+    type: string;
+    topic: string;
+    gid: string;
+    jid: string;
+}
+export interface ExportTransitions {
+    [key: string]: string[];
+}
+export interface ExportCycles {
+    [key: string]: string[];
+}
+export interface DurableJobExport {
+    data: StringAnyType;
+    dependencies: DependencyExport[];
+    state: StringAnyType;
+    status: string;
+    timeline: JobTimeline[];
+    transitions: ExportTransitions;
+    cycles: ExportCycles;
+}
+export interface JobExport {
+    dependencies: DependencyExport[];
+    process: StringAnyType;
+    status: string;
+}

package/build/types/exporter.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+;
+;
+;
+;
+;
+;

package/build/types/index.d.ts

@@ -4,6 +4,7 @@ export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
 export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, FindOptions, FindWhereOptions, FindWhereQuery, HookOptions, MeshOSActivityOptions, MeshOSWorkerOptions, MeshOSClassConfig, MeshOSConfig, MeshOSOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowContext, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
+export { ActivityAction, DependencyExport, DurableJobExport, ExportCycles, ExportItem, ExportOptions, ExportTransitions, JobAction, JobExport, JobActionExport, JobTimeline } from './exporter';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/build/types/quorum.d.ts

@@ -6,6 +6,8 @@ export interface QuorumProfile {
     worker_topic?: string;
     stream?: string;
     stream_depth?: number;
+    counts?: Record<string, number>;
+    timestamp?: string;
 }
 export interface PingMessage {
     type: 'ping';

package/build/types/task.d.ts

@@ -1 +1 @@
-export type WorkListTaskType = 'sleep' | 'expire' | 'interrupt' | 'delist';
+export type WorkListTaskType = 'sleep' | 'expire' | 'expire-child' | 'interrupt' | 'delist' | 'child';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.43",
+  "version": "0.0.45",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/services/activities/trigger.ts

@@ -186,11 +186,15 @@ class Trigger extends Activity {
   async registerJobDependency(multi?: RedisMulti): Promise<void> {
     const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
     let resolvedDepKey = depKey ? Pipe.resolve(depKey, this.context) : '';
+    const adjKey = this.config.stats?.adjacent;
+    let resolvedAdjKey = depKey ? Pipe.resolve(adjKey, this.context) : '';
     if (!resolvedDepKey) {
       resolvedDepKey = this.context.metadata.pj;
     }
     if (resolvedDepKey) {
+      const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
       await this.store.registerJobDependency(
+        isParentOrigin ? 'expire-child' : 'expire',
         resolvedDepKey,
         this.context.metadata.tpc,
         this.context.metadata.jid,
@@ -198,6 +202,16 @@ class Trigger extends Activity {
         multi,
       );
     }
+    if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
+      await this.store.registerJobDependency(
+        'child',
+        resolvedAdjKey,
+        this.context.metadata.tpc,
+        this.context.metadata.jid,
+        this.context.metadata.gid,
+        multi,
+      );
+    }
   }
 
   async setStats(multi?: RedisMulti): Promise<void> {

package/services/durable/client.ts

@@ -26,11 +26,10 @@ export class ClientService {
   }
 
   getHotMeshClient = async (workflowTopic: string, namespace?: string) => {
-    //use the cached instance
-    const instanceId = 'SINGLETON';
-    if (ClientService.instances.has(instanceId)) {
-      const hotMeshClient = await ClientService.instances.get(instanceId);
-      await this.verifyWorkflowActive(hotMeshClient, namespace ?? APP_ID);
+    const targetNS = namespace ?? APP_ID;
+    if (ClientService.instances.has(targetNS)) {
+      const hotMeshClient = await ClientService.instances.get(targetNS);
+      await this.verifyWorkflowActive(hotMeshClient, targetNS);
       if (!ClientService.topics.includes(workflowTopic)) {
         ClientService.topics.push(workflowTopic);
         await this.createStream(hotMeshClient, workflowTopic, namespace);
@@ -40,7 +39,7 @@ export class ClientService {
 
     //create and cache an instance
     const hotMeshClient = HotMesh.init({
-      appId: namespace ?? APP_ID,
+      appId: targetNS,
       logLevel: HMSH_LOGLEVEL,
       engine: {
         redis: {
@@ -49,9 +48,9 @@ export class ClientService {
         }
       }
     });
-    ClientService.instances.set(instanceId, hotMeshClient);
+    ClientService.instances.set(targetNS, hotMeshClient);
     await this.createStream(await hotMeshClient, workflowTopic, namespace);
-    await this.activateWorkflow(await hotMeshClient, namespace ?? APP_ID);
+    await this.activateWorkflow(await hotMeshClient, targetNS);
     return hotMeshClient;
   }
 

package/services/durable/exporter.ts

@@ -0,0 +1,408 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { StringStringType, Symbols } from "../../types";
+import { RedisClient, RedisMulti } from '../../types/redis';
+import {
+  ActivityAction,
+  DependencyExport,
+  ExportItem,
+  ExportOptions,
+  JobAction,
+  JobActionExport,
+  DurableJobExport,
+  JobTimeline } from '../../types/exporter';
+import { SerializerService } from '../serializer';
+import { restoreHierarchy } from '../../modules/utils';
+
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Splits, Inflates, and Sorts the job data for use in durable contexts
+ */
+class ExporterService {
+  appId: string;
+  logger: ILogger;
+  serializer: SerializerService
+  store: StoreService<RedisClient, RedisMulti>;
+  symbols: Promise<Symbols> | Symbols;
+
+  /**
+   * Friendly names for the activity ids
+   */
+  activitySymbols: Symbols = {
+    t1: 'trigger',
+    a1: 'pivot',
+    w1: 'worker',
+    a592: 'sleeper',
+    a594: 'awaiter',
+    a599: 'retryer',
+    c592: 'sleep_cycler',
+    c594: 'await_cycler',
+    c599: 'retry_cycler',
+    s5: 'scrubber',
+    sig: 'hook',
+    siga1: 'hook_pivot',
+    sigw1: 'hook_worker',
+    siga592: 'hook_sleeper',
+    siga594: 'hook_awaiter',
+    siga599: 'hook_retryer',
+    sigc592: 'hook_sleep_cycler',
+    sigc594: 'hook_await_cycler',
+    sigc599: 'hook_retry_cycler',
+  }
+
+  //adjacent transitions
+  transitions = {
+    trigger: ['pivot', 'hook'],
+    pivot: ['worker'],
+    worker: ['sleeper', 'awaiter', 'retryer', 'scrubber'],
+    sleeper: ['sleep_cycler'],
+    awaiter: ['await_cycler'],
+    retryer: ['retry_cycler'],
+    hook: ['hook_pivot'],
+    hook_pivot: ['hook_worker'],
+    hook_worker: ['hook_sleeper', 'hook_awaiter', 'hook_retryer'],
+    hook_sleeper: ['hook_sleep_cycler'],
+    hook_awaiter: ['hook_await_cycler'],
+    hook_retryer: ['hook_retry_cycler'],
+  };
+
+  //goto transitions
+  cycles = {
+    sleep_cycler: ['pivot'],
+    await_cycler: ['pivot'],
+    retry_cycler: ['pivot'],
+    hook_sleep_cycler: ['hook_pivot'],
+    hook_await_cycler: ['hook_pivot'],
+    hook_retry_cycler: ['hook_pivot'],
+  }
+
+  constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
+    this.appId = appId;
+    this.logger = logger;
+    this.store = store;
+    this.serializer = new SerializerService();
+  }
+
+  /**
+   * Convert the job hash and dependency list into a DurableJobExport object.
+   * This object contains various facets that describe the interaction
+   * in terms relevant to narrative storytelling.
+   */
+  async export(jobId: string, options: ExportOptions = {}): Promise<DurableJobExport> {
+    if (!this.symbols) {
+      this.symbols = this.store.getAllSymbols();
+      this.symbols = await this.symbols;
+    }
+    const depData = await this.store.getDependencies(jobId);
+    const jobData = await this.store.getRaw(jobId);
+    const jobExport = this.inflate(jobData, depData);
+    return jobExport;
+  }
+
+  /**
+   * Interleave actions into the replay timeline to create
+   * a time-ordered timeline of the entire interaction, beginning
+   * with the entry trigger and concluding with the scrubber
+   * activity. Using the returned timeline, it is possible to
+   * create an animated narrative of the job, highlighting
+   * activities in the graph according to the timeline's 
+   * activity-created (/ac) and activity-updated (/au) entries.
+   */
+  createTimeline(replay: ExportItem[], actions: JobActionExport): JobTimeline[] {
+    const timeline: JobTimeline[] = [];
+    replay.forEach((item) => {
+      const dimensions = item[0];
+      const parts = dimensions.split('/');
+      const activityName = item[1].split('/')[0];
+      const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
+      const timestamp = item[2];
+      const event: JobTimeline = {
+        activity: activityName,
+        duplex: duplex as 'entry' | 'exit',
+        dimension: dimensions,
+        timestamp,
+      };
+      timeline.push(event);
+
+      if (this.isMainEntry(item[1])) {
+        event.actions = [] as ActivityAction[];
+        this.interleaveActions(actions.main, event.actions);
+      } else if (this.isHookEntry(item[1])) {
+        const hookDimension =  `/${parts[1]}/${parts[2]}`;
+        const hookActions = actions.hooks[hookDimension];
+        event.actions = [] as ActivityAction[];
+        this.interleaveActions(hookActions, event.actions);
+      }
+    });
+    return timeline;
+  }
+
+  /**
+   * Interleave actions into the 'worker' and 'hook_worker'
+   * activities (between their /ac and /au entries)
+   */
+  interleaveActions(target: JobAction, actions: ActivityAction[]) {
+    if (target) {
+      for (let i = target.cursor + 1; i < target.items.length; i++) {
+        const [_, actionType, jobOrIndex] = target.items[i];
+        actions.push({ action: actionType, target: jobOrIndex });
+        target.cursor = i;
+        if (this.isPausingAction(actionType)) {
+          break;
+        }
+      }
+    }
+  }
+
+  isPausingAction(actionType: string): boolean {
+    return actionType === 'sleep' || actionType === 'waitForSignal';
+  }
+
+  isMainEntry(key: string): boolean {
+    return key.startsWith('worker/') && key.endsWith('/ac');
+  }
+
+  isHookEntry(key: string): boolean {
+    return key.startsWith('hook_worker/') && key.endsWith('/ac');
+  }
+
+  /**
+   * Inflates the key from Redis, 3-character symbol
+   * into a human-readable JSON path, reflecting the
+   * tree-like structure of the unidimensional Hash
+   */
+  inflateKey(key: string): string {
+    if (key in this.symbols) {
+      const path = this.symbols[key];
+      const parts = path.split('/');
+      if (parts[0] in this.activitySymbols) {
+        parts[0] = this.activitySymbols[parts[0]];
+      }
+      return parts.join('/');
+    }
+    return key;
+  }
+
+  /**
+   * Inflates the dependency data from Redis into a DurableJobExport object by
+   * organizing the dimensional isolate in sch a way asto interleave
+   * into a story
+   * @param data - the dependency data from Redis
+   * @returns - the organized dependency data
+   */
+  inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
+    //console.log('dependency data>', data);
+    const hookReg = /([0-9,]+)-(\d+)$/;
+    const flowReg = /-(\d+)$/;
+    return data.map((dependency, index: number): DependencyExport => {
+      const [action, topic, gid, ...jid] = dependency.split('::');
+      const jobId = jid.join('::');
+      const match = jobId.match(hookReg);
+      let prefix: string;
+      let type: 'hook' | 'flow' | 'other';
+      let dimensionKey: string = '';
+
+      if (match) {
+        //hook-originating dependency
+        const [_, dimension, counter] = match;
+        dimensionKey = dimension.split(',').join('/');
+        prefix = `${dimensionKey}[${counter}]`;
+        type = 'hook';
+      } else {
+        const match = jobId.match(flowReg);
+        if (match) {
+          //main workflow-originating dependency
+          const [_, counter] = match;
+          prefix = `[${counter}]`;
+          type = 'flow';
+        } else {
+          //'other' types like signal cleanup
+          prefix = '/';
+          type = 'other';
+        }
+      }
+      this.seedActions(
+        type,
+        action,
+        topic,
+        dependency,
+        prefix,
+        dimensionKey,
+        actions,
+        jobId,
+      );
+      return {
+        type: action,
+        topic,
+        gid,
+        jid: jobId,
+      } as unknown as DependencyExport;
+    });
+  }
+
+  /**
+   * Adds historical actions (proxyActivity, executeChild)
+   * using the `dependency list` to determine
+   * after-the-fact what happened within the 'black-box'
+   * worker function. This is necessary to interleave the
+   * actions into the replay timeline, given that it isn't
+   * really possible to know the inner-workings of the user's
+   * function
+   *  
+   */
+  seedActions(type: 'flow'|'hook'|'other', action: string, topic: string, dep: string, prefix: string, dimensionKey: string, actions: JobActionExport, jobId: string) {
+    if (type !== 'other' && action === 'expire-child') {
+      let depType: string;
+      if (topic == `${this.appId}.activity.execute`) {
+        depType = 'proxyActivity';
+      } else if (topic == `${this.appId}.execute`) {
+        depType = 'executeChild';
+      } else if (topic == `${this.appId}.wfsc.execute`) {
+        depType = 'waitForSignal';
+      }
+      
+      if (depType) {
+        if (type === 'flow') {
+          actions.main.items.push([prefix, depType, jobId]);
+        } else if (type === 'hook') {
+          if (!actions.hooks[dimensionKey]) {
+            actions.hooks[dimensionKey] = {
+              cursor: -1,
+              items: [],
+            };
+          }
+          actions.hooks[dimensionKey].items.push([prefix, depType, jobId]);
+        }
+      }
+    }
+  }
+
+  /**
+   * Inflates the job data from Redis into a DurableJobExport object
+   * @param jobHash - the job data from Redis
+   * @param dependencyList - the list of dependencies for the job
+   * @returns - the inflated job data
+   */
+  inflate(jobHash: StringStringType, dependencyList: string[]): DurableJobExport {
+    //the list of actions taken in the workflow and hook functions
+    const actions: JobActionExport = {
+      hooks: {},
+      main: { cursor: -1, items: [] },
+    };
+    const dependencies = this.inflateDependencyData(dependencyList, actions);
+    const state: StringStringType = {};
+    const data: StringStringType = {};
+    const other: ExportItem[] = [];
+    const replay: ExportItem[] = [];
+    const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
+
+    Object.entries(jobHash).forEach(([key, value]) => {
+      const match = key.match(regex);
+      if (match) {
+        //activity process state
+        this.inflateProcess(match, value, replay);
+      } else if (key.length === 3) {
+        //job state
+        state[this.inflateKey(key)] = this.serializer.fromString(value);
+      } else if (key.startsWith('_')) {
+        //job data
+        data[key.substring(1)] = value;
+      } else if (key.startsWith('-')) {
+        //actions with side effect (replayable)
+        this.inflateActions(key, value, actions);
+      } else {
+        //collator guids, etc
+        other.push([null, key, value]);
+      }
+    });
+
+    replay.sort(this.dateSort)
+    actions.main.items.sort(this.reverseSort);
+    Object.entries(actions.hooks).forEach(([key, value]) => {
+      value.items.sort(this.reverseSort);
+    });
+    
+    return {
+      data: restoreHierarchy(data),
+      dependencies,
+      state: Object.entries(restoreHierarchy(state))[0][1],
+      status: jobHash[':'],
+      timeline: this.createTimeline(replay, actions),
+      transitions: { ...this.transitions },
+      cycles: { ...this.cycles },
+    };
+  }
+
+  inflateProcess(match: RegExpMatchArray, value: string, replay: ExportItem[]) {
+    const [_, letters, numbers] = match;
+    const path = this.inflateKey(letters);
+    if (path.endsWith('/output/metadata/ac') ||
+        path.endsWith('/output/metadata/au')) {
+      const dimensions = `/${numbers.replace(/,/g, '/')}`;
+      const resolved = this.serializer.fromString(value);
+      replay.push([
+        dimensions,
+        path,
+        resolved,
+      ]);
+    }
+  }
+
+  inflateActions(key: string, value: string, actions: JobActionExport) {
+    let [_, dimensionalType, counter, subcounter] = key.split('-');
+    if (subcounter) {
+      counter = `${counter}.${subcounter}`;
+    }
+    const [type, ...dimensions] = dimensionalType.split(',');
+    let dimensionKey = '';
+    let isHook = false;
+    if (dimensions.length > 0) {
+      dimensionKey = `/${dimensions.join('/')}`;
+      isHook = true;
+    }
+    let targetList: ExportItem[];
+    if (isHook) {
+      if (!actions.hooks[dimensionKey]) {
+        actions.hooks[dimensionKey] = {
+          cursor: -1,
+          items: [],
+        };
+      }
+      targetList = actions.hooks[dimensionKey].items;
+    } else {
+      targetList = actions.main.items;
+    }
+    targetList.push([
+      `${dimensionKey}[${counter}]`,
+      type,
+      value,
+    ]);
+  }
+
+  reverseSort(aKey: ExportItem, bKey: ExportItem) {
+    if (aKey[0] > bKey[0]) {
+      return 1;
+    } else if (aKey[0] < bKey[0]) {
+      return -1;
+    } else {
+      if (aKey[1] > bKey[1]) {
+        return 1;
+      } else if (aKey[1] < bKey[1]) {
+        return -1;
+      }
+      return 0;
+    }
+  }
+
+  dateSort(aKey: ExportItem, bKey: ExportItem) {
+    if (aKey[2] > bKey[2]) {
+      return 1;
+    } else if (aKey[2] < bKey[2]) {
+      return -1;
+    } else {
+      return 0;
+    }
+  }
+}
+
+export { ExporterService };

package/services/durable/factory.ts

@@ -57,6 +57,7 @@ const getWorkflowYAML = (app: string, version: string) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
           job:
             maps:
               done: false
@@ -260,10 +261,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                   description: index will be appended later
             maps:
               signals: '{sigw1.output.data.signals}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-w']
-                  - ['{@string.concat}']
+              parentWorkflowId: '{$job.metadata.jid}'
               originJobId:
                 '@pipe':
                   - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
@@ -353,10 +351,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                   description: index will be appended later
             maps:
               signals: '{w1.output.data.signals}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-w']
-                  - ['{@string.concat}']
+              parentWorkflowId: '{$job.metadata.jid}'
               originJobId:
                 '@pipe':
                   - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
@@ -523,6 +518,7 @@ const getWorkflowYAML = (app: string, version: string) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         w1a:
           title: Activity Worker - Calls Activity Functions
@@ -563,61 +559,6 @@ const getWorkflowYAML = (app: string, version: string) => {
         t1a:
           - to: w1a
 
-    - subscribes: ${app}.sleep.execute
-      publishes: ${app}.sleep.executed
-
-      expire: 0
-
-      input:
-        schema:
-          type: object
-          properties:
-            parentWorkflowId:
-              type: string
-            originJobId:
-              type: string
-            workflowId:
-              type: string
-            duration:
-              type: number
-              description: in seconds
-            index:
-              type: number
-      output:
-        schema:
-          type: object
-          properties:
-            done:
-              type: boolean
-            duration:
-              type: number
-            index:
-              type: number
-
-      activities:
-        t1s:
-          title: Sleep Flow Trigger
-          type: trigger
-          stats:
-            id: '{$self.input.data.workflowId}'
-            key: '{$self.input.data.parentWorkflowId}'
-            parent: '{$self.input.data.originJobId}'
-
-        a1s:
-          title: Sleep for a duration
-          type: hook
-          sleep: '{t1s.output.data.duration}'
-          job:
-            maps:
-              done: true
-              duration: '{t1s.output.data.duration}'
-              index: '{t1s.output.data.index}'
-              workflowId: '{t1s.output.data.workflowId}'
-
-      transitions:
-        t1s:
-          - to: a1s
-
     - subscribes: ${app}.wfsc.execute
       publishes: ${app}.wfsc.executed
 
@@ -662,6 +603,7 @@ const getWorkflowYAML = (app: string, version: string) => {
           stats:
             id: '{$self.input.data.cycleWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         a1wc:
           title: Pivot - All Cycling Descendants Point Here
@@ -833,6 +775,7 @@ const getWorkflowYAML = (app: string, version: string) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         a1ww:
           title: WFS - signal entry point