@hotmeshio/hotmesh 0.0.10 → 0.0.11

package/build/services/pipe/functions/number.js

@@ -56,78 +56,5 @@ class NumberHandler {
     round(input) {
         return Math.round(input);
     }
-    add(...operands) {
-        // @ts-ignore
-        return operands.reduce((a, b) => {
-            if (Array.isArray(b)) {
-                return a + this.add(...b);
-            }
-            else {
-                return a + b;
-            }
-        }, 0);
-    }
-    subtract(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        let flatOperands = [];
-        operands.forEach((op) => {
-            if (Array.isArray(op)) {
-                flatOperands = [...flatOperands, ...op];
-            }
-            else {
-                flatOperands.push(op);
-            }
-        });
-        if (flatOperands.length === 0) {
-            throw new Error('At least one operand is required after flattening.');
-        }
-        const result = flatOperands.reduce((a, b, i) => {
-            return i === 0 ? a : a - b;
-        });
-        return result;
-    }
-    multiply(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        // @ts-ignore
-        return operands.reduce((a, b) => {
-            if (Array.isArray(b)) {
-                return a * this.multiply(...b);
-            }
-            else {
-                return a * b;
-            }
-        }, 1);
-    }
-    divide(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        let flatOperands = [];
-        operands.forEach((op) => {
-            if (Array.isArray(op)) {
-                flatOperands = [...flatOperands, ...op];
-            }
-            else {
-                flatOperands.push(op);
-            }
-        });
-        if (flatOperands.length === 0) {
-            throw new Error('At least one operand is required after flattening.');
-        }
-        const result = flatOperands.reduce((a, b, i) => {
-            if (b === 0) {
-                return NaN;
-            }
-            return i === 0 ? a : a / b;
-        });
-        if (isNaN(result)) {
-            return NaN;
-        }
-        return result;
-    }
 }
 exports.NumberHandler = NumberHandler;

package/build/services/signaler/stream.js

@@ -5,7 +5,7 @@ const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-const MAX_RETRIES = 4; //max delay (10s using exponential backoff);
+const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
 const MAX_TIMEOUT_MS = 60000;
 const GRADUATED_INTERVAL_MS = 5000;
 const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
@@ -158,8 +158,11 @@ class StreamSignaler {
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
         const tryCount = Math.min(input.metadata.try || 0, MAX_RETRIES);
-        if (maxRetries >= tryCount) {
-            return [true, Math.pow(10, tryCount)];
+        //only possible values for maxRetries are 1, 2, 3
+        //only possible values for tryCount are 0, 1, 2
+        if (maxRetries > tryCount) {
+            // 10ms, 100ms, or 1000ms delays between system retries
+            return [true, Math.pow(10, tryCount + 1)];
         }
         return [false, 0];
     }

package/build/types/durable.d.ts

@@ -7,7 +7,7 @@ type WorkflowOptions = {
     workflowTrace?: string;
     workflowSpan?: string;
 };
-type ActivityDataType = {
+type ActivityWorkflowDataType = {
     activityName: string;
     arguments: any[];
     workflowId: string;
@@ -35,6 +35,11 @@ type WorkerConfig = {
     namespace?: string;
     taskQueue: string;
     workflow: Function;
+    options?: WorkerOptions;
+};
+type WorkerOptions = {
+    maxSystemRetries?: number;
+    backoffExponent?: number;
 };
 type ContextType = {
     workflowId: string;
@@ -54,4 +59,4 @@ type ActivityConfig = {
         maximumInterval: string;
     };
 };
-export { ActivityConfig, ActivityDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkflowDataType, WorkflowOptions, };
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkerOptions, WorkflowDataType, WorkflowOptions, };

package/build/types/index.d.ts

@@ -3,6 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkerOptions, WorkflowDataType, WorkflowOptions, } from './durable';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/modules/errors.ts

@@ -12,6 +12,35 @@ class SetStateError extends Error {
   }
 }
 
+class DurableTimeoutError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 596;
+  }
+}
+class DurableMaxedError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 597;
+  }
+}
+class DurableFatalError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 598;
+  }
+}
+class DurableRetryError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 599;
+  }
+}
+
 class MapDataError extends Error {
   constructor() {
     super("Error occurred while mapping data");
@@ -52,4 +81,16 @@ class CollationError extends Error {
   }
 }
 
-export { CollationError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };
+export {
+  CollationError,
+  DurableTimeoutError,
+  DurableMaxedError,
+  DurableFatalError,
+  DurableRetryError,
+  DuplicateJobError,
+  GetStateError,
+  SetStateError,
+  MapDataError,
+  RegisterTimeoutError,
+  ExecActivityError
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "description": "Durable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -43,6 +43,7 @@
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:fatal": "NODE_ENV=test jest ./tests/durable/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
   },

package/services/activities/activity.ts

@@ -4,7 +4,6 @@ import {
   getValueByPath,
   restoreHierarchy } from '../../modules/utils';
 import { CollatorService } from '../collator';
-import { DimensionService } from '../dimension';
 import { EngineService } from '../engine';
 import { ILogger } from '../logger';
 import { MapperService } from '../mapper';
@@ -84,6 +83,7 @@ class Activity {
       if (this.doesHook()) {
         //sleep and wait to awaken upon a signal
         await this.registerHook(multi);
+        this.mapOutputData();
         this.mapJobData();
         await this.setState(multi);
         await CollatorService.authorizeReentry(this, multi);
@@ -94,6 +94,7 @@ class Activity {
       } else {
         //end the activity and transition to its children
         this.adjacencyList = await this.filterAdjacent();
+        this.mapOutputData();
         this.mapJobData();
         await this.setState(multi);
         await CollatorService.notarizeEarlyCompletion(this, multi);
@@ -322,6 +323,17 @@ class Activity {
     }
   }
 
+  mapOutputData(): void {
+    //activity YAML may include output map data that produces/extends activity output data.
+    if(this.config.output?.maps) {
+      const mapper = new MapperService(this.config.output.maps, this.context);
+      const actOutData = mapper.mapRules();
+      const activityId = this.metadata.aid;
+      const data = { ...this.context[activityId].output, ...actOutData };
+      this.context[activityId].output.data = data;
+    }
+  }
+
   async registerTimeout(): Promise<void> {
     //set timeout in support of hook and/or duplex
   }
@@ -526,7 +538,7 @@ class Activity {
 
   resolveAdjacentDad(): string {
     //concat self and child dimension (all children (leg 1) begin life at 0)
-    return `${this.resolveDad()}${DimensionService.getSeed(0)}`;
+    return `${this.resolveDad()}${CollatorService.getDimensionalSeed(0)}`;
   };
 
   async filterAdjacent(): Promise<StreamData[]> {

package/services/activities/cycle.ts

@@ -81,13 +81,18 @@ class Cycle extends Activity {
    * pattern allows for retries without violating the DAG.
    */
   async cycleAncestorActivity(multi: RedisMulti): Promise<string> {
+    //Cycle activity L1 is a standin for the target ancestor L1.
+    //Input data mapping (mapInputData) allows for the 
+    //next dimensonal thread to execute with different
+    //input data than the current dimensional thread
+    this.mapInputData();
     const streamData: StreamData = {
       metadata: {
         dad: CollatorService.resolveReentryDimension(this),
         jid: this.context.metadata.jid,
         aid: this.config.ancestor,
       },
-      data: {} //todo: verify immutability, before enabling: `this.context.data`
+      data: this.context.data
     };
     return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi)) as string;
   }

package/services/activities/trigger.ts

@@ -3,7 +3,6 @@ import { DuplicateJobError } from '../../modules/errors';
 import { formatISODate, getTimeSeries } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
-import { DimensionService } from '../dimension';
 import { EngineService } from '../engine';
 import { Pipe } from '../pipe';
 import { ReporterService } from '../reporter';
@@ -99,7 +98,7 @@ class Trigger extends Activity {
 
     const utc = formatISODate(new Date());
     const { id, version } = await this.engine.getVID();
-    this.initDimensionalAddress(DimensionService.getSeed());
+    this.initDimensionalAddress(CollatorService.getDimensionalSeed());
     const activityMetadata = {
       ...this.metadata,
       jid: jobId,
@@ -119,7 +118,7 @@ class Trigger extends Activity {
         trc: this.context.metadata.trc,
         spn: this.context.metadata.spn,
         jid: jobId,
-        dad: DimensionService.getSeed(), //top-level job implicitly uses `,0`
+        dad: CollatorService.getDimensionalSeed(), //top-level job implicitly uses `,0`
         key: jobKey,
         jc: utc,
         ju: utc,

package/services/collator/index.ts

@@ -85,7 +85,7 @@ class CollatorService {
   static async notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
     //1) ALWAYS actualize leg2 dimension (+1)
     //2) IF the activity is used in a cycle, don't close leg 2!
-    const decrement = activity.config.cycle ? 0 : -1_000_000_000_000;
+    const decrement = activity.config.cycle ? 0 : 1_000_000_000_000;
     return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - decrement, this.getDimensionalAddress(activity), multi);
   };
 
@@ -234,6 +234,17 @@ class CollatorService {
   static isActivityComplete(status: number): boolean {
     return (status - 0) <= 0;
   }
+
+  /**
+   * All activities exist on a dimensional plane. Zero
+   * is the default. A value of 
+   * `AxY,0,0,0,0,1,0,0` would reflect that
+   * an ancestor activity was dimensionalized beyond
+   * the default.
+   */
+  static getDimensionalSeed(index = 0): string {
+    return `,${index}`;
+  }
 }
 
 export { CollatorService };

package/services/durable/factory.ts

@@ -1,4 +1,21 @@
-const getWorkflowYAML = (topic: string, version = '1') => {
+/**
+ * 1) `maxSystemRetries` | can be 0 to 3 and represents milliseconds;
+ * if there is an error, the workflow will retry up to `maxSystemRetries` times
+ * delaying by 10, 100, and 1000ms; this is a system level retry
+ * and is not configurable. It exists to handle intermittent network
+ * errors. (NOTE: each retry spawns a new transition stream)
+ * 
+ * 2) `backoffExponent` | can be any number and represents `seconds` when applied;
+ * retries will happen indefinitely and adhere to the
+ * exponential backoff algorithm by multiplying by `backoffExponent`.
+ * For example, if `backoffExponent` is 10, the workflow will retry
+ * in 10s, 100s, 1000s, 10000s, etc.
+ *
+ * EXAMPLE | Using `maxSystemRetries = 3` and `backoffExponent = 10`, errant
+ * workflows will be retried on the following schedule (8 times in 27 hours):
+ * => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ */
+const getWorkflowYAML = (topic: string, version = '1', maxSystemRetries = 2, backoffExponent = 10) => {
   return `app:
   id: ${topic}
   version: '${version}'
@@ -30,10 +47,20 @@ const getWorkflowYAML = (topic: string, version = '1') => {
         a1:
           type: activity
           cycle: true
-  
+          output:
+            schema:
+              type: object
+              properties:
+                duration:
+                  type: number
+            maps:
+              duration: ${backoffExponent}
+
         w1:
           type: worker
           topic: ${topic}
+          retry:
+            '599': [${maxSystemRetries}]
           input:
             schema:
               type: object
@@ -55,18 +82,33 @@ const getWorkflowYAML = (topic: string, version = '1') => {
             maps:
               response: '{$self.output.data.response}'
 
+        a599:
+          title: Sleep before trying again
+          type: activity
+          sleep: "{a1.output.data.duration}"
+
         c1:
+          title: Goto Activity a1
           type: cycle
           ancestor: a1
+          input:
+            maps:
+              duration:
+                '@pipe':
+                  - ['{a1.output.data.duration}', ${backoffExponent}]
+                  - ['{@math.multiply}']
+
       transitions:
         t1:
           - to: a1
         a1:
           - to: w1
         w1:
-          - to: c1
+          - to: a599
             conditions:
-              code: 500
+              code: 599
+        a599:
+        - to: c1
 `;
 }
 

package/services/durable/handle.ts

@@ -16,22 +16,37 @@ export class WorkflowHandleService {
     let status = await this.hotMesh.getStatus(this.workflowId);
     const topic = `${this.workflowTopic}.${this.workflowId}`;
   
-    if (status == 0) {
-      return (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response;
-    }
-  
     return new Promise((resolve, reject) => {
       let isResolved = false;
       //common fulfill/unsubscribe
-      const complete = async (response?: any) => {
+      const complete = async (response?: any, err?: string) => {
         if (isResolved) return;
         isResolved = true;
         this.hotMesh.unsub(topic);
-        resolve(response || (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response);
+        if (err) {
+          return reject(JSON.parse(err));
+        } else if (!response) {
+          const state = await this.hotMesh.getState(this.workflowTopic, this.workflowId);
+          if (!state.data && state.metadata.err) {
+            return reject(JSON.parse(state.metadata.err));
+          }
+          response = state.data?.response;
+        }
+        resolve(response);
       };
-      this.hotMesh.sub(topic, async (topic: string, message: JobOutput) => {
-        await complete(message.data?.response);
+      //check for done
+      if (status == 0) {
+        return complete();
+      }
+      //subscribe to topic
+      this.hotMesh.sub(topic, async (topic: string, state: JobOutput) => {
+        if (!state.data && state.metadata.err) {
+          await complete(null, state.metadata.err);
+        } else {
+          await complete(state.data?.response);
+        }
       });
+      //resolve for race condition
       setTimeout(async () => {
         status = await this.hotMesh.getStatus(this.workflowId);
         if (status == 0) {

package/services/durable/worker.ts

@@ -2,8 +2,18 @@ import { asyncLocalStorage } from './asyncLocalStorage';
 import { HotMeshService as HotMesh } from '../hotmesh';
 import { RedisClass, RedisOptions } from '../../types/redis';
 import { StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
-import { ActivityDataType, Connection, Registry, WorkerConfig, WorkflowDataType } from "../../types/durable";
+import { ActivityWorkflowDataType,
+  Connection,
+  Registry,
+  WorkerConfig,
+  WorkerOptions,
+  WorkflowDataType } from "../../types/durable";
 import { getWorkflowYAML, getActivityYAML } from './factory';
+import {
+  DurableFatalError,
+  DurableMaxedError,
+  DurableRetryError,
+  DurableTimeoutError } from '../../modules/errors';
 
 /*
 Here is an example of how the methods in this file are used:
@@ -45,7 +55,7 @@ export class WorkerService {
   workflowRunner: HotMesh;
   activityRunner: HotMesh;
 
-  static getHotMesh = async (worflowTopic: string) => {
+  static getHotMesh = async (worflowTopic: string, options?: WorkerOptions) => {
     if (WorkerService.instances.has(worflowTopic)) {
       return await WorkerService.instances.get(worflowTopic);
     }
@@ -54,17 +64,17 @@ export class WorkerService {
       engine: { redis: { ...WorkerService.connection } }
     });
     WorkerService.instances.set(worflowTopic, hotMesh);
-    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML);
+    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML, options);
     return hotMesh;
   }
 
-  static async activateWorkflow(hotMesh: HotMesh, topic: string, factory: Function) {
+  static async activateWorkflow(hotMesh: HotMesh, topic: string, dagFactory: Function, options: WorkerOptions = {}) {
     const version = '1';
     const app = await hotMesh.engine.store.getApp(topic);
     const appVersion = app?.version;
     if(!appVersion) {
       try {
-        await hotMesh.deploy(factory(topic, version));
+        await hotMesh.deploy(dagFactory(topic, version, options.maxSystemRetries, options.backoffExponent));
         await hotMesh.activate(version);
       } catch (err) {
         hotMesh.engine.logger.error('durable-worker-deploy-activate-err', err);
@@ -117,7 +127,7 @@ export class WorkerService {
     worker.activityRunner = await worker.initActivityWorkflow(config, activityTopic);
     await WorkerService.activateWorkflow(worker.activityRunner, activityTopic, getActivityYAML);
     worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
-    await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML);
+    await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML, config.options);
     return worker;
   }
 
@@ -160,7 +170,7 @@ export class WorkerService {
     return async (data: StreamData): Promise<StreamDataResponse> => {
       try {
         //always run the activity function when instructed; return the response
-        const activityInput = data.data as unknown as ActivityDataType;
+        const activityInput = data.data as unknown as ActivityWorkflowDataType;
         const activityName = activityInput.activityName;
         const activityFunction = WorkerService.activityRegistry[activityName];
         const pojoResponse = await activityFunction.apply(this, activityInput.arguments);
@@ -172,12 +182,16 @@ export class WorkerService {
         };
       } catch (err) {
         this.activityRunner.engine.logger.error('durable-worker-activity-err', err);
+        if (!(err instanceof DurableTimeoutError) &&
+          !(err instanceof DurableMaxedError) &&
+          !(err instanceof DurableFatalError)) {
+          err = new DurableRetryError(err.message); 
+        }
         return {
           status: StreamStatus.ERROR,
-          code: 500,
-          message: err.message,
+          code: err.code,
           metadata: { ...data.metadata },
-          data: { error: err }
+          data: { message: err.message }
         } as StreamDataResponse;
       }
     }
@@ -257,11 +271,12 @@ export class WorkerService {
           data: { response: workflowResponse }
         };
       } catch (err) {
+        // 59* - Durable*Error
         return {
           status: StreamStatus.ERROR,
-          code: 500,
+          code: err.code || new DurableRetryError(err.message).code,
           metadata: { ...data.metadata },
-          data: { error: err }
+          data: { message: err.message, type: err.name }
         } as StreamDataResponse;
       }
     }

package/services/durable/workflow.ts

@@ -62,7 +62,7 @@ export class WorkflowService {
     const client = new Client({
       connection: await Connection.connect(WorkerService.connection),
     });
-    //todo: should I allow-cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
+    //todo: allow cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
     const handle = await client.workflow.start({
       ...options,
       workflowId: `${workflowId}${options.workflowId}`, //concat

package/services/pipe/functions/math.ts

@@ -1,4 +1,78 @@
 class MathHandler {
+  add(...operands: (number | number[])[]): number {
+    // @ts-ignore
+    return operands.reduce((a: number, b: number | number[]) => {
+      if (Array.isArray(b)) {
+        return a + this.add(...b);
+      } else {
+        return a + b;
+      }
+    }, 0);
+  }
+
+  subtract(...operands: (number | number[])[]): number {
+    if (operands.length === 0) {
+      throw new Error('At least one operand is required.');
+    }
+    let flatOperands: number[] = [];
+    operands.forEach((op: number | number[]) => {
+      if (Array.isArray(op)) {
+        flatOperands = [...flatOperands, ...op];
+      } else {
+        flatOperands.push(op);
+      }
+    });
+    if (flatOperands.length === 0) {
+      throw new Error('At least one operand is required after flattening.');
+    }
+    const result = flatOperands.reduce((a: number, b: number, i: number) => {
+      return i === 0 ? a : a - b;
+    });
+    return result;
+  }
+
+  multiply(...operands: (number | number[])[]): number {
+    if (operands.length === 0) {
+      throw new Error('At least one operand is required.');
+    }
+  
+    // @ts-ignore
+    return operands.reduce((a: number, b: number | number[]) => {
+      if (Array.isArray(b)) {
+        return a * this.multiply(...b);
+      } else {
+        return a * b;
+      }
+    }, 1);
+  }
+  
+  divide(...operands: (number | number[])[]): number {
+    if (operands.length === 0) {
+      throw new Error('At least one operand is required.');
+    }
+    let flatOperands: number[] = [];
+    operands.forEach((op: number | number[]) => {
+      if (Array.isArray(op)) {
+        flatOperands = [...flatOperands, ...op];
+      } else {
+        flatOperands.push(op);
+      }
+    });
+    if (flatOperands.length === 0) {
+      throw new Error('At least one operand is required after flattening.');
+    }
+    const result = flatOperands.reduce((a: number, b: number, i: number) => {
+      if (b === 0) {
+        return NaN;
+      }
+      return i === 0 ? a : a / b;
+    });
+    if (isNaN(result)) {
+      return NaN;
+    }
+    return result;
+  }
+
   abs(x: number): number {
     return Math.abs(x);
   }