@hotmeshio/hotmesh 0.0.10 → 0.0.12

package/build/services/hotmesh/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HotMeshService = void 0;
-const uuid_1 = require("uuid");
+const nanoid_1 = require("nanoid");
 const key_1 = require("../../modules/key");
 const engine_1 = require("../engine");
 const logger_1 = require("../logger");
@@ -39,7 +39,7 @@ class HotMeshService {
     }
     static async init(config) {
         const instance = new HotMeshService();
-        instance.guid = (0, uuid_1.v4)();
+        instance.guid = (0, nanoid_1.nanoid)();
         instance.verifyAndSetNamespace(config.namespace);
         instance.verifyAndSetAppId(config.appId);
         instance.logger = new logger_1.LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);

package/build/services/pipe/functions/math.d.ts

@@ -1,4 +1,8 @@
 declare class MathHandler {
+    add(...operands: (number | number[])[]): number;
+    subtract(...operands: (number | number[])[]): number;
+    multiply(...operands: (number | number[])[]): number;
+    divide(...operands: (number | number[])[]): number;
     abs(x: number): number;
     acos(x: number): number;
     acosh(x: number): number;

package/build/services/pipe/functions/math.js

@@ -2,6 +2,79 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MathHandler = void 0;
 class MathHandler {
+    add(...operands) {
+        // @ts-ignore
+        return operands.reduce((a, b) => {
+            if (Array.isArray(b)) {
+                return a + this.add(...b);
+            }
+            else {
+                return a + b;
+            }
+        }, 0);
+    }
+    subtract(...operands) {
+        if (operands.length === 0) {
+            throw new Error('At least one operand is required.');
+        }
+        let flatOperands = [];
+        operands.forEach((op) => {
+            if (Array.isArray(op)) {
+                flatOperands = [...flatOperands, ...op];
+            }
+            else {
+                flatOperands.push(op);
+            }
+        });
+        if (flatOperands.length === 0) {
+            throw new Error('At least one operand is required after flattening.');
+        }
+        const result = flatOperands.reduce((a, b, i) => {
+            return i === 0 ? a : a - b;
+        });
+        return result;
+    }
+    multiply(...operands) {
+        if (operands.length === 0) {
+            throw new Error('At least one operand is required.');
+        }
+        // @ts-ignore
+        return operands.reduce((a, b) => {
+            if (Array.isArray(b)) {
+                return a * this.multiply(...b);
+            }
+            else {
+                return a * b;
+            }
+        }, 1);
+    }
+    divide(...operands) {
+        if (operands.length === 0) {
+            throw new Error('At least one operand is required.');
+        }
+        let flatOperands = [];
+        operands.forEach((op) => {
+            if (Array.isArray(op)) {
+                flatOperands = [...flatOperands, ...op];
+            }
+            else {
+                flatOperands.push(op);
+            }
+        });
+        if (flatOperands.length === 0) {
+            throw new Error('At least one operand is required after flattening.');
+        }
+        const result = flatOperands.reduce((a, b, i) => {
+            if (b === 0) {
+                return NaN;
+            }
+            return i === 0 ? a : a / b;
+        });
+        if (isNaN(result)) {
+            return NaN;
+        }
+        return result;
+    }
     abs(x) {
         return Math.abs(x);
     }

package/build/services/pipe/functions/number.d.ts

@@ -17,9 +17,5 @@ declare class NumberHandler {
     min(...values: number[]): number;
     pow(base: number, exponent: number): number;
     round(input: number): number;
-    add(...operands: (number | number[])[]): number;
-    subtract(...operands: (number | number[])[]): number;
-    multiply(...operands: (number | number[])[]): number;
-    divide(...operands: (number | number[])[]): number;
 }
 export { NumberHandler };

package/build/services/pipe/functions/number.js

@@ -56,78 +56,5 @@ class NumberHandler {
     round(input) {
         return Math.round(input);
     }
-    add(...operands) {
-        // @ts-ignore
-        return operands.reduce((a, b) => {
-            if (Array.isArray(b)) {
-                return a + this.add(...b);
-            }
-            else {
-                return a + b;
-            }
-        }, 0);
-    }
-    subtract(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        let flatOperands = [];
-        operands.forEach((op) => {
-            if (Array.isArray(op)) {
-                flatOperands = [...flatOperands, ...op];
-            }
-            else {
-                flatOperands.push(op);
-            }
-        });
-        if (flatOperands.length === 0) {
-            throw new Error('At least one operand is required after flattening.');
-        }
-        const result = flatOperands.reduce((a, b, i) => {
-            return i === 0 ? a : a - b;
-        });
-        return result;
-    }
-    multiply(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        // @ts-ignore
-        return operands.reduce((a, b) => {
-            if (Array.isArray(b)) {
-                return a * this.multiply(...b);
-            }
-            else {
-                return a * b;
-            }
-        }, 1);
-    }
-    divide(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        let flatOperands = [];
-        operands.forEach((op) => {
-            if (Array.isArray(op)) {
-                flatOperands = [...flatOperands, ...op];
-            }
-            else {
-                flatOperands.push(op);
-            }
-        });
-        if (flatOperands.length === 0) {
-            throw new Error('At least one operand is required after flattening.');
-        }
-        const result = flatOperands.reduce((a, b, i) => {
-            if (b === 0) {
-                return NaN;
-            }
-            return i === 0 ? a : a / b;
-        });
-        if (isNaN(result)) {
-            return NaN;
-        }
-        return result;
-    }
 }
 exports.NumberHandler = NumberHandler;

package/build/services/signaler/stream.js

@@ -5,7 +5,7 @@ const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-const MAX_RETRIES = 4; //max delay (10s using exponential backoff);
+const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
 const MAX_TIMEOUT_MS = 60000;
 const GRADUATED_INTERVAL_MS = 5000;
 const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
@@ -158,8 +158,11 @@ class StreamSignaler {
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
         const tryCount = Math.min(input.metadata.try || 0, MAX_RETRIES);
-        if (maxRetries >= tryCount) {
-            return [true, Math.pow(10, tryCount)];
+        //only possible values for maxRetries are 1, 2, 3
+        //only possible values for tryCount are 0, 1, 2
+        if (maxRetries > tryCount) {
+            // 10ms, 100ms, or 1000ms delays between system retries
+            return [true, Math.pow(10, tryCount + 1)];
         }
         return [false, 0];
     }

package/build/types/durable.d.ts

@@ -7,7 +7,7 @@ type WorkflowOptions = {
     workflowTrace?: string;
     workflowSpan?: string;
 };
-type ActivityDataType = {
+type ActivityWorkflowDataType = {
     activityName: string;
     arguments: any[];
     workflowId: string;
@@ -35,6 +35,11 @@ type WorkerConfig = {
     namespace?: string;
     taskQueue: string;
     workflow: Function;
+    options?: WorkerOptions;
+};
+type WorkerOptions = {
+    maxSystemRetries?: number;
+    backoffExponent?: number;
 };
 type ContextType = {
     workflowId: string;
@@ -54,4 +59,4 @@ type ActivityConfig = {
         maximumInterval: string;
     };
 };
-export { ActivityConfig, ActivityDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkflowDataType, WorkflowOptions, };
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkerOptions, WorkflowDataType, WorkflowOptions, };

package/build/types/index.d.ts

@@ -3,6 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkerOptions, WorkflowDataType, WorkflowOptions, } from './durable';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/modules/errors.ts

@@ -12,6 +12,35 @@ class SetStateError extends Error {
   }
 }
 
+class DurableTimeoutError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 596;
+  }
+}
+class DurableMaxedError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 597;
+  }
+}
+class DurableFatalError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 598;
+  }
+}
+class DurableRetryError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 599;
+  }
+}
+
 class MapDataError extends Error {
   constructor() {
     super("Error occurred while mapping data");
@@ -52,4 +81,16 @@ class CollationError extends Error {
   }
 }
 
-export { CollationError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };
+export {
+  CollationError,
+  DurableTimeoutError,
+  DurableMaxedError,
+  DurableFatalError,
+  DurableRetryError,
+  DuplicateJobError,
+  GetStateError,
+  SetStateError,
+  MapDataError,
+  RegisterTimeoutError,
+  ExecActivityError
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.10",
+  "version": "0.0.12",
   "description": "Durable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -43,6 +43,7 @@
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:fatal": "NODE_ENV=test jest ./tests/durable/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
   },
@@ -54,7 +55,7 @@
     "@opentelemetry/api": "^1.4.1",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "uuid": "^9.0.1",
+    "nanoid": "^3.3.6",
     "winston": "^3.8.2"
   },
   "devDependencies": {

package/services/activities/activity.ts

@@ -4,7 +4,6 @@ import {
   getValueByPath,
   restoreHierarchy } from '../../modules/utils';
 import { CollatorService } from '../collator';
-import { DimensionService } from '../dimension';
 import { EngineService } from '../engine';
 import { ILogger } from '../logger';
 import { MapperService } from '../mapper';
@@ -84,6 +83,7 @@ class Activity {
       if (this.doesHook()) {
         //sleep and wait to awaken upon a signal
         await this.registerHook(multi);
+        this.mapOutputData();
         this.mapJobData();
         await this.setState(multi);
         await CollatorService.authorizeReentry(this, multi);
@@ -94,6 +94,7 @@ class Activity {
       } else {
         //end the activity and transition to its children
         this.adjacencyList = await this.filterAdjacent();
+        this.mapOutputData();
         this.mapJobData();
         await this.setState(multi);
         await CollatorService.notarizeEarlyCompletion(this, multi);
@@ -322,6 +323,17 @@ class Activity {
     }
   }
 
+  mapOutputData(): void {
+    //activity YAML may include output map data that produces/extends activity output data.
+    if(this.config.output?.maps) {
+      const mapper = new MapperService(this.config.output.maps, this.context);
+      const actOutData = mapper.mapRules();
+      const activityId = this.metadata.aid;
+      const data = { ...this.context[activityId].output, ...actOutData };
+      this.context[activityId].output.data = data;
+    }
+  }
+
   async registerTimeout(): Promise<void> {
     //set timeout in support of hook and/or duplex
   }
@@ -526,7 +538,7 @@ class Activity {
 
   resolveAdjacentDad(): string {
     //concat self and child dimension (all children (leg 1) begin life at 0)
-    return `${this.resolveDad()}${DimensionService.getSeed(0)}`;
+    return `${this.resolveDad()}${CollatorService.getDimensionalSeed(0)}`;
   };
 
   async filterAdjacent(): Promise<StreamData[]> {

package/services/activities/cycle.ts

@@ -81,13 +81,20 @@ class Cycle extends Activity {
    * pattern allows for retries without violating the DAG.
    */
   async cycleAncestorActivity(multi: RedisMulti): Promise<string> {
+    //Cycle activity L1 is a standin for the target ancestor L1.
+    //Input data mapping (mapInputData) allows for the 
+    //next dimensonal thread to execute with different
+    //input data than the current dimensional thread
+    this.mapInputData();
     const streamData: StreamData = {
       metadata: {
         dad: CollatorService.resolveReentryDimension(this),
         jid: this.context.metadata.jid,
         aid: this.config.ancestor,
+        spn: this.context['$self'].output.metadata?.l1s,
+        trc: this.context.metadata.trc,
       },
-      data: {} //todo: verify immutability, before enabling: `this.context.data`
+      data: this.context.data
     };
     return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi)) as string;
   }

package/services/activities/trigger.ts

@@ -1,9 +1,8 @@
-import { v4 as uuidv4 } from 'uuid';
+import { nanoid } from 'nanoid';
 import { DuplicateJobError } from '../../modules/errors';
 import { formatISODate, getTimeSeries } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
-import { DimensionService } from '../dimension';
 import { EngineService } from '../engine';
 import { Pipe } from '../pipe';
 import { ReporterService } from '../reporter';
@@ -99,7 +98,7 @@ class Trigger extends Activity {
 
     const utc = formatISODate(new Date());
     const { id, version } = await this.engine.getVID();
-    this.initDimensionalAddress(DimensionService.getSeed());
+    this.initDimensionalAddress(CollatorService.getDimensionalSeed());
     const activityMetadata = {
       ...this.metadata,
       jid: jobId,
@@ -119,7 +118,7 @@ class Trigger extends Activity {
         trc: this.context.metadata.trc,
         spn: this.context.metadata.spn,
         jid: jobId,
-        dad: DimensionService.getSeed(), //top-level job implicitly uses `,0`
+        dad: CollatorService.getDimensionalSeed(), //top-level job implicitly uses `,0`
         key: jobKey,
         jc: utc,
         ju: utc,
@@ -161,7 +160,7 @@ class Trigger extends Activity {
 
   resolveJobId(context: Partial<JobState>): string {
     const jobId = this.config.stats?.id;
-    return jobId ? Pipe.resolve(jobId, context) : uuidv4();
+    return jobId ? Pipe.resolve(jobId, context) : nanoid();
   }
 
   resolveJobKey(context: Partial<JobState>): string {

package/services/collator/index.ts

@@ -85,7 +85,7 @@ class CollatorService {
   static async notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
     //1) ALWAYS actualize leg2 dimension (+1)
     //2) IF the activity is used in a cycle, don't close leg 2!
-    const decrement = activity.config.cycle ? 0 : -1_000_000_000_000;
+    const decrement = activity.config.cycle ? 0 : 1_000_000_000_000;
     return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - decrement, this.getDimensionalAddress(activity), multi);
   };
 
@@ -234,6 +234,17 @@ class CollatorService {
   static isActivityComplete(status: number): boolean {
     return (status - 0) <= 0;
   }
+
+  /**
+   * All activities exist on a dimensional plane. Zero
+   * is the default. A value of 
+   * `AxY,0,0,0,0,1,0,0` would reflect that
+   * an ancestor activity was dimensionalized beyond
+   * the default.
+   */
+  static getDimensionalSeed(index = 0): string {
+    return `,${index}`;
+  }
 }
 
 export { CollatorService };

package/services/connector/index.ts

@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from 'uuid';
+import { nanoid } from 'nanoid';
 
 import { identifyRedisTypeFromClass } from '../../modules/utils';
 import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
@@ -23,14 +23,14 @@ export class ConnectorService {
       if (identifyRedisTypeFromClass(Redis) === 'redis') {
         for (let i = 1; i <= 3; i++) {
           instances.push(RedisConnection.connect(
-            uuidv4(),
+            nanoid(),
             Redis as RedisClassType,
             options as RedisClientOptions));
         }
       } else {
         for (let i = 1; i <= 3; i++) {
           instances.push(IORedisConnection.connect(
-            uuidv4(),
+            nanoid(),
             Redis as IORedisClassType,
             options as IORedisClientOptions));
         }

package/services/durable/client.ts

@@ -12,7 +12,7 @@ Here is an example of how the methods in this file are used:
 import { Durable } from '@hotmeshio/hotmesh';
 import Redis from 'ioredis';
 import { example } from './workflows';
-import { v4 as uuidv4 } from 'uuid';
+import { nanoid } from 'nanoid';
 
 async function run() {
   const connection = await Durable.Connection.connect({
@@ -31,7 +31,7 @@ async function run() {
     args: ['HotMesh'],
     taskQueue: 'hello-world',
     workflowName: 'example',
-    workflowId: 'workflow-' + uuidv4(),
+    workflowId: 'workflow-' + nanoid(),
   });
 
   console.log(`Started workflow ${handle.workflowId}`);

package/services/durable/connection.ts

@@ -7,7 +7,7 @@ Here is an example of how the methods in this file are used:
 
 import { Durable } from '@hotmeshio/hotmesh';
 import Redis from 'ioredis';
-import { v4 as uuidv4 } from 'uuid';
+import { nanoid } from 'nanoid';
 
 async function run() {
   const connection = await Durable.Connection.connect({
@@ -26,7 +26,7 @@ async function run() {
     taskQueue: 'hello-world',
     args: ['HotMesh'],
     workflowName: 'example',
-    workflowId: uuidv4(),
+    workflowId: nanoid(),
   });
 
   console.log(`Started workflow ${handle.workflowId}`);

package/services/durable/factory.ts

@@ -1,4 +1,21 @@
-const getWorkflowYAML = (topic: string, version = '1') => {
+/**
+ * 1) `maxSystemRetries` | can be 0 to 3 and represents milliseconds;
+ * if there is an error, the workflow will retry up to `maxSystemRetries` times
+ * delaying by 10, 100, and 1000ms; this is a system level retry
+ * and is not configurable. It exists to handle intermittent network
+ * errors. (NOTE: each retry spawns a new transition stream)
+ * 
+ * 2) `backoffExponent` | can be any number and represents `seconds` when applied;
+ * retries will happen indefinitely and adhere to the
+ * exponential backoff algorithm by multiplying by `backoffExponent`.
+ * For example, if `backoffExponent` is 10, the workflow will retry
+ * in 10s, 100s, 1000s, 10000s, etc.
+ *
+ * EXAMPLE | Using `maxSystemRetries = 3` and `backoffExponent = 10`, errant
+ * workflows will be retried on the following schedule (8 times in 27 hours):
+ * => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ */
+const getWorkflowYAML = (topic: string, version = '1', maxSystemRetries = 2, backoffExponent = 10) => {
   return `app:
   id: ${topic}
   version: '${version}'
@@ -30,10 +47,20 @@ const getWorkflowYAML = (topic: string, version = '1') => {
         a1:
           type: activity
           cycle: true
-  
+          output:
+            schema:
+              type: object
+              properties:
+                duration:
+                  type: number
+            maps:
+              duration: ${backoffExponent}
+
         w1:
           type: worker
           topic: ${topic}
+          retry:
+            '599': [${maxSystemRetries}]
           input:
             schema:
               type: object
@@ -55,18 +82,33 @@ const getWorkflowYAML = (topic: string, version = '1') => {
             maps:
               response: '{$self.output.data.response}'
 
+        a599:
+          title: Sleep before trying again
+          type: activity
+          sleep: "{a1.output.data.duration}"
+
         c1:
+          title: Goto Activity a1
           type: cycle
           ancestor: a1
+          input:
+            maps:
+              duration:
+                '@pipe':
+                  - ['{a1.output.data.duration}', ${backoffExponent}]
+                  - ['{@math.multiply}']
+
       transitions:
         t1:
           - to: a1
         a1:
           - to: w1
         w1:
-          - to: c1
+          - to: a599
             conditions:
-              code: 500
+              code: 599
+        a599:
+        - to: c1
 `;
 }
 

package/services/durable/handle.ts

@@ -16,22 +16,37 @@ export class WorkflowHandleService {
     let status = await this.hotMesh.getStatus(this.workflowId);
     const topic = `${this.workflowTopic}.${this.workflowId}`;
   
-    if (status == 0) {
-      return (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response;
-    }
-  
     return new Promise((resolve, reject) => {
       let isResolved = false;
       //common fulfill/unsubscribe
-      const complete = async (response?: any) => {
+      const complete = async (response?: any, err?: string) => {
         if (isResolved) return;
         isResolved = true;
         this.hotMesh.unsub(topic);
-        resolve(response || (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response);
+        if (err) {
+          return reject(JSON.parse(err));
+        } else if (!response) {
+          const state = await this.hotMesh.getState(this.workflowTopic, this.workflowId);
+          if (!state.data && state.metadata.err) {
+            return reject(JSON.parse(state.metadata.err));
+          }
+          response = state.data?.response;
+        }
+        resolve(response);
       };
-      this.hotMesh.sub(topic, async (topic: string, message: JobOutput) => {
-        await complete(message.data?.response);
+      //check for done
+      if (status == 0) {
+        return complete();
+      }
+      //subscribe to topic
+      this.hotMesh.sub(topic, async (topic: string, state: JobOutput) => {
+        if (!state.data && state.metadata.err) {
+          await complete(null, state.metadata.err);
+        } else {
+          await complete(state.data?.response);
+        }
       });
+      //resolve for race condition
       setTimeout(async () => {
         status = await this.hotMesh.getStatus(this.workflowId);
         if (status == 0) {