@hotmeshio/hotmesh 0.0.9 → 0.0.11

package/build/services/durable/worker.js

@@ -6,6 +6,7 @@ const asyncLocalStorage_1 = require("./asyncLocalStorage");
 const hotmesh_1 = require("../hotmesh");
 const stream_1 = require("../../types/stream");
 const factory_1 = require("./factory");
+const errors_1 = require("../../modules/errors");
 /*
 Here is an example of how the methods in this file are used:
 
@@ -26,7 +27,6 @@ async function run() {
   });
   const worker = await Worker.create({
     connection,
-    namespace: 'default',
     taskQueue: 'hello-world',
     workflow: workflows.example,
     activities,
@@ -40,13 +40,13 @@ run().catch((err) => {
 });
 */
 class WorkerService {
-    static async activateWorkflow(hotMesh, topic, factory) {
+    static async activateWorkflow(hotMesh, topic, dagFactory, options = {}) {
         const version = '1';
         const app = await hotMesh.engine.store.getApp(topic);
         const appVersion = app?.version;
         if (!appVersion) {
             try {
-                await hotMesh.deploy(factory(topic, version));
+                await hotMesh.deploy(dagFactory(topic, version, options.maxSystemRetries, options.backoffExponent));
                 await hotMesh.activate(version);
             }
             catch (err) {
@@ -100,7 +100,7 @@ class WorkerService {
         worker.activityRunner = await worker.initActivityWorkflow(config, activityTopic);
         await WorkerService.activateWorkflow(worker.activityRunner, activityTopic, factory_1.getActivityYAML);
         worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
-        await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, factory_1.getWorkflowYAML);
+        await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, factory_1.getWorkflowYAML, config.options);
         return worker;
     }
     static resolveWorkflowTarget(workflow) {
@@ -152,12 +152,16 @@ class WorkerService {
             }
             catch (err) {
                 this.activityRunner.engine.logger.error('durable-worker-activity-err', err);
+                if (!(err instanceof errors_1.DurableTimeoutError) &&
+                    !(err instanceof errors_1.DurableMaxedError) &&
+                    !(err instanceof errors_1.DurableFatalError)) {
+                    err = new errors_1.DurableRetryError(err.message);
+                }
                 return {
                     status: stream_1.StreamStatus.ERROR,
-                    code: 500,
-                    message: err.message,
+                    code: err.code,
                     metadata: { ...data.metadata },
-                    data: { error: err }
+                    data: { message: err.message }
                 };
             }
         };
@@ -228,11 +232,12 @@ class WorkerService {
                 };
             }
             catch (err) {
+                // 59* - Durable*Error
                 return {
                     status: stream_1.StreamStatus.ERROR,
-                    code: 500,
+                    code: err.code || new errors_1.DurableRetryError(err.message).code,
                     metadata: { ...data.metadata },
-                    data: { error: err }
+                    data: { message: err.message, type: err.name }
                 };
             }
         };
@@ -247,7 +252,7 @@ class WorkerService {
 _a = WorkerService;
 WorkerService.activityRegistry = {}; //user's activities
 WorkerService.instances = new Map();
-WorkerService.getHotMesh = async (worflowTopic) => {
+WorkerService.getHotMesh = async (worflowTopic, options) => {
     if (WorkerService.instances.has(worflowTopic)) {
         return await WorkerService.instances.get(worflowTopic);
     }
@@ -256,7 +261,7 @@ WorkerService.getHotMesh = async (worflowTopic) => {
         engine: { redis: { ...WorkerService.connection } }
     });
     WorkerService.instances.set(worflowTopic, hotMesh);
-    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, factory_1.getWorkflowYAML);
+    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, factory_1.getWorkflowYAML, options);
     return hotMesh;
 };
 WorkerService.Context = {

package/build/services/durable/workflow.js

@@ -62,7 +62,7 @@ class WorkflowService {
         const client = new client_1.ClientService({
             connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
         });
-        //todo: should I allow-cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
+        //todo: allow cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
         const handle = await client.workflow.start({
             ...options,
             workflowId: `${workflowId}${options.workflowId}`,

package/build/services/hotmesh/index.js

@@ -17,7 +17,7 @@ class HotMeshService {
     }
     verifyAndSetNamespace(namespace) {
         if (!namespace) {
-            this.namespace = key_1.PSNS;
+            this.namespace = key_1.HMNS;
         }
         else if (!namespace.match(/^[A-Za-z0-9-]+$/)) {
             throw new Error(`config.namespace [${namespace}] is invalid`);

package/build/services/pipe/functions/math.d.ts

@@ -1,4 +1,8 @@
 declare class MathHandler {
+    add(...operands: (number | number[])[]): number;
+    subtract(...operands: (number | number[])[]): number;
+    multiply(...operands: (number | number[])[]): number;
+    divide(...operands: (number | number[])[]): number;
     abs(x: number): number;
     acos(x: number): number;
     acosh(x: number): number;

package/build/services/pipe/functions/math.js

@@ -2,6 +2,79 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MathHandler = void 0;
 class MathHandler {
+    add(...operands) {
+        // @ts-ignore
+        return operands.reduce((a, b) => {
+            if (Array.isArray(b)) {
+                return a + this.add(...b);
+            }
+            else {
+                return a + b;
+            }
+        }, 0);
+    }
+    subtract(...operands) {
+        if (operands.length === 0) {
+            throw new Error('At least one operand is required.');
+        }
+        let flatOperands = [];
+        operands.forEach((op) => {
+            if (Array.isArray(op)) {
+                flatOperands = [...flatOperands, ...op];
+            }
+            else {
+                flatOperands.push(op);
+            }
+        });
+        if (flatOperands.length === 0) {
+            throw new Error('At least one operand is required after flattening.');
+        }
+        const result = flatOperands.reduce((a, b, i) => {
+            return i === 0 ? a : a - b;
+        });
+        return result;
+    }
+    multiply(...operands) {
+        if (operands.length === 0) {
+            throw new Error('At least one operand is required.');
+        }
+        // @ts-ignore
+        return operands.reduce((a, b) => {
+            if (Array.isArray(b)) {
+                return a * this.multiply(...b);
+            }
+            else {
+                return a * b;
+            }
+        }, 1);
+    }
+    divide(...operands) {
+        if (operands.length === 0) {
+            throw new Error('At least one operand is required.');
+        }
+        let flatOperands = [];
+        operands.forEach((op) => {
+            if (Array.isArray(op)) {
+                flatOperands = [...flatOperands, ...op];
+            }
+            else {
+                flatOperands.push(op);
+            }
+        });
+        if (flatOperands.length === 0) {
+            throw new Error('At least one operand is required after flattening.');
+        }
+        const result = flatOperands.reduce((a, b, i) => {
+            if (b === 0) {
+                return NaN;
+            }
+            return i === 0 ? a : a / b;
+        });
+        if (isNaN(result)) {
+            return NaN;
+        }
+        return result;
+    }
     abs(x) {
         return Math.abs(x);
     }

package/build/services/pipe/functions/number.d.ts

@@ -17,9 +17,5 @@ declare class NumberHandler {
     min(...values: number[]): number;
     pow(base: number, exponent: number): number;
     round(input: number): number;
-    add(...operands: (number | number[])[]): number;
-    subtract(...operands: (number | number[])[]): number;
-    multiply(...operands: (number | number[])[]): number;
-    divide(...operands: (number | number[])[]): number;
 }
 export { NumberHandler };

package/build/services/pipe/functions/number.js

@@ -56,78 +56,5 @@ class NumberHandler {
     round(input) {
         return Math.round(input);
     }
-    add(...operands) {
-        // @ts-ignore
-        return operands.reduce((a, b) => {
-            if (Array.isArray(b)) {
-                return a + this.add(...b);
-            }
-            else {
-                return a + b;
-            }
-        }, 0);
-    }
-    subtract(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        let flatOperands = [];
-        operands.forEach((op) => {
-            if (Array.isArray(op)) {
-                flatOperands = [...flatOperands, ...op];
-            }
-            else {
-                flatOperands.push(op);
-            }
-        });
-        if (flatOperands.length === 0) {
-            throw new Error('At least one operand is required after flattening.');
-        }
-        const result = flatOperands.reduce((a, b, i) => {
-            return i === 0 ? a : a - b;
-        });
-        return result;
-    }
-    multiply(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        // @ts-ignore
-        return operands.reduce((a, b) => {
-            if (Array.isArray(b)) {
-                return a * this.multiply(...b);
-            }
-            else {
-                return a * b;
-            }
-        }, 1);
-    }
-    divide(...operands) {
-        if (operands.length === 0) {
-            throw new Error('At least one operand is required.');
-        }
-        let flatOperands = [];
-        operands.forEach((op) => {
-            if (Array.isArray(op)) {
-                flatOperands = [...flatOperands, ...op];
-            }
-            else {
-                flatOperands.push(op);
-            }
-        });
-        if (flatOperands.length === 0) {
-            throw new Error('At least one operand is required after flattening.');
-        }
-        const result = flatOperands.reduce((a, b, i) => {
-            if (b === 0) {
-                return NaN;
-            }
-            return i === 0 ? a : a / b;
-        });
-        if (isNaN(result)) {
-            return NaN;
-        }
-        return result;
-    }
 }
 exports.NumberHandler = NumberHandler;

package/build/services/signaler/stream.js

@@ -5,7 +5,7 @@ const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-const MAX_RETRIES = 4; //max delay (10s using exponential backoff);
+const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
 const MAX_TIMEOUT_MS = 60000;
 const GRADUATED_INTERVAL_MS = 5000;
 const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
@@ -158,8 +158,11 @@ class StreamSignaler {
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
         const tryCount = Math.min(input.metadata.try || 0, MAX_RETRIES);
-        if (maxRetries >= tryCount) {
-            return [true, Math.pow(10, tryCount)];
+        //only possible values for maxRetries are 1, 2, 3
+        //only possible values for tryCount are 0, 1, 2
+        if (maxRetries > tryCount) {
+            // 10ms, 100ms, or 1000ms delays between system retries
+            return [true, Math.pow(10, tryCount + 1)];
         }
         return [false, 0];
     }

package/build/services/store/index.js

@@ -58,7 +58,7 @@ class StoreService {
         };
         this.redisClient = redisClient;
     }
-    async init(namespace = key_1.PSNS, appId, logger) {
+    async init(namespace = key_1.HMNS, appId, logger) {
         this.namespace = namespace;
         this.appId = appId;
         this.logger = logger;
@@ -111,7 +111,7 @@ class StoreService {
             if (bCreate) {
                 const packageJson = await Promise.resolve().then(() => __importStar(require('../../package.json')));
                 const version = packageJson['version'] || '0.0.0';
-                settings = { namespace: key_1.PSNS, version };
+                settings = { namespace: key_1.HMNS, version };
                 await this.setSettings(settings);
                 return settings;
             }

package/build/services/stream/clients/ioredis.js

@@ -7,7 +7,7 @@ class IORedisStreamService extends index_1.StreamService {
     constructor(redisClient) {
         super(redisClient);
     }
-    async init(namespace = key_1.PSNS, appId, logger) {
+    async init(namespace = key_1.HMNS, appId, logger) {
         this.namespace = namespace;
         this.logger = logger;
         this.appId = appId;

package/build/services/stream/clients/redis.js

@@ -7,7 +7,7 @@ class RedisStreamService extends index_1.StreamService {
     constructor(redisClient) {
         super(redisClient);
     }
-    async init(namespace = key_1.PSNS, appId, logger) {
+    async init(namespace = key_1.HMNS, appId, logger) {
         this.namespace = namespace;
         this.logger = logger;
         this.appId = appId;

package/build/services/sub/clients/ioredis.js

@@ -7,7 +7,7 @@ class IORedisSubService extends index_1.SubService {
     constructor(redisClient) {
         super(redisClient);
     }
-    async init(namespace = key_1.PSNS, appId, engineId, logger) {
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
         this.namespace = namespace;
         this.logger = logger;
         this.appId = appId;

package/build/services/sub/clients/redis.js

@@ -7,7 +7,7 @@ class RedisSubService extends index_1.SubService {
     constructor(redisClient) {
         super(redisClient);
     }
-    async init(namespace = key_1.PSNS, appId, engineId, logger) {
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
         this.namespace = namespace;
         this.logger = logger;
         this.appId = appId;

package/build/types/durable.d.ts

@@ -7,7 +7,7 @@ type WorkflowOptions = {
     workflowTrace?: string;
     workflowSpan?: string;
 };
-type ActivityDataType = {
+type ActivityWorkflowDataType = {
     activityName: string;
     arguments: any[];
     workflowId: string;
@@ -32,9 +32,14 @@ type Registry = {
 };
 type WorkerConfig = {
     connection: Connection;
-    namespace: string;
+    namespace?: string;
     taskQueue: string;
     workflow: Function;
+    options?: WorkerOptions;
+};
+type WorkerOptions = {
+    maxSystemRetries?: number;
+    backoffExponent?: number;
 };
 type ContextType = {
     workflowId: string;
@@ -54,4 +59,4 @@ type ActivityConfig = {
         maximumInterval: string;
     };
 };
-export { ActivityConfig, ActivityDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkflowDataType, WorkflowOptions, };
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkerOptions, WorkflowDataType, WorkflowOptions, };

package/build/types/index.d.ts

@@ -3,6 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkerOptions, WorkflowDataType, WorkflowOptions, } from './durable';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/modules/errors.ts

@@ -12,6 +12,35 @@ class SetStateError extends Error {
   }
 }
 
+class DurableTimeoutError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 596;
+  }
+}
+class DurableMaxedError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 597;
+  }
+}
+class DurableFatalError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 598;
+  }
+}
+class DurableRetryError extends Error {
+  code: number;
+  constructor(message: string) {
+    super(message);
+    this.code = 599;
+  }
+}
+
 class MapDataError extends Error {
   constructor() {
     super("Error occurred while mapping data");
@@ -52,4 +81,16 @@ class CollationError extends Error {
   }
 }
 
-export { CollationError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };
+export {
+  CollationError,
+  DurableTimeoutError,
+  DurableMaxedError,
+  DurableFatalError,
+  DurableRetryError,
+  DuplicateJobError,
+  GetStateError,
+  SetStateError,
+  MapDataError,
+  RegisterTimeoutError,
+  ExecActivityError
+};

package/modules/key.ts

@@ -27,7 +27,7 @@
  */
 
 //default namespace for hotmesh
-const PSNS = "hmsh";
+const HMNS = "hmsh";
 
 //these are the entity types that are stored in the key/value store
 enum KeyType {
@@ -126,4 +126,4 @@ class KeyService {
   }
 }
 
-export { KeyService, KeyType, KeyStoreParams, PSNS };
+export { KeyService, KeyType, KeyStoreParams, HMNS };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "description": "Durable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -43,6 +43,7 @@
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:fatal": "NODE_ENV=test jest ./tests/durable/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
   },

package/services/activities/activity.ts

@@ -4,7 +4,6 @@ import {
   getValueByPath,
   restoreHierarchy } from '../../modules/utils';
 import { CollatorService } from '../collator';
-import { DimensionService } from '../dimension';
 import { EngineService } from '../engine';
 import { ILogger } from '../logger';
 import { MapperService } from '../mapper';
@@ -84,6 +83,7 @@ class Activity {
       if (this.doesHook()) {
         //sleep and wait to awaken upon a signal
         await this.registerHook(multi);
+        this.mapOutputData();
         this.mapJobData();
         await this.setState(multi);
         await CollatorService.authorizeReentry(this, multi);
@@ -94,6 +94,7 @@ class Activity {
       } else {
         //end the activity and transition to its children
         this.adjacencyList = await this.filterAdjacent();
+        this.mapOutputData();
         this.mapJobData();
         await this.setState(multi);
         await CollatorService.notarizeEarlyCompletion(this, multi);
@@ -322,6 +323,17 @@ class Activity {
     }
   }
 
+  mapOutputData(): void {
+    //activity YAML may include output map data that produces/extends activity output data.
+    if(this.config.output?.maps) {
+      const mapper = new MapperService(this.config.output.maps, this.context);
+      const actOutData = mapper.mapRules();
+      const activityId = this.metadata.aid;
+      const data = { ...this.context[activityId].output, ...actOutData };
+      this.context[activityId].output.data = data;
+    }
+  }
+
   async registerTimeout(): Promise<void> {
     //set timeout in support of hook and/or duplex
   }
@@ -526,7 +538,7 @@ class Activity {
 
   resolveAdjacentDad(): string {
     //concat self and child dimension (all children (leg 1) begin life at 0)
-    return `${this.resolveDad()}${DimensionService.getSeed(0)}`;
+    return `${this.resolveDad()}${CollatorService.getDimensionalSeed(0)}`;
   };
 
   async filterAdjacent(): Promise<StreamData[]> {

package/services/activities/cycle.ts

@@ -81,13 +81,18 @@ class Cycle extends Activity {
    * pattern allows for retries without violating the DAG.
    */
   async cycleAncestorActivity(multi: RedisMulti): Promise<string> {
+    //Cycle activity L1 is a standin for the target ancestor L1.
+    //Input data mapping (mapInputData) allows for the 
+    //next dimensonal thread to execute with different
+    //input data than the current dimensional thread
+    this.mapInputData();
     const streamData: StreamData = {
       metadata: {
         dad: CollatorService.resolveReentryDimension(this),
         jid: this.context.metadata.jid,
         aid: this.config.ancestor,
       },
-      data: {} //todo: verify immutability, before enabling: `this.context.data`
+      data: this.context.data
     };
     return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi)) as string;
   }

package/services/activities/trigger.ts

@@ -3,7 +3,6 @@ import { DuplicateJobError } from '../../modules/errors';
 import { formatISODate, getTimeSeries } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
-import { DimensionService } from '../dimension';
 import { EngineService } from '../engine';
 import { Pipe } from '../pipe';
 import { ReporterService } from '../reporter';
@@ -99,7 +98,7 @@ class Trigger extends Activity {
 
     const utc = formatISODate(new Date());
     const { id, version } = await this.engine.getVID();
-    this.initDimensionalAddress(DimensionService.getSeed());
+    this.initDimensionalAddress(CollatorService.getDimensionalSeed());
     const activityMetadata = {
       ...this.metadata,
       jid: jobId,
@@ -119,7 +118,7 @@ class Trigger extends Activity {
         trc: this.context.metadata.trc,
         spn: this.context.metadata.spn,
         jid: jobId,
-        dad: DimensionService.getSeed(), //top-level job implicitly uses `,0`
+        dad: CollatorService.getDimensionalSeed(), //top-level job implicitly uses `,0`
         key: jobKey,
         jc: utc,
         ju: utc,

package/services/collator/index.ts

@@ -85,7 +85,7 @@ class CollatorService {
   static async notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
     //1) ALWAYS actualize leg2 dimension (+1)
     //2) IF the activity is used in a cycle, don't close leg 2!
-    const decrement = activity.config.cycle ? 0 : -1_000_000_000_000;
+    const decrement = activity.config.cycle ? 0 : 1_000_000_000_000;
     return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - decrement, this.getDimensionalAddress(activity), multi);
   };
 
@@ -234,6 +234,17 @@ class CollatorService {
   static isActivityComplete(status: number): boolean {
     return (status - 0) <= 0;
   }
+
+  /**
+   * All activities exist on a dimensional plane. Zero
+   * is the default. A value of 
+   * `AxY,0,0,0,0,1,0,0` would reflect that
+   * an ancestor activity was dimensionalized beyond
+   * the default.
+   */
+  static getDimensionalSeed(index = 0): string {
+    return `,${index}`;
+  }
 }
 
 export { CollatorService };