@hotmeshio/hotmesh 0.0.9 → 0.0.11

package/services/durable/factory.ts

@@ -1,4 +1,21 @@
-const getWorkflowYAML = (topic: string, version = '1') => {
+/**
+ * 1) `maxSystemRetries` | can be 0 to 3 and represents milliseconds;
+ * if there is an error, the workflow will retry up to `maxSystemRetries` times
+ * delaying by 10, 100, and 1000ms; this is a system level retry
+ * and is not configurable. It exists to handle intermittent network
+ * errors. (NOTE: each retry spawns a new transition stream)
+ * 
+ * 2) `backoffExponent` | can be any number and represents `seconds` when applied;
+ * retries will happen indefinitely and adhere to the
+ * exponential backoff algorithm by multiplying by `backoffExponent`.
+ * For example, if `backoffExponent` is 10, the workflow will retry
+ * in 10s, 100s, 1000s, 10000s, etc.
+ *
+ * EXAMPLE | Using `maxSystemRetries = 3` and `backoffExponent = 10`, errant
+ * workflows will be retried on the following schedule (8 times in 27 hours):
+ * => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ */
+const getWorkflowYAML = (topic: string, version = '1', maxSystemRetries = 2, backoffExponent = 10) => {
   return `app:
   id: ${topic}
   version: '${version}'
@@ -30,10 +47,20 @@ const getWorkflowYAML = (topic: string, version = '1') => {
         a1:
           type: activity
           cycle: true
-  
+          output:
+            schema:
+              type: object
+              properties:
+                duration:
+                  type: number
+            maps:
+              duration: ${backoffExponent}
+
         w1:
           type: worker
           topic: ${topic}
+          retry:
+            '599': [${maxSystemRetries}]
           input:
             schema:
               type: object
@@ -55,18 +82,33 @@ const getWorkflowYAML = (topic: string, version = '1') => {
             maps:
               response: '{$self.output.data.response}'
 
+        a599:
+          title: Sleep before trying again
+          type: activity
+          sleep: "{a1.output.data.duration}"
+
         c1:
+          title: Goto Activity a1
           type: cycle
           ancestor: a1
+          input:
+            maps:
+              duration:
+                '@pipe':
+                  - ['{a1.output.data.duration}', ${backoffExponent}]
+                  - ['{@math.multiply}']
+
       transitions:
         t1:
           - to: a1
         a1:
           - to: w1
         w1:
-          - to: c1
+          - to: a599
             conditions:
-              code: 500
+              code: 599
+        a599:
+        - to: c1
 `;
 }
 

package/services/durable/handle.ts

@@ -16,22 +16,37 @@ export class WorkflowHandleService {
     let status = await this.hotMesh.getStatus(this.workflowId);
     const topic = `${this.workflowTopic}.${this.workflowId}`;
   
-    if (status == 0) {
-      return (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response;
-    }
-  
     return new Promise((resolve, reject) => {
       let isResolved = false;
       //common fulfill/unsubscribe
-      const complete = async (response?: any) => {
+      const complete = async (response?: any, err?: string) => {
         if (isResolved) return;
         isResolved = true;
         this.hotMesh.unsub(topic);
-        resolve(response || (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response);
+        if (err) {
+          return reject(JSON.parse(err));
+        } else if (!response) {
+          const state = await this.hotMesh.getState(this.workflowTopic, this.workflowId);
+          if (!state.data && state.metadata.err) {
+            return reject(JSON.parse(state.metadata.err));
+          }
+          response = state.data?.response;
+        }
+        resolve(response);
       };
-      this.hotMesh.sub(topic, async (topic: string, message: JobOutput) => {
-        await complete(message.data?.response);
+      //check for done
+      if (status == 0) {
+        return complete();
+      }
+      //subscribe to topic
+      this.hotMesh.sub(topic, async (topic: string, state: JobOutput) => {
+        if (!state.data && state.metadata.err) {
+          await complete(null, state.metadata.err);
+        } else {
+          await complete(state.data?.response);
+        }
       });
+      //resolve for race condition
       setTimeout(async () => {
         status = await this.hotMesh.getStatus(this.workflowId);
         if (status == 0) {

package/services/durable/native.ts

@@ -21,7 +21,6 @@ async function run() {
   });
   const worker = await Worker.create({
     connection,
-    namespace: 'default',
     taskQueue: 'hello-world',
     workflow: workflows.example,
     activities,

package/services/durable/worker.ts

@@ -2,8 +2,18 @@ import { asyncLocalStorage } from './asyncLocalStorage';
 import { HotMeshService as HotMesh } from '../hotmesh';
 import { RedisClass, RedisOptions } from '../../types/redis';
 import { StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
-import { ActivityDataType, Connection, Registry, WorkerConfig, WorkflowDataType } from "../../types/durable";
+import { ActivityWorkflowDataType,
+  Connection,
+  Registry,
+  WorkerConfig,
+  WorkerOptions,
+  WorkflowDataType } from "../../types/durable";
 import { getWorkflowYAML, getActivityYAML } from './factory';
+import {
+  DurableFatalError,
+  DurableMaxedError,
+  DurableRetryError,
+  DurableTimeoutError } from '../../modules/errors';
 
 /*
 Here is an example of how the methods in this file are used:
@@ -25,7 +35,6 @@ async function run() {
   });
   const worker = await Worker.create({
     connection,
-    namespace: 'default',
     taskQueue: 'hello-world',
     workflow: workflows.example,
     activities,
@@ -46,7 +55,7 @@ export class WorkerService {
   workflowRunner: HotMesh;
   activityRunner: HotMesh;
 
-  static getHotMesh = async (worflowTopic: string) => {
+  static getHotMesh = async (worflowTopic: string, options?: WorkerOptions) => {
     if (WorkerService.instances.has(worflowTopic)) {
       return await WorkerService.instances.get(worflowTopic);
     }
@@ -55,17 +64,17 @@ export class WorkerService {
       engine: { redis: { ...WorkerService.connection } }
     });
     WorkerService.instances.set(worflowTopic, hotMesh);
-    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML);
+    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML, options);
     return hotMesh;
   }
 
-  static async activateWorkflow(hotMesh: HotMesh, topic: string, factory: Function) {
+  static async activateWorkflow(hotMesh: HotMesh, topic: string, dagFactory: Function, options: WorkerOptions = {}) {
     const version = '1';
     const app = await hotMesh.engine.store.getApp(topic);
     const appVersion = app?.version;
     if(!appVersion) {
       try {
-        await hotMesh.deploy(factory(topic, version));
+        await hotMesh.deploy(dagFactory(topic, version, options.maxSystemRetries, options.backoffExponent));
         await hotMesh.activate(version);
       } catch (err) {
         hotMesh.engine.logger.error('durable-worker-deploy-activate-err', err);
@@ -118,7 +127,7 @@ export class WorkerService {
     worker.activityRunner = await worker.initActivityWorkflow(config, activityTopic);
     await WorkerService.activateWorkflow(worker.activityRunner, activityTopic, getActivityYAML);
     worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
-    await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML);
+    await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML, config.options);
     return worker;
   }
 
@@ -161,7 +170,7 @@ export class WorkerService {
     return async (data: StreamData): Promise<StreamDataResponse> => {
       try {
         //always run the activity function when instructed; return the response
-        const activityInput = data.data as unknown as ActivityDataType;
+        const activityInput = data.data as unknown as ActivityWorkflowDataType;
         const activityName = activityInput.activityName;
         const activityFunction = WorkerService.activityRegistry[activityName];
         const pojoResponse = await activityFunction.apply(this, activityInput.arguments);
@@ -173,12 +182,16 @@ export class WorkerService {
         };
       } catch (err) {
         this.activityRunner.engine.logger.error('durable-worker-activity-err', err);
+        if (!(err instanceof DurableTimeoutError) &&
+          !(err instanceof DurableMaxedError) &&
+          !(err instanceof DurableFatalError)) {
+          err = new DurableRetryError(err.message); 
+        }
         return {
           status: StreamStatus.ERROR,
-          code: 500,
-          message: err.message,
+          code: err.code,
           metadata: { ...data.metadata },
-          data: { error: err }
+          data: { message: err.message }
         } as StreamDataResponse;
       }
     }
@@ -258,11 +271,12 @@ export class WorkerService {
           data: { response: workflowResponse }
         };
       } catch (err) {
+        // 59* - Durable*Error
         return {
           status: StreamStatus.ERROR,
-          code: 500,
+          code: err.code || new DurableRetryError(err.message).code,
           metadata: { ...data.metadata },
-          data: { error: err }
+          data: { message: err.message, type: err.name }
         } as StreamDataResponse;
       }
     }

package/services/durable/workflow.ts

@@ -62,7 +62,7 @@ export class WorkflowService {
     const client = new Client({
       connection: await Connection.connect(WorkerService.connection),
     });
-    //todo: should I allow-cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
+    //todo: allow cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
     const handle = await client.workflow.start({
       ...options,
       workflowId: `${workflowId}${options.workflowId}`, //concat

package/services/hotmesh/index.ts

@@ -1,5 +1,5 @@
 import { v4 as uuidv4 } from 'uuid';
-import { PSNS } from '../../modules/key';
+import { HMNS } from '../../modules/key';
 import { EngineService } from '../engine';
 import { LoggerService, ILogger } from '../logger';
 import { StreamSignaler } from '../signaler/stream';
@@ -33,7 +33,7 @@ class HotMeshService {
 
   verifyAndSetNamespace(namespace?: string) {
     if (!namespace) {
-      this.namespace = PSNS;
+      this.namespace = HMNS;
     } else if (!namespace.match(/^[A-Za-z0-9-]+$/)) {
       throw new Error(`config.namespace [${namespace}] is invalid`);
     } else {

package/services/pipe/functions/math.ts

@@ -1,4 +1,78 @@
 class MathHandler {
+  add(...operands: (number | number[])[]): number {
+    // @ts-ignore
+    return operands.reduce((a: number, b: number | number[]) => {
+      if (Array.isArray(b)) {
+        return a + this.add(...b);
+      } else {
+        return a + b;
+      }
+    }, 0);
+  }
+
+  subtract(...operands: (number | number[])[]): number {
+    if (operands.length === 0) {
+      throw new Error('At least one operand is required.');
+    }
+    let flatOperands: number[] = [];
+    operands.forEach((op: number | number[]) => {
+      if (Array.isArray(op)) {
+        flatOperands = [...flatOperands, ...op];
+      } else {
+        flatOperands.push(op);
+      }
+    });
+    if (flatOperands.length === 0) {
+      throw new Error('At least one operand is required after flattening.');
+    }
+    const result = flatOperands.reduce((a: number, b: number, i: number) => {
+      return i === 0 ? a : a - b;
+    });
+    return result;
+  }
+
+  multiply(...operands: (number | number[])[]): number {
+    if (operands.length === 0) {
+      throw new Error('At least one operand is required.');
+    }
+  
+    // @ts-ignore
+    return operands.reduce((a: number, b: number | number[]) => {
+      if (Array.isArray(b)) {
+        return a * this.multiply(...b);
+      } else {
+        return a * b;
+      }
+    }, 1);
+  }
+  
+  divide(...operands: (number | number[])[]): number {
+    if (operands.length === 0) {
+      throw new Error('At least one operand is required.');
+    }
+    let flatOperands: number[] = [];
+    operands.forEach((op: number | number[]) => {
+      if (Array.isArray(op)) {
+        flatOperands = [...flatOperands, ...op];
+      } else {
+        flatOperands.push(op);
+      }
+    });
+    if (flatOperands.length === 0) {
+      throw new Error('At least one operand is required after flattening.');
+    }
+    const result = flatOperands.reduce((a: number, b: number, i: number) => {
+      if (b === 0) {
+        return NaN;
+      }
+      return i === 0 ? a : a / b;
+    });
+    if (isNaN(result)) {
+      return NaN;
+    }
+    return result;
+  }
+
   abs(x: number): number {
     return Math.abs(x);
   }

package/services/pipe/functions/number.ts

@@ -70,81 +70,6 @@ class NumberHandler {
   round(input: number): number {
     return Math.round(input);
   }
-
-  add(...operands: (number | number[])[]): number {
-    // @ts-ignore
-    return operands.reduce((a: number, b: number | number[]) => {
-      if (Array.isArray(b)) {
-        return a + this.add(...b);
-      } else {
-        return a + b;
-      }
-    }, 0);
-  }
-
-  subtract(...operands: (number | number[])[]): number {
-    if (operands.length === 0) {
-      throw new Error('At least one operand is required.');
-    }
-    let flatOperands: number[] = [];
-    operands.forEach((op: number | number[]) => {
-      if (Array.isArray(op)) {
-        flatOperands = [...flatOperands, ...op];
-      } else {
-        flatOperands.push(op);
-      }
-    });
-    if (flatOperands.length === 0) {
-      throw new Error('At least one operand is required after flattening.');
-    }
-    const result = flatOperands.reduce((a: number, b: number, i: number) => {
-      return i === 0 ? a : a - b;
-    });
-    return result;
-  }
-  
-  
-  multiply(...operands: (number | number[])[]): number {
-    if (operands.length === 0) {
-      throw new Error('At least one operand is required.');
-    }
-  
-    // @ts-ignore
-    return operands.reduce((a: number, b: number | number[]) => {
-      if (Array.isArray(b)) {
-        return a * this.multiply(...b);
-      } else {
-        return a * b;
-      }
-    }, 1);
-  }
-  
-  divide(...operands: (number | number[])[]): number {
-    if (operands.length === 0) {
-      throw new Error('At least one operand is required.');
-    }
-    let flatOperands: number[] = [];
-    operands.forEach((op: number | number[]) => {
-      if (Array.isArray(op)) {
-        flatOperands = [...flatOperands, ...op];
-      } else {
-        flatOperands.push(op);
-      }
-    });
-    if (flatOperands.length === 0) {
-      throw new Error('At least one operand is required after flattening.');
-    }
-    const result = flatOperands.reduce((a: number, b: number, i: number) => {
-      if (b === 0) {
-        return NaN;
-      }
-      return i === 0 ? a : a / b;
-    });
-    if (isNaN(result)) {
-      return NaN;
-    }
-    return result;
-  }  
 }
 
 export { NumberHandler };

package/services/signaler/stream.ts

@@ -16,7 +16,7 @@ import {
   StreamStatus
 } from '../../types/stream';
 
-const MAX_RETRIES = 4; //max delay (10s using exponential backoff);
+const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
 const MAX_TIMEOUT_MS = 60000;
 const GRADUATED_INTERVAL_MS = 5000;
 const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
@@ -189,8 +189,11 @@ class StreamSignaler {
     const policy = policies?.[errorCode];
     const maxRetries = policy?.[0];
     const tryCount = Math.min(input.metadata.try || 0,  MAX_RETRIES);
-    if (maxRetries >= tryCount) {
-      return[true, Math.pow(10, tryCount)];
+    //only possible values for maxRetries are 1, 2, 3
+    //only possible values for tryCount are 0, 1, 2
+    if (maxRetries > tryCount) {
+      // 10ms, 100ms, or 1000ms delays between system retries
+      return[true, Math.pow(10, tryCount + 1)];
     }
     return [false, 0];
   }

package/services/store/index.ts

@@ -2,7 +2,7 @@ import {
   KeyService,
   KeyStoreParams,
   KeyType, 
-  PSNS} from '../../modules/key';
+  HMNS} from '../../modules/key';
 import { ILogger } from '../logger';
 import { MDATA_SYMBOLS, SerializerService as Serializer } from '../serializer';
 import { Cache } from './cache';
@@ -119,7 +119,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     this.redisClient = redisClient;
   }
 
-  async init(namespace = PSNS, appId: string, logger: ILogger): Promise<HotMeshApps> {
+  async init(namespace = HMNS, appId: string, logger: ILogger): Promise<HotMeshApps> {
     this.namespace = namespace;
     this.appId = appId;
     this.logger = logger;
@@ -178,7 +178,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       if (bCreate) {
         const packageJson = await import('../../package.json');
         const version: string = packageJson['version'] || '0.0.0';
-        settings = { namespace: PSNS, version } as HotMeshSettings;
+        settings = { namespace: HMNS, version } as HotMeshSettings;
         await this.setSettings(settings);
         return settings;
       }

package/services/stream/clients/ioredis.ts

@@ -1,4 +1,4 @@
-import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { KeyService, KeyStoreParams, KeyType, HMNS } from '../../../modules/key';
 import { ILogger } from '../../logger';
 import { StreamService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
@@ -14,7 +14,7 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
     super(redisClient);
   }
 
-  async init(namespace = PSNS, appId: string, logger: ILogger): Promise<void> {
+  async init(namespace = HMNS, appId: string, logger: ILogger): Promise<void> {
     this.namespace = namespace;
     this.logger = logger;
     this.appId = appId;

package/services/stream/clients/redis.ts

@@ -1,4 +1,4 @@
-import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { KeyService, KeyStoreParams, KeyType, HMNS } from '../../../modules/key';
 import { ILogger } from '../../logger';
 import { StreamService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
@@ -14,7 +14,7 @@ class RedisStreamService extends StreamService<RedisClientType, RedisMultiType>
     super(redisClient);
   }
 
-  async init(namespace = PSNS, appId: string, logger: ILogger): Promise<void> {
+  async init(namespace = HMNS, appId: string, logger: ILogger): Promise<void> {
     this.namespace = namespace;
     this.logger = logger;
     this.appId = appId;

package/services/sub/clients/ioredis.ts

@@ -1,4 +1,4 @@
-import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { KeyService, KeyStoreParams, KeyType, HMNS } from '../../../modules/key';
 import { ILogger } from '../../logger';
 import { SubService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
@@ -14,7 +14,7 @@ class IORedisSubService extends SubService<RedisClientType, RedisMultiType> {
     super(redisClient);
   }
 
-  async init(namespace = PSNS, appId: string, engineId: string, logger: ILogger): Promise<void> {
+  async init(namespace = HMNS, appId: string, engineId: string, logger: ILogger): Promise<void> {
     this.namespace = namespace;
     this.logger = logger;
     this.appId = appId;

package/services/sub/clients/redis.ts

@@ -1,4 +1,4 @@
-import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { KeyService, KeyStoreParams, KeyType, HMNS } from '../../../modules/key';
 import { ILogger } from '../../logger';
 import { SubService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
@@ -14,7 +14,7 @@ class RedisSubService extends SubService<RedisClientType, RedisMultiType> {
     super(redisClient);
   }
 
-  async init(namespace = PSNS, appId: string, engineId: string, logger: ILogger): Promise<void> {
+  async init(namespace = HMNS, appId: string, engineId: string, logger: ILogger): Promise<void> {
     this.namespace = namespace;
     this.logger = logger;
     this.appId = appId;

package/types/durable.ts

@@ -9,7 +9,7 @@ type WorkflowOptions = {
   workflowSpan?: string;
 }
 
-type ActivityDataType = {
+type ActivityWorkflowDataType = {
   activityName: string;
   arguments: any[];
   workflowId: string;
@@ -39,9 +39,15 @@ type Registry  = {
 
 type WorkerConfig = {
   connection: Connection;
-  namespace: string; //`appid` in the YAML (e.g, 'default')
+  namespace?: string; //`appid` in the YAML (e.g, 'default')
   taskQueue: string; //`subscribes` in the YAML (e.g, 'hello-world')
-  workflow: Function //target function to run
+  workflow: Function; //target function to run
+  options?: WorkerOptions;
+}
+
+type WorkerOptions = {
+  maxSystemRetries?: number; //1-3 (10ms, 100ms, 1_000ms)
+  backoffExponent?: number; //2-10ish
 }
 
 type ContextType = {
@@ -67,7 +73,7 @@ type ActivityConfig = {
 
 export {
   ActivityConfig,
-  ActivityDataType,
+  ActivityWorkflowDataType,
   ClientConfig,
   ContextType,
   ConnectionConfig,
@@ -76,6 +82,7 @@ export {
   ProxyType,
   Registry,
   WorkerConfig,
+  WorkerOptions,
   WorkflowDataType,
   WorkflowOptions,
-};
+};

package/types/index.ts

@@ -26,6 +26,21 @@ export { CacheMode } from './cache';
 export {
   CollationFaultType,
   CollationStage } from './collator';
+export {
+    ActivityConfig,
+    ActivityWorkflowDataType,
+    ClientConfig,
+    ContextType,
+    ConnectionConfig,
+    Connection,
+    NativeConnection,
+    ProxyType,
+    Registry,
+    WorkerConfig,
+    WorkerOptions,
+    WorkflowDataType,
+    WorkflowOptions,
+  }from './durable'
 export {
   HookCondition,
   HookConditions,

package/build/services/dimension/index.d.ts

@@ -1,29 +0,0 @@
-import { HotMeshGraph } from '../../types/hotmesh';
-declare class DimensionService {
-    static targetLength: number;
-    /**
-     * entry point for compiler-type activities. This is called by the compiler
-     * to bind the sorted activity IDs to the trigger activity. These are then used
-     * at runtime by the activities to track job/activity status.
-     * @param graphs
-     */
-    static compile(graphs: HotMeshGraph[]): void;
-    /**
-     * All activities exist on a dimensional plane. Zero
-     * is the default and is implied if no dimension is
-     * present in the hash item key. EVERY value in the
-     * job ledger is dimensionalized even if the dimension
-     * is not present. The key, `AaA`, might not contain
-     * a dimensional index, but it is still implicitly
-     * dimensionalized as `AaA,0` (assuming a trigger).
-     * A value of `AxY,0,0,0,0,1,0,0` would reflect that
-     * an ancestor activity was dimensionalized beyond
-     * the default. The dimensional string must
-     * be included if not zero. There is likely a preceding
-     * sibling dimension, so it would not need to include
-     * the suffix, so these addresses are equivalent:
-     * `AxY,0,0,0,0,0,0,0` == `AxY` for said sibling.
-     */
-    static getSeed(index?: number): string;
-}
-export { DimensionService };