@hotmeshio/hotmesh 0.0.15 → 0.0.17

package/README.md

@@ -22,7 +22,7 @@ The HotMesh SDK is designed to keep your code front-and-center. Write functions
     }
 
     export async function saludar(nombre: string): Promise<string> {
-      Math.random() > 0.5 && throw new Error('Random error');
+      if (Math.random() > 0.5) throw new Error('Random error');
       return `¡Hola, ${nombre}!`;
     }
     ```

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.15",
+    "version": "0.0.17",
     "description": "Unbreakable Workflows",
     "main": "build/index.js",
     "types": "build/index.d.ts",

package/build/services/activities/activity.js

@@ -159,8 +159,7 @@ class Activity {
             return jobStatus;
         }
         catch (error) {
-            if (error instanceof errors_1.CollationError) {
-                //caused by external over-signaling; the job is complete
+            if (error instanceof errors_1.CollationError && error.fault === 'inactive') {
                 this.logger.info('process-hook-event-inactive-error', { error });
                 return;
             }
@@ -206,6 +205,10 @@ class Activity {
             this.transitionAdjacent(multiResponse, telemetry);
         }
         catch (error) {
+            if (error instanceof errors_1.CollationError && error.fault === 'inactive') {
+                this.logger.info('process-event-inactive-error', { error });
+                return;
+            }
             this.logger.error('activity-process-event-error', { error });
             telemetry && telemetry.setActivityError(error.message);
             throw error;
@@ -493,7 +496,12 @@ class Activity {
     }
     async transition(adjacencyList, jobStatus) {
         let mIds = [];
-        if (jobStatus <= 0 || this.config.emit) {
+        //emit can be a mapping (allows emissions to be driven by the job state)
+        let emit = false;
+        if (this.config.emit) {
+            emit = pipe_1.Pipe.resolve(this.config.emit, this.context);
+        }
+        if (jobStatus <= 0 || emit) {
             //activity should not send 'emit' if the job is truly over
             const isTrueEmit = jobStatus > 0;
             await this.engine.runJobCompletionTasks(this.context, isTrueEmit);

package/build/services/activities/trigger.js

@@ -151,7 +151,7 @@ class Trigger extends activity_1.Activity {
     }
     async setStats(multi) {
         const md = this.context.metadata;
-        if (this.config.stats?.measures) {
+        if (md.key && this.config.stats?.measures) {
             const config = await this.engine.getVID();
             const reporter = new reporter_1.ReporterService(config, this.store, this.logger);
             await this.store.setStats(md.key, md.jid, md.ts, reporter.resolveTriggerStatistics(this.config, this.context), config, multi);

package/build/services/durable/client.d.ts

@@ -10,6 +10,7 @@ export declare class ClientService {
     workflow: {
         start: (options: WorkflowOptions) => Promise<WorkflowHandleService>;
         signal: (signalId: string, data: Record<any, any>) => Promise<string>;
+        getHandle: (taskQueue: string, workflowName: string, workflowId: string) => Promise<WorkflowHandleService>;
     };
     activateWorkflow(hotMesh: HotMesh, appId?: string, version?: string): Promise<void>;
     static shutdown(): Promise<void>;

package/build/services/durable/client.js

@@ -87,6 +87,7 @@ class ClientService {
                 const hotMeshClient = await this.getHotMeshClient(workflowTopic);
                 const payload = {
                     arguments: [...options.args],
+                    parentWorkflowId: options.parentWorkflowId,
                     workflowId: options.workflowId || (0, nanoid_1.nanoid)(),
                     workflowTopic: workflowTopic,
                     backoffCoefficient: options.config?.backoffCoefficient || factory_1.DEFAULT_COEFFICIENT,
@@ -97,6 +98,11 @@ class ClientService {
             },
             signal: async (signalId, data) => {
                 return await (await this.getHotMeshClient('durable.wfs.signal')).hook('durable.wfs.signal', { id: signalId, data });
+            },
+            getHandle: async (taskQueue, workflowName, workflowId) => {
+                const workflowTopic = `${taskQueue}-${workflowName}`;
+                const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+                return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
             }
         };
         this.connection = config.connection;

package/build/services/durable/factory.d.ts

@@ -2,6 +2,11 @@
  * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
  *       workflows will be retried on the following schedule (8 times in 27 hours):
  *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ * 593:
+ * 594: waitforsignal
+ * 595: sleep
+ * 596, 597, 598: fatal
+ * 599: retry
  */
 declare const getWorkflowYAML: (app: string, version: string) => string;
 declare const APP_VERSION = "1";

package/build/services/durable/factory.js

@@ -3,6 +3,11 @@
  * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
  *       workflows will be retried on the following schedule (8 times in 27 hours):
  *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ * 593:
+ * 594: waitforsignal
+ * 595: sleep
+ * 596, 597, 598: fatal
+ * 599: retry
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WFS_HOOK_ID = exports.WFSC_PUBLISHES_TOPIC = exports.WFSC_SUBSCRIBES_TOPIC = exports.WFS_PUBLISHES_TOPIC = exports.WFS_SUBSCRIBES_TOPIC = exports.DEFAULT_COEFFICIENT = exports.SLEEP_HOOK_ID = exports.ACTIVITY_HOOK_ID = exports.HOOK_ID = exports.PUBLISHES_TOPIC = exports.SUBSCRIBES_TOPIC = exports.SLEEP_PUBLISHES_TOPIC = exports.SLEEP_SUBSCRIBES_TOPIC = exports.ACTIVITY_PUBLISHES_TOPIC = exports.ACTIVITY_SUBSCRIBES_TOPIC = exports.APP_ID = exports.APP_VERSION = exports.getWorkflowYAML = void 0;
@@ -20,6 +25,8 @@ const getWorkflowYAML = (app, version) => {
         schema:
           type: object
           properties:
+            parentWorkflowId:
+              type: string
             workflowId:
               type: string
             arguments:
@@ -42,6 +49,11 @@ const getWorkflowYAML = (app, version) => {
           type: trigger
           stats:
             id: '{$self.input.data.workflowId}'
+            key: '{$self.input.data.parentWorkflowId}'
+            granularity: infinity
+            measures:
+              - measure: index
+                target: '{$self.input.data.parentWorkflowId}'
           job:
             maps:
               done: false
@@ -61,6 +73,7 @@ const getWorkflowYAML = (app, version) => {
         w1:
           type: worker
           topic: '{t1.output.data.workflowTopic}'
+          emit: '{$job.data.done}'
           retry:
             '599': [2]
           input:
@@ -118,6 +131,19 @@ const getWorkflowYAML = (app, version) => {
               response: '{$self.output.data.response}'
               done: '{$self.output.data.done}'
 
+        a2:
+          type: activity
+          title: Wait for cleanup signal
+          hook:
+            type: object
+            properties:
+              done:
+                type: boolean
+          job:
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+
+
         a594:
           title: Wait for signals
           type: await
@@ -275,7 +301,7 @@ const getWorkflowYAML = (app, version) => {
               done: true
 
         s2:
-          title: Awaken sleep flows so they end and self-clean
+          title: Awaken sleeping flows so they end and self-clean
           type: signal
           subtype: all
           key_name: parentWorkflowId
@@ -347,10 +373,55 @@ const getWorkflowYAML = (app, version) => {
                   type: boolean
             maps:
               done: true
+
+        s4:
+          title: Awaken child FLOWS so they end and self-clean
+          type: signal
+          subtype: all
+          key_name: parentWorkflowId
+          key_value:
+            '@pipe':
+              - ['{$job.metadata.jid}', '-f']
+              - ['{@string.concat}']
+          topic: ${app}.childflow.awaken
+          resolver:
+            schema:
+              type: object
+              properties:
+                data:
+                  type: object
+                  properties:
+                    parentWorkflowId:
+                      type: string
+                scrub:
+                  type: boolean
+            maps:
+              data:
+                parentWorkflowId:
+                  '@pipe':
+                    - ['{$job.metadata.jid}', '-f']
+                    - ['{@string.concat}']
+              scrub: true
+          signal:
+            schema:
+              type: object
+              properties:
+                done:
+                  type: boolean
+            maps:
+              done: true
     
       transitions:
         t1:
           - to: a1
+          - to: a2
+            conditions:
+              match:
+                - expected: true
+                  actual:
+                    '@pipe':
+                      - ['{$job.metadata.key}', true, false]
+                      - ['{@conditional.ternary}']
         a1:
           - to: w1
         w1:
@@ -372,11 +443,13 @@ const getWorkflowYAML = (app, version) => {
           - to: s3
             conditions:
               code: [200, 598, 597, 596]
+          - to: s4
+            conditions:
+              code: [200, 598, 597, 596]
         a594:
           - to: c594
             conditions:
               code: 202
-
         a595:
           - to: c595
             conditions:
@@ -384,6 +457,14 @@ const getWorkflowYAML = (app, version) => {
         a599:
           - to: c599
 
+      hooks:
+        ${app}.childflow.awaken:
+          - to: a2
+            conditions:
+              match:
+                - expected: '{t1.output.data.workflowId}'
+                  actual: '{$self.hook.data.id}'
+  
     - subscribes: ${app}.activity.execute
       publishes: ${app}.activity.executed
 

package/build/services/durable/workflow.js

@@ -51,17 +51,28 @@ class WorkflowService {
         const workflowId = store.get('workflowId');
         const workflowTrace = store.get('workflowTrace');
         const workflowSpan = store.get('workflowSpan');
+        const COUNTER = store.get('counter');
+        const execIndex = COUNTER.counter = COUNTER.counter + 1;
+        const childJobId = `${workflowId}-$${options.workflowName}-${execIndex}`;
+        const parentWorkflowId = `${workflowId}-f`;
         const client = new client_1.ClientService({
             connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
         });
-        const handle = await client.workflow.start({
-            ...options,
-            workflowId: `${workflowId}${options.workflowId}`,
-            workflowTrace,
-            workflowSpan,
-        });
-        const result = await handle.result();
-        return result;
+        let handle = await client.workflow.getHandle(options.taskQueue, options.workflowName, childJobId);
+        try {
+            return await handle.result();
+        }
+        catch (error) {
+            handle = await client.workflow.start({
+                ...options,
+                workflowId: childJobId,
+                parentWorkflowId,
+                workflowTrace,
+                workflowSpan,
+            });
+            const result = await handle.result();
+            return result;
+        }
     }
     static proxyActivities(options) {
         if (options.activities) {
@@ -168,7 +179,7 @@ class WorkflowService {
             const trc = store.get('workflowTrace');
             const spn = store.get('workflowSpan');
             const activityTopic = `${workflowTopic}-activity`;
-            const activityJobId = `${workflowId}-${activityName}-${execIndex}`;
+            const activityJobId = `${workflowId}-$${activityName}-${execIndex}`;
             let activityState;
             try {
                 const hotMeshClient = await worker_1.WorkerService.getHotMesh(activityTopic);

package/build/services/store/index.js

@@ -373,6 +373,9 @@ class StoreService {
     async getStatus(jobId, appId) {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
         const status = await this.redisClient[this.commands.hget](jobKey, ':');
+        if (status === null) {
+            throw new Error(`Job ${jobId} not found`);
+        }
         return Number(status);
     }
     async setState({ ...state }, status, jobId, symbolNames, dIds, multi) {

package/build/types/durable.d.ts

@@ -10,6 +10,7 @@ type WorkflowOptions = {
     args: any[];
     workflowId: string;
     workflowName?: string;
+    parentWorkflowId?: string;
     workflowTrace?: string;
     workflowSpan?: string;
     config?: WorkflowConfig;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.15",
+  "version": "0.0.17",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",

package/services/activities/activity.ts

@@ -209,8 +209,7 @@ class Activity {
       telemetry.setActivityAttributes(attrs);
       return jobStatus as number;
     } catch (error) {
-      if (error instanceof CollationError) {
-        //caused by external over-signaling; the job is complete
+      if (error instanceof CollationError && error.fault === 'inactive') {
         this.logger.info('process-hook-event-inactive-error', { error });
         return;
       }
@@ -256,6 +255,10 @@ class Activity {
       }
       this.transitionAdjacent(multiResponse, telemetry);
     } catch (error) {
+      if (error instanceof CollationError && error.fault === 'inactive') {
+        this.logger.info('process-event-inactive-error', { error });
+        return;
+      }
       this.logger.error('activity-process-event-error', { error });
       telemetry && telemetry.setActivityError(error.message);
       throw error;
@@ -581,7 +584,12 @@ class Activity {
 
   async transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]> {
     let mIds: string[] = [];
-     if (jobStatus <= 0 || this.config.emit) {
+    //emit can be a mapping (allows emissions to be driven by the job state)
+    let emit: boolean = false;
+    if (this.config.emit) {
+      emit = Pipe.resolve(this.config.emit, this.context);
+    }
+    if (jobStatus <= 0 || emit) {
       //activity should not send 'emit' if the job is truly over
       const isTrueEmit = jobStatus > 0;
       await this.engine.runJobCompletionTasks(this.context, isTrueEmit);

package/services/activities/trigger.ts

@@ -178,7 +178,7 @@ class Trigger extends Activity {
 
   async setStats(multi?: RedisMulti): Promise<void> {
     const md = this.context.metadata;
-    if (this.config.stats?.measures) {
+    if (md.key && this.config.stats?.measures) {
       const config = await this.engine.getVID();
       const reporter = new ReporterService(config, this.store, this.logger);
       await this.store.setStats(

package/services/durable/client.ts

@@ -101,6 +101,7 @@ export class ClientService {
       const hotMeshClient = await this.getHotMeshClient(workflowTopic);
       const payload = {
         arguments: [...options.args],
+        parentWorkflowId: options.parentWorkflowId,
         workflowId: options.workflowId || nanoid(),
         workflowTopic: workflowTopic,
         backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
@@ -115,6 +116,12 @@ export class ClientService {
 
     signal: async (signalId: string, data: Record<any, any>): Promise<string> => {
       return await (await this.getHotMeshClient('durable.wfs.signal')).hook('durable.wfs.signal', { id: signalId, data });
+    },
+
+    getHandle: async (taskQueue: string, workflowName: string, workflowId: string): Promise<WorkflowHandleService> => {
+      const workflowTopic = `${taskQueue}-${workflowName}`;
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+      return new WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
     }
   }
 

package/services/durable/factory.ts

@@ -2,6 +2,11 @@
  * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
  *       workflows will be retried on the following schedule (8 times in 27 hours):
  *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ * 593: 
+ * 594: waitforsignal
+ * 595: sleep
+ * 596, 597, 598: fatal
+ * 599: retry
  */
 
 //todo: getChildWorkflowYAML (includes key, so flow will cleanup)
@@ -19,6 +24,8 @@ const getWorkflowYAML = (app: string, version: string) => {
         schema:
           type: object
           properties:
+            parentWorkflowId:
+              type: string
             workflowId:
               type: string
             arguments:
@@ -41,6 +48,11 @@ const getWorkflowYAML = (app: string, version: string) => {
           type: trigger
           stats:
             id: '{$self.input.data.workflowId}'
+            key: '{$self.input.data.parentWorkflowId}'
+            granularity: infinity
+            measures:
+              - measure: index
+                target: '{$self.input.data.parentWorkflowId}'
           job:
             maps:
               done: false
@@ -60,6 +72,7 @@ const getWorkflowYAML = (app: string, version: string) => {
         w1:
           type: worker
           topic: '{t1.output.data.workflowTopic}'
+          emit: '{$job.data.done}'
           retry:
             '599': [2]
           input:
@@ -117,6 +130,19 @@ const getWorkflowYAML = (app: string, version: string) => {
               response: '{$self.output.data.response}'
               done: '{$self.output.data.done}'
 
+        a2:
+          type: activity
+          title: Wait for cleanup signal
+          hook:
+            type: object
+            properties:
+              done:
+                type: boolean
+          job:
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+
+
         a594:
           title: Wait for signals
           type: await
@@ -274,7 +300,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               done: true
 
         s2:
-          title: Awaken sleep flows so they end and self-clean
+          title: Awaken sleeping flows so they end and self-clean
           type: signal
           subtype: all
           key_name: parentWorkflowId
@@ -346,10 +372,55 @@ const getWorkflowYAML = (app: string, version: string) => {
                   type: boolean
             maps:
               done: true
+
+        s4:
+          title: Awaken child FLOWS so they end and self-clean
+          type: signal
+          subtype: all
+          key_name: parentWorkflowId
+          key_value:
+            '@pipe':
+              - ['{$job.metadata.jid}', '-f']
+              - ['{@string.concat}']
+          topic: ${app}.childflow.awaken
+          resolver:
+            schema:
+              type: object
+              properties:
+                data:
+                  type: object
+                  properties:
+                    parentWorkflowId:
+                      type: string
+                scrub:
+                  type: boolean
+            maps:
+              data:
+                parentWorkflowId:
+                  '@pipe':
+                    - ['{$job.metadata.jid}', '-f']
+                    - ['{@string.concat}']
+              scrub: true
+          signal:
+            schema:
+              type: object
+              properties:
+                done:
+                  type: boolean
+            maps:
+              done: true
     
       transitions:
         t1:
           - to: a1
+          - to: a2
+            conditions:
+              match:
+                - expected: true
+                  actual:
+                    '@pipe':
+                      - ['{$job.metadata.key}', true, false]
+                      - ['{@conditional.ternary}']
         a1:
           - to: w1
         w1:
@@ -371,11 +442,13 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: s3
             conditions:
               code: [200, 598, 597, 596]
+          - to: s4
+            conditions:
+              code: [200, 598, 597, 596]
         a594:
           - to: c594
             conditions:
               code: 202
-
         a595:
           - to: c595
             conditions:
@@ -383,6 +456,14 @@ const getWorkflowYAML = (app: string, version: string) => {
         a599:
           - to: c599
 
+      hooks:
+        ${app}.childflow.awaken:
+          - to: a2
+            conditions:
+              match:
+                - expected: '{t1.output.data.workflowId}'
+                  actual: '{$self.hook.data.id}'
+  
     - subscribes: ${app}.activity.execute
       publishes: ${app}.activity.executed
 

package/services/durable/workflow.ts

@@ -8,7 +8,6 @@ import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable"
 import { JobOutput, JobState } from '../../types';
 import { ACTIVITY_PUBLISHES_TOPIC, ACTIVITY_SUBSCRIBES_TOPIC, SLEEP_SUBSCRIBES_TOPIC, WFS_SUBSCRIBES_TOPIC } from './factory';
 import { DurableIncompleteSignalError, DurableSleepError, DurableWaitForSignalError } from '../../modules/errors';
-import { stringify } from 'querystring';
 
 /*
 `proxyActivities` returns a wrapped instance of the 
@@ -52,18 +51,34 @@ export class WorkflowService {
     const workflowId = store.get('workflowId');
     const workflowTrace = store.get('workflowTrace');
     const workflowSpan = store.get('workflowSpan');
+    const COUNTER = store.get('counter');
+    const execIndex = COUNTER.counter = COUNTER.counter + 1;
+    const childJobId = `${workflowId}-$${options.workflowName}-${execIndex}`;
+    const parentWorkflowId = `${workflowId}-f`;
 
     const client = new Client({
       connection: await Connection.connect(WorkerService.connection),
     });
-    const handle = await client.workflow.start({
-      ...options,
-      workflowId: `${workflowId}${options.workflowId}`, //concat (caller MUST PROVIDE)
-      workflowTrace,
-      workflowSpan,
-    });
-    const result = await handle.result();
-    return result as T;
+
+    let handle = await client.workflow.getHandle(
+      options.taskQueue,
+      options.workflowName,
+      childJobId
+    );
+
+    try {
+      return await handle.result() as T;
+    } catch (error) {
+      handle = await client.workflow.start({
+        ...options,
+        workflowId: childJobId,
+        parentWorkflowId,
+        workflowTrace,
+        workflowSpan,
+      });
+      const result = await handle.result();
+      return result as T;
+    }
   }
 
   static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT> {
@@ -172,7 +187,7 @@ export class WorkflowService {
       const trc = store.get('workflowTrace');
       const spn = store.get('workflowSpan');
       const activityTopic = `${workflowTopic}-activity`;
-      const activityJobId = `${workflowId}-${activityName}-${execIndex}`;
+      const activityJobId = `${workflowId}-$${activityName}-${execIndex}`;
 
       let activityState: JobOutput
       try {

package/services/store/index.ts

@@ -452,6 +452,9 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
   async getStatus(jobId: string, appId: string): Promise<number> {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId, jobId });
     const status = await this.redisClient[this.commands.hget](jobKey, ':');
+    if (status === null) {
+      throw new Error(`Job ${jobId} not found`);
+    }
     return Number(status);
   }
 

package/types/durable.ts

@@ -12,6 +12,7 @@ type WorkflowOptions = {
   args: any[];          //input arguments to pass in
   workflowId: string;   //execution id (the job id)
   workflowName?: string; //the name of the user's workflow function
+  parentWorkflowId?: string;  //system reserved; the id of the parent; if present the flow will not self-clean until the parent that spawned it self-cleans
   workflowTrace?: string;
   workflowSpan?: string;
   config?: WorkflowConfig;