@hotmeshio/hotmesh 0.0.16 → 0.0.18

package/services/durable/factory.ts

@@ -24,6 +24,8 @@ const getWorkflowYAML = (app: string, version: string) => {
         schema:
           type: object
           properties:
+            parentWorkflowId:
+              type: string
             workflowId:
               type: string
             arguments:
@@ -46,12 +48,17 @@ const getWorkflowYAML = (app: string, version: string) => {
           type: trigger
           stats:
             id: '{$self.input.data.workflowId}'
+            key: '{$self.input.data.parentWorkflowId}'
+            granularity: infinity
+            measures:
+              - measure: index
+                target: '{$self.input.data.parentWorkflowId}'
           job:
             maps:
               done: false
 
         a1:
-          type: activity
+          type: hook
           cycle: true
           output:
             schema:
@@ -65,6 +72,7 @@ const getWorkflowYAML = (app: string, version: string) => {
         w1:
           type: worker
           topic: '{t1.output.data.workflowTopic}'
+          emit: '{$job.data.done}'
           retry:
             '599': [2]
           input:
@@ -122,6 +130,19 @@ const getWorkflowYAML = (app: string, version: string) => {
               response: '{$self.output.data.response}'
               done: '{$self.output.data.done}'
 
+        a2:
+          type: hook
+          title: Wait for cleanup signal
+          hook:
+            type: object
+            properties:
+              done:
+                type: boolean
+          job:
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+
+
         a594:
           title: Wait for signals
           type: await
@@ -226,7 +247,7 @@ const getWorkflowYAML = (app: string, version: string) => {
 
         a599:
           title: Sleep exponentially longer before retrying
-          type: activity
+          type: hook
           sleep: '{a1.output.data.duration}'
 
         c599:
@@ -279,7 +300,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               done: true
 
         s2:
-          title: Awaken sleep flows so they end and self-clean
+          title: Awaken sleeping flows so they end and self-clean
           type: signal
           subtype: all
           key_name: parentWorkflowId
@@ -351,10 +372,55 @@ const getWorkflowYAML = (app: string, version: string) => {
                   type: boolean
             maps:
               done: true
+
+        s4:
+          title: Awaken child FLOWS so they end and self-clean
+          type: signal
+          subtype: all
+          key_name: parentWorkflowId
+          key_value:
+            '@pipe':
+              - ['{$job.metadata.jid}', '-f']
+              - ['{@string.concat}']
+          topic: ${app}.childflow.awaken
+          resolver:
+            schema:
+              type: object
+              properties:
+                data:
+                  type: object
+                  properties:
+                    parentWorkflowId:
+                      type: string
+                scrub:
+                  type: boolean
+            maps:
+              data:
+                parentWorkflowId:
+                  '@pipe':
+                    - ['{$job.metadata.jid}', '-f']
+                    - ['{@string.concat}']
+              scrub: true
+          signal:
+            schema:
+              type: object
+              properties:
+                done:
+                  type: boolean
+            maps:
+              done: true
     
       transitions:
         t1:
           - to: a1
+          - to: a2
+            conditions:
+              match:
+                - expected: true
+                  actual:
+                    '@pipe':
+                      - ['{$job.metadata.key}', true, false]
+                      - ['{@conditional.ternary}']
         a1:
           - to: w1
         w1:
@@ -376,11 +442,13 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: s3
             conditions:
               code: [200, 598, 597, 596]
+          - to: s4
+            conditions:
+              code: [200, 598, 597, 596]
         a594:
           - to: c594
             conditions:
               code: 202
-
         a595:
           - to: c595
             conditions:
@@ -388,6 +456,14 @@ const getWorkflowYAML = (app: string, version: string) => {
         a599:
           - to: c599
 
+      hooks:
+        ${app}.childflow.awaken:
+          - to: a2
+            conditions:
+              match:
+                - expected: '{t1.output.data.workflowId}'
+                  actual: '{$self.hook.data.id}'
+  
     - subscribes: ${app}.activity.execute
       publishes: ${app}.activity.executed
 
@@ -463,7 +539,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               done: true
 
         s1a:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object
@@ -483,6 +559,7 @@ const getWorkflowYAML = (app: string, version: string) => {
       hooks:
         ${app}.activity.awaken:
           - to: s1a
+            keep_alive: true
             conditions:
               match:
                 - expected: '{t1a.output.data.workflowId}'
@@ -529,13 +606,13 @@ const getWorkflowYAML = (app: string, version: string) => {
                 target: '{$self.input.data.parentWorkflowId}'
 
         a1s:
-          type: activity
+          type: hook
           title: Sleep for a duration
           sleep: '{t1s.output.data.duration}'
           emit: true
 
         a2s:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object
@@ -607,7 +684,7 @@ const getWorkflowYAML = (app: string, version: string) => {
 
         a1wc:
           title: Split signal data
-          type: activity
+          type: hook
           cycle: true
           output:
             schema:
@@ -645,7 +722,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                   - ['{t1wc.output.data.signals}', 1]
                   - ['{@array.slice}']
         a2wc:
-          type: activity
+          type: hook
           output:
             schema:
               type: object
@@ -772,7 +849,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                 target: '{$self.input.data.parentWorkflowId}'
 
         a1ww:
-          type: activity
+          type: hook
           title: Wait for custom signal
           emit: true
           hook:
@@ -787,7 +864,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               signalId: '{t1ww.output.data.signalId}'
 
         a2ww:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object

package/services/durable/search.ts

@@ -0,0 +1,54 @@
+import { HotMeshService as HotMesh } from '../hotmesh'
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { StoreService } from '../store';
+import { KeyService, KeyType } from '../../modules/key';
+
+export class Search {
+  jobId: string;
+  hotMeshClient: HotMesh;
+  store: StoreService<RedisClient, RedisMulti> | null;
+
+  safeKey(key:string): string {
+    //note: protect the execution namespace with a prefix,
+    //so its design never conflicts with the hotmesh keyspace
+    return `_${key}`;
+  }
+
+  constructor(workflowId: string, hotMeshClient: HotMesh) {
+    const keyParams = {
+      appId: hotMeshClient.appId,
+      jobId: ''
+    }
+    const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+    this.jobId = `${hotMeshPrefix}${workflowId}`;
+    this.hotMeshClient = hotMeshClient;
+    this.store = hotMeshClient.engine.store as StoreService<RedisClient, RedisMulti>;
+  }
+
+  async set(key: string, value: string): Promise<void> {
+    await this.store.exec('HSET', this.jobId, this.safeKey(key), value.toString());
+  }
+
+  async get(key: string): Promise<string> {
+    try {
+      return await this.store.exec('HGET',this.jobId, this.safeKey(key)) as string;
+    } catch (err) {
+      this.hotMeshClient.logger.error('durable-search-get-error', { err });
+      return '';
+    }
+  }
+
+  async del(key: string): Promise<void> {
+    await this.store.exec('HDEL', this.jobId, this.safeKey(key));
+  }
+
+  async incr(key: string, val: number): Promise<number> {
+    return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string);
+  }
+
+  async mult(key: string, val: number): Promise<number> {
+    const log = Math.log(val);
+    const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string);
+    return Math.exp(logTotal);
+  }
+}

package/services/durable/workflow.ts

@@ -4,10 +4,11 @@ import { asyncLocalStorage } from './asyncLocalStorage';
 import { WorkerService } from './worker';
 import { ClientService as Client } from './client';
 import { ConnectionService as Connection } from './connection';
-import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable";
+import { ActivityConfig, ProxyType, WorkflowConfig, WorkflowOptions, WorkflowSearchOptions } from "../../types/durable";
 import { JobOutput, JobState } from '../../types';
 import { ACTIVITY_PUBLISHES_TOPIC, ACTIVITY_SUBSCRIBES_TOPIC, SLEEP_SUBSCRIBES_TOPIC, WFS_SUBSCRIBES_TOPIC } from './factory';
 import { DurableIncompleteSignalError, DurableSleepError, DurableWaitForSignalError } from '../../modules/errors';
+import { Search } from './search';
 
 /*
 `proxyActivities` returns a wrapped instance of the 
@@ -54,6 +55,7 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const childJobId = `${workflowId}-$${options.workflowName}-${execIndex}`;
+    const parentWorkflowId = `${workflowId}-f`;
 
     const client = new Client({
       connection: await Connection.connect(WorkerService.connection),
@@ -71,6 +73,7 @@ export class WorkflowService {
       handle = await client.workflow.start({
         ...options,
         workflowId: childJobId,
+        parentWorkflowId,
         workflowTrace,
         workflowSpan,
       });
@@ -95,6 +98,36 @@ export class WorkflowService {
     return proxy;
   }
 
+  static async data(command: 'del' | 'get' | 'set' | 'incr' | 'mult', ...args: string[]): Promise<number | boolean | string> {
+    const store = asyncLocalStorage.getStore();
+    if (!store) {
+      throw new Error('durable-store-not-found');
+    }
+    const workflowId = store.get('workflowId');
+    const workflowTopic = store.get('workflowTopic');
+
+    try {
+      const hotMeshClient = await WorkerService.getHotMesh(workflowTopic);
+      const search = new Search(workflowId, hotMeshClient);
+      if (command === 'get') {
+        return await search.get(args[0]) as string;
+      } else if (command === 'set') {
+        await search.set(args[0], args[1]);
+        return true;
+      } else if (command === 'del') {
+        await search.del(args[0]);
+        return true;
+      } else if (command === 'incr') {
+        return await search.incr(args[0], Number(args[1])) as number;
+      } else if (command === 'mult') {
+        return await search.mult(args[0], Number(args[1])) as number;
+      }
+    } catch (e) {
+      console.error(e);
+      return '';
+    }
+  }
+
   static async sleep(duration: string): Promise<number> {
     const seconds = ms(duration) / 1000;
 

package/services/engine/index.ts

@@ -3,10 +3,13 @@ import {
   formatISODate,
   getSubscriptionTopic,
   identifyRedisType,
+  polyfill,
   restoreHierarchy } from '../../modules/utils';
 import Activities from '../activities';
-import { Activity } from '../activities/activity';
 import { Await } from '../activities/await';
+import { Cycle } from '../activities/cycle';
+import { Hook } from '../activities/hook';
+import { Signal } from '../activities/signal';
 import { Worker } from '../activities/worker';
 import { Trigger } from '../activities/trigger';
 import { CompilerService } from '../compiler';
@@ -235,9 +238,10 @@ class EngineService {
   }
 
   // ************* METADATA/MODEL METHODS *************
-  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Activity> {
+  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Await|Cycle|Hook|Signal|Trigger|Worker> {
     const [activityId, schema] = await this.getSchema(topic);
-    const ActivityHandler = Activities[schema.type];
+    polyfill
+    const ActivityHandler = Activities[polyfill.resolveActivityType(schema.type)];
     if (ActivityHandler) {
       const utc = formatISODate(new Date());
       const metadata: ActivityMetadata = {
@@ -334,7 +338,7 @@ class EngineService {
       data: streamData.data,
     };
     if (streamData.type === StreamDataType.TIMEHOOK || streamData.type === StreamDataType.WEBHOOK || streamData.type === StreamDataType.TRANSITION) {
-      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Activity;
+      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Hook;
       if (streamData.type === StreamDataType.TIMEHOOK) {
         await activityHandler.processTimeHookEvent(streamData.metadata.jid);
       } else if (streamData.type === StreamDataType.TRANSITION) {

package/services/store/clients/ioredis.ts

@@ -5,6 +5,7 @@ import { Cache } from '../cache';
 import { StoreService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
 import { ReclaimedMessageType } from '../../../types/stream';
+import { type } from 'os';
 
 class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
   redisClient: RedisClientType;
@@ -22,6 +23,18 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
     return this.redisClient.multi();
   }
 
+  async exec(...args: any[]): Promise<string|string[]|string[][]> {
+    const response = await this.redisClient.call.apply(this.redisClient, args as any);
+    if (typeof response === 'string') {
+      return response as string;
+    } else if (Array.isArray(response)) {
+      if (Array.isArray(response[0])) {
+        return response as string[][];
+      }
+      return response as string[];
+    }
+  }
+
   hGetAllResult(result: any) {
     //ioredis response signature is [null, {}] or [null, null]
     return result[1];

package/services/store/clients/redis.ts

@@ -50,6 +50,10 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
     return multi as unknown as RedisMultiType;
   }
 
+  async exec(...args: any[]): Promise<string|string[]|string[][]> {
+    return await this.redisClient.sendCommand(args);
+  }
+
   async publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean> {
     const topic = this.mintKey(keyType, { appId, engineId });
     const status: number = await this.redisClient.publish(topic, JSON.stringify(message));

package/services/store/index.ts

@@ -70,6 +70,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
   //todo: standardize signatures and move concrete methods to this class
   abstract getMulti(): U;
+  abstract exec(...args: any[]): Promise<string|string[]|string[][]>;
   abstract publish(
     keyType: KeyType.QUORUM,
     message: Record<string, any>,

package/services/telemetry/index.ts

@@ -18,6 +18,7 @@ import {
   Context,
   context, 
   SpanStatusCode } from '../../types/telemetry';
+import { polyfill } from '../../modules/utils';
 
 class TelemetryService {
   span: Span;
@@ -253,7 +254,7 @@ class TelemetryService {
     if (config.type === 'trigger') {
       state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
       state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
-    } else if (config.type === 'activity' && leg === 1) {
+    } else if (polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
       //activities run non-duplexed and only have a single leg
       state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
       state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;

package/types/activity.ts

@@ -1,7 +1,7 @@
 import { MetricTypes } from "./stats";
 import { StreamRetryPolicy } from "./stream";
 
-type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal';
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal' | 'hook';
 
 type Consumes = Record<string, string[]>;
 
@@ -65,6 +65,10 @@ interface CycleActivity extends BaseActivity {
   ancestor: string; //ancestor activity id
 }
 
+interface HookActivity extends BaseActivity {
+  type: 'hook';
+}
+
 interface SignalActivity extends BaseActivity {
   type: 'signal';         //signal activities call hook/hookAll
   subtype: 'one' | 'all'; //trigger: hook(One) or hookAll
@@ -80,7 +84,7 @@ interface IterateActivity extends BaseActivity {
   type: 'iterate';
 }
 
-type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity;
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity | HookActivity;
 
 type ActivityData = Record<string, any>;
 type ActivityMetadata = {
@@ -124,6 +128,7 @@ export {
   TriggerActivityStats,
   AwaitActivity,
   CycleActivity,
+  HookActivity,
   SignalActivity,
   BaseActivity,
   IterateActivity,

package/types/durable.ts

@@ -7,13 +7,21 @@ type WorkflowConfig = {
   initialInterval?: string; //default 1s
 }
 
+type WorkflowSearchOptions = {
+  index: string;         //FT index name (myapp:myindex)
+  prefix: string[];      //FT prefixes (['myapp:myindex:prefix1', 'myapp:myindex:prefix2'])
+  schema: Record<string, {type: 'TEXT' | 'NUMERIC' | 'TAG', sortable: boolean}>;
+}
+
 type WorkflowOptions = {
   taskQueue: string;
   args: any[];          //input arguments to pass in
   workflowId: string;   //execution id (the job id)
   workflowName?: string; //the name of the user's workflow function
+  parentWorkflowId?: string;  //system reserved; the id of the parent; if present the flow will not self-clean until the parent that spawned it self-cleans
   workflowTrace?: string;
   workflowSpan?: string;
+  search?: WorkflowSearchOptions
   config?: WorkflowConfig;
 }
 
@@ -100,6 +108,7 @@ export {
   WorkerConfig,
   WorkflowConfig,
   WorkerOptions,
+  WorkflowSearchOptions,
   WorkflowDataType,
   WorkflowOptions,
 };

package/types/hook.ts

@@ -16,6 +16,7 @@ interface HookConditions {
 
 interface HookRule {
   to: string;
+  keep_alive?: boolean; //if true, the hook will not be deleted after use
   conditions: HookConditions;
 }
 

package/types/index.ts

@@ -10,6 +10,7 @@ export {
   AwaitActivity,
   BaseActivity,
   CycleActivity,
+  HookActivity,
   WorkerActivity,
   IterateActivity,
   SignalActivity,
@@ -39,6 +40,7 @@ export {
     WorkflowConfig,
     WorkerConfig,
     WorkerOptions,
+    WorkflowSearchOptions,
     WorkflowDataType,
     WorkflowOptions,
   }from './durable'