@hotmeshio/hotmesh 0.0.17 → 0.0.19

package/services/durable/factory.ts

@@ -58,7 +58,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               done: false
 
         a1:
-          type: activity
+          type: hook
           cycle: true
           output:
             schema:
@@ -131,7 +131,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               done: '{$self.output.data.done}'
 
         a2:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object
@@ -247,7 +247,7 @@ const getWorkflowYAML = (app: string, version: string) => {
 
         a599:
           title: Sleep exponentially longer before retrying
-          type: activity
+          type: hook
           sleep: '{a1.output.data.duration}'
 
         c599:
@@ -539,7 +539,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               done: true
 
         s1a:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object
@@ -559,6 +559,7 @@ const getWorkflowYAML = (app: string, version: string) => {
       hooks:
         ${app}.activity.awaken:
           - to: s1a
+            keep_alive: true
             conditions:
               match:
                 - expected: '{t1a.output.data.workflowId}'
@@ -605,13 +606,13 @@ const getWorkflowYAML = (app: string, version: string) => {
                 target: '{$self.input.data.parentWorkflowId}'
 
         a1s:
-          type: activity
+          type: hook
           title: Sleep for a duration
           sleep: '{t1s.output.data.duration}'
           emit: true
 
         a2s:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object
@@ -683,7 +684,7 @@ const getWorkflowYAML = (app: string, version: string) => {
 
         a1wc:
           title: Split signal data
-          type: activity
+          type: hook
           cycle: true
           output:
             schema:
@@ -721,7 +722,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                   - ['{t1wc.output.data.signals}', 1]
                   - ['{@array.slice}']
         a2wc:
-          type: activity
+          type: hook
           output:
             schema:
               type: object
@@ -848,7 +849,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                 target: '{$self.input.data.parentWorkflowId}'
 
         a1ww:
-          type: activity
+          type: hook
           title: Wait for custom signal
           emit: true
           hook:
@@ -863,7 +864,7 @@ const getWorkflowYAML = (app: string, version: string) => {
               signalId: '{t1ww.output.data.signalId}'
 
         a2ww:
-          type: activity
+          type: hook
           title: Wait for cleanup signal
           hook:
             type: object

package/services/durable/search.ts

@@ -0,0 +1,54 @@
+import { HotMeshService as HotMesh } from '../hotmesh'
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { StoreService } from '../store';
+import { KeyService, KeyType } from '../../modules/key';
+
+export class Search {
+  jobId: string;
+  hotMeshClient: HotMesh;
+  store: StoreService<RedisClient, RedisMulti> | null;
+
+  safeKey(key:string): string {
+    //note: protect the execution namespace with a prefix,
+    //so its design never conflicts with the hotmesh keyspace
+    return `_${key}`;
+  }
+
+  constructor(workflowId: string, hotMeshClient: HotMesh) {
+    const keyParams = {
+      appId: hotMeshClient.appId,
+      jobId: ''
+    }
+    const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+    this.jobId = `${hotMeshPrefix}${workflowId}`;
+    this.hotMeshClient = hotMeshClient;
+    this.store = hotMeshClient.engine.store as StoreService<RedisClient, RedisMulti>;
+  }
+
+  async set(key: string, value: string): Promise<void> {
+    await this.store.exec('HSET', this.jobId, this.safeKey(key), value.toString());
+  }
+
+  async get(key: string): Promise<string> {
+    try {
+      return await this.store.exec('HGET',this.jobId, this.safeKey(key)) as string;
+    } catch (err) {
+      this.hotMeshClient.logger.error('durable-search-get-error', { err });
+      return '';
+    }
+  }
+
+  async del(key: string): Promise<void> {
+    await this.store.exec('HDEL', this.jobId, this.safeKey(key));
+  }
+
+  async incr(key: string, val: number): Promise<number> {
+    return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string);
+  }
+
+  async mult(key: string, val: number): Promise<number> {
+    const log = Math.log(val);
+    const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string);
+    return Math.exp(logTotal);
+  }
+}

package/services/durable/worker.ts

@@ -15,43 +15,14 @@ import {
   Registry,
   WorkerConfig,
   WorkerOptions,
-  WorkflowDataType } from "../../types/durable";
+  WorkflowDataType, 
+  WorkflowSearchOptions} from "../../types/durable";
 import { RedisClass, RedisOptions } from '../../types/redis';
 import {
   StreamData,
   StreamDataResponse,
   StreamStatus } from '../../types/stream';
-
-/*
-Here is an example of how the methods in this file are used:
-
-./worker.ts
-
-import { Durable } from '@hotmeshio/hotmesh';
-import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
-
-import * as workflows from './workflows';
-
-async function run() {
-  const worker = await Durable.Worker.create({
-    connection: {
-      class: Redis,
-      options: {
-        host: 'localhost',
-        port: 6379,
-      },
-    },
-    taskQueue: 'hello-world',
-    workflow: workflows.example,
-  });
-  await worker.run();
-}
-
-run().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});
-*/
+import { KeyService, KeyType } from '../../modules/key';
 
 export class WorkerService {
   static activityRegistry: Registry = {}; //user's activities
@@ -112,6 +83,38 @@ export class WorkerService {
     return WorkerService.activityRegistry;
   }
 
+  /**
+   * For those deployments with a redis stack backend (with the FT module),
+   * this method will configure the search index for the workflow.
+   */
+  //todo: bind this to the Search service; update constructor to expect hotMeshClient as first param (id is optional
+  //refactor and delete other one as well)
+  static async configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> {
+    if (search?.schema) {
+      const store = hotMeshClient.engine.store;
+      const schema: string[] = [];
+      for (const [key, value] of Object.entries(search.schema)) {
+        //prefix with a comma (avoids collisions with hotmesh reserved words)
+        schema.push(`_${key}`);
+        schema.push(value.type);
+        if (value.sortable) {
+          schema.push('SORTABLE');
+        }
+      }
+      try {
+        const keyParams = {
+          appId: hotMeshClient.appId,
+          jobId: ''
+        }
+        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
+        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
+      } catch (err) {
+        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
+      }
+    }
+  }
+
   static async create(config: WorkerConfig) {
     //always call `registerActivities` before `import`
     WorkerService.connection = config.connection;
@@ -125,6 +128,7 @@ export class WorkerService {
     const worker = new WorkerService();
     worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
     worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
+    WorkerService.configureSearchIndex(worker.workflowRunner, config.search)
     await WorkerService.activateWorkflow(worker.workflowRunner);
     return worker;
   }

package/services/durable/workflow.ts

@@ -4,39 +4,11 @@ import { asyncLocalStorage } from './asyncLocalStorage';
 import { WorkerService } from './worker';
 import { ClientService as Client } from './client';
 import { ConnectionService as Connection } from './connection';
-import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable";
+import { ActivityConfig, ProxyType, WorkflowConfig, WorkflowOptions, WorkflowSearchOptions } from "../../types/durable";
 import { JobOutput, JobState } from '../../types';
 import { ACTIVITY_PUBLISHES_TOPIC, ACTIVITY_SUBSCRIBES_TOPIC, SLEEP_SUBSCRIBES_TOPIC, WFS_SUBSCRIBES_TOPIC } from './factory';
 import { DurableIncompleteSignalError, DurableSleepError, DurableWaitForSignalError } from '../../modules/errors';
-
-/*
-`proxyActivities` returns a wrapped instance of the 
-target activity, so that when the workflow calls a
-proxied activity, it is actually calling the proxy
-function, which in turn calls the activity function.
-
-Here is an example of how the methods in this file are used:
-
-./workflows.ts
-
-import { Durable } from '@hotmeshio/hotmesh';
-import * as activities from './activities';
-
-const { greet } = Durable.workflow.proxyActivities<typeof activities>({
-  activities: activities,
-  startToCloseTimeout: '1 minute',
-  retryPolicy: {
-    initialInterval: '5 seconds',  // Initial delay between retries
-    maximumAttempts: 3,            // Max number of retry attempts
-    backoffCoefficient: 2.0,       // Backoff factor for delay between retries: delay = initialInterval * (backoffCoefficient ^ retry_attempt)
-    maximumInterval: '30 seconds', // Max delay between retries
-  },
-});
-
-export async function example(name: string): Promise<string> {
-  return await greet(name);
-}
-*/
+import { Search } from './search';
 
 export class WorkflowService {
 
@@ -97,6 +69,18 @@ export class WorkflowService {
     return proxy;
   }
 
+  static async search(): Promise<Search> {
+    const store = asyncLocalStorage.getStore();
+    if (!store) {
+      throw new Error('durable-store-not-found');
+    }
+    const workflowId = store.get('workflowId');
+    const workflowTopic = store.get('workflowTopic');
+
+    const hotMeshClient = await WorkerService.getHotMesh(workflowTopic);
+    return new Search(workflowId, hotMeshClient);
+  }
+
   static async sleep(duration: string): Promise<number> {
     const seconds = ms(duration) / 1000;
 

package/services/engine/index.ts

@@ -3,10 +3,13 @@ import {
   formatISODate,
   getSubscriptionTopic,
   identifyRedisType,
+  polyfill,
   restoreHierarchy } from '../../modules/utils';
 import Activities from '../activities';
-import { Activity } from '../activities/activity';
 import { Await } from '../activities/await';
+import { Cycle } from '../activities/cycle';
+import { Hook } from '../activities/hook';
+import { Signal } from '../activities/signal';
 import { Worker } from '../activities/worker';
 import { Trigger } from '../activities/trigger';
 import { CompilerService } from '../compiler';
@@ -235,9 +238,10 @@ class EngineService {
   }
 
   // ************* METADATA/MODEL METHODS *************
-  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Activity> {
+  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Await|Cycle|Hook|Signal|Trigger|Worker> {
     const [activityId, schema] = await this.getSchema(topic);
-    const ActivityHandler = Activities[schema.type];
+    polyfill
+    const ActivityHandler = Activities[polyfill.resolveActivityType(schema.type)];
     if (ActivityHandler) {
       const utc = formatISODate(new Date());
       const metadata: ActivityMetadata = {
@@ -334,7 +338,7 @@ class EngineService {
       data: streamData.data,
     };
     if (streamData.type === StreamDataType.TIMEHOOK || streamData.type === StreamDataType.WEBHOOK || streamData.type === StreamDataType.TRANSITION) {
-      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Activity;
+      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Hook;
       if (streamData.type === StreamDataType.TIMEHOOK) {
         await activityHandler.processTimeHookEvent(streamData.metadata.jid);
       } else if (streamData.type === StreamDataType.TRANSITION) {

package/services/store/clients/ioredis.ts

@@ -5,6 +5,7 @@ import { Cache } from '../cache';
 import { StoreService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
 import { ReclaimedMessageType } from '../../../types/stream';
+import { type } from 'os';
 
 class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
   redisClient: RedisClientType;
@@ -22,6 +23,18 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
     return this.redisClient.multi();
   }
 
+  async exec(...args: any[]): Promise<string|string[]|string[][]> {
+    const response = await this.redisClient.call.apply(this.redisClient, args as any);
+    if (typeof response === 'string') {
+      return response as string;
+    } else if (Array.isArray(response)) {
+      if (Array.isArray(response[0])) {
+        return response as string[][];
+      }
+      return response as string[];
+    }
+  }
+
   hGetAllResult(result: any) {
     //ioredis response signature is [null, {}] or [null, null]
     return result[1];

package/services/store/clients/redis.ts

@@ -50,6 +50,10 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
     return multi as unknown as RedisMultiType;
   }
 
+  async exec(...args: any[]): Promise<string|string[]|string[][]> {
+    return await this.redisClient.sendCommand(args);
+  }
+
   async publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean> {
     const topic = this.mintKey(keyType, { appId, engineId });
     const status: number = await this.redisClient.publish(topic, JSON.stringify(message));

package/services/store/index.ts

@@ -70,6 +70,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
   //todo: standardize signatures and move concrete methods to this class
   abstract getMulti(): U;
+  abstract exec(...args: any[]): Promise<string|string[]|string[][]>;
   abstract publish(
     keyType: KeyType.QUORUM,
     message: Record<string, any>,

package/services/telemetry/index.ts

@@ -18,6 +18,7 @@ import {
   Context,
   context, 
   SpanStatusCode } from '../../types/telemetry';
+import { polyfill } from '../../modules/utils';
 
 class TelemetryService {
   span: Span;
@@ -253,7 +254,7 @@ class TelemetryService {
     if (config.type === 'trigger') {
       state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
       state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
-    } else if (config.type === 'activity' && leg === 1) {
+    } else if (polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
       //activities run non-duplexed and only have a single leg
       state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
       state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;

package/types/activity.ts

@@ -1,7 +1,7 @@
 import { MetricTypes } from "./stats";
 import { StreamRetryPolicy } from "./stream";
 
-type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal';
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal' | 'hook';
 
 type Consumes = Record<string, string[]>;
 
@@ -65,6 +65,10 @@ interface CycleActivity extends BaseActivity {
   ancestor: string; //ancestor activity id
 }
 
+interface HookActivity extends BaseActivity {
+  type: 'hook';
+}
+
 interface SignalActivity extends BaseActivity {
   type: 'signal';         //signal activities call hook/hookAll
   subtype: 'one' | 'all'; //trigger: hook(One) or hookAll
@@ -80,7 +84,7 @@ interface IterateActivity extends BaseActivity {
   type: 'iterate';
 }
 
-type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity;
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity | HookActivity;
 
 type ActivityData = Record<string, any>;
 type ActivityMetadata = {
@@ -124,6 +128,7 @@ export {
   TriggerActivityStats,
   AwaitActivity,
   CycleActivity,
+  HookActivity,
   SignalActivity,
   BaseActivity,
   IterateActivity,

package/types/durable.ts

@@ -7,6 +7,13 @@ type WorkflowConfig = {
   initialInterval?: string; //default 1s
 }
 
+type WorkflowSearchOptions = {
+  index?: string;         //FT index name (myapp:myindex)
+  prefix?: string[];      //FT prefixes (['myapp:myindex:prefix1', 'myapp:myindex:prefix2'])
+  schema?: Record<string, {type: 'TEXT' | 'NUMERIC' | 'TAG', sortable: boolean}>;
+  data?: Record<string, string>;
+}
+
 type WorkflowOptions = {
   taskQueue: string;
   args: any[];          //input arguments to pass in
@@ -15,6 +22,7 @@ type WorkflowOptions = {
   parentWorkflowId?: string;  //system reserved; the id of the parent; if present the flow will not self-clean until the parent that spawned it self-cleans
   workflowTrace?: string;
   workflowSpan?: string;
+  search?: WorkflowSearchOptions
   config?: WorkflowConfig;
 }
 
@@ -59,6 +67,7 @@ type WorkerConfig = {
   taskQueue: string; //`subscribes` in the YAML (e.g, 'hello-world')
   workflow: Function; //target function to run
   options?: WorkerOptions;
+  search?: WorkflowSearchOptions;
 }
 
 type WorkerOptions = {
@@ -101,6 +110,7 @@ export {
   WorkerConfig,
   WorkflowConfig,
   WorkerOptions,
+  WorkflowSearchOptions,
   WorkflowDataType,
   WorkflowOptions,
 };

package/types/hook.ts

@@ -16,6 +16,7 @@ interface HookConditions {
 
 interface HookRule {
   to: string;
+  keep_alive?: boolean; //if true, the hook will not be deleted after use
   conditions: HookConditions;
 }
 

package/types/index.ts

@@ -10,6 +10,7 @@ export {
   AwaitActivity,
   BaseActivity,
   CycleActivity,
+  HookActivity,
   WorkerActivity,
   IterateActivity,
   SignalActivity,
@@ -39,6 +40,7 @@ export {
     WorkflowConfig,
     WorkerConfig,
     WorkerOptions,
+    WorkflowSearchOptions,
     WorkflowDataType,
     WorkflowOptions,
   }from './durable'