@hotmeshio/hotmesh 0.0.17 → 0.0.18

package/services/durable/search.ts

@@ -0,0 +1,54 @@
+import { HotMeshService as HotMesh } from '../hotmesh'
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { StoreService } from '../store';
+import { KeyService, KeyType } from '../../modules/key';
+
+export class Search {
+  jobId: string;
+  hotMeshClient: HotMesh;
+  store: StoreService<RedisClient, RedisMulti> | null;
+
+  safeKey(key:string): string {
+    //note: protect the execution namespace with a prefix,
+    //so its design never conflicts with the hotmesh keyspace
+    return `_${key}`;
+  }
+
+  constructor(workflowId: string, hotMeshClient: HotMesh) {
+    const keyParams = {
+      appId: hotMeshClient.appId,
+      jobId: ''
+    }
+    const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+    this.jobId = `${hotMeshPrefix}${workflowId}`;
+    this.hotMeshClient = hotMeshClient;
+    this.store = hotMeshClient.engine.store as StoreService<RedisClient, RedisMulti>;
+  }
+
+  async set(key: string, value: string): Promise<void> {
+    await this.store.exec('HSET', this.jobId, this.safeKey(key), value.toString());
+  }
+
+  async get(key: string): Promise<string> {
+    try {
+      return await this.store.exec('HGET',this.jobId, this.safeKey(key)) as string;
+    } catch (err) {
+      this.hotMeshClient.logger.error('durable-search-get-error', { err });
+      return '';
+    }
+  }
+
+  async del(key: string): Promise<void> {
+    await this.store.exec('HDEL', this.jobId, this.safeKey(key));
+  }
+
+  async incr(key: string, val: number): Promise<number> {
+    return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string);
+  }
+
+  async mult(key: string, val: number): Promise<number> {
+    const log = Math.log(val);
+    const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string);
+    return Math.exp(logTotal);
+  }
+}

package/services/durable/workflow.ts

@@ -4,10 +4,11 @@ import { asyncLocalStorage } from './asyncLocalStorage';
 import { WorkerService } from './worker';
 import { ClientService as Client } from './client';
 import { ConnectionService as Connection } from './connection';
-import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable";
+import { ActivityConfig, ProxyType, WorkflowConfig, WorkflowOptions, WorkflowSearchOptions } from "../../types/durable";
 import { JobOutput, JobState } from '../../types';
 import { ACTIVITY_PUBLISHES_TOPIC, ACTIVITY_SUBSCRIBES_TOPIC, SLEEP_SUBSCRIBES_TOPIC, WFS_SUBSCRIBES_TOPIC } from './factory';
 import { DurableIncompleteSignalError, DurableSleepError, DurableWaitForSignalError } from '../../modules/errors';
+import { Search } from './search';
 
 /*
 `proxyActivities` returns a wrapped instance of the 
@@ -97,6 +98,36 @@ export class WorkflowService {
     return proxy;
   }
 
+  static async data(command: 'del' | 'get' | 'set' | 'incr' | 'mult', ...args: string[]): Promise<number | boolean | string> {
+    const store = asyncLocalStorage.getStore();
+    if (!store) {
+      throw new Error('durable-store-not-found');
+    }
+    const workflowId = store.get('workflowId');
+    const workflowTopic = store.get('workflowTopic');
+
+    try {
+      const hotMeshClient = await WorkerService.getHotMesh(workflowTopic);
+      const search = new Search(workflowId, hotMeshClient);
+      if (command === 'get') {
+        return await search.get(args[0]) as string;
+      } else if (command === 'set') {
+        await search.set(args[0], args[1]);
+        return true;
+      } else if (command === 'del') {
+        await search.del(args[0]);
+        return true;
+      } else if (command === 'incr') {
+        return await search.incr(args[0], Number(args[1])) as number;
+      } else if (command === 'mult') {
+        return await search.mult(args[0], Number(args[1])) as number;
+      }
+    } catch (e) {
+      console.error(e);
+      return '';
+    }
+  }
+
   static async sleep(duration: string): Promise<number> {
     const seconds = ms(duration) / 1000;
 

package/services/engine/index.ts

@@ -3,10 +3,13 @@ import {
   formatISODate,
   getSubscriptionTopic,
   identifyRedisType,
+  polyfill,
   restoreHierarchy } from '../../modules/utils';
 import Activities from '../activities';
-import { Activity } from '../activities/activity';
 import { Await } from '../activities/await';
+import { Cycle } from '../activities/cycle';
+import { Hook } from '../activities/hook';
+import { Signal } from '../activities/signal';
 import { Worker } from '../activities/worker';
 import { Trigger } from '../activities/trigger';
 import { CompilerService } from '../compiler';
@@ -235,9 +238,10 @@ class EngineService {
   }
 
   // ************* METADATA/MODEL METHODS *************
-  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Activity> {
+  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Await|Cycle|Hook|Signal|Trigger|Worker> {
     const [activityId, schema] = await this.getSchema(topic);
-    const ActivityHandler = Activities[schema.type];
+    polyfill
+    const ActivityHandler = Activities[polyfill.resolveActivityType(schema.type)];
     if (ActivityHandler) {
       const utc = formatISODate(new Date());
       const metadata: ActivityMetadata = {
@@ -334,7 +338,7 @@ class EngineService {
       data: streamData.data,
     };
     if (streamData.type === StreamDataType.TIMEHOOK || streamData.type === StreamDataType.WEBHOOK || streamData.type === StreamDataType.TRANSITION) {
-      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Activity;
+      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Hook;
       if (streamData.type === StreamDataType.TIMEHOOK) {
         await activityHandler.processTimeHookEvent(streamData.metadata.jid);
       } else if (streamData.type === StreamDataType.TRANSITION) {

package/services/store/clients/ioredis.ts

@@ -5,6 +5,7 @@ import { Cache } from '../cache';
 import { StoreService } from '../index';
 import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
 import { ReclaimedMessageType } from '../../../types/stream';
+import { type } from 'os';
 
 class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
   redisClient: RedisClientType;
@@ -22,6 +23,18 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
     return this.redisClient.multi();
   }
 
+  async exec(...args: any[]): Promise<string|string[]|string[][]> {
+    const response = await this.redisClient.call.apply(this.redisClient, args as any);
+    if (typeof response === 'string') {
+      return response as string;
+    } else if (Array.isArray(response)) {
+      if (Array.isArray(response[0])) {
+        return response as string[][];
+      }
+      return response as string[];
+    }
+  }
+
   hGetAllResult(result: any) {
     //ioredis response signature is [null, {}] or [null, null]
     return result[1];

package/services/store/clients/redis.ts

@@ -50,6 +50,10 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
     return multi as unknown as RedisMultiType;
   }
 
+  async exec(...args: any[]): Promise<string|string[]|string[][]> {
+    return await this.redisClient.sendCommand(args);
+  }
+
   async publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean> {
     const topic = this.mintKey(keyType, { appId, engineId });
     const status: number = await this.redisClient.publish(topic, JSON.stringify(message));

package/services/store/index.ts

@@ -70,6 +70,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
   //todo: standardize signatures and move concrete methods to this class
   abstract getMulti(): U;
+  abstract exec(...args: any[]): Promise<string|string[]|string[][]>;
   abstract publish(
     keyType: KeyType.QUORUM,
     message: Record<string, any>,

package/services/telemetry/index.ts

@@ -18,6 +18,7 @@ import {
   Context,
   context, 
   SpanStatusCode } from '../../types/telemetry';
+import { polyfill } from '../../modules/utils';
 
 class TelemetryService {
   span: Span;
@@ -253,7 +254,7 @@ class TelemetryService {
     if (config.type === 'trigger') {
       state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
       state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
-    } else if (config.type === 'activity' && leg === 1) {
+    } else if (polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
       //activities run non-duplexed and only have a single leg
       state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
       state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;

package/types/activity.ts

@@ -1,7 +1,7 @@
 import { MetricTypes } from "./stats";
 import { StreamRetryPolicy } from "./stream";
 
-type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal';
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal' | 'hook';
 
 type Consumes = Record<string, string[]>;
 
@@ -65,6 +65,10 @@ interface CycleActivity extends BaseActivity {
   ancestor: string; //ancestor activity id
 }
 
+interface HookActivity extends BaseActivity {
+  type: 'hook';
+}
+
 interface SignalActivity extends BaseActivity {
   type: 'signal';         //signal activities call hook/hookAll
   subtype: 'one' | 'all'; //trigger: hook(One) or hookAll
@@ -80,7 +84,7 @@ interface IterateActivity extends BaseActivity {
   type: 'iterate';
 }
 
-type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity;
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity | HookActivity;
 
 type ActivityData = Record<string, any>;
 type ActivityMetadata = {
@@ -124,6 +128,7 @@ export {
   TriggerActivityStats,
   AwaitActivity,
   CycleActivity,
+  HookActivity,
   SignalActivity,
   BaseActivity,
   IterateActivity,

package/types/durable.ts

@@ -7,6 +7,12 @@ type WorkflowConfig = {
   initialInterval?: string; //default 1s
 }
 
+type WorkflowSearchOptions = {
+  index: string;         //FT index name (myapp:myindex)
+  prefix: string[];      //FT prefixes (['myapp:myindex:prefix1', 'myapp:myindex:prefix2'])
+  schema: Record<string, {type: 'TEXT' | 'NUMERIC' | 'TAG', sortable: boolean}>;
+}
+
 type WorkflowOptions = {
   taskQueue: string;
   args: any[];          //input arguments to pass in
@@ -15,6 +21,7 @@ type WorkflowOptions = {
   parentWorkflowId?: string;  //system reserved; the id of the parent; if present the flow will not self-clean until the parent that spawned it self-cleans
   workflowTrace?: string;
   workflowSpan?: string;
+  search?: WorkflowSearchOptions
   config?: WorkflowConfig;
 }
 
@@ -101,6 +108,7 @@ export {
   WorkerConfig,
   WorkflowConfig,
   WorkerOptions,
+  WorkflowSearchOptions,
   WorkflowDataType,
   WorkflowOptions,
 };

package/types/hook.ts

@@ -16,6 +16,7 @@ interface HookConditions {
 
 interface HookRule {
   to: string;
+  keep_alive?: boolean; //if true, the hook will not be deleted after use
   conditions: HookConditions;
 }
 

package/types/index.ts

@@ -10,6 +10,7 @@ export {
   AwaitActivity,
   BaseActivity,
   CycleActivity,
+  HookActivity,
   WorkerActivity,
   IterateActivity,
   SignalActivity,
@@ -39,6 +40,7 @@ export {
     WorkflowConfig,
     WorkerConfig,
     WorkerOptions,
+    WorkflowSearchOptions,
     WorkflowDataType,
     WorkflowOptions,
   }from './durable'