@hotmeshio/hotmesh 0.0.16 → 0.0.18

package/services/activities/activity.ts

@@ -1,4 +1,4 @@
-import { CollationError, GetStateError } from '../../modules/errors';
+import { CollationError } from '../../modules/errors';
 import {
   formatISODate,
   getValueByPath,
@@ -9,7 +9,6 @@ import { ILogger } from '../logger';
 import { MapperService } from '../mapper';
 import { Pipe } from '../pipe';
 import { MDATA_SYMBOLS } from '../serializer';
-import { StoreSignaler } from '../signaler/store';
 import { StoreService } from '../store';
 import { TelemetryService } from '../telemetry';
 import { 
@@ -30,7 +29,6 @@ import {
   StreamDataType,
   StreamStatus } from '../../types/stream';
 import { TransitionRule } from '../../types/transition';
-import { HookRule } from '../../types/hook';
 
 /**
  * The base class for all activities
@@ -48,7 +46,7 @@ class Activity {
   code: StreamCode = 200;
   leg: ActivityLeg;
   adjacencyList: StreamData[];
-  adjacentIndex = 0; //can be updated by leg2 using 'as' metadata hincrby output
+  adjacentIndex = 0;
 
   constructor(
     config: ActivityType,
@@ -67,171 +65,21 @@ class Activity {
       this.store = engine.store;
   }
 
-  //********  INITIAL ENTRY POINT (A)  ********//
-  async process(): Promise<string> {
-    this.logger.debug('activity-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
-    let telemetry: TelemetryService;
-    try {
-      this.setLeg(1);
-      await CollatorService.notarizeEntry(this);
-
-      await this.getState();
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-      let multiResponse: MultiResponseFlags;
-
-      const multi = this.store.getMulti();
-      if (this.doesHook()) {
-        //sleep and wait to awaken upon a signal
-        await this.registerHook(multi);
-        this.mapOutputData();
-        this.mapJobData();
-        await this.setState(multi);
-        await CollatorService.authorizeReentry(this, multi);
-
-        await this.setStatus(0, multi);
-        await multi.exec();
-        telemetry.mapActivityAttributes();
-      } else {
-        //end the activity and transition to its children
-        this.adjacencyList = await this.filterAdjacent();
-        this.mapOutputData();
-        this.mapJobData();
-        await this.setState(multi);
-        await CollatorService.notarizeEarlyCompletion(this, multi);
-
-        await this.setStatus(this.adjacencyList.length - 1, multi);
-        multiResponse = await multi.exec() as MultiResponseFlags;
-        telemetry.mapActivityAttributes();
-        const jobStatus = this.resolveStatus(multiResponse);
-        const attrs: StringScalarType = { 'app.job.jss': jobStatus };
-        const messageIds = await this.transition(this.adjacencyList, jobStatus);
-        if (messageIds.length) {
-          attrs['app.activity.mids'] = messageIds.join(',')
-        }
-        telemetry.setActivityAttributes(attrs);
-      }
-
-      return this.context.metadata.aid;
-    } catch (error) {
-      if (error instanceof GetStateError) {
-        this.logger.error('activity-get-state-error', { error });
-      } else {
-        this.logger.error('activity-process-error', { error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry.endActivitySpan();
-      this.logger.debug('activity-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
-    }
-  }
-
   setLeg(leg: ActivityLeg): void {
     this.leg = leg;
   }
 
-  //********  SIGNAL RE-ENTRY POINT  ********//
-  doesHook(): boolean {
-    return !!(this.config.hook?.topic || this.config.sleep);
-  }
-
-  async getHookRule(topic: string): Promise<HookRule | undefined> {
-    const rules = await this.store.getHookRules();
-    return rules?.[topic]?.[0] as HookRule;
-  }
-
-  async registerHook(multi?: RedisMulti): Promise<string | void> {
-    if (this.config.hook?.topic) {
-      const signaler = new StoreSignaler(this.store, this.logger);
-      return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
-    } else if (this.config.sleep) {
-      const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
-      const jobId = this.context.metadata.jid;
-      const activityId = this.metadata.aid;
-      const dId = this.metadata.dad;
-      await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
-      return jobId;
-    }
-  }
-
-  async processWebHookEvent(): Promise<JobStatus | void> {
-    this.logger.debug('engine-process-web-hook-event', {
-      topic: this.config.hook.topic,
-      aid: this.metadata.aid
-    });
-    const signaler = new StoreSignaler(this.store, this.logger);
-    const data = { ...this.data };
-    const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
-    if (jobId) {
-      await this.processHookEvent(jobId);
-      await signaler.deleteWebHookSignal(this.config.hook.topic, data);
-    } //else => already resolved
-  }
-
-  async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
-    this.logger.debug('engine-process-time-hook-event', {
-      jid: jobId,
-      aid: this.metadata.aid
-    });
-    return await this.processHookEvent(jobId);
-  }
-
-  async processHookEvent(jobId: string): Promise<JobStatus | void> {
-    this.logger.debug('activity-process-hook-event', { jobId });
-    let telemetry: TelemetryService;
-    try {
-      this.setLeg(2);
-      await this.getState(jobId);
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-      const aState = await CollatorService.notarizeReentry(this);
-      this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
-
-      this.bindActivityData('hook');
-      this.mapJobData();
-      this.adjacencyList = await this.filterAdjacent();
-
-      const multi = this.engine.store.getMulti();
-      await this.setState(multi);
-      await CollatorService.notarizeCompletion(this, multi);
-
-      await this.setStatus(this.adjacencyList.length - 1, multi);
-      const multiResponse = await multi.exec() as MultiResponseFlags;
-
-      telemetry.mapActivityAttributes();
-      const jobStatus = this.resolveStatus(multiResponse);
-      const attrs: StringScalarType = { 'app.job.jss': jobStatus };
-      const messageIds = await this.transition(this.adjacencyList, jobStatus);
-      if (messageIds.length) {
-        attrs['app.activity.mids'] = messageIds.join(',')
-      }
-      telemetry.setActivityAttributes(attrs);
-      return jobStatus as number;
-    } catch (error) {
-      if (error instanceof CollationError && error.fault === 'inactive') {
-        this.logger.info('process-hook-event-inactive-error', { error });
-        return;
-      }
-      this.logger.error('engine-process-hook-event-error', { error });
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry.endActivitySpan();
-    }
-  }
-
   //********  DUPLEX RE-ENTRY POINT  ********//
-  async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<void> {
+  async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output', jobId?: string): Promise<void> {
     this.setLeg(2);
-    const jid = this.context.metadata.jid;
+    const jid = this.context.metadata.jid || jobId;
     const aid = this.metadata.aid;
     this.status = status;
     this.code = code;
     this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
     let telemetry: TelemetryService;
     try {
-      await this.getState();
+      await this.getState(jobId);
       const aState = await CollatorService.notarizeReentry(this);
       this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
 
@@ -247,18 +95,19 @@ class Activity {
       telemetry.startActivitySpan(this.leg);
       let multiResponse: MultiResponseFlags;
       if (status === StreamStatus.PENDING) {
-        multiResponse = await this.processPending(telemetry);
+        multiResponse = await this.processPending(telemetry, type);
       } else if (status === StreamStatus.SUCCESS) {
-        multiResponse = await this.processSuccess(telemetry);
+        multiResponse = await this.processSuccess(telemetry, type);
       } else {
-        multiResponse = await this.processError(telemetry);
+        multiResponse = await this.processError(telemetry, type);
       }
       this.transitionAdjacent(multiResponse, telemetry);
     } catch (error) {
-      if (error instanceof CollationError && error.fault === 'inactive') {
+      if (error instanceof CollationError) {
         this.logger.info('process-event-inactive-error', { error });
         return;
       }
+      console.error(error);
       this.logger.error('activity-process-event-error', { error });
       telemetry && telemetry.setActivityError(error.message);
       throw error;
@@ -268,8 +117,8 @@ class Activity {
     }
   }
 
-  async processPending(telemetry: TelemetryService): Promise<MultiResponseFlags> {
-    this.bindActivityData('output');
+  async processPending(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags> {
+    this.bindActivityData(type);
     this.adjacencyList = await this.filterAdjacent();
     this.mapJobData();
     const multi = this.store.getMulti();
@@ -280,8 +129,8 @@ class Activity {
     return await multi.exec() as MultiResponseFlags;
   }
 
-  async processSuccess(telemetry: TelemetryService): Promise<MultiResponseFlags> {
-    this.bindActivityData('output');
+  async processSuccess(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags> {
+    this.bindActivityData(type);
     this.adjacencyList = await this.filterAdjacent();
     this.mapJobData();
     const multi = this.store.getMulti();
@@ -292,7 +141,7 @@ class Activity {
     return await multi.exec() as MultiResponseFlags;
   }
 
-  async processError(telemetry: TelemetryService): Promise<MultiResponseFlags> {
+  async processError(telemetry: TelemetryService, type: string): Promise<MultiResponseFlags> {
     this.bindActivityError(this.data);
     this.adjacencyList = await this.filterAdjacent();
     const multi = this.store.getMulti();
@@ -584,7 +433,11 @@ class Activity {
 
   async transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]> {
     let mIds: string[] = [];
-     if (jobStatus <= 0 || this.config.emit) {
+    let emit: boolean = false;
+    if (this.config.emit) {
+      emit = Pipe.resolve(this.config.emit, this.context);
+    }
+    if (jobStatus <= 0 || emit) {
       //activity should not send 'emit' if the job is truly over
       const isTrueEmit = jobStatus > 0;
       await this.engine.runJobCompletionTasks(this.context, isTrueEmit);

package/services/activities/hook.ts

@@ -0,0 +1,149 @@
+import { CollationError, GetStateError } from '../../modules/errors';
+import { CollatorService } from '../collator';
+import { EngineService } from '../engine';
+import { Pipe } from '../pipe';
+import { StoreSignaler } from '../signaler/store';
+import { TelemetryService } from '../telemetry';
+import { 
+  ActivityData,
+  ActivityMetadata,
+  ActivityType,
+  HookActivity} from '../../types/activity';
+import { JobState, JobStatus } from '../../types/job';
+import {
+  MultiResponseFlags,
+  RedisMulti } from '../../types/redis';
+import { StringScalarType } from '../../types/serializer';
+import { HookRule } from '../../types/hook';
+import { Activity } from './activity';
+import { StreamStatus } from '../../types';
+
+/**
+ * Listens for `webhook`, `timehook`, and `cycle` (repeat) signals
+ */
+class Hook extends Activity {
+  config: HookActivity;
+
+  constructor(
+    config: ActivityType,
+    data: ActivityData,
+    metadata: ActivityMetadata,
+    hook: ActivityData | null,
+    engine: EngineService,
+    context?: JobState) {
+      super(config, data, metadata, hook, engine, context);
+  }
+
+  //********  INITIAL ENTRY POINT (A)  ********//
+  async process(): Promise<string> {
+    this.logger.debug('hook-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    let telemetry: TelemetryService;
+    try {
+      this.setLeg(1);
+      await CollatorService.notarizeEntry(this);
+
+      await this.getState();
+      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+      telemetry.startActivitySpan(this.leg);
+      let multiResponse: MultiResponseFlags;
+
+      const multi = this.store.getMulti();
+      if (this.doesHook()) {
+        //sleep and wait to awaken upon a signal
+        await this.registerHook(multi);
+        this.mapOutputData();
+        this.mapJobData();
+        await this.setState(multi);
+        await CollatorService.authorizeReentry(this, multi);
+
+        await this.setStatus(0, multi);
+        await multi.exec();
+        telemetry.mapActivityAttributes();
+      } else {
+        //end the activity and transition to its children
+        this.adjacencyList = await this.filterAdjacent();
+        this.mapOutputData();
+        this.mapJobData();
+        await this.setState(multi);
+        await CollatorService.notarizeEarlyCompletion(this, multi);
+
+        await this.setStatus(this.adjacencyList.length - 1, multi);
+        multiResponse = await multi.exec() as MultiResponseFlags;
+        telemetry.mapActivityAttributes();
+        const jobStatus = this.resolveStatus(multiResponse);
+        const attrs: StringScalarType = { 'app.job.jss': jobStatus };
+        const messageIds = await this.transition(this.adjacencyList, jobStatus);
+        if (messageIds.length) {
+          attrs['app.activity.mids'] = messageIds.join(',')
+        }
+        telemetry.setActivityAttributes(attrs);
+      }
+
+      return this.context.metadata.aid;
+    } catch (error) {
+      if (error instanceof GetStateError) {
+        this.logger.error('hook-get-state-error', { error });
+      } else {
+        this.logger.error('hook-process-error', { error });
+      }
+      telemetry.setActivityError(error.message);
+      throw error;
+    } finally {
+      telemetry.endActivitySpan();
+      this.logger.debug('hook-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    }
+  }
+
+  //********  SIGNAL RE-ENTRY POINT  ********//
+  doesHook(): boolean {
+    //does this activity use a time-hook or web-hook
+    return !!(this.config.hook?.topic || this.config.sleep);
+  }
+
+  async getHookRule(topic: string): Promise<HookRule | undefined> {
+    const rules = await this.store.getHookRules();
+    return rules?.[topic]?.[0] as HookRule;
+  }
+
+  async registerHook(multi?: RedisMulti): Promise<string | void> {
+    if (this.config.hook?.topic) {
+      const signaler = new StoreSignaler(this.store, this.logger);
+      return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
+    } else if (this.config.sleep) {
+      const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
+      const jobId = this.context.metadata.jid;
+      const activityId = this.metadata.aid;
+      const dId = this.metadata.dad;
+      await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
+      return jobId;
+    }
+  }
+
+  async processWebHookEvent(): Promise<JobStatus | void> {
+    this.logger.debug('hook-process-web-hook-event', {
+      topic: this.config.hook.topic,
+      aid: this.metadata.aid
+    });
+    const signaler = new StoreSignaler(this.store, this.logger);
+    const data = { ...this.data };
+    const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
+    if (jobId) {
+      //if a webhook signal is sent that includes 'keep_alive' the hook will remain open
+      const code = data.keep_alive ? 202 : 200;
+      await this.processEvent(StreamStatus.SUCCESS, code, 'hook', jobId);
+      if (code === 200) {
+        await signaler.deleteWebHookSignal(this.config.hook.topic, data);
+      }
+    } //else => already resolved
+  }
+
+  async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
+    this.logger.debug('hook-process-time-hook-event', {
+      jid: jobId,
+      aid: this.metadata.aid
+    });
+    await this.processEvent(StreamStatus.SUCCESS, 200, 'hook');
+  }
+}
+
+export { Hook };

package/services/activities/index.ts

@@ -1,6 +1,7 @@
 import { Activity } from './activity';
 import { Await } from './await';
 import { Cycle } from './cycle';
+import { Hook } from './hook';
 import { Iterate } from './iterate';
 import { Signal } from './signal';
 import { Trigger } from './trigger';
@@ -10,6 +11,7 @@ export default {
   activity: Activity,
   await: Await,
   cycle: Cycle,
+  hook: Hook,
   iterate: Iterate,
   signal: Signal,
   trigger: Trigger,

package/services/activities/trigger.ts

@@ -178,7 +178,7 @@ class Trigger extends Activity {
 
   async setStats(multi?: RedisMulti): Promise<void> {
     const md = this.context.metadata;
-    if (this.config.stats?.measures) {
+    if (md.key && this.config.stats?.measures) {
       const config = await this.engine.getVID();
       const reporter = new ReporterService(config, this.store, this.logger);
       await this.store.setStats(

package/services/collator/index.ts

@@ -54,7 +54,6 @@ class CollatorService {
     //set second digit to 8, allowing for re-entry
     //decrement by -10_000_000_000_000
     const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10_000_000_000_000, this.getDimensionalAddress(activity), multi);
-    //this.verifyInteger(amount, 1, 'exit');
     return amount;
   }
 

package/services/compiler/deployer.ts

@@ -25,10 +25,12 @@ class Deployer {
   async deploy(store: StoreService<RedisClient, RedisMulti>) {
     this.store = store;
     CollatorService.compile(this.manifest.app.graphs);
+    this.convertActivitiesToHooks();
     this.convertTopicsToTypes();
     this.copyJobSchemas();
     this.bindBackRefs();
     this.bindParents();
+    this.bindCycleTarget();
     this.resolveMappingDependencies();
     this.resolveJobMapsPaths();
     await this.generateSymKeys();
@@ -148,8 +150,22 @@ class Deployer {
     }
   }
 
-  //more intuitive for SDK users to use 'topic',
-  //but the compiler is desiged to be generic and uses 'subtypes'
+  //the cycle/goto activity includes and ancestor target;
+  //update with the cycle flag, so it can be rerun
+  bindCycleTarget() {
+    for (const graph of this.manifest!.app.graphs) {
+      const activities = graph.activities;
+      for (const activityKey in activities) {
+        const activity = activities[activityKey];
+        if (activity.type === 'cycle') {
+          activities[activity.ancestor].cycle = true;
+        }
+      }
+    }
+  }
+
+  //it's more intuitive for SDK users to use 'topic',
+  //but the compiler is desiged to be generic and uses the attribute, 'subtypes'
   convertTopicsToTypes() {
     for (const graph of this.manifest!.app.graphs) {
       const activities = graph.activities;
@@ -162,6 +178,20 @@ class Deployer {
     }
   }
 
+  //legacy; remove at beta (assume no legacy refs to 'activity' at that point)
+  convertActivitiesToHooks() {
+    for (const graph of this.manifest!.app.graphs) {
+      const activities = graph.activities;
+      for (const activityKey in activities) {
+        const activity = activities[activityKey];
+        if (['activity'].includes(activity.type)) {
+          activity.type = 'hook';
+        }
+      }
+    }
+  }
+
+
   async bindParents() {
     const graphs = this.manifest.app.graphs;
     for (const graph of graphs) {  

package/services/durable/client.ts

@@ -5,9 +5,10 @@ import { HotMeshService as HotMesh } from '../hotmesh';
 import {
   ClientConfig,
   Connection,
-  WorkflowOptions } from '../../types/durable';
+  WorkflowOptions, 
+  WorkflowSearchOptions} from '../../types/durable';
 import { JobState } from '../../types/job';
-import { KeyType } from '../../modules/key';
+import { KeyService, KeyType } from '../../modules/key';
 
 /*
 Here is an example of how the methods in this file are used:
@@ -90,6 +91,41 @@ export class ClientService {
     return hotMeshClient;
   }
 
+  /**
+   * For those deployments with a redis stack backend (with the FT module),
+   * this method will configure the search index for the workflow.
+   */
+  configureSearchIndex = async (hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> => {
+    if (search) {
+      const store = hotMeshClient.engine.store;
+      const schema: string[] = [];
+      for (const [key, value] of Object.entries(search.schema)) {
+        //prefix with a comma (avoids collisions with hotmesh reserved words)
+        schema.push(`_${key}`);
+        schema.push(value.type);
+        if (value.sortable) {
+          schema.push('SORTABLE');
+        }
+      }
+      try {
+        const keyParams = {
+          appId: hotMeshClient.appId,
+          jobId: ''
+        }
+        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
+        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
+      } catch (err) {
+        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
+      }
+    }
+  }
+
+  search = async (hotMeshClient: HotMesh, index: string, query: string[]): Promise<string[]> => {
+    const store = hotMeshClient.engine.store;
+    return await store.exec('FT.SEARCH', index, ...query) as string[];
+  }
+
   workflow = {
     start: async (options: WorkflowOptions): Promise<WorkflowHandleService> => {
       const taskQueueName = options.taskQueue;
@@ -99,8 +135,10 @@ export class ClientService {
       //topic is concat of taskQueue and workflowName
       const workflowTopic = `${taskQueueName}-${workflowName}`;
       const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+      this.configureSearchIndex(hotMeshClient, options.search)
       const payload = {
         arguments: [...options.args],
+        parentWorkflowId: options.parentWorkflowId,
         workflowId: options.workflowId || nanoid(),
         workflowTopic: workflowTopic,
         backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
@@ -121,6 +159,17 @@ export class ClientService {
       const workflowTopic = `${taskQueue}-${workflowName}`;
       const hotMeshClient = await this.getHotMeshClient(workflowTopic);
       return new WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
+    },
+
+    search: async (taskQueue: string, workflowName: string, index: string, ...query: string[]): Promise<string[]> => {
+      const workflowTopic = `${taskQueue}-${workflowName}`;
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+      try {
+        return await this.search(hotMeshClient, index, query);
+      } catch (err) {
+        hotMeshClient.engine.logger.error('durable-client-search-err', { err });
+        throw err;
+      }
     }
   }