@hotmeshio/hotmesh 0.0.12 → 0.0.14

package/services/activities/await.ts

@@ -1,5 +1,6 @@
 import { GetStateError } from '../../modules/errors';
 import { Activity } from './activity';
+import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
 import {
   ActivityData,
@@ -7,15 +8,10 @@ import {
   AwaitActivity,
   ActivityType } from '../../types/activity';
 import { JobState } from '../../types/job';
-import { MultiResponseFlags } from '../../types/redis';
-import { StringScalarType } from '../../types/serializer';
-import {
-  StreamCode,
-  StreamData,
-  StreamDataType,
-  StreamStatus } from '../../types/stream';
+import { MultiResponseFlags, RedisMulti } from '../../types/redis';
+import { StreamData, StreamDataType } from '../../types/stream';
 import { TelemetryService } from '../telemetry';
-import { CollatorService } from '../collator';
+import { Pipe } from '../pipe';
 
 class Await extends Activity {
   config: AwaitActivity;
@@ -35,24 +31,26 @@ class Await extends Activity {
     this.logger.debug('await-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     let telemetry: TelemetryService;
     try {
+      //confirm entry is allowed and restore state
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
-
       await this.getState();
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
       this.mapInputData();
 
+      //save state and authorize reentry
       const multi = this.store.getMulti();
       //todo: await this.registerTimeout();
+      const messageId = await this.execActivity(multi);
       await CollatorService.authorizeReentry(this, multi);
       await this.setState(multi);
       await this.setStatus(0, multi);
       const multiResponse = await multi.exec() as MultiResponseFlags;
 
+      //telemetry
       telemetry.mapActivityAttributes();
       const jobStatus = this.resolveStatus(multiResponse);
-      const messageId = await this.execActivity();
       telemetry.setActivityAttributes({
         'app.activity.mid': messageId,
         'app.job.jss': jobStatus
@@ -61,9 +59,9 @@ class Await extends Activity {
     } catch (error) {
       telemetry.setActivityError(error.message);
       if (error instanceof GetStateError) {
-        this.logger.error('await-get-state-error', error);
+        this.logger.error('await-get-state-error', { error });
       } else {
-        this.logger.error('await-process-error', error);
+        this.logger.error('await-process-error', { error });
       }
       throw error;
     } finally {
@@ -72,14 +70,14 @@ class Await extends Activity {
     }
   }
 
-
-  async execActivity(): Promise<string> {
+  async execActivity(multi: RedisMulti): Promise<string> {
+    const topic = Pipe.resolve(this.config.subtype, this.context);
     const streamData: StreamData = {
       metadata: {
         jid: this.context.metadata.jid,
         dad: this.metadata.dad,
         aid: this.metadata.aid,
-        topic: this.config.subtype,
+        topic,
         spn: this.context['$self'].output.metadata?.l1s,
         trc: this.context.metadata.trc,
       },
@@ -91,81 +89,7 @@ class Await extends Activity {
         retry: this.config.retry
       };
     }
-    return (await this.engine.streamSignaler?.publishMessage(null, streamData)) as string;
-  }
-
-
-  //********  `RESOLVE` ENTRY POINT (B)  ********//
-  //this method is invoked when the job spawned by this job ends;
-  //`this.data` is the job data produced by the spawned job
-  async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<void> {
-    this.setLeg(2);
-    const jid = this.context.metadata.jid;
-    const aid = this.metadata.aid;
-    if (!jid) {
-      throw new Error('await-process-event-error');
-    }
-    this.logger.debug('await-resolve-await', { jid, aid, status, code });
-    this.status = status;
-    this.code = code;
-    let telemetry: TelemetryService;
-    try {
-      await this.getState();
-      const aState = await CollatorService.notarizeReentry(this);
-      this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-
-      let multiResponse: MultiResponseFlags = [];
-      if (status === StreamStatus.SUCCESS) {
-        this.bindActivityData('output');
-        this.adjacencyList = await this.filterAdjacent();
-        multiResponse = await this.processSuccessResponse(this.adjacencyList);
-      } else {
-        this.bindActivityError(this.data);
-        this.adjacencyList = await this.filterAdjacent();
-        multiResponse = await this.processErrorResponse(this.adjacencyList);
-      }
-
-      telemetry.mapActivityAttributes();
-      const jobStatus = this.resolveStatus(multiResponse);
-      const attrs: StringScalarType = { 'app.job.jss': jobStatus };
-      const messageIds = await this.transition(this.adjacencyList, jobStatus);
-      if (messageIds.length) {
-        attrs['app.activity.mids'] = messageIds.join(',')
-      }
-      telemetry.setActivityAttributes(attrs);
-    } catch (error) {
-      this.logger.error('await-resolve-await-error', error);
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry.endActivitySpan();
-      this.logger.debug('await-resolve-await-end', { jid, aid, status, code });
-    }
-  }
-
-  async processSuccessResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags> {
-    this.mapJobData();
-    const multi = this.store.getMulti();
-    await this.setState(multi);
-    await CollatorService.notarizeCompletion(this, multi);
-
-    await this.setStatus(adjacencyList.length - 1, multi);
-    return await multi.exec() as MultiResponseFlags;
-  }
-
-  async processErrorResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags> {
-    //todo: if adjacencyList.length == 0, then map to the job output
-    //      this method would be added to Base activity class 
-    //this.mapJobData();
-    const multi = this.store.getMulti();
-    await this.setState(multi);
-    await CollatorService.notarizeCompletion(this, multi);
-
-    await this.setStatus(adjacencyList.length - 1, multi);
-    return await multi.exec() as MultiResponseFlags;
+    return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi)) as string;
   }
 }
 

package/services/activities/cycle.ts

@@ -61,9 +61,9 @@ class Cycle extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof GetStateError) {
-        this.logger.error('cycle-get-state-error', error);
+        this.logger.error('cycle-get-state-error', { error });
       } else {
-        this.logger.error('cycle-process-error', error);
+        this.logger.error('cycle-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;

package/services/activities/index.ts

@@ -1,8 +1,8 @@
 import { Activity } from './activity';
 import { Await } from './await';
 import { Cycle } from './cycle';
-import { Emit } from './emit';
 import { Iterate } from './iterate';
+import { Signal } from './signal';
 import { Trigger } from './trigger';
 import { Worker } from './worker';
 
@@ -11,7 +11,7 @@ export default {
   await: Await,
   cycle: Cycle,
   iterate: Iterate,
-  emit: Emit,
+  signal: Signal,
   trigger: Trigger,
   worker: Worker,
 };

package/services/activities/signal.ts

@@ -0,0 +1,124 @@
+import { GetStateError } from '../../modules/errors';
+import { Activity, ActivityType } from './activity';
+import { CollatorService } from '../collator';
+import { EngineService } from '../engine';
+import { MapperService } from '../mapper';
+import { Pipe } from '../pipe';
+import { TelemetryService } from '../telemetry';
+import {
+  ActivityData,
+  ActivityMetadata,
+  SignalActivity } from '../../types/activity';
+import { JobState } from '../../types/job';
+import { MultiResponseFlags, RedisMulti } from '../../types/redis';
+import { StringScalarType } from '../../types/serializer';
+import { JobStatsInput } from '../../types/stats';
+
+class Signal extends Activity {
+  config: SignalActivity;
+
+  constructor(
+    config: ActivityType,
+    data: ActivityData,
+    metadata: ActivityMetadata,
+    hook: ActivityData | null,
+    engine: EngineService,
+    context?: JobState) {
+      super(config, data, metadata, hook, engine, context);
+  }
+
+
+  //********  LEG 1 ENTRY  ********//
+  async process(): Promise<string> {
+    this.logger.debug('signal-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    let telemetry: TelemetryService;
+    try {
+      //verify entry is allowed
+      this.setLeg(1);
+      await CollatorService.notarizeEntry(this);
+      await this.getState();
+      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+      telemetry.startActivitySpan(this.leg);
+
+      //save state and notarize early completion (signals only run leg1)
+      const multi = this.store.getMulti();
+      this.adjacencyList = await this.filterAdjacent();
+      this.mapOutputData();
+      this.mapJobData();
+      await this.setState(multi);
+      await CollatorService.notarizeEarlyCompletion(this, multi);
+      await this.setStatus(this.adjacencyList.length - 1, multi);
+      const multiResponse = await multi.exec() as MultiResponseFlags;
+
+      //signal to awaken all paused jobs that share the targeted job key
+      await this.hookAll();
+
+      //transition to adjacent activities
+      const jobStatus = this.resolveStatus(multiResponse);
+      const attrs: StringScalarType = { 'app.job.jss': jobStatus };
+      const messageIds = await this.transition(this.adjacencyList, jobStatus);
+      if (messageIds.length) {
+        attrs['app.activity.mids'] = messageIds.join(',')
+      }
+      telemetry.mapActivityAttributes();
+      telemetry.setActivityAttributes(attrs);
+
+      return this.context.metadata.aid;
+    } catch (error) {
+      if (error instanceof GetStateError) {
+        this.logger.error('signal-get-state-error', { error });
+      } else {
+        this.logger.error('signal-process-error', { error });
+      }
+      telemetry.setActivityError(error.message);
+      throw error;
+    } finally {
+      telemetry.endActivitySpan();
+      this.logger.debug('signal-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    }
+  }
+
+  mapSignalData(): Record<string, any> {
+    if(this.config.signal?.maps) {
+      const mapper = new MapperService(this.config.signal.maps, this.context);
+      return mapper.mapRules();
+    }
+  }
+
+  mapResolverData(): Record<string, any> {
+    if(this.config.resolver?.maps) {
+      const mapper = new MapperService(this.config.resolver.maps, this.context);
+      return mapper.mapRules();
+    }
+  }
+
+  /**
+   * The signal activity will hook all paused jobs that share the same job key.
+   */
+  async hookAll(): Promise<string[]> {
+    //prep 1) generate `input signal data` (essentially the webhook payload)
+    const signalInputData = this.mapSignalData();
+
+    //prep 2) generate data that resolves the job key (per the YAML config)
+    const keyResolverData = this.mapResolverData() as JobStatsInput;
+    if (this.config.scrub) {
+      //self-clean the indexes upon use if configured
+      keyResolverData.scrub = true;
+    }
+
+    //prep 3) jobKeys can contain multiple indexes (per the YAML config)
+    const key_name = Pipe.resolve(this.config.key_name, this.context);
+    const key_value = Pipe.resolve(this.config.key_value, this.context);
+    const indexQueryFacets = [`${key_name}:${key_value}`];
+
+    //execute: `hookAll` will now resume all paused jobs that share the same job key
+    return await this.engine.hookAll(
+      this.config.topic,
+      signalInputData,
+      keyResolverData,
+      indexQueryFacets
+    );
+  }
+}
+
+export { Signal };

package/services/activities/trigger.ts

@@ -61,9 +61,9 @@ class Trigger extends Activity {
       return this.context.metadata.jid;
     } catch (error) {
       if (error instanceof DuplicateJobError) {
-        this.logger.error('duplicate-job-error', error);
+        this.logger.error('duplicate-job-error', { error });
       } else {
-        this.logger.error('trigger-process-error', error);
+        this.logger.error('trigger-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
@@ -140,6 +140,7 @@ class Trigger extends Activity {
        },
     };
     this.context['$self'] = this.context[this.metadata.aid];
+    this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
   }
 
   bindJobMetadataPaths(): string[] {
@@ -151,7 +152,7 @@ class Trigger extends Activity {
   }
 
   resolveGranularity(): string {
-    return ReporterService.DEFAULT_GRANULARITY;
+    return this.config.stats?.granularity || ReporterService.DEFAULT_GRANULARITY;
   }
 
   getJobStatus(): number {

package/services/activities/worker.ts

@@ -8,13 +8,10 @@ import {
   ActivityType,
   WorkerActivity } from '../../types/activity';
 import { JobState } from '../../types/job';
-import { MultiResponseFlags } from '../../types/redis';
-import { StringScalarType } from '../../types/serializer';
-import {
-  StreamCode,
-  StreamData,
-  StreamStatus } from '../../types/stream';
+import { MultiResponseFlags, RedisMulti } from '../../types/redis';
+import { StreamData} from '../../types/stream';
 import { TelemetryService } from '../telemetry';
+import { Pipe } from '../pipe';
 
 class Worker extends Activity {
   config: WorkerActivity;
@@ -34,24 +31,26 @@ class Worker extends Activity {
     this.logger.debug('worker-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     let telemetry: TelemetryService;
     try {
+      //confirm entry is allowed and restore state
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
-
       await this.getState();
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
       this.mapInputData();
 
+      //save state and authorize reentry
       const multi = this.store.getMulti();
       //todo: await this.registerTimeout();
+      const messageId = await this.execActivity(multi);
       await CollatorService.authorizeReentry(this, multi);
       await this.setState(multi);
       await this.setStatus(0, multi);
       const multiResponse = await multi.exec() as MultiResponseFlags;
 
+      //telemetry
       telemetry.mapActivityAttributes();
       const jobStatus = this.resolveStatus(multiResponse);
-      const messageId = await this.execActivity();
       telemetry.setActivityAttributes({
         'app.activity.mid': messageId,
         'app.job.jss': jobStatus
@@ -60,9 +59,9 @@ class Worker extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof GetStateError) {
-        this.logger.error('worker-get-state-error', error);
+        this.logger.error('worker-get-state-error', { error });
       } else {
-        this.logger.error('worker-process-error', error);
+        this.logger.error('worker-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
@@ -72,13 +71,14 @@ class Worker extends Activity {
     }
   }
 
-  async execActivity(): Promise<string> {
+  async execActivity(multi: RedisMulti): Promise<string> {
+    const topic = Pipe.resolve(this.config.subtype, this.context);
     const streamData: StreamData = {
       metadata: {
         jid: this.context.metadata.jid,
         dad: this.metadata.dad,
         aid: this.metadata.aid,
-        topic: this.config.subtype,
+        topic,
         spn: this.context['$self'].output.metadata.l1s,
         trc: this.context.metadata.trc,
       },
@@ -89,7 +89,7 @@ class Worker extends Activity {
         retry: this.config.retry
       };
     }
-    return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData)) as string;
+    return (await this.engine.streamSignaler?.publishMessage(topic, streamData, multi)) as string;
   }
 }
 

package/services/compiler/deployer.ts

@@ -8,6 +8,7 @@ import { HookRule } from '../../types/hook';
 import { HotMeshGraph, HotMeshManifest } from '../../types/hotmesh';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { StringAnyType, Symbols } from '../../types/serializer';
+import { Pipe } from '../pipe';
 
 const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
 const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
@@ -425,7 +426,8 @@ class Deployer {
       const activities = graph.activities;
       for (const activityKey in activities) {
         const activity = activities[activityKey];
-        if (activity.type === 'worker') {
+        //only precreate if the topic is concrete and not `mappable`
+        if (activity.type === 'worker' && Pipe.resolve(activity.subtype, {}) === activity.subtype) {
           params.topic = activity.subtype;
           const key = this.store.mintKey(KeyType.STREAMS, params);
           //create one worker group per unique activity subtype (the topic)

package/services/durable/client.ts

@@ -1,8 +1,13 @@
+import { nanoid } from 'nanoid';
+import { APP_ID, APP_VERSION, DEFAULT_COEFFICIENT, SUBSCRIBES_TOPIC, getWorkflowYAML } from './factory';
 import { WorkflowHandleService } from './handle';
 import { HotMeshService as HotMesh } from '../hotmesh';
-import { ClientConfig, Connection, WorkflowOptions } from '../../types/durable';
-import { getWorkflowYAML } from './factory';
+import {
+  ClientConfig,
+  Connection,
+  WorkflowOptions } from '../../types/durable';
 import { JobState } from '../../types/job';
+import { KeyType } from '../../modules/key';
 
 /*
 Here is an example of how the methods in this file are used:
@@ -55,13 +60,14 @@ export class ClientService {
     this.connection = config.connection;
   }
 
-  getHotMesh = async (worflowTopic: string) => {
+  getHotMeshClient = async (worflowTopic: string) => {
+    //NOTE: every unique topic inits a new engine
     if (ClientService.instances.has(worflowTopic)) {
       return await ClientService.instances.get(worflowTopic);
     }
 
-    const hotMesh = HotMesh.init({
-      appId: worflowTopic,
+    const hotMeshClient = HotMesh.init({
+      appId: APP_ID,
       engine: {
         redis: {
           class: this.connection.class,
@@ -69,9 +75,19 @@ export class ClientService {
         }
       }
     });
-    ClientService.instances.set(worflowTopic, hotMesh);
-    await this.activateWorkflow(await hotMesh, worflowTopic);
-    return hotMesh;
+    ClientService.instances.set(worflowTopic, hotMeshClient);
+
+    //since the YAML topic is dynamic, it MUST be manually created before use
+    const store = (await hotMeshClient).engine.store;
+    const params = { appId: APP_ID, topic: worflowTopic };
+    const streamKey = store.mintKey(KeyType.STREAMS, params);
+    try {
+      await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
+    } catch (err) {
+      //ignore if already exists
+    }
+    await this.activateWorkflow(await hotMeshClient);
+    return hotMeshClient;
   }
 
   workflow = {
@@ -80,36 +96,45 @@ export class ClientService {
       const workflowName = options.workflowName;
       const trc = options.workflowTrace;
       const spn = options.workflowSpan;
+      //topic is concat of taskQueue and workflowName
       const workflowTopic = `${taskQueueName}-${workflowName}`;
-      const hotMesh = await this.getHotMesh(workflowTopic);
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic);
       const payload = {
         arguments: [...options.args],
-        workflowId: options.workflowId,
+        workflowId: options.workflowId || nanoid(),
+        workflowTopic: workflowTopic,
+        backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
       }
       const context = { metadata: { trc, spn }, data: {}};
-      const jobId = await hotMesh.pub(workflowTopic, payload, context as JobState);
-      return new WorkflowHandleService(hotMesh, workflowTopic, jobId);
+      const jobId = await hotMeshClient.pub(
+        SUBSCRIBES_TOPIC,
+        payload,
+        context as JobState);
+      return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
     },
-  };
 
-  async activateWorkflow(hotMesh: HotMesh, workflowTopic: string): Promise<void> {
-    const version = '1';
-    const app = await hotMesh.engine.store.getApp(workflowTopic);
+    signal: async (signalId: string, data: Record<any, any>): Promise<string> => {
+      return await (await this.getHotMeshClient('durable.wfs.signal')).hook('durable.wfs.signal', { id: signalId, data });
+    }
+  }
+
+  async activateWorkflow(hotMesh: HotMesh, appId = APP_ID, version = APP_VERSION): Promise<void> {
+    const app = await hotMesh.engine.store.getApp(appId);
     const appVersion = app?.version as unknown as number;
     if(isNaN(appVersion)) {
       try {
-        await hotMesh.deploy(getWorkflowYAML(workflowTopic, version));
+        await hotMesh.deploy(getWorkflowYAML(appId, version));
         await hotMesh.activate(version);
-      } catch (err) {
-        hotMesh.engine.logger.error('durable-client-deploy-activate-err', err);
-        throw err;
+      } catch (error) {
+        hotMesh.engine.logger.error('durable-client-deploy-activate-err', { error });
+        throw error;
       }
     } else if(app && !app.active) {
       try {
         await hotMesh.activate(version);
-      } catch (err) {
-        hotMesh.engine.logger.error('durable-client-activate-err', err);
-        throw err;
+      } catch (error) {
+        hotMesh.engine.logger.error('durable-client-activate-err', { error});
+        throw error;
       }
     }
   }