@hotmeshio/hotmesh 0.0.34 → 0.0.36

package/services/durable/handle.ts

@@ -1,5 +1,7 @@
-import { JobOutput } from '../../types/job';
+import { STATUS_CODE_INTERRUPT } from '../../modules/enums';
 import { HotMeshService as HotMesh } from '../hotmesh';
+import { JobInterruptOptions, JobOutput } from '../../types/job';
+import { StreamError } from '../../types/stream';
 
 export class WorkflowHandleService {
   hotMesh: HotMesh;
@@ -12,10 +14,22 @@ export class WorkflowHandleService {
     this.hotMesh = hotMesh;
   }
 
+  /**
+   * Sends a signal to the workflow. This is a way to send
+   * a message to a workflow that is paused due to having
+   * executed a `waitForSignal` workflow extension. Awakens
+   * the workflow if no other signals are pending.
+   */
   async signal(signalId: string, data: Record<any, any>): Promise<void> {
     await this.hotMesh.hook(`${this.hotMesh.appId}.wfs.signal`, { id: signalId, data });
   }
 
+  /**
+   * Returns the job state of the workflow. If the workflow has completed
+   * this is also the job output. If the workflow is still running, this
+   * is the current state of the job, but it may change depending upon
+   * the activities that remain.
+   */
   async state(metadata = false): Promise<Record<string, any>> {
     const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
     if (!state.data && state.metadata.err) {
@@ -24,14 +38,41 @@ export class WorkflowHandleService {
     return metadata ? state : state.data;
   }
 
+  /**
+   * Returns the current search state of the workflow. This is
+   * different than the job state or individual activity state.
+   * Search state represents name/value pairs that were added
+   * to the workflow. As the workflow is stored in a Redis hash,
+   * this is a way to store additional data that is indexed
+   * and searchable using the RediSearch module.
+   */
   async queryState(fields: string[]): Promise<Record<string, any>> {
     return await this.hotMesh.getQueryState(this.workflowId, fields);
   }
 
+  /**
+   * Returns the current status of the workflow. This is a semaphore
+   * value that represents the current state of the workflow, where
+   * 0 is complete and a negative value represents that the flow was
+   * interrupted.
+   */
   async status(): Promise<number> {
     return await this.hotMesh.getStatus(this.workflowId);
   }
 
+  /**
+   * Interrupts a running workflow. Standard Job Completion tasks will
+   * run. Subscribers will be notified and the job hash will be expired. 
+   */
+  async interrupt(options?: JobInterruptOptions): Promise<string> {
+    return await this.hotMesh.interrupt(`${this.hotMesh.appId}.execute`, this.workflowId, options);
+  }
+
+  /**
+   * Awaits for the workflow to complete and returns the result. If
+   * the workflow thows and error, this method will likewise throw
+   * an error.
+   */
   async result(loadState?: boolean): Promise<any> {
     if (loadState) {
       const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
@@ -58,29 +99,34 @@ export class WorkflowHandleService {
           return reject(JSON.parse(err));
         } else if (!response) {
           const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
-          if (!state.data && state.metadata.err) {
-            return reject(JSON.parse(state.metadata.err));
+          if (state.metadata.err) {
+            const error = JSON.parse(state.metadata.err) as StreamError;
+            if (error.code === STATUS_CODE_INTERRUPT || !state.data) {
+              return reject({ ...error, job_id: this.workflowId });
+            }
           }
           response = state.data?.response;
         }
         resolve(response);
       };
       //check for done
-      if (status == 0) {
+      if (status <= 0) {
         return complete();
       }
       //subscribe to topic
       this.hotMesh.sub(topic, async (topic: string, state: JobOutput) => {
-        if (!state.data && state.metadata.err) {
-          await complete(null, state.metadata.err);
-        } else {
-          await complete(state.data?.response);
+        if (state.metadata.err) {
+          const error = JSON.parse(state.metadata.err) as StreamError;
+          if (error.code === STATUS_CODE_INTERRUPT || !state.data) {
+            return await complete(null, state.metadata.err);
+          }
         }
+        await complete(state.data?.response);
       });
       //resolve for race condition
       setTimeout(async () => {
         status = await this.hotMesh.getStatus(this.workflowId);
-        if (status == 0) {
+        if (status <= 0) {
           await complete();
         }
       }, 0);

package/services/durable/meshos.ts

@@ -1,5 +1,3 @@
-import { nanoid } from 'nanoid';
-
 import { Durable } from '.';
 import { asyncLocalStorage } from './asyncLocalStorage';
 import { ClientService as Client } from './client';
@@ -18,6 +16,7 @@ import {
   WorkflowSearchOptions } from '../../types/durable';
 import { RedisOptions, RedisClass } from '../../types/redis';
 import { StringAnyType } from '../../types/serializer';
+import { guid } from '../../modules/utils';
 
 /**
  * The base class for running MeshOS workflows.
@@ -117,7 +116,7 @@ export class MeshOSService {
    */
   static mintGuid(): string {
     const my = new this();
-    return `${my.search?.prefix?.[0]}${nanoid()}`;
+    return `${my.search?.prefix?.[0]}${guid()}`;
   }
 
   /**

package/services/durable/worker.ts

@@ -201,14 +201,21 @@ export class WorkerService {
         //incoming data payload has arguments and workflowId
         const workflowInput = data.data as unknown as WorkflowDataType;
         const context = new Map();
+        context.set('raw', data);
         context.set('namespace', config.namespace ?? APP_ID);
         context.set('counter', counter);
         context.set('workflowId', workflowInput.workflowId);
-        if (data.data.workflowDimension) {
+        context.set('workflowId', workflowInput.workflowId);
+        if (workflowInput.originJobId) {
+          //if present there is an origin job to which this job is subordinated; 
+          // garbage collect (expire) this job when originJobId is expired
+          context.set('originJobId', workflowInput.originJobId);
+        }
+        if (workflowInput.workflowDimension) {
           //every hook function runs in an isolated dimension controlled
           //by the index assigned when the signal was received; even if the
           //hook function re-runs, its scope will always remain constant
-          context.set('workflowDimension', data.data.workflowDimension);
+          context.set('workflowDimension', workflowInput.workflowDimension);
         }
         context.set('workflowTopic', workflowTopic);
         context.set('workflowName', workflowTopic.split('-').pop());

package/services/durable/workflow.ts

@@ -19,7 +19,7 @@ import {
   ProxyType,
   WorkflowContext,
   WorkflowOptions } from "../../types/durable";
-import { JobOutput, JobState } from '../../types/job';
+import { JobInterruptOptions, JobOutput, JobState } from '../../types/job';
 import { StreamStatus } from '../../types/stream';
 import { deterministicRandom } from '../../modules/utils';
 
@@ -35,6 +35,7 @@ export class WorkflowService {
     const store = asyncLocalStorage.getStore();
     const namespace = store.get('namespace');
     const workflowId = store.get('workflowId');
+    const originJobId = store.get('originJobId');
     const workflowDimension = store.get('workflowDimension') ?? '';
     const workflowTrace = store.get('workflowTrace');
     const workflowSpan = store.get('workflowSpan');
@@ -65,6 +66,7 @@ export class WorkflowService {
         ...options,
         namespace,
         workflowId: childJobId,
+        originJobId: originJobId ?? workflowId, 
         parentWorkflowId,
         workflowTrace,
         workflowSpan,
@@ -289,6 +291,66 @@ export class WorkflowService {
     }
   }
 
+  static getLocalState() {
+    const store = asyncLocalStorage.getStore();
+    return {
+      workflowId: store.get('workflowId'),
+      namespace: store.get('namespace'),
+      workflowTopic: store.get('workflowTopic'),
+      workflowDimension: store.get('workflowDimension') ?? '',
+      counter: store.get('counter'),
+    }
+  }
+
+  /**
+   * Executes a function once and caches the result. If the function is called
+   * again, the cached result is returned. This is useful for wrapping
+   * expensive activity calls that should only be run once, but which might
+   * not require the configuration nuance/expense provided by proxyActivities.
+   * @template T - the result type
+   */
+  static async once<T>(fn: (...args: any[]) => Promise<T>, ...args: any[]): Promise<T> {
+    const {
+      workflowId,
+      namespace,
+      workflowTopic,
+      workflowDimension,
+      counter: COUNTER,
+    } = WorkflowService.getLocalState();
+    const execIndex = COUNTER.counter = COUNTER.counter + 1;
+    const sessionId = `-once${workflowDimension}-${execIndex}-`;
+    const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
+    const keyParams = {
+      appId: hotMeshClient.appId,
+      jobId: workflowId
+    }
+    const workflowGuid = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+    const value = await hotMeshClient.engine.store.exec('HGET', workflowGuid, sessionId) as string;
+    if (value) {
+      return JSON.parse(value) as T;
+    }
+    const response = await fn(...args);
+    await hotMeshClient.engine.store.exec('HSET', workflowGuid, sessionId, JSON.stringify(response));
+    return response;
+  }
+
+  /** 
+   * Interrupts a running job
+   * 
+   * @param {string} jobId - the target job id
+   * @param {JobInterruptOptions} options - the interrupt options
+   * @returns {Promise<string>} - the stream id
+   */
+  static async interrupt(jobId: string, options: JobInterruptOptions = {}): Promise<string | void> {
+    const store = asyncLocalStorage.getStore();
+    const workflowTopic = store.get('workflowTopic');
+    const namespace = store.get('namespace');
+    const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
+    if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'interrupt')) {
+      return await hotMeshClient.interrupt(`${hotMeshClient.appId}.execute`, jobId, options);
+    }
+  }
+
   /**
    * Sleeps the workflow for a duration. As the function is reentrant, 
    * upon reentry, the function will traverse prior execution paths up
@@ -407,6 +469,7 @@ export class WorkflowService {
       //increment by state (not value) to avoid race conditions
       const execIndex = COUNTER.counter = COUNTER.counter + 1;
       const workflowId = store.get('workflowId');
+      const originJobId = store.get('originJobId');
       const workflowDimension = store.get('workflowDimension') ?? '';
       const workflowTopic = store.get('workflowTopic');
       const trc = store.get('workflowTrace');
@@ -439,7 +502,8 @@ export class WorkflowService {
         const duration = ms(options?.startToCloseTimeout || '1 minute');
         const payload = {
           arguments: Array.from(arguments),
-          //the parent id is provided to categorize this activity for later cleanup
+          //when the origin job is removed
+          originJobId: originJobId ?? workflowId,
           parentWorkflowId: `${workflowId}-a`,
           workflowId: activityJobId,
           workflowTopic: activityTopic,

package/services/engine/index.ts

@@ -1,7 +1,14 @@
 import { KeyType } from '../../modules/key';
+import {
+  OTT_WAIT_TIME,
+  STATUS_CODE_SUCCESS,
+  STATUS_CODE_PENDING,
+  STATUS_CODE_TIMEOUT, 
+  DURABLE_EXPIRE_SECONDS} from '../../modules/enums';
 import {
   formatISODate,
   getSubscriptionTopic,
+  guid,
   identifyRedisType,
   polyfill,
   restoreHierarchy } from '../../modules/utils';
@@ -9,6 +16,7 @@ import Activities from '../activities';
 import { Await } from '../activities/await';
 import { Cycle } from '../activities/cycle';
 import { Hook } from '../activities/hook';
+import { Interrupt } from '../activities/interrupt';
 import { Signal } from '../activities/signal';
 import { Worker } from '../activities/worker';
 import { Trigger } from '../activities/trigger';
@@ -41,7 +49,9 @@ import {
   JobMetadata,
   JobOutput,
   PartialJobState,
-  JobStatus } from '../../types/job';
+  JobStatus, 
+  JobInterruptOptions,
+  JobCompletionOptions } from '../../types/job';
 import {
   HotMeshApps,
   HotMeshConfig,
@@ -69,12 +79,6 @@ import {
   StreamRole,
   StreamStatus } from '../../types/stream';
 
-//wait time to see if a job is complete
-const OTT_WAIT_TIME = 1000;
-const STATUS_CODE_SUCCESS = 200;
-const STATUS_CODE_PENDING = 202;
-const STATUS_CODE_TIMEOUT = 504;
-
 class EngineService {
   namespace: string;
   apps: HotMeshApps | null;
@@ -239,7 +243,7 @@ class EngineService {
   }
 
   // ************* METADATA/MODEL METHODS *************
-  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Await|Cycle|Hook|Signal|Trigger|Worker> {
+  async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Await|Cycle|Hook|Signal|Trigger|Worker|Interrupt> {
     const [activityId, schema] = await this.getSchema(topic);
     polyfill
     const ActivityHandler = Activities[polyfill.resolveActivityType(schema.type)];
@@ -329,6 +333,7 @@ class EngineService {
       aid: streamData.metadata.aid,
       status: streamData.status || StreamStatus.SUCCESS,
       code: streamData.code || 200,
+      type: streamData.type,
     });
     const context: PartialJobState = {
       metadata: {
@@ -380,6 +385,7 @@ class EngineService {
       const spn = context['$self']?.output?.metadata?.l2s || context['$self']?.output?.metadata?.l1s;
       const streamData: StreamData = {
         metadata: {
+          guid: guid(),
           jid: context.metadata.pj,
           dad: context.metadata.pd,
           aid: context.metadata.pa,
@@ -404,17 +410,28 @@ class EngineService {
     }
   }
   hasParentJob(context: JobState): boolean {
-    //todo: include the dimensional address (pd)
     return Boolean(context.metadata.pj && context.metadata.pa);
   }
   resolveError(metadata: JobMetadata): StreamError | undefined {
     if (metadata && metadata.err) {
       return JSON.parse(metadata.err) as StreamError;
-    }
+    } 
+  }
+
+  // ****************** `INTERRUPT` ACTIVE JOBS *****************
+  async interrupt(topic: string, jobId: string, options: JobInterruptOptions = {}): Promise<string> {
+    await this.store.interrupt(topic, jobId, options);
+    const context = await this.getState(topic, jobId) as JobState;
+    const completionOpts: JobCompletionOptions = {
+      interrupt: options.descend,
+      expire: options.expire,
+    };
+    return await this.runJobCompletionTasks(context, completionOpts) as string;
   }
 
   // ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
   async scrub(jobId: string) {
+    //todo: do not allow scrubbing of non-existent or actively running job
     await this.store.scrub(jobId);
   }
 
@@ -427,6 +444,7 @@ class EngineService {
       status,
       code,
       metadata: {
+        guid: guid(),
         aid,
         topic
       },
@@ -434,13 +452,23 @@ class EngineService {
     };
     return await this.streamSignaler.publishMessage(null, streamData) as string;
   }
-  async hookTime(jobId: string, activityId: string): Promise<JobStatus | void> {
-    //the activityid is concatenated with its dimensional address (dad); split to resolve
+  async hookTime(jobId: string, activityId: string, type?: 'sleep'|'expire'|'interrupt'): Promise<string | void> {
+    if (type === 'interrupt') {
+      return await this.interrupt(
+        activityId, //note: 'activityId' is the actually job topic
+        jobId,
+        { suppress: true, expire: 1 },
+      );
+    } else if (type === 'expire') {
+      return await this.store.expireJob(jobId, 1);
+    }
+    //'sleep': parse the activityId into parts
     const [aid, ...dimensions] = activityId.split(',');
     const dad = `,${dimensions.join(',')}`;
     const streamData: StreamData = {
       type: StreamDataType.TIMEHOOK,
       metadata: {
+        guid: guid(),
         jid: jobId,
         aid,
         dad,
@@ -540,7 +568,7 @@ class EngineService {
       }, timeout);
     });
   }
-  async resolveOneTimeSubscription(context: JobState, jobOutput: JobOutput, emit = false) {
+  async pubOneTimeSubs(context: JobState, jobOutput: JobOutput, emit = false) {
     //todo: subscriber should query for the job...only publish minimum context needed
     if (this.hasOneTimeSubscription(context)) {
       const message: JobMessage = {
@@ -557,7 +585,7 @@ class EngineService {
     const schema = await this.store.getSchema(activityId, config);
     return schema.publishes;
   }
-  async resolvePersistentSubscriptions(context: JobState, jobOutput: JobOutput, emit = false) {
+  async pubPermSubs(context: JobState, jobOutput: JobOutput, emit = false) {
     const topic = await this.getPublishesTopic(context);
     if (topic) {
       const message: JobMessage = {
@@ -584,22 +612,42 @@ class EngineService {
 
 
   // ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
-  async runJobCompletionTasks(context: JobState, emit = false) {
-    //if 'emit' is true, the job isn't done. it's just emitting
+  async runJobCompletionTasks(context: JobState, options: JobCompletionOptions = {}): Promise<string | void> {
+    //'emit' indicates the job is still active
     const isAwait = this.hasParentJob(context);
-    const isOneTimeSubscription = this.hasOneTimeSubscription(context);
+    const isOneTimeSub = this.hasOneTimeSubscription(context);
     const topic = await this.getPublishesTopic(context);
-    if (isAwait || isOneTimeSubscription || topic) {
-      const jobOutput = await this.getState(context.metadata.tpc, context.metadata.jid);
-      //always wait for stream pub/sub
-      await this.execAdjacentParent(context, jobOutput, emit);
-      //no need to wait for standard pub/sub
-      this.resolveOneTimeSubscription(context, jobOutput, emit);
-      this.resolvePersistentSubscriptions(context, jobOutput, emit);
+    let msgId: string;
+    if (isAwait || isOneTimeSub || topic) {
+      const jobOutput = await this.getState(
+        context.metadata.tpc,
+        context.metadata.jid,
+      );
+      msgId = await this.execAdjacentParent(
+        context,
+        jobOutput,
+        options.emit,
+       );
+      this.pubOneTimeSubs(context, jobOutput, options.emit);
+      this.pubPermSubs(context, jobOutput, options.emit);
     }
-    if (!emit) {
-      this.task.registerJobForCleanup(context.metadata.jid, context.metadata.expire);
+    if (!options.emit) {
+      this.task.registerJobForCleanup(
+        context.metadata.jid,
+        this.resolveExpires(context, options),
+        options,
+      );
     }
+    return msgId;
+  }
+
+  /**
+   * Job hash expiration is typically reliant on the metadata field
+   * if the activity concludes normally. However, if the job is `interrupted`,
+   * it will be expired immediately.
+   */
+  resolveExpires(context: JobState, options: JobCompletionOptions): number {
+    return context.metadata.expire ?? options.expire ?? DURABLE_EXPIRE_SECONDS;
   }
 
 

package/services/hotmesh/index.ts

@@ -1,5 +1,5 @@
-import { nanoid } from 'nanoid';
 import { HMNS } from '../../modules/key';
+import { guid } from '../../modules/utils';
 import { RedisConnection } from '../connector/clients/redis';
 import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
 import { EngineService } from '../engine';
@@ -11,7 +11,8 @@ import {
   JobState,
   JobData,
   JobOutput, 
-  JobStatus } from '../../types/job';
+  JobStatus, 
+  JobInterruptOptions} from '../../types/job';
 import {
   HotMeshConfig,
   HotMeshManifest } from '../../types/hotmesh';
@@ -58,7 +59,7 @@ class HotMeshService {
 
   static async init(config: HotMeshConfig) {
     const instance = new HotMeshService();
-    instance.guid = nanoid();
+    instance.guid = guid();
     instance.verifyAndSetNamespace(config.namespace);
     instance.verifyAndSetAppId(config.appId);
     instance.logger = new LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
@@ -69,7 +70,7 @@ class HotMeshService {
   }
 
   static guid(): string {
-    return nanoid();
+    return guid();
   }
 
   async initEngine(config: HotMeshConfig, logger: ILogger): Promise<void> {
@@ -146,6 +147,10 @@ class HotMeshService {
     //activation is a quorum operation
     return await this.quorum?.activate(version, delay);
   }
+  async inventory(version: string, delay?: number): Promise<number> {
+    //get count of all peers
+    return await this.quorum?.inventory(delay);
+  }
 
   // ************* REPORTER METHODS *************
   async getStats(topic: string, query: JobStatsInput): Promise<StatsResponse> {
@@ -167,6 +172,11 @@ class HotMeshService {
     return await this.engine?.resolveQuery(topic, query);
   }
 
+  // ****************** `INTERRUPT` ACTIVE JOBS *****************
+  async interrupt(topic: string, jobId: string, options: JobInterruptOptions = {}): Promise<string> {
+    return await this.engine?.interrupt(topic, jobId, options);
+  }
+
   // ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
   async scrub(jobId: string) {
     await this.engine?.scrub(jobId);

package/services/quorum/index.ts

@@ -117,6 +117,8 @@ class QuorumService {
         self.engine.processWebHooks()
       } else if (message.type === 'job') {
         self.engine.routeToSubscribers(message.topic, message.job)
+      } else if (message.type === 'cron') {
+        self.engine.processTimeHooks();
       }
       //if there are any callbacks, call them
       if (self.callbacks.length > 0) {
@@ -164,6 +166,13 @@ class QuorumService {
 
 
   // ************* COMPILER METHODS *************
+  async inventory(delay = QUORUM_DELAY): Promise<number> {
+    await this.requestQuorum(delay);
+    const q1 = await this.requestQuorum(delay);
+    const q2 = await this.requestQuorum(delay);
+    const q3 = await this.requestQuorum(delay);
+    return Math.round((q1 + q2 + q3) / 3);
+  }
   async activate(version: string, delay = QUORUM_DELAY): Promise<boolean> {
     version = version.toString();
     const config = await this.engine.getVID();