@hotmeshio/hotmesh 0.0.33 → 0.0.35

package/services/activities/interrupt.ts

@@ -0,0 +1,159 @@
+import { EngineService } from '../engine';
+import { Activity, ActivityType } from './activity';
+import {
+  ActivityData,
+  ActivityMetadata,
+  InterruptActivity } from '../../types/activity';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
+import { MultiResponseFlags } from '../../types';
+import { CollatorService } from '../collator';
+import { JobInterruptOptions, JobState } from '../../types/job';
+import { TelemetryService } from '../telemetry';
+import { Pipe } from '../pipe';
+
+class Interrupt extends Activity {
+  config: InterruptActivity;
+
+  constructor(
+    config: ActivityType,
+    data: ActivityData,
+    metadata: ActivityMetadata,
+    hook: ActivityData | null,
+    engine: EngineService,
+    context?: JobState
+    ) {
+      super(config, data, metadata, hook, engine, context);
+  }
+
+
+  //********  LEG 1 ENTRY  ********//
+  async process(): Promise<string> {
+    this.logger.debug('interrupt-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    let telemetry: TelemetryService;
+    try {
+      this.setLeg(1);
+      await CollatorService.notarizeEntry(this);
+      await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid); // Ensure job active
+      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+      telemetry.startActivitySpan(this.leg);
+  
+      if (this.isInterruptingSelf()) {
+        return await this.interruptSelf(telemetry);
+      } else {
+        return await this.interruptAnother(telemetry);
+      }
+    } catch (error) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('interrupt-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
+        this.logger.error('interrupt-get-state-error', { error });
+        return;
+      } else {
+        this.logger.error('interrupt-process-error', { error });
+      }
+      telemetry.setActivityError(error.message);
+      throw error;
+    } finally {
+      telemetry?.endActivitySpan();
+      this.logger.debug('interrupt-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    }
+  }
+
+  async interruptSelf(telemetry: TelemetryService): Promise<string> {
+    // Apply final updates to THIS job's state
+    if (this.config.job?.maps) {
+      this.mapJobData();
+      await this.setState();
+    }
+  
+    // Interrupt THIS job
+    const messageId = await this.interrupt();
+  
+    // Notarize completion and log
+    telemetry.mapActivityAttributes();
+    const multi = this.store.getMulti();
+    await CollatorService.notarizeEarlyCompletion(this, multi);
+    await this.setStatus(-1, multi);
+    const multiResponse = await multi.exec() as MultiResponseFlags;
+    const jobStatus = this.resolveStatus(multiResponse);
+    telemetry.setActivityAttributes({
+      'app.activity.mid': messageId,
+      'app.job.jss': jobStatus
+    });
+  
+    return this.context.metadata.aid;
+  }
+
+  async interruptAnother(telemetry: TelemetryService): Promise<string> {
+    // Interrupt ANOTHER job
+    const messageId = await this.interrupt();
+    const attrs = { 'app.activity.mid': messageId };
+  
+    // Apply updates to THIS job's state
+    telemetry.mapActivityAttributes();
+    this.adjacencyList = await this.filterAdjacent();
+    if (this.config.job?.maps || this.config.output?.maps) {
+      this.mapOutputData();
+      this.mapJobData();
+      const multi = this.store.getMulti();
+      await this.setState(multi);
+    }
+  
+    // Notarize completion
+    const multi = this.store.getMulti();
+    await CollatorService.notarizeEarlyCompletion(this, multi);
+    await this.setStatus(this.adjacencyList.length - 1, multi);
+    const multiResponse = await multi.exec() as MultiResponseFlags;
+    const jobStatus = this.resolveStatus(multiResponse);
+    attrs['app.job.jss'] = jobStatus;
+  
+    // Transition next generation and log
+    const messageIds = await this.transition(this.adjacencyList, jobStatus);
+    if (messageIds.length) {
+      attrs['app.activity.mids'] = messageIds.join(',');
+    }
+    telemetry.setActivityAttributes(attrs);
+  
+    return this.context.metadata.aid;
+  }
+  
+
+  isInterruptingSelf(): boolean {
+    if (!this.config.target) {
+      return true;
+    }
+    const resolvedJob = Pipe.resolve(this.config.target, this.context);
+    return resolvedJob == this.context.metadata.jid;
+  }
+
+  resolveInterruptOptions(): JobInterruptOptions {
+    return {
+      reason: this.config.reason !== undefined 
+        ? Pipe.resolve(this.config.reason, this.context) 
+        : undefined,
+      throw: this.config.throw !== undefined 
+        ? Pipe.resolve(this.config.throw, this.context) 
+        : undefined,
+      descend: this.config.descend !== undefined 
+        ? Pipe.resolve(this.config.descend, this.context) 
+        : undefined,
+    };
+  }
+
+  async interrupt(): Promise<string> {
+    const options = this.resolveInterruptOptions();
+    return await this.engine.interrupt(
+      this.config.topic !== undefined 
+        ? Pipe.resolve(this.config.topic, this.context) 
+        : this.context.metadata.tpc,
+      this.config.target !== undefined 
+        ? Pipe.resolve(this.config.target, this.context) 
+        : this.context.metadata.jid,
+      options as JobInterruptOptions,
+    );
+  }
+}
+
+export { Interrupt };

package/services/activities/signal.ts

@@ -1,4 +1,4 @@
-import { GetStateError } from '../../modules/errors';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
 import { Activity, ActivityType } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -37,6 +37,7 @@ class Signal extends Activity {
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
 
@@ -50,6 +51,7 @@ class Signal extends Activity {
       await this.setStatus(this.adjacencyList.length - 1, multi);
       const multiResponse = await multi.exec() as MultiResponseFlags;
 
+      //todo: this should execute BEFORE the status is decremented
       if (this.config.subtype === 'all') {
         await this.hookAll();
       } else {
@@ -68,15 +70,19 @@ class Signal extends Activity {
 
       return this.context.metadata.aid;
     } catch (error) {
-      if (error instanceof GetStateError) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('signal-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
         this.logger.error('signal-get-state-error', { error });
+        return;
       } else {
         this.logger.error('signal-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('signal-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     }
   }

package/services/activities/trigger.ts

@@ -1,6 +1,5 @@
-import { nanoid } from 'nanoid';
 import { DuplicateJobError } from '../../modules/errors';
-import { formatISODate, getTimeSeries } from '../../modules/utils';
+import { formatISODate, getTimeSeries, guid } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -47,6 +46,7 @@ class Trigger extends Activity {
       const multi = this.store.getMulti();
       await this.setState(multi);
       await this.setStats(multi);
+      await this.setDependency(multi);
       await multi.exec();
 
       telemetry.mapActivityAttributes();
@@ -68,8 +68,8 @@ class Trigger extends Activity {
       telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endJobSpan();
-      telemetry.endActivitySpan();
+      telemetry?.endJobSpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('trigger-process-end', { subscribes: this.config.subscribes, jid: this.context.metadata.jid });
     }
   }
@@ -161,7 +161,7 @@ class Trigger extends Activity {
 
   resolveJobId(context: Partial<JobState>): string {
     const jobId = this.config.stats?.id;
-    return jobId ? Pipe.resolve(jobId, context) : nanoid();
+    return jobId ? Pipe.resolve(jobId, context) : guid();
   }
 
   resolveJobKey(context: Partial<JobState>): string {
@@ -176,6 +176,22 @@ class Trigger extends Activity {
     }
   }
 
+  /**
+   * Registers this job as a dependent of the parent job
+   */
+  async setDependency(multi?: RedisMulti): Promise<void> {
+    const depKey = this.config.stats?.parent;
+    const resolvedDepKey = depKey ? Pipe.resolve(depKey, this.context) : '';
+    if (resolvedDepKey) {
+      await this.store.setDependency(
+        resolvedDepKey,
+        this.context.metadata.tpc,
+        this.context.metadata.jid,
+        multi,
+      );
+    }
+  }
+
   async setStats(multi?: RedisMulti): Promise<void> {
     const md = this.context.metadata;
     if (md.key && this.config.stats?.measures) {

package/services/activities/worker.ts

@@ -1,4 +1,4 @@
-import { GetStateError } from '../../modules/errors';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -12,6 +12,7 @@ import { MultiResponseFlags, RedisMulti } from '../../types/redis';
 import { StreamData} from '../../types/stream';
 import { TelemetryService } from '../telemetry';
 import { Pipe } from '../pipe';
+import { guid } from '../../modules/utils';
 
 class Worker extends Activity {
   config: WorkerActivity;
@@ -35,6 +36,7 @@ class Worker extends Activity {
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
       this.mapInputData();
@@ -58,15 +60,19 @@ class Worker extends Activity {
 
       return this.context.metadata.aid;
     } catch (error) {
-      if (error instanceof GetStateError) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('await-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
         this.logger.error('worker-get-state-error', { error });
+        return;
       } else {
         this.logger.error('worker-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('worker-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     }
   }
@@ -75,6 +81,7 @@ class Worker extends Activity {
     const topic = Pipe.resolve(this.config.subtype, this.context);
     const streamData: StreamData = {
       metadata: {
+        guid: guid(),
         jid: this.context.metadata.jid,
         dad: this.metadata.dad,
         aid: this.metadata.aid,

package/services/collator/index.ts

@@ -1,4 +1,4 @@
-import { CollationError } from '../../modules/errors';
+import { CollationError, InactiveJobError } from '../../modules/errors';
 import { RedisMulti } from '../../types/redis';
 import { CollationFaultType, CollationStage } from '../../types/collator';
 import { ActivityDuplex } from '../../types/activity';
@@ -11,6 +11,15 @@ class CollatorService {
   //max int digit count that supports `hincrby`
   static targetLength = 15;
 
+  /**
+   * Upon re/entry, verify that the job status is active
+   */
+  static assertJobActive(status: number, jobId: string, activityId: string): void {
+    if (status <= 0) {
+      throw new InactiveJobError(jobId, status, activityId);
+    }
+  }
+
   /**
    * returns the dimensional address (dad) for the target; due
    * to the nature of the notary system, the dad for leg 2 entry

package/services/compiler/deployer.ts

@@ -143,9 +143,7 @@ class Deployer {
         if (graph.publishes) {
           activities[activityKey].publishes = graph.publishes;
         }
-        if (graph.expire) {
-          activities[activityKey].expire = graph.expire;
-        }
+        activities[activityKey].expire = graph.expire ?? undefined;
       }
     }
   }

package/services/connector/index.ts

@@ -1,6 +1,4 @@
-import { nanoid } from 'nanoid';
-
-import { identifyRedisTypeFromClass } from '../../modules/utils';
+import { guid, identifyRedisTypeFromClass } from '../../modules/utils';
 import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
 import { RedisConnection } from '../connector/clients/redis';
 import {
@@ -23,14 +21,14 @@ export class ConnectorService {
       if (identifyRedisTypeFromClass(Redis) === 'redis') {
         for (let i = 1; i <= 3; i++) {
           instances.push(RedisConnection.connect(
-            nanoid(),
+            guid(),
             Redis as RedisClassType,
             options as RedisClientOptions));
         }
       } else {
         for (let i = 1; i <= 3; i++) {
           instances.push(IORedisConnection.connect(
-            nanoid(),
+            guid(),
             Redis as IORedisClassType,
             options as IORedisClientOptions));
         }

package/services/durable/client.ts

@@ -11,6 +11,7 @@ import { JobState } from '../../types/job';
 import { KeyService, KeyType } from '../../modules/key';
 import { Search } from './search';
 import { StreamStatus } from '../../types';
+import { DURABLE_EXPIRE_SECONDS } from '../../modules/enums';
 
 export class ClientService {
 
@@ -96,12 +97,15 @@ export class ClientService {
       const workflowName = options.entity ?? options.workflowName;
       const trc = options.workflowTrace;
       const spn = options.workflowSpan;
-      //topic is concat of taskQueue and workflowName
+      //NOTE: HotMesh 'workflowTopic' is a created by concatenating
+      //     the taskQueue and workflowName used by the Durable module
       const workflowTopic = `${taskQueueName}-${workflowName}`;
       const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
-      this.configureSearchIndex(hotMeshClient, options.search)
+      this.configureSearchIndex(hotMeshClient, options.search);
       const payload = {
         arguments: [...options.args],
+        originJobId: options.originJobId,
+        expire: options.expire ?? DURABLE_EXPIRE_SECONDS,
         parentWorkflowId: options.parentWorkflowId,
         workflowId: options.workflowId || HotMesh.guid(),
         workflowTopic: workflowTopic,
@@ -113,12 +117,12 @@ export class ClientService {
         payload,
         context as JobState
       );
-      // Seed search data if present
+      // Seed search data
       if (jobId && options.search?.data) {
         const searchSessionId = `-search-0`;
         const search = new Search(jobId, hotMeshClient, searchSessionId);
         const entries = Object.entries(options.search.data).flat();
-        search.set(...entries);
+        await search.set(...entries);
       }
       return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
     },
@@ -198,9 +202,8 @@ export class ClientService {
   }
 
   static async shutdown(): Promise<void> {
-    for (const [key, value] of ClientService.instances) {
-      const hotMesh = await value;
-      await hotMesh.stop();
+    for (const [_, hotMeshInstance] of ClientService.instances) {
+      (await hotMeshInstance).stop();
     }
   }
 }