@hotmeshio/hotmesh 0.0.34 → 0.0.36

package/build/types/stream.d.ts

@@ -22,10 +22,13 @@ export declare enum StreamDataType {
     RESULT = "result",
     WORKER = "worker",
     RESPONSE = "response",
-    TRANSITION = "transition"
+    TRANSITION = "transition",
+    SIGNAL = "signal",
+    INTERRUPT = "interrupt"
 }
 export interface StreamData {
     metadata: {
+        guid: string;
         topic?: string;
         jid?: string;
         dad?: string;

package/build/types/stream.js

@@ -16,6 +16,8 @@ var StreamDataType;
     StreamDataType["WORKER"] = "worker";
     StreamDataType["RESPONSE"] = "response";
     StreamDataType["TRANSITION"] = "transition";
+    StreamDataType["SIGNAL"] = "signal";
+    StreamDataType["INTERRUPT"] = "interrupt";
 })(StreamDataType = exports.StreamDataType || (exports.StreamDataType = {}));
 var StreamRole;
 (function (StreamRole) {

package/modules/enums.ts

@@ -0,0 +1,32 @@
+// Engine Constants
+export const STATUS_CODE_SUCCESS = 200;
+export const STATUS_CODE_PENDING = 202;
+export const STATUS_CODE_TIMEOUT = 504;
+export const STATUS_CODE_INTERRUPT = 410;
+export const OTT_WAIT_TIME = 1000;
+
+// Stream Constants
+export const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
+export const MAX_TIMEOUT_MS = 60000;
+export const GRADUATED_INTERVAL_MS = 5000;
+
+export const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
+export const TEST_BLOCK_DURATION = 1000; //Set to `1000` so tests can interrupt quickly
+export const BLOCK_TIME_MS = process.env.NODE_ENV === 'test' ? TEST_BLOCK_DURATION : BLOCK_DURATION;
+
+export const XCLAIM_DELAY_MS = 1000 * 60; //max time a message can be unacked before it is claimed by another
+export const XCLAIM_COUNT = 3; //max number of times a message can be claimed by another before it is dead-lettered
+export const XPENDING_COUNT = 10;
+
+export const STATUS_CODE_UNACKED = 999;
+export const STATUS_CODE_UNKNOWN = 500;
+export const STATUS_MESSAGE_UNKNOWN = 'unknown';
+
+// HotMesh Constants
+export const EXPIRE_DURATION = 15;        // default expire in seconds; once job state semaphore reaches '0', this is applied to set Redis to expire the job HASH
+export const BASE_FIDELITY_SECONDS = 15;  // granularity resolution window size
+export const TEST_FIDELITY_SECONDS = 5;
+export const FIDELITY_SECONDS = process.env.NODE_ENV === 'test' ? TEST_FIDELITY_SECONDS : BASE_FIDELITY_SECONDS
+
+// DURABLE CONSTANTS
+export const DURABLE_EXPIRE_SECONDS = 1;

package/modules/errors.ts

@@ -2,8 +2,11 @@ import { ActivityDuplex } from "../types/activity";
 import { CollationFaultType, CollationStage } from "../types/collator";
 
 class GetStateError extends Error {
-  constructor() {
-    super("Error occurred while getting job state");
+  jobId: string;
+  code: 404;
+  constructor(jobId: string) {
+    super(`${jobId} Not Found`);
+    this.jobId = jobId;
   }
 }
 class SetStateError extends Error {
@@ -107,7 +110,18 @@ class DuplicateJobError extends Error {
     this.message = `Duplicate job: ${jobId}`;
   }
 }
-
+class InactiveJobError extends Error {
+  jobId: string;
+  activityId: string;
+  status: number; //non-positive integer
+  constructor(jobId: string, status: number, activityId: string) {
+    super("Inactive job");
+    this.jobId = jobId;
+    this.activityId = activityId;
+    this.message = `Inactive job: ${jobId}`;
+    this.status = status;
+  }
+}
 class ExecActivityError extends Error {
   constructor() {
     super("Error occurred while executing activity");
@@ -131,18 +145,19 @@ class CollationError extends Error {
 
 export {
   CollationError,
-  DurableTimeoutError,
-  DurableMaxedError,
   DurableFatalError,
-  DurableRetryError,
-  DurableWaitForSignalError,
   DurableIncompleteSignalError,
+  DurableMaxedError,
+  DurableRetryError,
   DurableSleepError,
   DurableSleepForError,
+  DurableTimeoutError,
+  DurableWaitForSignalError,
   DuplicateJobError,
+  ExecActivityError,
   GetStateError,
-  SetStateError,
+  InactiveJobError,
   MapDataError,
   RegisterTimeoutError,
-  ExecActivityError
+  SetStateError,
 };

package/modules/key.ts

@@ -34,6 +34,7 @@ enum KeyType {
   APP,
   ENGINE_ID,
   HOOKS,
+  JOB_DEPENDENTS,
   JOB_STATE,
   JOB_STATS_GENERAL,
   JOB_STATS_MEDIAN,
@@ -93,7 +94,9 @@ class KeyService {
       case KeyType.QUORUM:
         return `${namespace}:${params.appId}:q:${params.engineId || ''}`;
       case KeyType.JOB_STATE:
-        return `${namespace}:${params.appId}:j:${params.jobId}`;
+          return `${namespace}:${params.appId}:j:${params.jobId}`;
+      case KeyType.JOB_DEPENDENTS:
+        return `${namespace}:${params.appId}:d:${params.jobId}`;
       case KeyType.JOB_STATS_GENERAL:
         return `${namespace}:${params.appId}:s:${params.jobKey}:${params.dateTime}`;
       case KeyType.JOB_STATS_MEDIAN:

package/modules/utils.ts

@@ -1,3 +1,4 @@
+import { nanoid } from "nanoid";
 import { StoreService } from "../services/store";
 import { AppSubscriptions, AppTransitions, AppVID } from "../types/app";
 import { RedisClient, RedisMulti } from "../types/redis";
@@ -8,6 +9,10 @@ export async function sleepFor(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+export function guid(): string {
+  return nanoid();
+}
+
 export function deterministicRandom(seed: number): number {
   let x = Math.sin(seed) * 10000;
   return x - Math.floor(x);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.34",
+  "version": "0.0.36",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -30,6 +30,7 @@
     "test:emit": "NODE_ENV=test jest ./tests/functional/emit/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:hook": "NODE_ENV=test jest ./tests/functional/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:signal": "NODE_ENV=test jest ./tests/functional/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:interrupt": "NODE_ENV=test jest ./tests/functional/interrupt/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:parallel": "NODE_ENV=test jest ./tests/functional/parallel/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:sequence": "NODE_ENV=test jest ./tests/functional/sequence/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:quorum": "NODE_ENV=test jest ./tests/functional/quorum/index.test.ts --detectOpenHandles --forceExit --verbose",
@@ -46,6 +47,7 @@
     "test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:meshos": "NODE_ENV=test jest ./tests/durable/meshos/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:interrupt": "NODE_ENV=test jest ./tests/durable/interrupt/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hook": "NODE_ENV=test jest ./tests/durable/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",

package/services/activities/activity.ts

@@ -1,7 +1,8 @@
-import { CollationError } from '../../modules/errors';
+import { CollationError, GetStateError, InactiveJobError } from '../../modules/errors';
 import {
   formatISODate,
   getValueByPath,
+  guid,
   restoreHierarchy } from '../../modules/utils';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -29,6 +30,7 @@ import {
   StreamDataType,
   StreamStatus } from '../../types/stream';
 import { TransitionRule } from '../../types/transition';
+import { EXPIRE_DURATION } from '../../modules/enums';
 
 /**
  * The base class for all activities
@@ -84,6 +86,7 @@ class Activity {
     let telemetry: TelemetryService;
     try {
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       const aState = await CollatorService.notarizeReentry(this);
       this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
 
@@ -110,12 +113,18 @@ class Activity {
       if (error instanceof CollationError) {
         this.logger.info('process-event-inactive-error', { error });
         return;
+      } else if (error instanceof InactiveJobError) {
+        this.logger.info('process-event-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
+        this.logger.info('process-event-get-job-error', { error });
+        return;
       }
       this.logger.error('activity-process-event-error', { error });
       telemetry && telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry && telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('activity-process-event-end', { jid, aid });
     }
   }
@@ -159,6 +168,7 @@ class Activity {
     telemetry.mapActivityAttributes();
     const jobStatus = this.resolveStatus(multiResponse);
     const attrs: StringScalarType = { 'app.job.jss': jobStatus };
+    //adjacencyList membership has already been set at this point (according to activity status)
     const messageIds = await this.transition(this.adjacencyList, jobStatus);
     if (messageIds.length) {
       attrs['app.activity.mids'] = messageIds.join(',')
@@ -380,12 +390,16 @@ class Activity {
       self.hook = { };
     }
     context['$self'] = self;
-    context['$job'] = context; //NEVER call STRINGIFY! (circular)
+    context['$job'] = context; //NEVER call STRINGIFY! (now circular)
     return context as JobState;
   }
 
   initPolicies(context: JobState) {
-    context.metadata.expire = this.config.expire;
+    const expire = Pipe.resolve(
+      this.config.expire ?? EXPIRE_DURATION,
+      context
+    );
+    context.metadata.expire = expire;
   }
 
   bindActivityData(type: 'output' | 'hook'): void {
@@ -418,6 +432,7 @@ class Activity {
         if (MapperService.evaluate(transitionRule, this.context, this.code)) {
           adjacencyList.push({
             metadata: {
+              guid: guid(),
               jid: this.context.metadata.jid,
               dad: adjacentDad,
               aid: toActivityId,
@@ -434,17 +449,20 @@ class Activity {
   }
 
   async transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]> {
+    if (this.jobWasInterrupted(jobStatus)) {
+      return;
+    }
     let mIds: string[] = [];
     let emit: boolean = false;
     if (this.config.emit) {
       emit = Pipe.resolve(this.config.emit, this.context);
     }
     if (jobStatus <= 0 || emit) {
-      //activity should not send 'emit' if the job is truly over
-      const isTrueEmit = jobStatus > 0;
-      await this.engine.runJobCompletionTasks(this.context, isTrueEmit);
+      await this.engine.runJobCompletionTasks(
+        this.context,
+        { emit: jobStatus > 0 },
+      );
     }
-
     if (adjacencyList.length && jobStatus > 0) {
       const multi = this.store.getMulti();
       for (const execSignal of adjacencyList) {
@@ -454,6 +472,14 @@ class Activity {
     }
     return mIds;
   }
+
+  /**
+   * A job with a vale < -100_000_000 is considered interrupted,
+   * as the interruption event decrements the job status by 1billion.
+   */
+  jobWasInterrupted(jobStatus: JobStatus): boolean {
+    return jobStatus < -100_000_000;
+  }
 }
 
 export { Activity, ActivityType };

package/services/activities/await.ts

@@ -1,4 +1,4 @@
-import { GetStateError } from '../../modules/errors';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -12,6 +12,7 @@ import { MultiResponseFlags, RedisMulti } from '../../types/redis';
 import { StreamData, StreamDataType } from '../../types/stream';
 import { TelemetryService } from '../telemetry';
 import { Pipe } from '../pipe';
+import { guid } from '../../modules/utils';
 
 class Await extends Activity {
   config: AwaitActivity;
@@ -35,6 +36,7 @@ class Await extends Activity {
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
       this.mapInputData();
@@ -57,15 +59,19 @@ class Await extends Activity {
       });
       return this.context.metadata.aid;
     } catch (error) {
-      telemetry.setActivityError(error.message);
-      if (error instanceof GetStateError) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('await-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
         this.logger.error('await-get-state-error', { error });
+        return;
       } else {
         this.logger.error('await-process-error', { error });
       }
+      telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('await-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     }
   }
@@ -74,6 +80,7 @@ class Await extends Activity {
     const topic = Pipe.resolve(this.config.subtype, this.context);
     const streamData: StreamData = {
       metadata: {
+        guid: guid(),
         jid: this.context.metadata.jid,
         dad: this.metadata.dad,
         aid: this.metadata.aid,

package/services/activities/cycle.ts

@@ -1,4 +1,4 @@
-import { GetStateError } from '../../modules/errors';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
 import { Activity, ActivityType } from './activity';
@@ -10,6 +10,7 @@ import { JobState } from '../../types/job';
 import { MultiResponseFlags, RedisMulti } from '../../types/redis';
 import { StreamData } from '../../types/stream';
 import { TelemetryService } from '../telemetry';
+import { guid } from '../../modules/utils';
 
 class Cycle extends Activity {
   config: CycleActivity;
@@ -34,6 +35,7 @@ class Cycle extends Activity {
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
       this.mapInputData();
@@ -60,15 +62,19 @@ class Cycle extends Activity {
 
       return this.context.metadata.aid;
     } catch (error) {
-      if (error instanceof GetStateError) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('cycle-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
         this.logger.error('cycle-get-state-error', { error });
+        return;
       } else {
         this.logger.error('cycle-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('cycle-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     }
   }
@@ -88,6 +94,7 @@ class Cycle extends Activity {
     this.mapInputData();
     const streamData: StreamData = {
       metadata: {
+        guid: guid(),
         dad: CollatorService.resolveReentryDimension(this),
         jid: this.context.metadata.jid,
         aid: this.config.ancestor,

package/services/activities/hook.ts

@@ -1,4 +1,4 @@
-import { GetStateError } from '../../modules/errors';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -43,6 +43,7 @@ class Hook extends Activity {
       await CollatorService.notarizeEntry(this);
 
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
       let multiResponse: MultiResponseFlags;
@@ -81,15 +82,19 @@ class Hook extends Activity {
 
       return this.context.metadata.aid;
     } catch (error) {
-      if (error instanceof GetStateError) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('hook-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
         this.logger.error('hook-get-state-error', { error });
+        return;
       } else {
         this.logger.error('hook-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('hook-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     }
   }

package/services/activities/index.ts

@@ -2,7 +2,7 @@ import { Activity } from './activity';
 import { Await } from './await';
 import { Cycle } from './cycle';
 import { Hook } from './hook';
-import { Iterate } from './iterate';
+import { Interrupt } from './interrupt';
 import { Signal } from './signal';
 import { Trigger } from './trigger';
 import { Worker } from './worker';
@@ -12,7 +12,7 @@ export default {
   await: Await,
   cycle: Cycle,
   hook: Hook,
-  iterate: Iterate,
+  interrupt: Interrupt,
   signal: Signal,
   trigger: Trigger,
   worker: Worker,

package/services/activities/interrupt.ts

@@ -0,0 +1,159 @@
+import { EngineService } from '../engine';
+import { Activity, ActivityType } from './activity';
+import {
+  ActivityData,
+  ActivityMetadata,
+  InterruptActivity } from '../../types/activity';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
+import { MultiResponseFlags } from '../../types';
+import { CollatorService } from '../collator';
+import { JobInterruptOptions, JobState } from '../../types/job';
+import { TelemetryService } from '../telemetry';
+import { Pipe } from '../pipe';
+
+class Interrupt extends Activity {
+  config: InterruptActivity;
+
+  constructor(
+    config: ActivityType,
+    data: ActivityData,
+    metadata: ActivityMetadata,
+    hook: ActivityData | null,
+    engine: EngineService,
+    context?: JobState
+    ) {
+      super(config, data, metadata, hook, engine, context);
+  }
+
+
+  //********  LEG 1 ENTRY  ********//
+  async process(): Promise<string> {
+    this.logger.debug('interrupt-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    let telemetry: TelemetryService;
+    try {
+      this.setLeg(1);
+      await CollatorService.notarizeEntry(this);
+      await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid); // Ensure job active
+      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+      telemetry.startActivitySpan(this.leg);
+  
+      if (this.isInterruptingSelf()) {
+        return await this.interruptSelf(telemetry);
+      } else {
+        return await this.interruptAnother(telemetry);
+      }
+    } catch (error) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('interrupt-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
+        this.logger.error('interrupt-get-state-error', { error });
+        return;
+      } else {
+        this.logger.error('interrupt-process-error', { error });
+      }
+      telemetry.setActivityError(error.message);
+      throw error;
+    } finally {
+      telemetry?.endActivitySpan();
+      this.logger.debug('interrupt-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+    }
+  }
+
+  async interruptSelf(telemetry: TelemetryService): Promise<string> {
+    // Apply final updates to THIS job's state
+    if (this.config.job?.maps) {
+      this.mapJobData();
+      await this.setState();
+    }
+  
+    // Interrupt THIS job
+    const messageId = await this.interrupt();
+  
+    // Notarize completion and log
+    telemetry.mapActivityAttributes();
+    const multi = this.store.getMulti();
+    await CollatorService.notarizeEarlyCompletion(this, multi);
+    await this.setStatus(-1, multi);
+    const multiResponse = await multi.exec() as MultiResponseFlags;
+    const jobStatus = this.resolveStatus(multiResponse);
+    telemetry.setActivityAttributes({
+      'app.activity.mid': messageId,
+      'app.job.jss': jobStatus
+    });
+  
+    return this.context.metadata.aid;
+  }
+
+  async interruptAnother(telemetry: TelemetryService): Promise<string> {
+    // Interrupt ANOTHER job
+    const messageId = await this.interrupt();
+    const attrs = { 'app.activity.mid': messageId };
+  
+    // Apply updates to THIS job's state
+    telemetry.mapActivityAttributes();
+    this.adjacencyList = await this.filterAdjacent();
+    if (this.config.job?.maps || this.config.output?.maps) {
+      this.mapOutputData();
+      this.mapJobData();
+      const multi = this.store.getMulti();
+      await this.setState(multi);
+    }
+  
+    // Notarize completion
+    const multi = this.store.getMulti();
+    await CollatorService.notarizeEarlyCompletion(this, multi);
+    await this.setStatus(this.adjacencyList.length - 1, multi);
+    const multiResponse = await multi.exec() as MultiResponseFlags;
+    const jobStatus = this.resolveStatus(multiResponse);
+    attrs['app.job.jss'] = jobStatus;
+  
+    // Transition next generation and log
+    const messageIds = await this.transition(this.adjacencyList, jobStatus);
+    if (messageIds.length) {
+      attrs['app.activity.mids'] = messageIds.join(',');
+    }
+    telemetry.setActivityAttributes(attrs);
+  
+    return this.context.metadata.aid;
+  }
+  
+
+  isInterruptingSelf(): boolean {
+    if (!this.config.target) {
+      return true;
+    }
+    const resolvedJob = Pipe.resolve(this.config.target, this.context);
+    return resolvedJob == this.context.metadata.jid;
+  }
+
+  resolveInterruptOptions(): JobInterruptOptions {
+    return {
+      reason: this.config.reason !== undefined 
+        ? Pipe.resolve(this.config.reason, this.context) 
+        : undefined,
+      throw: this.config.throw !== undefined 
+        ? Pipe.resolve(this.config.throw, this.context) 
+        : undefined,
+      descend: this.config.descend !== undefined 
+        ? Pipe.resolve(this.config.descend, this.context) 
+        : undefined,
+    };
+  }
+
+  async interrupt(): Promise<string> {
+    const options = this.resolveInterruptOptions();
+    return await this.engine.interrupt(
+      this.config.topic !== undefined 
+        ? Pipe.resolve(this.config.topic, this.context) 
+        : this.context.metadata.tpc,
+      this.config.target !== undefined 
+        ? Pipe.resolve(this.config.target, this.context) 
+        : this.context.metadata.jid,
+      options as JobInterruptOptions,
+    );
+  }
+}
+
+export { Interrupt };

package/services/activities/signal.ts

@@ -1,4 +1,4 @@
-import { GetStateError } from '../../modules/errors';
+import { GetStateError, InactiveJobError } from '../../modules/errors';
 import { Activity, ActivityType } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -37,6 +37,7 @@ class Signal extends Activity {
       this.setLeg(1);
       await CollatorService.notarizeEntry(this);
       await this.getState();
+      CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
       telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
       telemetry.startActivitySpan(this.leg);
 
@@ -50,6 +51,7 @@ class Signal extends Activity {
       await this.setStatus(this.adjacencyList.length - 1, multi);
       const multiResponse = await multi.exec() as MultiResponseFlags;
 
+      //todo: this should execute BEFORE the status is decremented
       if (this.config.subtype === 'all') {
         await this.hookAll();
       } else {
@@ -68,15 +70,19 @@ class Signal extends Activity {
 
       return this.context.metadata.aid;
     } catch (error) {
-      if (error instanceof GetStateError) {
+      if (error instanceof InactiveJobError) {
+        this.logger.error('signal-inactive-job-error', { error });
+        return;
+      } else if (error instanceof GetStateError) {
         this.logger.error('signal-get-state-error', { error });
+        return;
       } else {
         this.logger.error('signal-process-error', { error });
       }
       telemetry.setActivityError(error.message);
       throw error;
     } finally {
-      telemetry.endActivitySpan();
+      telemetry?.endActivitySpan();
       this.logger.debug('signal-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
     }
   }