@hotmeshio/hotmesh 0.0.34 → 0.0.35

package/services/signaler/stream.ts

@@ -1,5 +1,16 @@
+import {
+  BLOCK_TIME_MS,
+  MAX_RETRIES,
+  MAX_TIMEOUT_MS,
+  GRADUATED_INTERVAL_MS,
+  STATUS_CODE_UNACKED,
+  STATUS_CODE_UNKNOWN,
+  STATUS_MESSAGE_UNKNOWN,
+  XCLAIM_COUNT,
+  XCLAIM_DELAY_MS,
+  XPENDING_COUNT } from '../../modules/enums';
 import { KeyType } from '../../modules/key';
-import { XSleepFor, sleepFor } from '../../modules/utils';
+import { XSleepFor, guid, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { StreamService } from '../stream';
@@ -16,19 +27,6 @@ import {
   StreamStatus
 } from '../../types/stream';
 
-const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
-const MAX_TIMEOUT_MS = 60000;
-const GRADUATED_INTERVAL_MS = 5000;
-const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
-const TEST_BLOCK_DURATION = 1000; //Set to `1000` so tests can interrupt quickly
-const BLOCK_TIME_MS = process.env.NODE_ENV === 'test' ? TEST_BLOCK_DURATION : BLOCK_DURATION;
-const SYSTEM_STATUS_CODE = 999;
-const UNKNOWN_STATUS_CODE = 500;
-const UNKNOWN_STATUS_MESSAGE = 'unknown';
-const XCLAIM_DELAY_MS = 1000 * 60; //max time a message can be unacked before it is claimed by another
-const XCLAIM_COUNT = 3; //max number of times a message can be claimed by another before it is dead-lettered
-const XPENDING_COUNT = 10;
-
 class StreamSignaler {
   static signalers: Set<StreamSignaler> = new Set();
   appId: string;
@@ -131,7 +129,7 @@ class StreamSignaler {
       telemetry.startStreamSpan(input, this.role);  
       output = await this.execStreamLeg(input, stream, id, callback.bind(this));
       if (output?.status === StreamStatus.ERROR) {
-        telemetry.setStreamError(`Function Status Code ${ output.code || UNKNOWN_STATUS_CODE }`);
+        telemetry.setStreamError(`Function Status Code ${ output.code || STATUS_CODE_UNKNOWN }`);
       }
       this.errorCount = 0;
     } catch (err) {
@@ -171,12 +169,17 @@ class StreamSignaler {
           await sleepFor(timeout);
           return await this.publishMessage(input.metadata.topic, { 
             data: input.data,
+            //note: retain guid (this is a retry attempt)
             metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
             policies: input.policies,
           }) as string;
         } else {
           output = this.structureError(input, output);
         }
+      } else if (typeof output.metadata !== 'object') {
+        output.metadata = { ...input.metadata, guid: guid() };
+      } else {
+        output.metadata.guid = guid();
       }
       output.type = StreamDataType.RESPONSE;
       return await this.publishMessage(null, output as StreamDataResponse) as string;
@@ -203,7 +206,7 @@ class StreamSignaler {
     if (typeof err.message === 'string') {
       error.message = err.message;
     } else {
-      error.message = UNKNOWN_STATUS_MESSAGE;
+      error.message = STATUS_MESSAGE_UNKNOWN;
     }
     if (typeof err.stack === 'string') {
       error.stack = err.stack;
@@ -213,18 +216,18 @@ class StreamSignaler {
     }
     return {
       status: 'error',
-      code: UNKNOWN_STATUS_CODE,
-      metadata: { ...input.metadata },
+      code: STATUS_CODE_UNKNOWN,
+      metadata: { ...input.metadata, guid: guid() },
       data: error as StreamError
     } as StreamDataResponse;
   }
 
   structureUnacknowledgedError(input: StreamData) {
     const message = 'stream message max delivery count exceeded';
-    const code = SYSTEM_STATUS_CODE;
+    const code = STATUS_CODE_UNACKED;
     const data: StreamError = { message, code };
     const output: StreamDataResponse = { 
-      metadata: { ...input.metadata },
+      metadata: { ...input.metadata, guid: guid() },
       status: StreamStatus.ERROR,
       code,
       data,
@@ -235,9 +238,9 @@ class StreamSignaler {
   }
 
   structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse {
-    const message = output.data?.message ? output.data?.message.toString() : UNKNOWN_STATUS_MESSAGE;
+    const message = output.data?.message ? output.data?.message.toString() : STATUS_MESSAGE_UNKNOWN;
     const statusCode = output.code || output.data?.code;
-    const code = isNaN(statusCode as number) ? UNKNOWN_STATUS_CODE : parseInt(statusCode.toString());
+    const code = isNaN(statusCode as number) ? STATUS_CODE_UNKNOWN : parseInt(statusCode.toString());
     const data: StreamError = { message, code };
     if (typeof output.data?.error === 'object') {
       data.error = { ...output.data.error };
@@ -245,7 +248,7 @@ class StreamSignaler {
     return {
       status: StreamStatus.ERROR,
       code,
-      metadata: { ...input.metadata },
+      metadata: { ...input.metadata, guid: guid() },
       data
     } as StreamDataResponse;
   }
@@ -307,7 +310,7 @@ class StreamSignaler {
     //     ii) corrupt hardware/network/transport/etc
     // 3b) system error: Redis unable to accept `xadd` request
     // 4c) system error: Redis unable to accept `xdel`/`xack` request
-    this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: SYSTEM_STATUS_CODE, count });
+    this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: STATUS_CODE_UNACKED, count });
     const streamData = reclaimedMessage[0]?.[1]?.[1];
 
     //fatal risk point 1 of 3): json is corrupt

package/services/store/index.ts

@@ -28,6 +28,9 @@ import {
 import { Transitions } from '../../types/transition';
 import { formatISODate, getSymKey } from '../../modules/utils';
 import { ReclaimedMessageType } from '../../types/stream';
+import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
+import { STATUS_CODE_INTERRUPT } from '../../modules/enums';
+import { GetStateError } from '../../modules/errors';
 
 interface AbstractRedisClient {
   exec(): any;
@@ -371,6 +374,22 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return await this.redisClient[this.commands.hset](key, payload as any);
   }
 
+  /**
+   * Registers jobId with the originJobId that spawned it. In the future,
+   * when originJobId is interrupted or expired, the items in the
+   * list (added via RPUSH) are LPOPed. If origin was expired, then
+   * LPOPed items from the list are likewise expired;
+   */
+  async setDependency(originJobId: string, topic: string, jobId: string, multi? : U): Promise<any> {
+    const privateMulti = multi || this.getMulti();
+    const depParams = { appId: this.appId, jobId: originJobId };
+    const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
+    privateMulti[this.commands.rpush](depKey, `expire::${topic}::${jobId}`);
+    if (!multi) {
+      return await privateMulti.exec();
+    }
+  }
+
   async setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi? : U): Promise<any> {
     const params: KeyStoreParams = { appId: appVersion.id, jobId, jobKey, dateTime };
     const privateMulti = multi || this.getMulti();
@@ -477,8 +496,8 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
   }
 
   /**
-   * returns custom search fields and values. The fields param
-   * should not prefix items with an underscore.
+   * Returns custom search fields and values.
+   * NOTE: The `fields` param should NOT prefix items with an underscore.
    */
   async getQueryState(jobId: string, fields: string[]): Promise<StringAnyType> {
     const key = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
@@ -520,7 +539,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       }
       return [state, status];
     } else {
-      throw new Error(`Job ${jobId} not found`);
+      throw new GetStateError(jobId);
     }
   }
 
@@ -745,7 +764,27 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
-  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'cron', deletionTime: number, multi?: U): Promise<void> {
+  /**
+   * register the descendants of an expired origin flow to be 
+   * expired at a future date; options indicate whether this
+   * is a standard `expire` or an `interrupt`
+   */
+  async registerExpireJob(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void> {
+    const depParams = { appId: this.appId, jobId };
+    const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
+    const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
+    const depKeyContext = `::${context}::${depKey}`;
+    const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
+    await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
+  }
+
+  /**
+   * registers a hook activity to be awakened (uses ZSET to
+   * store the 'sleep group' and LIST to store the events
+   * for the given sleep group. Sleep groups are
+   * organized into 'n'-second blocks (LISTS))
+   */
+  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', deletionTime: number, multi?: U): Promise<void> {
     const listKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
     const timeEvent = `${type}::${activityId}::${jobId}`
     const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
@@ -755,21 +794,90 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
-  async getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string] | void> {
+  async getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt'] | void> {
     const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
-    const now = Date.now();
-    listKey = listKey || await this.zRangeByScore(zsetKey, 0, now);
+    listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
     if (listKey) {
-      const timeEvent = await this.redisClient[this.commands.lpop](listKey);
+      const [pType, pKey] = this.resolveKeyContext(listKey);
+      const timeEvent = await this.redisClient[this.commands.lpop](pKey);
       if (timeEvent) {
-        //placeholder: there are 3 time-related event triggers: sleep, expire, cron
-        const [type, activityId, ...jobId] = timeEvent.split('::');
-        return [listKey, jobId.join('::'), activityId];
+        //there are 3 time-related event triggers: sleep, expire, interrupt
+        const [_type, activityId, ...jobId] = timeEvent.split('::');
+        return [listKey, jobId.join('::'), activityId, pType];
       }
       await this.redisClient[this.commands.zrem](zsetKey, listKey);
     }
   }
 
+  /**
+   * when processing time jobs, the target LIST ID returned
+   * from the ZSET query can be prefixed to denote what to
+   * do with the work list. (not everything is known in advance,
+   * so the ZSET key defines HOW to approach the work in the
+   * generic LIST (lists typically contain target job ids)
+   * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+   */
+  resolveKeyContext(listKey: string): [('sleep'|'expire'|'interrupt'), string] {
+    if (listKey.startsWith('::INTERRUPT')) {
+      return ['interrupt', listKey.split('::')[2]];
+    } else if (listKey.startsWith('::EXPIRE')) {
+      return ['expire', listKey.split('::')[2]];
+    } else {
+      return ['sleep', listKey];
+    }
+  }
+
+  /**
+   * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
+   * This method is called by the engine and not by an activity and is
+   * followed by a call to execute job completion/cleanup tasks
+   * associated with a job completion event.
+   */
+  async interrupt(topic: string, jobId: string, options: JobInterruptOptions = {}): Promise<void> {
+    try {
+      //verify job exists
+      const status = await this.getStatus(jobId, this.appId);
+      if (status <= 0) {
+        //verify still active; job already completed
+        throw new Error(`Job ${jobId} already completed`);
+      }
+      //decrement job status (:) by 1bil
+      const amount = -1_000_000_000;
+      const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+      const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
+      if (result <= amount) {
+        //verify active state; job already interrupted
+        throw new Error(`Job ${jobId} already completed`);
+      }
+      //persist the error unless specifically told not to
+      if (options.throw !== false) {
+        const errKey = `metadata/err`;       //job errors are stored at the path `metadata/err`
+        const symbolNames = [`$${topic}`];   //the symbol for `metadata/err` is in redis and stored using the job topic
+        const symKeys = await this.getSymbolKeys(symbolNames);
+        const symVals = await this.getSymbolValues();
+        this.serializer.resetSymbols(symKeys, symVals, {});
+
+        //persists the standard 410 error (job is `gone`)
+        const err = JSON.stringify({
+          code: STATUS_CODE_INTERRUPT,
+          message: options.reason ?? `job [${jobId}] interrupted`,
+          job_id: jobId
+        });
+
+        const payload = { [errKey]: amount.toString() }
+        const hashData = this.serializer.package(payload, symbolNames);
+        const errSymbol = Object.keys(hashData)[0];
+        await this.redisClient[this.commands.hset](jobKey, errSymbol, err);
+      }
+    } catch (e) {
+      if (!options.suppress) {
+        throw e;
+      } else {
+        this.logger.debug('suppressed-interrupt', { message: e.message })
+      }
+    }
+  }
+
   async scrub(jobId: string) {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     await this.redisClient[this.commands.del](jobKey);

package/services/task/index.ts

@@ -1,13 +1,12 @@
+import {
+  EXPIRE_DURATION,
+  FIDELITY_SECONDS } from '../../modules/enums';
+import { XSleepFor, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
-import { RedisClient, RedisMulti } from '../../types/redis';
 import { HookInterface } from '../../types/hook';
-import { XSleepFor, sleepFor } from '../../modules/utils';
-
-//system timer granularity limit (task queues organize)
-const FIDELITY_SECONDS = 15; //note: this can be reduced using 'watch' or scout role
-//default resolution/fidelity when expiring
-const EXPIRATION_FIDELITY_SECONDS = 60;
+import { JobCompletionOptions } from '../../types/job';
+import { RedisClient, RedisMulti } from '../../types/redis';
 
 class TaskService {
   store: StoreService<RedisClient, RedisMulti>;
@@ -43,25 +42,26 @@ class TaskService {
     await this.store.addTaskQueues(keys);
   }
 
-  async registerJobForCleanup(jobId: string, inSeconds = EXPIRATION_FIDELITY_SECONDS): Promise<void> {
-    if (inSeconds > -1) {
+  async registerJobForCleanup(jobId: string, inSeconds = EXPIRE_DURATION, options: JobCompletionOptions): Promise<void> {
+    if (inSeconds > 0) {
       await this.store.expireJob(jobId, inSeconds);
+      const expireTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
+      await this.store.registerExpireJob(jobId, expireTimeSlot, options);
     }
   }
 
-  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'cron', inSeconds = FIDELITY_SECONDS, multi?: RedisMulti): Promise<void> {
-    const awakenTimeSlot = Math.floor((Date.now() + inSeconds * 1000) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
+  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', inSeconds = FIDELITY_SECONDS, multi?: RedisMulti): Promise<void> {
+    const awakenTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
     await this.store.registerTimeHook(jobId, activityId, type, awakenTimeSlot, multi);
   }
 
-  //todo: need 'scout' role in quorum to check for this and then alert the quorum to get to work
-  async processTimeHooks(timeEventCallback: (jobId: string, activityId: string) => Promise<void>, listKey?: string): Promise<void> {
+  async processTimeHooks(timeEventCallback: (jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt') => Promise<void>, listKey?: string): Promise<void> {
     try {
-      const job = await this.store.getNextTimeJob(listKey);
-      if (job) {
-        const [listKey, jobId, activityId] = job;
-        await timeEventCallback(jobId, activityId);
-        await sleepFor(0);
+      const timeJob = await this.store.getNextTimeJob(listKey);
+      if (timeJob) {
+        const [listKey, jobId, activityId, type] = timeJob;
+        await timeEventCallback(jobId, activityId, type);
+        await sleepFor(0);          
         this.processTimeHooks(timeEventCallback, listKey);
       } else {
         let sleep = XSleepFor(FIDELITY_SECONDS * 1000);

package/types/activity.ts

@@ -1,7 +1,7 @@
 import { MetricTypes } from "./stats";
 import { StreamRetryPolicy } from "./stream";
 
-type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal' | 'hook';
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'interrupt' | 'cycle' | 'signal' | 'hook';
 
 type Consumes = Record<string, string[]>;
 
@@ -17,7 +17,7 @@ interface BaseActivity {
   telemetry?: Record<string, any>;
   emit?: boolean;                      //if true, the activity will emit a message to the `publishes` topic immediately before transitioning to adjacent activities
   sleep?: number;                      //@pipe /in seconds
-  expire?: number;                     //-1 forever (15 seconds default); todo: make globally configurable
+  expire?: number;                     //-1 forever; 0 persists the flow until the parent flow that expired it is dismissed; 15 seconds is the default
   retry?: StreamRetryPolicy
   cycle?: boolean;                     //if true, the `notary` will leave leg 2 open, so it can be re/cycled
   collationInt?: number;               //compiler
@@ -37,10 +37,30 @@ interface Measure {
 }
 
 interface TriggerActivityStats {
+  /**
+   * parent job; including this allows the parent's 
+   * expiration/interruption events to cascade; set 
+   * `expire` in the YAML for the dependent graph 
+   * to 0 and provide the parent for dependent,
+   * cascading interruption and cleanup
+   */
+  parent?: string;
   id?: { [key: string]: unknown } | string;
   key?: { [key: string]: unknown } | string;
-  granularity?: string; //return 'infinity' to disable; default behavior is to always segment keys by time to ensure indexes (Redis LIST) never grow unbounded as a default behavior; for now, 5m is default and infinity can be set to override
-  measures?: Measure[]; //what to capture
+  /**
+   * @deprecated
+   * return 'infinity' to disable; default behavior
+   * is to always segment keys by time to ensure
+   * indexes (Redis LIST) never grow unbounded
+   * as a default behavior; for now, 5m is default
+   * and infinity can be set to override
+   */
+  granularity?: string;
+  /**
+   * @deprecated
+   * what to capture
+   */
+  measures?: Measure[];
 }
 
 interface TriggerActivity extends BaseActivity {
@@ -82,11 +102,21 @@ interface SignalActivity extends BaseActivity {
   code?: number;                  //202, 200 (default)
 }
 
-interface IterateActivity extends BaseActivity {
-  type: 'iterate';
+interface InterruptActivity extends BaseActivity {
+  type: 'interrupt';
+  /** Optional Reason; will be used as the error `message` when thrown; NOTE: 410 is the error `code` */
+  reason?: string;
+  /** default is `true` (throw JobInterrupted error upon interrupting) */
+  throw?: boolean;
+  /** TODO: // default is `false` (do not interrupt child jobs) */
+  descend?: boolean;
+  /** target job id (if not present the current job will be targeted) */
+  target?: string;
+  /** topic to publish the interrupt message (if not present the current job topic will be used) */
+  topic?: string;
 }
 
-type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity | HookActivity;
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | InterruptActivity | HookActivity | SignalActivity | CycleActivity;
 
 type ActivityData = Record<string, any>;
 type ActivityMetadata = {
@@ -133,7 +163,7 @@ export {
   HookActivity,
   SignalActivity,
   BaseActivity,
-  IterateActivity,
+  InterruptActivity,
   TriggerActivity,
   WorkerActivity
 };

package/types/durable.ts

@@ -60,10 +60,12 @@ type WorkflowOptions = {
   entity?: string;            //If invoking a workflow, passing 'entity' will apply the value as the workflowName, taskQueue, and prefix, ensuring the FT.SEARCH index is properly scoped. This is a convenience method but limits options.
   workflowName?: string;      //the name of the user's workflow function; optional if 'entity' is provided
   parentWorkflowId?: string;  //system reserved; the id of the parent; if present the flow will not self-clean until the parent that spawned it self-cleans
+  originJobId?: string;       //system reserved;
   workflowTrace?: string;
   workflowSpan?: string;
   search?: WorkflowSearchOptions
   config?: WorkflowConfig;
+  expire?: number;            //default is 3seconds; time before completed jobs and dependents are expired/scrubbed/removed
 }
 
 type HookOptions = {
@@ -80,8 +82,8 @@ type HookOptions = {
 type SignalOptions = {
   taskQueue: string;
   data: Record<string, any>; //input data (any serializable object)
-  workflowId: string;   //execution id (the job id)
-  workflowName?: string; //the name of the user's workflow function
+  workflowId: string;        //execution id (the job id)
+  workflowName?: string;     //the name of the user's workflow function
 }
 
 type ActivityWorkflowDataType = {
@@ -95,6 +97,8 @@ type WorkflowDataType = {
   arguments: any[];
   workflowId: string;
   workflowTopic: string;
+  workflowDimension?: string; //is present if hook (not main workflow)
+  originJobId?: string;       //is present if there is an originating ancestor job (should rename to originJobId)
 }
 
 type MeshOSClassConfig = {
@@ -105,8 +109,8 @@ type MeshOSClassConfig = {
 }
 
 type MeshOSConfig = {
-  id?: string; //guid for the workflow when instancing
-  await?: boolean; //default is false; must explicitly send true to await the final result
+  id?: string;        //guid for the workflow when instancing
+  await?: boolean;    //default is false; must explicitly send true to await the final result
   taskQueue?: string; //optional target queue isolate for the function
 }
 

package/types/index.ts

@@ -12,7 +12,7 @@ export {
   CycleActivity,
   HookActivity,
   WorkerActivity,
-  IterateActivity,
+  InterruptActivity,
   SignalActivity,
   TriggerActivity,
   TriggerActivityStats } from './activity';

package/types/job.ts

@@ -44,6 +44,19 @@ type JobState = {
   };
 };
 
+type JobInterruptOptions = {
+  /** Optional reason when throwing the error  */
+  reason?: string;
+  /** default is `true` when `undefined` (throw JobInterrupted/410 error) */
+  throw?: boolean;
+  /** default behavior is `false` when `undefined` (do NOT interrupt child jobs) */
+  descend?: boolean;
+  /** default is false; if true, errors related to inactivation (like overage...already inactive) are suppressed/ignored */
+  suppress?: boolean;
+  /** how long to wait in seconds before fully expiring/removing the hash from Redis; the job is inactive, but can remain in the cache indefinitely. minimum 1 second.*/
+  expire?: number;
+};
+
 //format when publishing job meta/data on the wire when it completes
 type JobOutput = {
   metadata: JobMetadata;
@@ -56,4 +69,20 @@ type PartialJobState = {
   data: JobData;
 };
 
-export { JobState, JobStatus, JobData, JobsData, JobMetadata, PartialJobState, JobOutput };
+type JobCompletionOptions = {
+  emit?: boolean;      //default false
+  interrupt?: boolean;  //default undefined
+  expire?: number;      // in seconds to wait before deleting/expiring job hash
+}
+
+export {
+  JobCompletionOptions,
+  JobInterruptOptions,
+  JobData,
+  JobsData,
+  JobMetadata,
+  JobOutput,
+  JobState,
+  JobStatus,
+  PartialJobState,
+};

package/types/quorum.ts

@@ -1,13 +1,5 @@
 import { JobOutput } from "./job";
 
-/**
- * The types in this file are used to define those messages that are sent
- * to hotmesh client instances when a new version is about to be activated.
- * These messages serve to coordinate the cache invalidation and switch-over
- * to the new version without any downtime and a coordinating parent server.
- */
-export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage;
-
 //used for coordination like version activation
 export interface PingMessage {
   type: 'ping';
@@ -19,6 +11,11 @@ export interface WorkMessage {
   originator: string; //guid
 }
 
+export interface CronMessage {
+  type: 'cron';
+  originator: string; //guid
+}
+
 //used for coordination like version activation
 export interface PongMessage {
   type: 'pong';
@@ -57,3 +54,11 @@ export interface SubscriptionCallback {
 export interface QuorumMessageCallback {
   (topic: string, message: QuorumMessage): void;
 }
+
+/**
+ * The types in this file are used to define those messages that are sent
+ * to hotmesh client instances when a new version is about to be activated.
+ * These messages serve to coordinate the cache invalidation and switch-over
+ * to the new version without any downtime and a coordinating parent server.
+ */
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;

package/types/stream.ts

@@ -27,10 +27,13 @@ export enum StreamDataType {
   WORKER = 'worker',
   RESPONSE = 'response', //worker response
   TRANSITION = 'transition',
+  SIGNAL = 'signal',
+  INTERRUPT = 'interrupt',
 }
 
 export interface StreamData {
   metadata: {
+    guid: string;   //every message is minted with a guid to distinguish retries from new messages
     topic?: string;
     jid?: string; //is optonal if type is WEBHOOK
     dad?: string; //dimensional address

package/build/services/activities/iterate.d.ts

@@ -1,9 +0,0 @@
-import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
-import { ActivityData, ActivityMetadata, IterateActivity } from '../../types/activity';
-declare class Iterate extends Activity {
-    config: IterateActivity;
-    constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService);
-    mapInputData(): Promise<void>;
-}
-export { Iterate };

package/build/services/activities/iterate.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Iterate = void 0;
-const activity_1 = require("./activity");
-class Iterate extends activity_1.Activity {
-    constructor(config, data, metadata, hook, engine) {
-        super(config, data, metadata, hook, engine);
-    }
-    async mapInputData() {
-        this.logger.info('iterate-map-input-data');
-    }
-}
-exports.Iterate = Iterate;

package/services/activities/iterate.ts

@@ -1,26 +0,0 @@
-import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
-import {
-  ActivityData,
-  ActivityMetadata,
-  IterateActivity } from '../../types/activity';
-
-class Iterate extends Activity {
-  config: IterateActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService
-    ) {
-      super(config, data, metadata, hook, engine);
-  }
-
-  async mapInputData(): Promise<void> {
-    this.logger.info('iterate-map-input-data');
-  }
-}
-
-export { Iterate };