@hotmeshio/hotmesh 0.0.53 → 0.0.55

package/build/types/index.d.ts

@@ -4,11 +4,12 @@ export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
 export { ActivityConfig, ActivityWorkflowDataType, ChildResponseType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyResponseType, ProxyType, Registry, SignalOptions, FindOptions, FindWhereOptions, FindWhereQuery, HookOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowContext, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
+export { DurableChildErrorType, DurableProxyErrorType, DurableSleepErrorType, DurableWaitForAllErrorType, DurableWaitForErrorType } from "./error";
 export { ActivityAction, DependencyExport, DurableJobExport, ExportCycles, ExportItem, ExportOptions, ExportTransitions, JobAction, JobExport, JobActionExport, JobTimeline } from './exporter';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';
-export { JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState } from './job';
+export { JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState, ExtensionType } from './job';
 export { MappingStatements } from './map';
 export { Pipe, PipeContext, PipeItem, PipeItems, PipeObject, ReduceObject } from './pipe';
 export { HotMesh, HotMeshApp, HotMeshApps, HotMeshConfig, HotMeshEngine, RedisConfig, HotMeshGraph, HotMeshManifest, HotMeshSettings, HotMeshWorker, KeyStoreParams, KeyType } from './hotmesh';

package/build/types/job.d.ts

@@ -1,3 +1,4 @@
+import { StringStringType } from "./serializer";
 type JobData = Record<string, unknown | Record<string, unknown>>;
 type JobsData = Record<string, unknown>;
 type ActivityData = {
@@ -55,6 +56,28 @@ type JobMetadata = {
     /** process data expire policy */
     expire?: number;
 };
+/**
+ * User-defined (extended) types for job data. Users may interleave
+ * data into the job hash safely by using the `ExtensionType` interface.
+ * The data will be prefixed as necessary using an underscore or
+ * dash to ensure it is not confused with system process data.
+ */
+type ExtensionType = {
+    /**
+     * Custom search data field (name/value pairs) to seed the Hash.
+     * Every field will be prefixed with an underscore before being
+     * stored with the initial Hash data set along side system
+     * process data.
+     */
+    search?: StringStringType;
+    /**
+     * Custom marker data field used for adding a searchable marker to the job.
+     * markers always begin with a dash (-). Any field that does not
+     * begin with a dash will be removed and will not be inserted with
+     * the initial data set.
+     */
+    marker?: StringStringType;
+};
 /**
  * job_status semaphore
  */
@@ -132,4 +155,4 @@ type JobCompletionOptions = {
      */
     expire?: number;
 };
-export { JobCompletionOptions, JobInterruptOptions, JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState, };
+export { JobCompletionOptions, JobInterruptOptions, JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState, ExtensionType, };

package/modules/errors.ts

@@ -1,5 +1,3 @@
-import { ActivityDuplex } from "../types/activity";
-import { CollationFaultType, CollationStage } from "../types/collator";
 import {
   HMSH_CODE_DURABLE_MAXED,
   HMSH_CODE_DURABLE_TIMEOUT,
@@ -11,6 +9,14 @@ import {
   HMSH_CODE_DURABLE_CHILD,
   HMSH_CODE_DURABLE_ALL,
   HMSH_CODE_DURABLE_SLEEP } from "./enums";
+import { ActivityDuplex } from "../types/activity";
+import { CollationFaultType, CollationStage } from "../types/collator";
+import {
+  DurableChildErrorType,
+  DurableProxyErrorType,
+  DurableSleepErrorType,
+  DurableWaitForAllErrorType,
+  DurableWaitForErrorType } from "../types/error";
 
 class GetStateError extends Error {
   jobId: string;
@@ -32,13 +38,8 @@ class DurableWaitForError extends Error {
   workflowId: string;
   index: number;
   workflowDimension: string; //hook workflowDimension (e.g., ',0,1,0') (use empty string for `null`)
-  constructor(params: {
-    signalId: string,
-    index: number,
-    workflowDimension: string
-    workflowId: string;
-  }) {
-    super(`Durable WaitFor Error [${params.workflowId}]`);
+  constructor(params: DurableWaitForErrorType) {
+    super(`WaitFor Interruption`);
     this.signalId = params.signalId;
     this.index = params.index;
     this.workflowDimension = params.workflowDimension;
@@ -59,20 +60,8 @@ class DurableProxyError extends Error {
   workflowDimension: string;
   workflowId: string;
   workflowTopic: string;
-  constructor(params: {
-    arguments: string[],
-    activityName: string,
-    backoffCoefficient?: number,
-    index: number,
-    maximumAttempts?: number,
-    maximumInterval?: number,
-    originJobId: string | null,
-    parentWorkflowId: string,
-    workflowDimension: string,
-    workflowId: string,
-    workflowTopic: string,
-  }) {
-    super(`Durable Proxy Activity Error [${params.parentWorkflowId}] => [${params.workflowId}]`);
+  constructor(params: DurableProxyErrorType) {
+    super(`ProxyActivity Interruption`);
     this.arguments = params.arguments;
     this.workflowId = params.workflowId;
     this.workflowTopic = params.workflowTopic;
@@ -101,20 +90,8 @@ class DurableChildError extends Error {
   parentWorkflowId: string;
   workflowId: string;
   workflowTopic: string;
-  constructor(params: {
-    arguments: string[],
-    await?: boolean,
-    backoffCoefficient?: number,
-    index: number,
-    maximumAttempts?: number,
-    maximumInterval?: number,
-    originJobId: string | null,
-    parentWorkflowId: string,
-    workflowDimension: string,
-    workflowId: string,
-    workflowTopic: string,
-  }) {
-    super(`Durable Child Error [${params.parentWorkflowId}] => [${params.workflowId}]`);
+  constructor(params: DurableChildErrorType) {
+    super(`ExecChild Interruption`);
     this.arguments = params.arguments;
     this.workflowId = params.workflowId;
     this.workflowTopic = params.workflowTopic;
@@ -140,17 +117,8 @@ class DurableWaitForAllError extends Error {
   parentWorkflowId: string;
   workflowId: string;
   workflowTopic: string;
-  constructor(params: {
-    items: string[],
-    workflowId: string,
-    workflowTopic: string,
-    parentWorkflowId: string,
-    originJobId: string | null,
-    size: number,
-    index: number,
-    workflowDimension: string
-  }) {
-    super(`Durable Wait for All Error [${params.parentWorkflowId}] => [${params.workflowId}]`);
+  constructor(params: DurableWaitForAllErrorType) {
+    super(`Collation Interruption`);
     this.items = params.items;
     this.size = params.size;
     this.workflowId = params.workflowId;
@@ -169,13 +137,8 @@ class DurableSleepError extends Error {
   duration: number; //seconds
   index: number;
   workflowDimension: string; //empty string for null
-  constructor(params: {
-    duration: number,
-    index: number,
-    workflowDimension: string,
-    workflowId: string,
-  }) {
-    super(`Durable Sleep Error [${params.workflowId}]`);
+  constructor(params: DurableSleepErrorType) {
+    super(`SleepFor Interruption`);
     this.duration = params.duration;
     this.workflowId = params.workflowId;
     this.index = params.index;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.53",
+  "version": "0.0.55",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/services/activities/hook.ts

@@ -86,7 +86,14 @@ class Hook extends Activity {
    * does this activity use a time-hook or web-hook
    */
   doesHook(): boolean {
-    return !!(this.config.hook?.topic || this.config.sleep);
+    if (this.config.sleep) {
+      const duration = Pipe.resolve(
+        this.config.sleep,
+        this.context,
+      );
+      return !isNaN(duration) && Number(duration) > 0
+    }
+    return !!this.config.hook?.topic;
   }
 
   async doHook(telemetry: TelemetryService) {

package/services/activities/trigger.ts

@@ -15,7 +15,7 @@ import {
   ActivityMetadata,
   ActivityType,
   TriggerActivity } from '../../types/activity';
-import { JobState } from '../../types/job';
+import { JobState, ExtensionType } from '../../types/job';
 import { RedisMulti } from '../../types/redis';
 import { StringScalarType } from '../../types/serializer';
 import { WorkListTaskType } from '../../types/task';
@@ -33,7 +33,7 @@ class Trigger extends Activity {
       super(config, data, metadata, hook, engine, context);
   }
 
-  async process(): Promise<string> {
+  async process(options?: ExtensionType): Promise<string> {
     this.logger.debug('trigger-process', { subscribes: this.config.subscribes });
     let telemetry: TelemetryService;
     try {
@@ -48,6 +48,9 @@ class Trigger extends Activity {
       this.adjacencyList = await this.filterAdjacent();
       await this.setStatus(this.adjacencyList.length);
 
+      this.bindSearchData(options);
+      this.bindMarkerData(options);
+
       const multi = this.store.getMulti();
       await this.setState(multi);
       await this.setStats(multi);
@@ -83,6 +86,28 @@ class Trigger extends Activity {
     }
   }
 
+  safeKey(key:string): string {
+    return `_${key}`;
+  }
+
+  bindSearchData(options?: ExtensionType): void {
+    if (options?.search) {
+      Object.keys(options.search).forEach((key) => {
+        this.context.data[this.safeKey(key)] = options.search[key].toString();
+      });
+    }
+  }
+
+  bindMarkerData(options?: ExtensionType): void {
+    if (options?.marker) {
+      Object.keys(options.marker).forEach((key) => {
+        if (key.startsWith('-')) {
+          this.context.data[key] = options.marker[key].toString();
+        }
+      });
+    }
+  }
+
   async setStatus(amount: number): Promise<void> {
     this.context.metadata.js = amount;
   }

package/services/durable/client.ts

@@ -164,15 +164,9 @@ export class ClientService {
       const jobId = await hotMeshClient.pub(
         `${options.namespace ?? APP_ID}.execute`,
         payload,
-        context as JobState
+        context as JobState,
+        { search: options?.search?.data, marker: options?.marker},
       );
-      // Seed search data
-      if (jobId && options.search?.data) {
-        const searchSessionId = `-search-0`;
-        const search = new Search(jobId, hotMeshClient, searchSessionId);
-        const entries = Object.entries(options.search.data).flat();
-        await search.set(...entries);
-      }
       return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
     },
 

package/services/durable/exporter.ts

@@ -1,22 +1,22 @@
-import { VALSEP } from '../../modules/key';
 import { restoreHierarchy } from '../../modules/utils';
 import { ILogger } from '../logger';
 import { SerializerService } from '../serializer';
 import { StoreService } from '../store';
 import {
-  ExportItem,
   ExportOptions,
   DurableJobExport, 
-  IdemType,
-  TimelineEntry } from '../../types/exporter';
+  TimelineType,
+  TransitionType, 
+  ExportFields} from '../../types/exporter';
 import { RedisClient, RedisMulti } from '../../types/redis';
-import { StringStringType, Symbols } from "../../types/serializer";
+import { StringAnyType, StringStringType, Symbols } from "../../types/serializer";
 
 class ExporterService {
   appId: string;
   logger: ILogger;
   store: StoreService<RedisClient, RedisMulti>;
   symbols: Promise<Symbols> | Symbols;
+  private static symbols: Map<string, Symbols> = new Map();
 
   constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
     this.appId = appId;
@@ -29,173 +29,129 @@ class ExporterService {
    * facets that describe the workflow in terms relevant to narrative storytelling.
    */
   async export(jobId: string, options: ExportOptions = {}): Promise<DurableJobExport> {
-    if (!this.symbols) {
-      this.symbols = this.store.getAllSymbols();
-      this.symbols = await this.symbols;
+    if (!ExporterService.symbols.has(this.appId)) {
+      const symbols: Symbols | Promise<Symbols> = this.store.getAllSymbols();
+      ExporterService.symbols.set(this.appId, await symbols);
     }
     const jobData = await this.store.getRaw(jobId);
-    const jobExport = this.inflate(jobData/*, depData*/);
+    const jobExport = this.inflate(jobData, options);
     return jobExport;
   }
 
-  /**
-   * Inflates the key from Redis, 3-character symbol
-   * into a human-readable JSON path, reflecting the
-   * tree-like structure of the unidimensional Hash
-   */
-  inflateKey(key: string): string {
-    if (key in this.symbols) {
-      const path = this.symbols[key];
-      const parts = path.split('/');
-      return parts.join('/');
-    }
-    return key;
-  }
-
-  /**
-   * Inflates the dependency data from Redis into a DurableJobExport object by
-   * organizing the dimensional isolate in sch a way asto interleave
-   * into a story
-   * @param data - the dependency data from Redis
-   * @returns - the organized dependency data
-   */
-  inflateDependencyData(data: string[]): Record<string, any>[] {
-    return data.map((dependency, index: number): Record<string, any> => {
-      const [action, topic, gid, dimension, ...jid] = dependency.split(VALSEP);
-      const job_id = jid.join(VALSEP);
-      return {
-        index,
-        action,
-        topic,
-        gid,
-        dimension,
-        job_id,
-      }
-    });
-  }
-
   /**
    * Inflates the job data from Redis into a DurableJobExport object
    * @param jobHash - the job data from Redis
    * @param dependencyList - the list of dependencies for the job
    * @returns - the inflated job data
    */
-  inflate(jobHash: StringStringType): DurableJobExport {
-    const idempotents: IdemType[] = [];
-    const state: StringStringType = {};
+  inflate(jobHash: StringStringType, options: ExportOptions): DurableJobExport {
+    const timeline: TimelineType[] = [];
+    const state: StringAnyType = {};
     const data: StringStringType = {};
-    const other: ExportItem[] = [];
-    const replay: Record<string, TimelineEntry> = {};
+    const transitionsObject: Record<string, TransitionType> = {};
     const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
 
     Object.entries(jobHash).forEach(([key, value]) => {
       const match = key.match(regex);
+
       if (match) {
-        //activity process state
-        this.inflateProcess(match, value, replay);
+        //transitions
+        this.inflateTransition(match, value, transitionsObject);
+
       } else if (key.length === 3) {
-        //job state
+        //state
         state[this.inflateKey(key)] = SerializerService.fromString(value);
+
       } else if (key.startsWith('_')) {
-        //job data
+        //data
         data[key.substring(1)] = value;
+
       } else if (key.startsWith('-')) {
-        //actions with side effect (replayable)
-        idempotents.push({
+        //timeline
+        const keyParts = this.keyToObject(key); //key parts have meaning
+        timeline.push({
+          ...keyParts,
           key,
-          value: SerializerService.fromString(value),
-          parts: extractParts(key),
+          value: this.resolveValue(value, options.values),
         });
-      } else {
-        //collator guids, etc
-        other.push([null, key, value]);
       }
     });
 
-    const sortEntriesByCreated = (obj: { [key: string]: TimelineEntry }): TimelineEntry[]  => {
-      const entriesArray: TimelineEntry[] = Object.values(obj);
-      entriesArray.sort((a, b) => {
-        return (a.created || a.updated).localeCompare(b.created || b.updated);
-      });
-      return entriesArray;
-    }
+    return this.filterFields({
+      data: restoreHierarchy(data),
+      state: Object.entries(restoreHierarchy(state))[0][1],
+      status: parseInt(jobHash[':'], 10),
+      timeline: this.sortParts(timeline),
+      transitions: this.sortEntriesByCreated(transitionsObject),
+    }, options.block, options.allow);
+  }
 
-    /**
-     * idem list has a complicated sort order based on indexes and dimensions
-     */
-    const sortParts = (parts: IdemType[]): IdemType[]=> {
-      return parts.sort((a, b) => {
-        const { dimension: aDim, index: aIdx, secondary: aSec } = a.parts;
-        const { dimension: bDim, index: bIdx, secondary: bSec } = b.parts;
-    
-        if (aDim === undefined && bDim !== undefined) return -1;
-        if (aDim !== undefined && bDim === undefined) return 1;
-        if (aDim !== undefined && bDim !== undefined) {
-          if (aDim < bDim) return -1;
-          if (aDim > bDim) return 1;
-        }
-    
-        if (aIdx < bIdx) return -1;
-        if (aIdx > bIdx) return 1;
-    
-        if (aSec === undefined && bSec !== undefined) return -1;
-        if (aSec !== undefined && bSec === undefined) return 1;
-        if (aSec !== undefined && bSec !== undefined) {
-          if (aSec < bSec) return -1;
-          if (aSec > bSec) return 1;
-        }
-    
-        return 0;
-      });
-    };
-    
-    function extractParts(key: string): {index: number, dimension?: string, secondary?: number} {
-      function extractDimension(label: string): string {
-        const parts = label.split(',');
-        if (parts.length > 1) {
-          parts.shift();
-          return parts.join(',');
-        }
+  resolveValue(raw: string, withValues: boolean): Record<string, any> | string | number | null {
+    const resolved = SerializerService.fromString(raw);
+    if (withValues !== false) {
+      return resolved;
+    }
+    if (resolved && typeof resolved === 'object') {
+      if ('data' in resolved) {
+        resolved.data = {};
+      }
+      if ('$error' in resolved) {
+        resolved.$error = {};
       }
+    }
+    return resolved;
+  }
 
-      const parts = key.split('-');
-      if (parts.length === 4) {
-        //-proxy-5-     -search-1-1-
-        return {
-          index: parseInt(parts[2], 10),
-          dimension: extractDimension(parts[1]),
+  /**
+   * Inflates the key from Redis, 3-character symbol
+   * into a human-readable JSON path, reflecting the
+   * tree-like structure of the unidimensional Hash
+   * @private
+   */
+  inflateKey(key: string): string {
+    const symbols = ExporterService.symbols.get(this.appId);
+    if (key in symbols) {
+      const path = symbols[key];
+      const parts = path.split('/');
+      return parts.join('/');
+    }
+    return key;
+  }
+
+ filterFields(fullObject: DurableJobExport, block: ExportFields[] = [], allow: ExportFields[] = []): Partial<DurableJobExport> {
+    let result: Partial<DurableJobExport> = {};
+    if (allow && allow.length > 0) {
+      allow.forEach(field => {
+        if (field in fullObject) {
+          result[field] = fullObject[field] as StringAnyType & number & TimelineType[] & TransitionType[];
         }
-      } else {
-        //-search,0,0-1-1-    -proxy,0,0-1-
-        return {
-          index: parseInt(parts[2], 10),
-          secondary: parseInt(parts[3], 10),
-          dimension: extractDimension(parts[1]),
+      });
+    } else {
+      result = { ...fullObject };
+    }
+    if (block && block.length > 0) {
+      block.forEach(field => {
+        if (field in result) {
+          delete result[field];
         }
-      }
+      });
     }
-
-    return {
-      data: restoreHierarchy(data),
-      idempotents: sortParts(idempotents),
-      state: Object.entries(restoreHierarchy(state))[0][1],
-      status: jobHash[':'],
-      replay: sortEntriesByCreated(replay),
-    };
+    return result as DurableJobExport;
   }
 
-  inflateProcess(match: RegExpMatchArray, value: string, replay: Record<string, Record<string, any>>) {
+  inflateTransition(match: RegExpMatchArray, value: string, transitionsObject: Record<string, TransitionType>) {
     const [_, letters, dimensions] = match;
     const path = this.inflateKey(letters);
     const parts = path.split('/');
     const activity = parts[0];
     const isCreate = path.endsWith('/output/metadata/ac');
     const isUpdate = path.endsWith('/output/metadata/au');
+    //for now only export activity start/stop; activity data would also be interesting
     if (isCreate || isUpdate) {
       const targetName = `${activity},${dimensions}`;
-      let target = replay[targetName];
+      let target = transitionsObject[targetName];
       if (!target) {
-        replay[targetName] = {
+        transitionsObject[targetName] = {
           activity,
           dimensions,
           created: isCreate ? value : null,
@@ -206,6 +162,71 @@ class ExporterService {
       }
     }
   }
+
+  sortEntriesByCreated(obj: { [key: string]: TransitionType }): TransitionType[] {
+    const entriesArray: TransitionType[] = Object.values(obj);
+    entriesArray.sort((a, b) => {
+      return (a.created || a.updated).localeCompare(b.created || b.updated);
+    });
+    return entriesArray;
+  }
+
+  /**
+   * marker names are overloaded with details like sequence, type, etc
+   */
+  keyToObject(key: string): {index: number, dimension?: string, secondary?: number} {
+    function extractDimension(label: string): string {
+      const parts = label.split(',');
+      if (parts.length > 1) {
+        parts.shift();
+        return parts.join(',');
+      }
+    }
+    const parts = key.split('-');
+    if (parts.length === 4) {
+      //-proxy-5-     -search-1-1-
+      return {
+        index: parseInt(parts[2], 10),
+        dimension: extractDimension(parts[1]),
+      }
+    } else {
+      //-search,0,0-1-1-    -proxy,0,0-1-
+      return {
+        index: parseInt(parts[2], 10),
+        secondary: parseInt(parts[3], 10),
+        dimension: extractDimension(parts[1]),
+      }
+    }
+  }
+
+  /**
+   * idem list has a complicated sort order based on indexes and dimensions
+   */
+  sortParts(parts: TimelineType[]): TimelineType[] {
+    return parts.sort((a, b) => {
+      const { dimension: aDim, index: aIdx, secondary: aSec } = a;
+      const { dimension: bDim, index: bIdx, secondary: bSec } = b;
+  
+      if (aDim === undefined && bDim !== undefined) return -1;
+      if (aDim !== undefined && bDim === undefined) return 1;
+      if (aDim !== undefined && bDim !== undefined) {
+        if (aDim < bDim) return -1;
+        if (aDim > bDim) return 1;
+      }
+  
+      if (aIdx < bIdx) return -1;
+      if (aIdx > bIdx) return 1;
+  
+      if (aSec === undefined && bSec !== undefined) return -1;
+      if (aSec !== undefined && bSec === undefined) return 1;
+      if (aSec !== undefined && bSec !== undefined) {
+        if (aSec < bSec) return -1;
+        if (aSec > bSec) return 1;
+      }
+  
+      return 0;
+    });
+  };
 }
 
 export { ExporterService };

package/services/durable/handle.ts

@@ -1,6 +1,6 @@
 import { ExporterService } from './exporter';
 import { HotMeshService as HotMesh } from '../hotmesh';
-import { DurableJobExport } from '../../types/exporter';
+import { DurableJobExport, ExportOptions } from '../../types/exporter';
 import { JobInterruptOptions, JobOutput } from '../../types/job';
 import { StreamError } from '../../types/stream';
 
@@ -21,8 +21,8 @@ export class WorkflowHandleService {
     );
   }
 
-  async export(): Promise<DurableJobExport>  {
-    return this.exporter.export(this.workflowId);
+  async export(options?: ExportOptions): Promise<DurableJobExport>  {
+    return this.exporter.export(this.workflowId, options);
   }
 
   /**

package/services/durable/worker.ts

@@ -288,7 +288,7 @@ export class WorkerService {
           const workflowInput = data.data as unknown as WorkflowDataType;
           const execIndex = counter.counter - interruptionRegistry.length + 1;
           const { workflowId, workflowTopic, workflowDimension, originJobId } = workflowInput;
-          const collatorFlowId = `-${workflowId}-$COLLATOR${workflowDimension || ''}-${execIndex}`;
+          const collatorFlowId = `-${workflowId}-$${workflowDimension || ''}-$${execIndex}`;
           return {
             status: StreamStatus.SUCCESS,
             code: HMSH_CODE_DURABLE_ALL,