@hotmeshio/hotmesh 0.0.52 → 0.0.54

package/services/engine/index.ts

@@ -51,7 +51,8 @@ import {
   PartialJobState,
   JobStatus, 
   JobInterruptOptions,
-  JobCompletionOptions } from '../../types/job';
+  JobCompletionOptions, 
+  ExtensionType} from '../../types/job';
 import {
   HotMeshApps,
   HotMeshConfig,
@@ -450,6 +451,7 @@ class EngineService {
         streamData.status = StreamStatus.ERROR;
         streamData.data = error;
         streamData.code = error.code;
+        streamData.stack = error.stack;
       } else if (emit) {
         streamData.status = StreamStatus.PENDING;
         streamData.code = HMSH_CODE_PENDING;
@@ -474,7 +476,10 @@ class EngineService {
 
   // ****************** `INTERRUPT` ACTIVE JOBS *****************
   async interrupt(topic: string, jobId: string, options: JobInterruptOptions = {}): Promise<string> {
+    //immediately interrupt the job, going directly to the data source
     await this.store.interrupt(topic, jobId, options);
+
+    //now that the job is interrupted, we can clean up
     const context = await this.getState(topic, jobId) as JobState;
     const completionOpts: JobCompletionOptions = {
       interrupt: options.descend,
@@ -562,10 +567,10 @@ class EngineService {
 
   // ********************** PUB/SUB ENTRY POINT **********************
   //publish (returns just the job id)
-  async pub(topic: string, data: JobData, context?: JobState): Promise<string> {
+  async pub(topic: string, data: JobData, context?: JobState, extended?: ExtensionType): Promise<string> {
     const activityHandler = await this.initActivity(topic, data, context);
     if (activityHandler) {
-      return await activityHandler.process();
+      return await activityHandler.process(extended);
     } else {
       throw new Error(`unable to process activity for topic ${topic}`);
     }

package/services/exporter/index.ts

@@ -1,18 +1,18 @@
+import { VALSEP } from '../../modules/key';
 import { ILogger } from '../logger';
+import { restoreHierarchy } from '../../modules/utils';
+import { SerializerService } from '../serializer';
 import { StoreService } from '../store';
-import {
-  StringAnyType,
-  StringStringType,
-  Symbols } from "../../types/serializer";
-import { RedisClient, RedisMulti } from '../../types/redis';
 import {
   DependencyExport,
   ExportOptions,
   JobActionExport,
   JobExport } from '../../types/exporter';
-import { SerializerService } from '../serializer';
-import { restoreHierarchy } from '../../modules/utils';
-import { VALSEP } from '../../modules/key';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import {
+  StringAnyType,
+  StringStringType,
+  Symbols } from "../../types/serializer";
 
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
@@ -21,7 +21,6 @@ import { VALSEP } from '../../modules/key';
 class ExporterService {
   appId: string;
   logger: ILogger;
-  serializer: SerializerService
   store: StoreService<RedisClient, RedisMulti>;
   symbols: Promise<Symbols> | Symbols;
 
@@ -29,7 +28,6 @@ class ExporterService {
     this.appId = appId;
     this.logger = logger;
     this.store = store;
-    this.serializer = new SerializerService();
   }
 
   /**
@@ -83,11 +81,11 @@ class ExporterService {
         const [_, letters, numbers] = match;
         const path = this.inflateKey(letters);
         const dimensions = `${numbers.replace(/,/g, '/')}`;
-        const resolved = this.serializer.fromString(value);
+        const resolved = SerializerService.fromString(value);
         process[`${dimensions}/${path}`] = resolved;
       } else if (key.length === 3) {
         //job state
-        process[this.inflateKey(key)] = this.serializer.fromString(value);
+        process[this.inflateKey(key)] = SerializerService.fromString(value);
       }
     });
     

package/services/hotmesh/index.ts

@@ -12,7 +12,8 @@ import {
   JobData,
   JobOutput, 
   JobStatus, 
-  JobInterruptOptions} from '../../types/job';
+  JobInterruptOptions,
+  ExtensionType} from '../../types/job';
 import {
   HotMeshConfig,
   HotMeshManifest } from '../../types/hotmesh';
@@ -115,8 +116,8 @@ class HotMeshService {
   }
 
   // ************* PUB/SUB METHODS *************
-  async pub(topic: string, data: JobData = {}, context?: JobState): Promise<string> {
-    return await this.engine?.pub(topic, data, context);
+  async pub(topic: string, data: JobData = {}, context?: JobState, extended?: ExtensionType): Promise<string> {
+    return await this.engine?.pub(topic, data, context, extended);
   }
   async sub(topic: string, callback: JobMessageCallback): Promise<void> {
     return await this.engine?.sub(topic, callback);

package/services/mapper/index.ts

@@ -36,7 +36,7 @@ class MapperService {
   }
 
   /**
-   * resolve a pipe expression of the form: { @pipe: [["{data.foo.bar}", 2, false, "hello world"]] }
+   * resolves a pipe expression of the form: { @pipe: [["{data.foo.bar}", 2, false, "hello world"]] }
    * @param value 
    * @returns 
    */
@@ -46,7 +46,7 @@ class MapperService {
   }
 
   /**
-   * resolve a simple mapping expression in the form: "{data.foo.bar}" or 2 or false or "hello world"
+   * resolves a mapping expression in the form: "{data.foo.bar}" or 2 or false or "hello world"
    * @param value 
    * @returns 
    */
@@ -55,6 +55,10 @@ class MapperService {
     return pipe.process();
   }
 
+  /**
+   * Evaluates a transition rule against the current job state and incoming Stream message
+   * to determine which (if any) transition should be taken.
+   */
   static evaluate(transitionRule: TransitionRule | boolean, context: JobState, code: StreamCode): boolean {
     if (typeof transitionRule === 'boolean') {
       return transitionRule;

package/services/pipe/functions/array.ts

@@ -1,6 +1,6 @@
 class ArrayHandler {
   get(array: any[], index: number): any {
-    return array[index];
+    return array?.[index || 0];
   }
 
   length(array: any[]): any {
@@ -11,26 +11,6 @@ class ArrayHandler {
     return array1.concat(array2);
   }
 
-  every(array: any[], callback: (value: any, index: number, array: any[]) => boolean): boolean {
-    return array.every(callback);
-  }
-
-  filter(array: any[], callback: (value: any, index: number, array: any[]) => boolean): any[] {
-    return array.filter(callback);
-  }
-
-  find(array: any[], callback: (value: any, index: number, array: any[]) => boolean): any {
-    return array.find(callback);
-  }
-
-  findIndex(array: any[], callback: (value: any, index: number, array: any[]) => boolean): number {
-    return array.findIndex(callback);
-  }
-
-  forEach(array: any[], callback: (value: any, index: number, array: any[]) => void): void {
-    array.forEach(callback);
-  }
-
   indexOf(array: any[], searchElement: any, fromIndex?: number): number {
     return array.indexOf(searchElement, fromIndex);
   }
@@ -43,20 +23,13 @@ class ArrayHandler {
     return array.lastIndexOf(searchElement, fromIndex);
   }
 
-  map(array: any[], callback: (value: any, index: number, array: any[]) => any): any[] {
-    return array.map(callback);
-  }
-
   pop(array: any[]): any {
     return array.pop();
   }
 
-  push(array: any[], ...items: any[]): number {
-    return array.push(...items);
-  }
-
-  reduce(array: any[], callback: (accumulator: any, currentValue: any, currentIndex: number, array: any[]) => any, initialValue?: any): any {
-    return array.reduce(callback, initialValue);
+  push(array: any[], ...items: any[]): any[] {
+    array.push(...items);
+    return array;
   }
 
   reverse(array: any[]): any[] {
@@ -71,12 +44,26 @@ class ArrayHandler {
     return array.slice(start, end);
   }
 
-  some(array: any[], callback: (value: any, index: number, array: any[]) => boolean): boolean {
-    return array.some(callback);
-  }
-
-  sort(array: any[], compareFunction?: (a: any, b: any) => number): any[] {
-    return array.sort(compareFunction);
+  sort(array: any[], order: 'ASCENDING' | 'DESCENDING' = 'ASCENDING'): any[] {
+    return array.sort((a, b) => {
+      if (order === 'ASCENDING') {
+        if (a === b) return 0;
+        if (a === null || a === undefined) return -1;
+        if (b === null || b === undefined) return 1;
+        if (typeof a === 'string' && typeof b === 'string') {
+          return a.localeCompare(b);
+        }
+        return a < b ? -1 : 1;
+      } else {
+        if (a === b) return 0;
+        if (a === null || a === undefined) return 1;
+        if (b === null || b === undefined) return -1;
+        if (typeof a === 'string' && typeof b === 'string') {
+          return b.localeCompare(a);
+        }
+        return a > b ? -1 : 1;
+      }
+    });
   }
 
   splice(array: any[], start: number, deleteCount?: number, ...items: any[]): any[] {

package/services/pipe/functions/conditional.ts

@@ -26,6 +26,10 @@ class ConditionalHandler {
   less_than_or_equal(value1: number, value2: number): boolean {
     return value1 <= value2;
   }
+
+  nullish(value1: any, value2: any): any {
+    return value1 ?? value2;
+  }
 }
 
 export { ConditionalHandler };

package/services/pipe/functions/date.ts

@@ -1,3 +1,5 @@
+import { formatISODate } from "../../../modules/utils";
+
 type DateInput = Date | string | number;
 
 class DateHandler {
@@ -181,6 +183,10 @@ class DateHandler {
     return DateHandler.getDateInstance(date).toISOString();
   }
 
+  toISOXString(date?: DateInput): string {
+    return formatISODate(date ? DateHandler.getDateInstance(date) : new Date());
+  }
+
   toJSON(date: DateInput): string {
     return DateHandler.getDateInstance(date).toJSON();
   }

package/services/pipe/functions/index.ts

@@ -3,6 +3,7 @@ import { BitwiseHandler } from './bitwise';
 import { ConditionalHandler } from './conditional';
 import { DateHandler } from './date';
 import { JsonHandler } from './json';
+import { LogicalHandler } from './logical';
 import { MathHandler } from './math';
 import { NumberHandler } from './number';
 import { ObjectHandler } from './object';
@@ -11,11 +12,12 @@ import { SymbolHandler } from './symbol';
 import { UnaryHandler } from './unary';
 
 export default { 
-  array: new ArrayHandler(), 
-  bitwise: new BitwiseHandler(), 
-  conditional: new ConditionalHandler(), 
-  date: new DateHandler(), 
-  json: new JsonHandler(), 
+  array: new ArrayHandler(),
+  bitwise: new BitwiseHandler(),
+  conditional: new ConditionalHandler(),
+  date: new DateHandler(),
+  json: new JsonHandler(),
+  logical: new LogicalHandler(),
   math: new MathHandler(),
   number: new NumberHandler(),
   object: new ObjectHandler(),

package/services/pipe/functions/logical.ts

@@ -0,0 +1,11 @@
+class LogicalHandler {
+  and(firstValue: boolean, secondValue: boolean): boolean {
+    return firstValue && secondValue;
+  }
+
+  or(firstValue: boolean, secondValue: boolean): boolean {
+    return firstValue || secondValue;
+  }
+}
+
+export { LogicalHandler };

package/services/pipe/functions/object.ts

@@ -1,14 +1,33 @@
 class ObjectHandler {
+  get(obj: object, prop: string | symbol): any {
+    return obj?.[prop];
+  }
+
+  set(obj: object, prop: string | symbol, value: any): any {
+    if (!obj) obj = {};
+    obj[prop] = value;
+    return obj;
+  }
+
+  create(...args: any[]): object {
+    const obj = {};
+    if (args.length === 0) return obj;
+    for (let i = 0; i < args.length; i += 2) {
+      obj[args[i]] = args[i + 1];
+    }
+    return obj;
+  }
+
   keys(obj: object): string[] {
-    return Object.keys(obj);
+    return obj && Object.keys(obj) || [];
   }
 
   values(obj: object): any[] {
-    return Object.values(obj);
+    return obj && Object.values(obj) || [];
   }
 
   entries(obj: object): [string, any][] {
-    return Object.entries(obj);
+    return obj && Object.entries(obj) || [];
   }
 
   fromEntries(iterable: Iterable<[string, any]>): object {
@@ -16,19 +35,19 @@ class ObjectHandler {
   }
 
   assign(target: object, ...sources: object[]): object {
-    return Object.assign(target, ...sources);
+    return Object.assign(target || {}, ...sources);
   }
 
   getOwnPropertyNames(obj: object): string[] {
-    return Object.getOwnPropertyNames(obj);
+    return Object.getOwnPropertyNames(obj || {});
   }
 
   getOwnPropertySymbols(obj: object): symbol[] {
-    return Object.getOwnPropertySymbols(obj);
+    return Object.getOwnPropertySymbols(obj || {});
   }
 
   getOwnPropertyDescriptor(obj: object, prop: string | symbol): PropertyDescriptor | undefined {
-    return Object.getOwnPropertyDescriptor(obj, prop);
+    return Object.getOwnPropertyDescriptor(obj || {}, prop);
   }
 
   defineProperty(obj: object, prop: string | symbol, descriptor: PropertyDescriptor): object {

package/services/pipe/index.ts

@@ -1,21 +1,34 @@
 import FUNCTIONS from './functions'
 import { JobState, JobData, JobsData } from '../../types/job';
-import { PipeItem, PipeItems, Pipe as PipeType } from '../../types/pipe';
+import {
+  PipeContext,
+  PipeItem,
+  PipeItems,
+  PipeObject,
+  Pipe as PipeType,
+  ReduceObject } from '../../types/pipe';
 
 class Pipe {
   rules: PipeType;
   jobData: JobData;
+  context: PipeContext;
 
-  constructor(rules: PipeType, jobData: JobData) {
+  constructor(rules: PipeType, jobData: JobData, context?: PipeContext) {
     this.rules = rules;
     this.jobData = jobData;
+    this.context = context;
   }
 
-  private isPipeType(currentRow: PipeItem[]|PipeType): currentRow is PipeType {
+  private isPipeType(currentRow: PipeItem[] | PipeType | PipeObject | ReduceObject): currentRow is PipeType {
     return !Array.isArray(currentRow) && '@pipe' in currentRow;
   }
 
-  static isPipeObject(obj: { [key: string]: unknown }|PipeItem): boolean {
+  private isreduceType(currentRow: PipeItem[] | PipeType | PipeObject | ReduceObject): currentRow is PipeType {
+    return !Array.isArray(currentRow) && '@reduce' in currentRow;
+  }
+
+
+  static isPipeObject(obj: { [key: string]: unknown } | PipeItem): boolean {
     return typeof obj === 'object' && obj !== null && !Array.isArray(obj) && '@pipe' in obj;
   }
 
@@ -33,36 +46,82 @@ class Pipe {
    * loop through each PipeItem row in this Pipe, resolving and transforming line by line
    * @returns {any} the result of the pipe
    */
-  process(): any {
-    let resolved = this.processCells(this.rules[0] as PipeItem[]);
+  process(resolved: unknown[] | null = null): any {
+    let index = 0;
+    if (!(resolved || this.isPipeType(this.rules[0]) || this.isreduceType(this.rules[0]))) {
+      resolved = this.processCells(this.rules[0] as PipeItem[]); // Add type assertion
+      index = 1;
+    }
     const len = this.rules.length;
-    for (let i = 1; i < len; i++) {
-      resolved = this.processRow(this.rules[i], resolved, []);
+    const subPipeQueue = [];
+    for (let i = index; i < len; i++) {
+      resolved = this.processRow(this.rules[i], resolved, subPipeQueue);
     }
     return resolved[0];
   }
 
-  private processRow(currentRow: PipeItem[]|PipeType, resolvedPriorRow: unknown[]|null, subPipeQueue: unknown[]): PipeItem[] {
-    if (this.isPipeType(currentRow)) {
-      //currentRow is a recursive subPipe
-      const subPipe = new Pipe(currentRow['@pipe'], this.jobData);
+  /**
+   * Transforms iterable `input` into a single value. Vars $output, $item, $key
+   * and $input are available. The final statement in the iterator (the reduction)
+   * is assumed to be the return value. A default $output object may be provided
+   * to the iterator by placing the the second cell of the preceding row. Otherwise,
+   * construct the object during first run and ensure it is the first cell of the
+   * last row of the iterator, so it is returned as the $output for the next cycle
+   * @param {unknown[]} input 
+   * @returns {unknown}
+   * @private
+   */
+  reduce(input: Array<unknown[]>): unknown {
+    let resolved = input[1] ?? null;
+
+    if (Array.isArray(input[0])) {
+      for (let index = 0; index < input[0].length; index++) {
+        this.context = { $input: input[0], $output: resolved, $item: input[0][index], $key: index.toString(), $index: index };
+        resolved = this.process([resolved]);
+      }
+    } else {
+      let index = -1;
+      for (let $key in (input[0] as Record<string, unknown>)) {
+        index++;
+        this.context = { $input: input[0], $output: resolved, $item: input[0][$key], $key, $index: index };
+        resolved = this.process([resolved]);
+      }
+    }
+    return [resolved];
+  }
+
+  private processRow(currentRow: PipeItem[] | PipeType | PipeObject | ReduceObject , resolvedPriorRow: unknown[]|null, subPipeQueue: unknown[]): PipeItem[] {
+    if (resolvedPriorRow && this.isreduceType(currentRow)) {
+      //reduce the resolvedPriorRow and return the output from the reducer
+      const subPipe = new Pipe(currentRow['@reduce'], this.jobData);
+      const reduced = subPipe.reduce(resolvedPriorRow as Array<unknown[]>)
+      return reduced as PipeItem[];
+    } else if (this.isPipeType(currentRow)) {
+      //process subPipe and push to subPipeQueue; echo resolvedPriorRow
+      const subPipe = new Pipe(currentRow['@pipe'], this.jobData, this.context);
       subPipeQueue.push(subPipe.process());
-      //return prior row as if nothing happened
       return resolvedPriorRow as PipeItem[];
     } else {
+      //pivot the subPipeQueue into the arguments array (resolvedPriorRow)
       if (subPipeQueue.length > 0) {
-        //if items in subPipeQueue, flush and use as resolvedPriorRow
         resolvedPriorRow = [...subPipeQueue];
         subPipeQueue.length = 0;
-      } else if (!resolvedPriorRow) {
+      }
+
+      if (!resolvedPriorRow) {
         //if no prior row, use current row as prior row
-        return [].concat(this.processCells([...currentRow]));
+        return [].concat(this.processCells(Array.isArray(currentRow) ? [...currentRow] as PipeItem[] : []));
       } else {
-        const [functionName, ...params] = currentRow;
+        const [functionName, ...params] = currentRow as PipeItem[]; // Add type assertion
         //use resolved values from prior row (n - 1) as input params to cell 1 function
-        const resolvedValue = Pipe.resolveFunction(functionName as string)(...resolvedPriorRow);
+        let resolvedValue: unknown;
+        if (this.isContextVariable(functionName)) {
+          resolvedValue = this.resolveContextValue(functionName as string);
+        } else {
+          resolvedValue = Pipe.resolveFunction(functionName as string)(...resolvedPriorRow);
+        }
         //resolve remaining cells in row and return concatenated with resolvedValue
-        return [resolvedValue].concat(this.processCells([...params]));
+        return [(resolvedValue as PipeItem)].concat(this.processCells([...params]));
       }
     }
   }
@@ -83,8 +142,10 @@ class Pipe {
 
   processCells(cells: PipeItems): unknown[] {
     const resolved = [];
-    for (const currentCell of cells) {
-      resolved.push(this.resolveCellValue(currentCell));
+    if (Array.isArray(cells)) {
+      for (const currentCell of cells) {
+        resolved.push(this.resolveCellValue(currentCell));
+      }
     }
     return resolved;
   }
@@ -93,6 +154,12 @@ class Pipe {
     return typeof currentCell === 'string' && currentCell.startsWith('{@') && currentCell.endsWith('}');
   }
 
+  private isContextVariable(currentCell: PipeItem): boolean {
+    if(typeof currentCell === 'string' && currentCell.endsWith('}')) {
+      return currentCell.startsWith('{$item') || currentCell.startsWith('{$key') || currentCell.startsWith('{$index') || currentCell.startsWith('{$input') || currentCell.startsWith('{$output');
+    }
+  }
+
   private isMappable(currentCell: PipeItem): boolean {
     return typeof currentCell === 'string' &&  currentCell.startsWith('{') && currentCell.endsWith('}');
   }
@@ -101,6 +168,8 @@ class Pipe {
     if (this.isFunction(currentCell)) {
       const fn = Pipe.resolveFunction(currentCell as string);
       return fn.call();
+    } else if (this.isContextVariable(currentCell)) {
+      return this.resolveContextValue(currentCell as string);
     } else if (this.isMappable(currentCell)) {
       return this.resolveMappableValue(currentCell as string);
     } else {
@@ -126,6 +195,15 @@ class Pipe {
     return this.getNestedProperty(this.jobData, term);
   }
 
+  resolveContextValue(currentCell: string): unknown {
+    const term = this.resolveContextTerm(currentCell);
+    return this.getNestedProperty(this.context, term);
+  }
+
+  resolveContextTerm(currentCell: string): string {
+    return currentCell.substring(1, currentCell.length - 1);
+  }
+
   resolveFunctionTerm(currentCell: string): string {
     return currentCell.substring(2, currentCell.length - 1);
   }

package/services/quorum/index.ts

@@ -25,9 +25,7 @@ import {
   QuorumMessageCallback,
   QuorumProfile,
   RollCallMessage,
-  SubscriptionCallback,
-  ThrottleMessage
-} from '../../types/quorum';
+  SubscriptionCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { RedisClientType } from '../../types/redisclient';
 

package/services/router/index.ts

@@ -194,7 +194,7 @@ class Router {
     try {
       output = await callback(input);
     } catch (error) {
-      this.logger.error(`stream-call-function-error`, { error });
+      this.logger.error(`stream-call-function-error`, { ...error, input: input, stack: error.stack, message: error.message, name: error.name, stream, id });
       output = this.structureUnhandledError(input, error);
     }
     return output as StreamDataResponse;
@@ -233,10 +233,19 @@ class Router {
   }
 
   shouldRetry(input: StreamData, output: StreamDataResponse): [boolean, number] {
+    //const isUnhandledEngineError = output.code === 500;
     const policies = input.policies?.retry;
     const errorCode = output.code.toString();
     const policy = policies?.[errorCode];
     const maxRetries = policy?.[0];
+    // if (isUnhandledEngineError && !policy) {
+    //   //if main goes down, replicas take over within 5s
+    //   //if this is not system/platform related, the exponential
+    //   //backoff will be applied and eventually slow to a crawl while
+    //   //the root cause is identified
+    //   input.policies = { retry: { [errorCode]: [10] } };
+    //   return [true, 0];
+    // }
     const tryCount = Math.min(input.metadata.try || 0,  HMSH_MAX_RETRIES);
     //only possible values for maxRetries are 1, 2, 3
     //only possible values for tryCount are 0, 1, 2
@@ -287,13 +296,15 @@ class Router {
     const message = output.data?.message ? output.data?.message.toString() : HMSH_STATUS_UNKNOWN;
     const statusCode = output.code || output.data?.code;
     const code = isNaN(statusCode as number) ? HMSH_CODE_UNKNOWN : parseInt(statusCode.toString());
-    const data: StreamError = { message, code };
+    const stack = output.data?.stack ? output.data?.stack.toString() : undefined;
+    const data: StreamError = { message, code, stack };
     if (typeof output.data?.error === 'object') {
       data.error = { ...output.data.error };
     }
     return {
       status: StreamStatus.ERROR,
       code,
+      stack,
       metadata: { ...input.metadata, guid: guid() },
       data
     } as StreamDataResponse;
@@ -362,7 +373,7 @@ class Router {
     //The stream activity was not processed within established limits. Possibilities Include:
     // 1) user error: the workers were not properly configured and are timing out
     // 2a) system error: JSON is corrupt
-    //     i) bad/unwitting actor
+    //     i) unwitting actor
     //     ii) corrupt hardware/network/transport/etc
     // 3b) system error: Redis unable to accept `xadd` request
     // 4c) system error: Redis unable to accept `xdel`/`xack` request