@hotmeshio/hotmesh 0.0.12 → 0.0.14

package/services/pipe/functions/array.ts

@@ -3,6 +3,10 @@ class ArrayHandler {
     return array[index];
   }
 
+  length(array: any[]): any {
+    return array?.length;
+  }
+
   concat(array1: any[], array2: any[]): any[] {
     return array1.concat(array2);
   }

package/services/reporter/index.ts

@@ -45,12 +45,17 @@ class ReporterService {
   }
 
   private validateOptions(options: GetStatsOptions): void {
-    const { start, end, range } = options;
-    if (start && end && range || !start && !end && !range) {
+    const { start, end, range, granularity } = options;
+    if (granularity !== 'infinity' && (start && end && range || !start && !end && !range)) {
       throw new Error('Invalid combination of start, end, and range values. Provide either start+end, end+range, or start+range.');
     }
   }
+
   private generateDateTimeSets(granularity: string, range: string|undefined, end: string, start?: string): string[] {
+    if (granularity === 'infinity') {
+      //if granularity is infinity, it means a date/time sequence/slice is not used to further segment the statistics
+      return ['0'];
+    }
     if (!range) {
       //pluck just a single value when no range provided
       range = '0m';
@@ -190,6 +195,9 @@ class ReporterService {
   }
 
   private isoTimestampFromKeyTimestamp(hashKey: string): string {
+    if (hashKey.endsWith(':')) {
+      return '0';
+    }
     const keyTimestamp = hashKey.slice(-12);
     const year = keyTimestamp.slice(0, 4);
     const month = keyTimestamp.slice(4, 6);

package/services/signaler/store.ts

@@ -3,6 +3,7 @@ import { StoreService } from '../store';
 import { HookRule, HookSignal } from '../../types/hook';
 import { JobState } from '../../types/job';
 import { RedisClient, RedisMulti } from '../../types/redis';
+import { Pipe } from '../pipe';
 
 class StoreSignaler {
   store: StoreService<RedisClient, RedisMulti>;
@@ -21,10 +22,12 @@ class StoreSignaler {
   async registerWebHook(topic: string, context: JobState, multi?: RedisMulti): Promise<string> {
     const hookRule = await this.getHookRule(topic);
     if (hookRule) {
+      const mapExpression = hookRule.conditions.match[0].expected;
+      const resolved = Pipe.resolve(mapExpression, context);
       const jobId = context.metadata.jid;
       const hook: HookSignal = {
         topic,
-        resolved: jobId,
+        resolved,
         jobId,
       }
       await this.store.setHookSignal(hook, multi);
@@ -37,8 +40,10 @@ class StoreSignaler {
   async processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<string> {
     const hookRule = await this.getHookRule(topic);
     if (hookRule) {
-      //todo: use the rule to generate `resolved`
-      const resolved = (data as { id: string}).id;
+      //NOTE: both formats are supported: $self.hook.data OR $hook.data
+      const context = { $self: { hook: { data }}, $hook: { data }};
+      const mapExpression = hookRule.conditions.match[0].actual;
+      const resolved = Pipe.resolve(mapExpression, context);
       const jobId = await this.store.getHookSignal(topic, resolved);
       return jobId;
     } else {

package/services/signaler/stream.ts

@@ -149,9 +149,9 @@ class StreamSignaler {
     let output: StreamDataResponse | void;
     try {
       output = await callback(input);
-    } catch (err) {
-      this.logger.error(`stream-call-function-error`, { stream, id, err });
-      output = this.structureUnhandledError(input, err);
+    } catch (error) {
+      this.logger.error(`stream-call-function-error`, { error });
+      output = this.structureUnhandledError(input, error);
     }
     return output as StreamDataResponse;
   }

package/services/store/clients/ioredis.ts

@@ -63,9 +63,9 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
   async xadd(key: string, id: string, messageId: string, messageValue: string, multi?: RedisMultiType): Promise<string | RedisMultiType> {
     try {
       return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
-    } catch (err) {
-      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+      throw error;
     }
   }
 
@@ -79,9 +79,9 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
   ): Promise<[string, string, number, [string, number][]][] | [string, string, number, number] | unknown[]> {
     try {
       return await this.redisClient.xpending(key, group, start, end, count, consumer);
-    } catch (err) {
-      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { error });
+      throw error;
     }
   }
 
@@ -95,27 +95,27 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
   ): Promise<ReclaimedMessageType> {
     try {
       return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args) as unknown as ReclaimedMessageType;
-    } catch (err) {
-      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
   async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
     try {
       return await (multi || this.redisClient).xack(key, group, id);
-    } catch (err) {
-      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
   async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
     try {
       return await (multi || this.redisClient).xdel(key, id);
-    } catch (err) {
-      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { error });
+      throw error;
     }
   }
 }

package/services/store/clients/redis.ts

@@ -80,10 +80,10 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
     const args = mkStream === 'MKSTREAM' ? ['MKSTREAM'] : [];
     try {
       return (await this.redisClient.sendCommand(['XGROUP', 'CREATE', key, groupName, id, ...args])) === 1;
-    } catch (err) {
+    } catch (error) {
       const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
-      this.logger.warn(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, err);
-      throw err;
+      this.logger.warn(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { error });
+      throw error;
     }
   }
 
@@ -94,9 +94,9 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
     }
     try {
       return await (multi || this.redisClient).XADD(key, id, { [args[0]]: args[1] });
-    } catch (err) {
-      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+      throw error;
     }
   }
 
@@ -115,9 +115,9 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       if (count !== undefined) args.push(count.toString());
       if (consumer) args.push(consumer);
       return await this.redisClient.sendCommand(['XPENDING', ...args]);
-    } catch (err) {
-      this.logger.error(`Error in retrieving pending messages for group: ${group} in key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in retrieving pending messages for group: ${group} in key: ${key}`, { error });
+      throw error;
     }
   }
   
@@ -131,9 +131,9 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
   ): Promise<ReclaimedMessageType> {
     try {
       return await this.redisClient.sendCommand(['XCLAIM', key, group, consumer, minIdleTime.toString(), id, ...args]) as unknown as ReclaimedMessageType;
-    } catch (err) {
-      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
@@ -145,9 +145,9 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       } else {
         return await this.redisClient[this.commands.xack](key, group, id);
       }
-    } catch (err) {
-      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
@@ -159,9 +159,9 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       } else {
         return await this.redisClient[this.commands.xdel](key, id);
       }
-    } catch (err) {
-      this.logger.error(`Error in deleting messages with ids: ${id} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in deleting messages with ids: ${id} for key: ${key}`, { error });
+      throw error;
     }
   }
 }

package/services/store/index.ts

@@ -431,7 +431,10 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     const output: IdsData = {};
     for (const [index, result] of results.entries()) {
       const key = indexKeys[index];
-      const idsList: string[] = result[1];
+
+      //todo: resolve this discrepancy between redis/ioredis
+      const idsList: string[] = result[1] || result;
+
       if (idsList && idsList.length > 0) {
         output[key] = idsList;
       } else {
@@ -695,11 +698,17 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return workItemKey;
   }
 
-  async deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string): Promise<void> {
+  async deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub = false): Promise<void> {
     const zsetKey = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId });
     const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
     if (didRemove) {
-      await this.redisClient[this.commands.rename](processedKey, key);
+      if (scrub) {
+        //indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
+        this.redisClient[this.commands.expire](processedKey, 0);
+        this.redisClient[this.commands.expire](key.split(":").slice(0, 5).join(":"), 0);
+      } else {
+        await this.redisClient[this.commands.rename](processedKey, key);
+      }
     }
     this.cache.removeWorkItem(this.appId);
   }

package/services/task/index.ts

@@ -25,15 +25,15 @@ class TaskService {
   async processWebHooks(hookEventCallback: HookInterface): Promise<void> {
     const workItemKey = await this.store.getActiveTaskQueue();
     if (workItemKey) {
-      const [topic, sourceKey, ...sdata] = workItemKey.split('::');
+      const [topic, sourceKey, scrub, ...sdata] = workItemKey.split('::');
       const data = JSON.parse(sdata.join('::'));
       const destinationKey = `${sourceKey}:processed`;
       const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
       if (jobId) {
+        //todo: don't use 'id', make configurable using hook rule
         await hookEventCallback(topic, { ...data, id: jobId });
-        //todo: do final checksum count (values are tracked in the stats hash)
       } else {
-        await this.store.deleteProcessedTaskQueue(workItemKey, sourceKey, destinationKey);
+        await this.store.deleteProcessedTaskQueue(workItemKey, sourceKey, destinationKey, scrub === 'true');
       }
       setImmediate(() => this.processWebHooks(hookEventCallback));
     }

package/types/activity.ts

@@ -1,7 +1,7 @@
 import { MetricTypes } from "./stats";
 import { StreamRetryPolicy } from "./stream";
 
-type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle';
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal';
 
 type Consumes = Record<string, string[]>;
 
@@ -15,6 +15,7 @@ interface BaseActivity {
   job?: Record<string, any>;
   hook?: Record<string, any>;
   telemetry?: Record<string, any>;
+  emit?: boolean;                      //if true, the activity will emit a message to the `publishes` topic immediately before transitioning to adjacent activities
   sleep?: number;                      //@pipe /in seconds
   expire?: number;                     //-1 forever (15 seconds default); todo: make globally configurable
   retry?: StreamRetryPolicy
@@ -38,6 +39,7 @@ interface Measure {
 interface TriggerActivityStats {
   id?: { [key: string]: unknown } | string;
   key?: { [key: string]: unknown } | string;
+  granularity?: string; //return 'infinity' to disable; default behavior is to always segment keys by time to ensure indexes (Redis LIST) never grow unbounded as a default behavior; for now, 5m is default and infinity can be set to override
   measures?: Measure[]; //what to capture
 }
 
@@ -58,20 +60,27 @@ interface WorkerActivity extends BaseActivity {
   timeout: number;
 }
 
-interface EmitActivity extends BaseActivity {
-  type: 'emit';
-}
-
 interface CycleActivity extends BaseActivity {
   type: 'cycle';
   ancestor: string; //ancestor activity id
 }
 
+interface SignalActivity extends BaseActivity {
+  type: 'signal';         //signal activities call hook/hookAll
+  subtype: 'one' | 'all'; //trigger: hook(One) or hookAll
+  topic: string;          //e.g., 'hook.resume'
+  key_name: string;       //e.g., 'parent_job_id'
+  key_value: string;      //e.g., '1234567890'
+  scrub: boolean;         //if true, the index will be deleted after use
+  signal?: Record<string, any>;   //used to define/map the signal input data
+  resolver?: Record<string, any>; //used to define/map the signal key resolver
+}
+
 interface IterateActivity extends BaseActivity {
   type: 'iterate';
 }
 
-type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | EmitActivity | IterateActivity;
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity;
 
 type ActivityData = Record<string, any>;
 type ActivityMetadata = {
@@ -115,8 +124,8 @@ export {
   TriggerActivityStats,
   AwaitActivity,
   CycleActivity,
+  SignalActivity,
   BaseActivity,
-  EmitActivity,
   IterateActivity,
   TriggerActivity,
   WorkerActivity

package/types/durable.ts

@@ -1,5 +1,12 @@
 import { RedisClass, RedisOptions } from './redis';
 
+type WorkflowConfig = {
+  backoffCoefficient?: number; //default 10
+  maximumAttempts?: number; //default 2
+  maximumInterval?: string; //default 30s
+  initialInterval?: string; //default 1s
+}
+
 type WorkflowOptions = {
   taskQueue: string;
   args: any[];          //input arguments to pass in
@@ -7,6 +14,14 @@ type WorkflowOptions = {
   workflowName?: string; //the name of the user's workflow function
   workflowTrace?: string;
   workflowSpan?: string;
+  config?: WorkflowConfig;
+}
+
+type SignalOptions = {
+  taskQueue: string;
+  data: Record<string, any>; //input data (any serializable object)
+  workflowId: string;   //execution id (the job id)
+  workflowName?: string; //the name of the user's workflow function
 }
 
 type ActivityWorkflowDataType = {
@@ -47,7 +62,7 @@ type WorkerConfig = {
 
 type WorkerOptions = {
   maxSystemRetries?: number; //1-3 (10ms, 100ms, 1_000ms)
-  backoffExponent?: number; //2-10ish
+  backoffCoefficient?: number; //2-10ish
 }
 
 type ContextType = {
@@ -81,7 +96,9 @@ export {
   NativeConnection,
   ProxyType,
   Registry,
+  SignalOptions,
   WorkerConfig,
+  WorkflowConfig,
   WorkerOptions,
   WorkflowDataType,
   WorkflowOptions,

package/types/index.ts

@@ -10,9 +10,9 @@ export {
   AwaitActivity,
   BaseActivity,
   CycleActivity,
-  EmitActivity,
   WorkerActivity,
   IterateActivity,
+  SignalActivity,
   TriggerActivity,
   TriggerActivityStats } from './activity';
 export {
@@ -36,6 +36,7 @@ export {
     NativeConnection,
     ProxyType,
     Registry,
+    WorkflowConfig,
     WorkerConfig,
     WorkerOptions,
     WorkflowDataType,

package/types/stats.ts

@@ -39,6 +39,7 @@ interface JobStatsInput {
   start?: string;
   end?: string;
   sparse?: boolean;
+  scrub?: boolean; //remove all indexes upon completion
 }
 
 interface GetStatsOptions {

package/build/services/activities/emit.d.ts

@@ -1,9 +0,0 @@
-import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
-import { ActivityData, ActivityMetadata, EmitActivity } from '../../types/activity';
-declare class Emit extends Activity {
-    config: EmitActivity;
-    constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService);
-    mapInputData(): Promise<void>;
-}
-export { Emit };

package/build/services/activities/emit.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Emit = void 0;
-const activity_1 = require("./activity");
-class Emit extends activity_1.Activity {
-    constructor(config, data, metadata, hook, engine) {
-        super(config, data, metadata, hook, engine);
-    }
-    async mapInputData() {
-        this.logger.info('emit-map-input-data');
-    }
-}
-exports.Emit = Emit;

package/services/activities/emit.ts

@@ -1,25 +0,0 @@
-import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
-import {
-  ActivityData,
-  ActivityMetadata,
-  EmitActivity } from '../../types/activity';
-
-class Emit extends Activity {
-  config: EmitActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService) {
-    super(config, data, metadata, hook, engine);
-  }
-
-  async mapInputData(): Promise<void> {
-    this.logger.info('emit-map-input-data');
-  }
-}
-
-export { Emit };