@hotmeshio/hotmesh 0.0.43 → 0.0.44

package/services/engine/index.ts

@@ -21,6 +21,7 @@ import { Signal } from '../activities/signal';
 import { Worker } from '../activities/worker';
 import { Trigger } from '../activities/trigger';
 import { CompilerService } from '../compiler';
+import { ExporterService } from '../exporter';
 import { ILogger } from '../logger';
 import { ReporterService } from '../reporter';
 import { Router } from '../router';
@@ -78,12 +79,14 @@ import {
   StreamRole,
   StreamStatus } from '../../types/stream';
 import { WorkListTaskType } from '../../types/task';
+import { JobExport } from '../../types/exporter';
 
 class EngineService {
   namespace: string;
   apps: HotMeshApps | null;
   appId: string;
   guid: string;
+  exporter: ExporterService | null;
   router: Router | null;
   store: StoreService<RedisClient, RedisMulti> | null;
   stream: StreamService<RedisClient, RedisMulti> | null;
@@ -109,8 +112,8 @@ class EngineService {
       await instance.initStoreChannel(config.engine.store);
       await instance.initSubChannel(config.engine.sub);
       await instance.initStreamChannel(config.engine.stream);
-      instance.router = instance.initRouter(config);
 
+      instance.router = instance.initRouter(config);
       instance.router.consumeMessages(
         instance.stream.mintKey(
           KeyType.STREAMS,
@@ -121,9 +124,15 @@ class EngineService {
         instance.processStreamMessage.bind(instance)
       );
 
-      //the task service is used by the engine to process `webhooks` and `timehooks`
-      instance.taskService = new TaskService(instance.store, logger);
-
+      instance.taskService = new TaskService(
+        instance.store,
+        logger
+      );
+      instance.exporter = new ExporterService(
+        instance.appId,
+        instance.store,
+        logger,
+      );
       return instance;
     }
   }
@@ -690,9 +699,15 @@ class EngineService {
 
 
   // ****** GET JOB STATE/COLLATION STATUS BY ID *********
+  async export(jobId: string): Promise<JobExport> {
+    return await this.exporter.export(jobId);
+  } 
+  async getRaw(jobId: string): Promise<StringStringType> {
+    return await this.store.getRaw(jobId);
+  }
   async getStatus(jobId: string): Promise<JobStatus> {
     const { id: appId } = await this.getVID();
-    return this.store.getStatus(jobId, appId);
+    return await this.store.getStatus(jobId, appId);
   }
   //todo: add 'options' parameter;
   //      (e.g, if {dimensions:true}, use hscan to deliver

package/services/exporter/index.ts

@@ -0,0 +1,147 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import {
+  StringAnyType,
+  StringStringType,
+  Symbols } from "../../types/serializer";
+import { RedisClient, RedisMulti } from '../../types/redis';
+import {
+  DependencyExport,
+  ExportOptions,
+  JobActionExport,
+  JobExport } from '../../types/exporter';
+import { SerializerService } from '../serializer';
+import { restoreHierarchy } from '../../modules/utils';
+
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Expands process data and includes dependency list
+ */
+class ExporterService {
+  appId: string;
+  logger: ILogger;
+  serializer: SerializerService
+  store: StoreService<RedisClient, RedisMulti>;
+  symbols: Promise<Symbols> | Symbols;
+
+  constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
+    this.appId = appId;
+    this.logger = logger;
+    this.store = store;
+    this.serializer = new SerializerService();
+  }
+
+  /**
+   * Convert the job hash and dependency list into a JobExport object.
+   * This object contains various facets that describe the interaction
+   * in terms relevant to narrative storytelling.
+   */
+  async export(jobId: string, options: ExportOptions = {}): Promise<JobExport> {
+    if (!this.symbols) {
+      this.symbols = this.store.getAllSymbols();
+      this.symbols = await this.symbols;
+    }
+    const depData = await this.store.getDependencies(jobId);
+    const jobData = await this.store.getRaw(jobId);
+    const jobExport = this.inflate(jobData, depData);
+    return jobExport;
+  }
+
+  /**
+   * Inflates the key from Redis, 3-character symbol
+   * into a human-readable JSON path, reflecting the
+   * tree-like structure of the unidimensional Hash
+   */
+  inflateKey(key: string): string {
+    return (key in this.symbols) ? this.symbols[key] : key;
+  }
+
+  /**
+   * Inflates the job data from Redis into a JobExport object
+   * @param jobHash - the job data from Redis
+   * @param dependencyList - the list of dependencies for the job
+   * @returns - the inflated job data
+   */
+  inflate(jobHash: StringStringType, dependencyList: string[]): JobExport {
+    //the list of actions taken in the workflow and hook functions
+    const actions: JobActionExport = {
+      hooks: {},
+      main: {
+        cursor: -1,
+        items: []
+      }
+    };
+    const process: StringAnyType = {};
+    const dependencies = this.inflateDependencyData(dependencyList, actions);
+    const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
+
+    Object.entries(jobHash).forEach(([key, value]) => {
+      const match = key.match(regex);
+      if (match) {
+        //activity process state
+        const [_, letters, numbers] = match;
+        const path = this.inflateKey(letters);
+        const dimensions = `${numbers.replace(/,/g, '/')}`;
+        const resolved = this.serializer.fromString(value);
+        process[`${dimensions}/${path}`] = resolved;
+      } else if (key.length === 3) {
+        //job state
+        process[this.inflateKey(key)] = this.serializer.fromString(value);
+      }
+    });
+    
+    return {
+      dependencies,
+      process: restoreHierarchy(process),
+      status: jobHash[':'],
+    };
+  }
+
+  /**
+   * Inflates the dependency data from Redis into a JobExport object by
+   * organizing the dimensional isolate in sch a way asto interleave
+   * into a story
+   * @param data - the dependency data from Redis
+   * @returns - the organized dependency data
+   */
+  inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
+    const hookReg = /([0-9,]+)-(\d+)$/;
+    const flowReg = /-(\d+)$/;
+    return data.map((dependency, index: number): DependencyExport => {
+      const [action, topic, gid, ...jid] = dependency.split('::');
+      const jobId = jid.join('::');
+      const match = jobId.match(hookReg);
+      let prefix: string;
+      let type: 'hook' | 'flow' | 'other';
+      let dimensionKey: string = '';
+
+      if (match) {
+        //hook-originating dependency
+        const [_, dimension, counter] = match;
+        dimensionKey = dimension.split(',').join('/');
+        prefix = `${dimensionKey}[${counter}]`;
+        type = 'hook';
+      } else {
+        const match = jobId.match(flowReg);
+        if (match) {
+          //main workflow-originating dependency
+          const [_, counter] = match;
+          prefix = `[${counter}]`;
+          type = 'flow';
+        } else {
+          //'other' types like signal cleanup
+          prefix = '/';
+          type = 'other';
+        }
+      }
+      return {
+        type: action,
+        topic,
+        gid,
+        jid: jobId,
+      } as unknown as DependencyExport;
+    });
+  }
+}
+
+export { ExporterService };

package/services/hotmesh/index.ts

@@ -24,7 +24,8 @@ import {
   StatsResponse } from '../../types/stats';
 import { ConnectorService } from '../connector';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
-import { StringAnyType } from '../../types/serializer';
+import { StringAnyType, StringStringType } from '../../types/serializer';
+import { JobExport } from '../../types/exporter';
 
 class HotMeshService {
   namespace: string;
@@ -152,6 +153,12 @@ class HotMeshService {
   }
 
   // ************* REPORTER METHODS *************
+  async export(jobId: string): Promise<JobExport> {
+    return await this.engine?.export(jobId);
+  }
+  async getRaw(jobId: string): Promise<StringStringType> {
+    return await this.engine?.getRaw(jobId);
+  }
   async getStats(topic: string, query: JobStatsInput): Promise<StatsResponse> {
     return await this.engine?.getStats(topic, query);
   }

package/services/quorum/index.ts

@@ -145,7 +145,8 @@ class QuorumService {
         engine_id: this.guid,
         namespace: this.namespace,
         app_id: this.appId,
-        stream: this.engine.stream.mintKey(KeyType.STREAMS, { appId: this.appId })
+        stream: this.engine.stream.mintKey(KeyType.STREAMS, { appId: this.appId }),
+        counts: this.engine.router.counts,
       };
     }
     this.store.publish(

package/services/router/index.ts

@@ -40,6 +40,7 @@ class Router {
   logger: ILogger;
   throttle = 0;
   errorCount = 0;
+  counts: { [key: string]: number } = {};
   currentTimerId: NodeJS.Timeout | null = null;
   shouldConsume: boolean;
 
@@ -183,6 +184,8 @@ class Router {
       } else {
         output.metadata.guid = guid();
       }
+      const code = output.code || 200;
+      this.counts[code] = (this.counts[code] || 0) + 1;
       output.type = StreamDataType.RESPONSE;
       return await this.publishMessage(null, output as StreamDataResponse) as string;
     }

package/services/store/index.ts

@@ -241,6 +241,36 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
+  async getAllSymbols(): Promise<Symbols> {
+    //get hash with all reserved symbol ranges
+    const rangeKey = this.mintKey(KeyType.SYMKEYS, { appId: this.appId });
+    const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
+    const rangeKeys = Object.keys(ranges).sort();
+    delete rangeKeys[':cursor'];
+    const multi = this.getMulti();
+    for (const rangeKey of rangeKeys) {
+      const symbolKey = this.mintKey(KeyType.SYMKEYS, { activityId: rangeKey, appId: this.appId });
+      multi[this.commands.hgetall](symbolKey);
+    }
+    const results = await multi.exec() as Array<[null, Symbols]> | Array<Symbols>;
+
+    const symbolSets: Symbols = {};
+    results.forEach((result: [null, Symbols] | Symbols, index: number) => {
+      if (result) {
+        let vals: Symbols;
+        if (Array.isArray(result) && result.length === 2) {
+          vals = result[1];
+        } else {
+          vals = result as Symbols;
+        }
+        for (const [key, value] of Object.entries(vals)) {
+          symbolSets[value as string] = key.startsWith(rangeKeys[index]) ? key : `${rangeKeys[index]}/${key}`;
+        }        
+      }
+    });
+    return symbolSets;
+  }
+
   async getSymbols(activityId: string): Promise<Symbols> {
     let symbols: Symbols = this.cache.getSymbols(this.appId, activityId);
     if (symbols) {
@@ -396,7 +426,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * when `originJobId` is interrupted/expired, the items in the
    * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
    */
-  async registerJobDependency(originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
+  async registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
     const dependencyParams = {
       appId: this.appId,
@@ -406,8 +436,8 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       KeyType.JOB_DEPENDENTS,
       dependencyParams,
     );
-    //tasks have '4' segments
-    const expireTask = `expire::${topic}::${gId}::${jobId}`;
+    //items listed as job dependencies have different relationships
+    const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
     privateMulti[this.commands.rpush](depKey, expireTask);
     if (!multi) {
       return await privateMulti.exec();
@@ -589,6 +619,15 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
+  async getRaw(jobId: string): Promise<StringStringType> {
+    const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+    const job = await this.redisClient[this.commands.hgetall](jobKey);
+    if (!job) {
+      throw new GetStateError(jobId);
+    }
+    return job;
+  }
+
   /**
    * collate is a generic method for incrementing a value in a hash
    * in order to track their progress during processing.
@@ -839,6 +878,12 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
   }
 
+  async getDependencies(jobId: string): Promise<string[]> {
+    const depParams = { appId: this.appId, jobId };
+    const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
+    return this.redisClient[this.commands.lrange](depKey, 0, -1);
+  }
+
   /**
    * registers a hook activity to be awakened (uses ZSET to
    * store the 'sleep group' and LIST to store the events
@@ -862,11 +907,15 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       let [pType, pKey] = this.resolveTaskKeyContext(listKey);
       const timeEvent = await this.redisClient[this.commands.lpop](pKey);
       if (timeEvent) {
-        //there are 4 time-related task
-        //1) sleep (awaken), 2) expire, 3) interrupt, 4) delist
-        const [type, activityId, gId, ...jobId] = timeEvent.split('::');
+        //there are task types
+        //1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
+        let [type, activityId, gId, ...jobId] = timeEvent.split('::');
         if (type === 'delist') {
           pType = 'delist';
+        } else if (type === 'child') {
+          pType = 'child';
+        } else if (type === 'expire-child') {
+          type = 'expire'; //use the same logic as 'expire'
         }
         return [listKey, jobId.join('::'), gId, activityId, pType];
       }
@@ -884,7 +933,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * generic LIST (lists typically contain target job ids)
    * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
    */
-  resolveTaskKeyContext(listKey: string): [('sleep'|'expire'|'interrupt'|'delist'), string] {
+  resolveTaskKeyContext(listKey: string): [WorkListTaskType, string] {
     if (listKey.startsWith('::INTERRUPT')) {
       return ['interrupt', listKey.split('::')[2]];
     } else if (listKey.startsWith('::EXPIRE')) {

package/services/task/index.ts

@@ -117,7 +117,10 @@ class TaskService {
 
         if (Array.isArray(workListTask)) {
           const [listKey, target, gId, activityId, type] = workListTask;
-          if (type === 'delist') {
+          if (type === 'child') {
+            //continue; this child is listed here for convenience, but
+            //  will be expired by an origin ancestor and is listed there
+          } else if (type === 'delist') {
             //delist the signalKey (target)
             const key = this.store.mintKey(KeyType.SIGNALS, { appId: this.store.appId });
             await this.store.redisClient[this.store.commands.hdel](key, target);

package/services/worker/index.ts

@@ -1,17 +1,19 @@
 import { KeyType } from "../../modules/key";
+import { identifyRedisType } from "../../modules/utils";
+import { ConnectorService } from "../connector";
 import { ILogger } from "../logger";
 import { Router } from "../router";
 import { StoreService } from '../store';
-import { RedisStoreService as RedisStore } from '../store/clients/redis';
 import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
+import { RedisStoreService as RedisStore } from '../store/clients/redis';
 import { StreamService } from '../stream';
-import { RedisStreamService as RedisStream } from '../stream/clients/redis';
 import { IORedisStreamService as IORedisStream } from '../stream/clients/ioredis';
+import { RedisStreamService as RedisStream } from '../stream/clients/redis';
 import { SubService } from '../sub';
 import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
 import { RedisSubService as RedisSub } from '../sub/clients/redis';
-import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
 import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
+import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
 import {
   QuorumMessage,
   QuorumProfile,
@@ -19,8 +21,6 @@ import {
 import { RedisClient, RedisMulti } from "../../types/redis";
 import { RedisClientType } from '../../types/redisclient';
 import { StreamRole } from "../../types/stream";
-import { identifyRedisType } from "../../modules/utils";
-import { ConnectorService } from "../connector";
 
 class WorkerService {
   namespace: string;
@@ -174,6 +174,7 @@ class WorkerService {
         app_id: this.appId,
         worker_topic: this.topic,
         stream: this.stream.mintKey(KeyType.STREAMS, params),
+        counts: this.router.counts,
       };
     }
     this.store.publish(

package/types/activity.ts

@@ -38,13 +38,18 @@ interface Measure {
 
 interface TriggerActivityStats {
   /**
-   * parent job; including this allows the parent's 
+   * dependent parent job id; including this allows the parent's 
    * expiration/interruption events to cascade; set 
    * `expire` in the YAML for the dependent graph 
    * to 0 and provide the parent for dependent,
    * cascading interruption and cleanup
    */
   parent?: string;
+  /**
+   * adjacent parent job id; this is the actual adjacent
+   * parent in the graph, but it is not used for cascading expiration
+   */
+  adjacent?: string;
   id?: { [key: string]: unknown } | string;
   key?: { [key: string]: unknown } | string;
   /**

package/types/exporter.ts

@@ -0,0 +1,61 @@
+import { StringAnyType, StringStringType } from "./serializer";
+
+export type ExportItem = [(string | null), string, any];
+
+export interface ExportOptions {};
+
+export type JobAction = {
+  cursor: number;
+  items: ExportItem[];
+};
+
+export interface JobActionExport {
+  hooks: {
+    [key: string]: JobAction;
+  };
+  main: JobAction;
+};
+
+export interface ActivityAction {
+  action: string;
+  target: string;
+}
+
+export interface JobTimeline {
+  activity: string; //activity name
+  dimension: string; //dimensional isolate path
+  duplex: 'entry' | 'exit'; //activity entry or exit
+  timestamp: string; //actually a number but too many digits for JS
+  actions?: ActivityAction[];
+}
+
+export interface DependencyExport {
+  type: string;
+  topic: string;
+  gid: string;
+  jid: string;
+}
+
+export interface ExportTransitions {
+  [key: string]: string[];
+};
+
+export interface ExportCycles {
+  [key: string]: string[];
+};
+
+export interface DurableJobExport {
+  data: StringAnyType;
+  dependencies: DependencyExport[];
+  state: StringAnyType;
+  status: string;
+  timeline: JobTimeline[];
+  transitions: ExportTransitions;
+  cycles: ExportCycles;
+};
+
+export interface JobExport {
+  dependencies: DependencyExport[];
+  process: StringAnyType;
+  status: string;
+};

package/types/index.ts

@@ -53,7 +53,19 @@ export {
     WorkflowSearchOptions,
     WorkflowDataType,
     WorkflowOptions,
-  } from './durable'
+  } from './durable';
+export {
+  ActivityAction,
+  DependencyExport,
+  DurableJobExport,
+  ExportCycles,
+  ExportItem,
+  ExportOptions,
+  ExportTransitions,
+  JobAction,
+  JobExport,
+  JobActionExport,
+  JobTimeline } from './exporter';
 export {
   HookCondition,
   HookConditions,

package/types/quorum.ts

@@ -9,6 +9,7 @@ export interface QuorumProfile {
   worker_topic?: string;
   stream?: string;
   stream_depth?: number;
+  counts?: Record<string, number>;
 }
 
 export interface PingMessage {

package/types/task.ts

@@ -1 +1 @@
-export type WorkListTaskType = 'sleep' | 'expire' | 'interrupt' | 'delist';
+export type WorkListTaskType = 'sleep' | 'expire' | 'expire-child' | 'interrupt' | 'delist' | 'child';