@hotmeshio/hotmesh 0.0.37 → 0.0.39

package/modules/errors.ts

@@ -1,9 +1,18 @@
 import { ActivityDuplex } from "../types/activity";
 import { CollationFaultType, CollationStage } from "../types/collator";
+import {
+  HMSH_CODE_DURABLE_MAXED,
+  HMSH_CODE_DURABLE_TIMEOUT,
+  HMSH_CODE_DURABLE_FATAL,
+  HMSH_CODE_DURABLE_INCOMPLETE,
+  HMSH_CODE_NOTFOUND,
+  HMSH_CODE_DURABLE_RETRYABLE,
+  HMSH_CODE_DURABLE_SLEEPFOR,
+  HMSH_CODE_DURABLE_WAITFOR } from "./enums";
 
 class GetStateError extends Error {
   jobId: string;
-  code: 404;
+  code = HMSH_CODE_NOTFOUND;
   constructor(jobId: string) {
     super(`${jobId} Not Found`);
     this.jobId = jobId;
@@ -21,7 +30,7 @@ class DurableIncompleteSignalError extends Error {
   code: number;
   constructor(message: string) {
     super(message);
-    this.code = 593;
+    this.code = HMSH_CODE_DURABLE_INCOMPLETE;
   }
 }
 
@@ -32,7 +41,7 @@ class DurableWaitForSignalError extends Error {
   constructor(message: string, signals: {signal: string, index: number}[]) {
     super(message);
     this.signals = signals;
-    this.code = 594;
+    this.code = HMSH_CODE_DURABLE_WAITFOR;
   }
 }
 
@@ -60,35 +69,35 @@ class DurableSleepForError extends Error {
     this.duration = duration;
     this.index = index;
     this.dimension = dimension;
-    this.code = 592;
+    this.code = HMSH_CODE_DURABLE_SLEEPFOR;
   }
 }
 class DurableTimeoutError extends Error {
   code: number;
   constructor(message: string) {
     super(message);
-    this.code = 596;
+    this.code = HMSH_CODE_DURABLE_TIMEOUT;
   }
 }
 class DurableMaxedError extends Error {
   code: number;
   constructor(message: string) {
     super(message);
-    this.code = 597;
+    this.code = HMSH_CODE_DURABLE_MAXED;
   }
 }
 class DurableFatalError extends Error {
   code: number;
   constructor(message: string) {
     super(message);
-    this.code = 598;
+    this.code = HMSH_CODE_DURABLE_FATAL;
   }
 }
 class DurableRetryError extends Error {
   code: number;
   constructor(message: string) {
     super(message);
-    this.code = 599;
+    this.code = HMSH_CODE_DURABLE_RETRYABLE;
   }
 }
 

package/modules/key.ts

@@ -1,3 +1,5 @@
+import { KeyStoreParams, KeyType } from '../types/hotmesh';
+
 /**
  * Keys
  * 
@@ -26,46 +28,7 @@
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
 
-//default namespace for hotmesh
-const HMNS = "hmsh";
-
-//these are the entity types that are stored in the key/value store
-enum KeyType {
-  APP = 'APP',
-  ENGINE_ID = 'ENGINE',
-  HOOKS = 'HOOKS',
-  JOB_DEPENDENTS = 'JOB_DEPENDENTS',
-  JOB_STATE = 'JOB_STATE',
-  JOB_STATS_GENERAL = 'JOB_STATS_GENERAL',
-  JOB_STATS_MEDIAN = 'JOB_STATS_MEDIAN',
-  JOB_STATS_INDEX = 'JOB_STATS_INDEX',
-  HOTMESH = 'HOTMESH',
-  QUORUM = 'QUORUM',
-  SCHEMAS = 'SCHEMAS',
-  SIGNALS = 'SIGNALS',
-  STREAMS = 'STREAMS',
-  SUBSCRIPTIONS = 'SUBSCRIPTIONS',
-  SUBSCRIPTION_PATTERNS = 'SUBSCRIPTION_PATTERNS',
-  SYMKEYS = 'SYMKEYS',
-  SYMVALS = 'SYMVALS',
-  TIME_RANGE = 'TIME_RANGE',
-  WORK_ITEMS = 'WORK_ITEMS',
-}
-
-//when minting a key, the following parameters are used to create a unique key per entity
-type KeyStoreParams = {
-  appId?: string;       //app id is a uuid for a hotmesh app
-  engineId?: string;    //unique auto-generated guid for an ephemeral engine instance
-  appVersion?: string;  //(e.g. "1.0.0", "1", "1.0")
-  jobId?: string;       //a customer-defined id for job; must be unique for the entire app
-  activityId?: string;  //activity id is a uuid for a given hotmesh app
-  jobKey?: string;      //a customer-defined label for a job that serves to categorize events 
-  dateTime?: string;    //UTC date time: YYYY-MM-DDTHH:MM (20203-04-12T00:00); serves as a time-series bucket for the job_key
-  facet?: string;       //data path starting at root with values separated by colons (e.g. "object/type:bar")
-  topic?: string;       //topic name (e.g., "foo" or "" for top-level)
-  timeValue?: number;   //time value (rounded to minute) (for delete range)
-  scoutType?: 'signal' | 'time'; //a single member of the quorum serves as the 'scout' for the group, triaging tasks for the collective
-};
+const HMNS = "hmsh"; //default
 
 class KeyService {
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.37",
+  "version": "0.0.39",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",

package/services/activities/activity.ts

@@ -1,4 +1,4 @@
-import { EXPIRE_DURATION } from '../../modules/enums';
+import { HMSH_EXPIRE_DURATION } from '../../modules/enums';
 import {
   CollationError,
   GenerationalError,
@@ -435,7 +435,7 @@ class Activity {
 
   initPolicies(context: JobState) {
     const expire = Pipe.resolve(
-      this.config.expire ?? EXPIRE_DURATION,
+      this.config.expire ?? HMSH_EXPIRE_DURATION,
       context
     );
     context.metadata.expire = expire;

package/services/activities/hook.ts

@@ -131,16 +131,25 @@ class Hook extends Activity {
 
   async registerHook(multi?: RedisMulti): Promise<string | void> {
     if (this.config.hook?.topic) {
-      const taskService = new TaskService(this.store, this.logger);
-      return await taskService.registerWebHook(this.config.hook.topic, this.context, this.resolveDad(), multi);
+      return await this.engine.taskService.registerWebHook(
+        this.config.hook.topic,
+        this.context,
+        this.resolveDad(),
+        multi
+      );
     } else if (this.config.sleep) {
-      const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
-      const jobId = this.context.metadata.jid;
-      const gId = this.context.metadata.gid;
-      const activityId = this.metadata.aid;
-      const dId = this.metadata.dad;
-      await this.engine.taskService.registerTimeHook(jobId, gId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
-      return jobId;
+      const duration = Pipe.resolve(
+        this.config.sleep,
+        this.context,
+      );
+      await this.engine.taskService.registerTimeHook(
+        this.context.metadata.jid,
+        this.context.metadata.gid,
+        `${this.metadata.aid}${this.metadata.dad || ''}`,
+        'sleep',
+        duration,
+      );
+      return this.context.metadata.jid;
     }
   }
 

package/services/activities/trigger.ts

@@ -47,7 +47,7 @@ class Trigger extends Activity {
       const multi = this.store.getMulti();
       await this.setState(multi);
       await this.setStats(multi);
-      await this.setDependency(multi);
+      await this.registerJobDependency(multi);
       await multi.exec();
 
       telemetry.mapActivityAttributes();
@@ -180,13 +180,17 @@ class Trigger extends Activity {
   }
 
   /**
-   * Registers this job as a dependent of the parent job
+   * Registers this job as a dependent of the parent job; when the
+   * parent job is interrupted, this job will be interrupted
    */
-  async setDependency(multi?: RedisMulti): Promise<void> {
-    const depKey = this.config.stats?.parent;
-    const resolvedDepKey = depKey ? Pipe.resolve(depKey, this.context) : '';
+  async registerJobDependency(multi?: RedisMulti): Promise<void> {
+    const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
+    let resolvedDepKey = depKey ? Pipe.resolve(depKey, this.context) : '';
+    if (!resolvedDepKey) {
+      resolvedDepKey = this.context.metadata.pj;
+    }
     if (resolvedDepKey) {
-      await this.store.setDependency(
+      await this.store.registerJobDependency(
         resolvedDepKey,
         this.context.metadata.tpc,
         this.context.metadata.jid,

package/services/durable/client.ts

@@ -11,11 +11,12 @@ import { JobState } from '../../types/job';
 import { KeyService, KeyType } from '../../modules/key';
 import { Search } from './search';
 import { StreamStatus } from '../../types';
-import { DURABLE_EXPIRE_SECONDS } from '../../modules/enums';
+import { HMSH_LOGLEVEL, HMSH_EXPIRE_JOB_SECONDS } from '../../modules/enums';
 
 export class ClientService {
 
   connection: Connection;
+  topics: string[] = [];
   options: WorkflowOptions;
   static instances = new Map<string, HotMesh | Promise<HotMesh>>();
 
@@ -23,14 +24,22 @@ export class ClientService {
     this.connection = config.connection;
   }
 
-  getHotMeshClient = async (worflowTopic: string, namespace?: string) => {
-    //NOTE: every unique topic inits a new engine
-    if (ClientService.instances.has(worflowTopic)) {
-      return await ClientService.instances.get(worflowTopic);
+  getHotMeshClient = async (workflowTopic: string, namespace?: string) => {
+    //use the cached instance
+    const instanceId = 'SINGLETON';
+    if (ClientService.instances.has(instanceId)) {
+      const hotMeshClient = await ClientService.instances.get(instanceId);
+      if (!this.topics.includes(workflowTopic)) {
+        this.topics.push(workflowTopic);
+        await this.createStream(hotMeshClient, workflowTopic, namespace);
+      }
+      return hotMeshClient;
     }
 
+    //create and cache an instance
     const hotMeshClient = HotMesh.init({
       appId: namespace ?? APP_ID,
+      logLevel: HMSH_LOGLEVEL,
       engine: {
         redis: {
           class: this.connection.class,
@@ -38,19 +47,27 @@ export class ClientService {
         }
       }
     });
-    ClientService.instances.set(worflowTopic, hotMeshClient);
+    ClientService.instances.set(instanceId, hotMeshClient);
+    await this.createStream(await hotMeshClient, workflowTopic, namespace);
+    await this.activateWorkflow(await hotMeshClient, namespace ?? APP_ID);
+    return hotMeshClient;
+  }
 
-    //since the YAML topic is dynamic, it MUST be manually created before use
-    const store = (await hotMeshClient).engine.store;
-    const params = { appId: namespace ?? APP_ID, topic: worflowTopic };
+  /**
+   * Creates a stream (Redis `XGROUP.CREATE`) where events can be published (XADD).
+   * It is possible that the worker that will read from this stream channel
+   * has not yet been initialized, so this call ensures that the channel
+   * exists and is ready to serve as a container for events.
+   */
+  createStream = async(hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => {
+    const store = hotMeshClient.engine.store;
+    const params = { appId: namespace ?? APP_ID, topic: workflowTopic };
     const streamKey = store.mintKey(KeyType.STREAMS, params);
     try {
       await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
     } catch (err) {
       //ignore if already exists
     }
-    await this.activateWorkflow(await hotMeshClient, namespace ?? APP_ID);
-    return hotMeshClient;
   }
 
   /**
@@ -105,7 +122,7 @@ export class ClientService {
       const payload = {
         arguments: [...options.args],
         originJobId: options.originJobId,
-        expire: options.expire ?? DURABLE_EXPIRE_SECONDS,
+        expire: options.expire ?? HMSH_EXPIRE_JOB_SECONDS,
         parentWorkflowId: options.parentWorkflowId,
         workflowId: options.workflowId || HotMesh.guid(),
         workflowTopic: workflowTopic,
@@ -155,9 +172,8 @@ export class ClientService {
       if (options.search?.data) {
         const searchSessionId = `-search-${HotMesh.guid()}-0`;
         const search = new Search(options.workflowId, hotMeshClient, searchSessionId);
-        for (const [key, value] of Object.entries(options.search.data)) {
-          search.set(key, value);
-        }
+        const entries = Object.entries(options.search.data).flat();
+        await search.set(...entries);
       }
       return msgId;
     },

package/services/durable/handle.ts

@@ -1,4 +1,4 @@
-import { STATUS_CODE_INTERRUPT } from '../../modules/enums';
+import { HMSH_CODE_INTERRUPT } from '../../modules/enums';
 import { HotMeshService as HotMesh } from '../hotmesh';
 import { JobInterruptOptions, JobOutput } from '../../types/job';
 import { StreamError } from '../../types/stream';
@@ -101,7 +101,7 @@ export class WorkflowHandleService {
           const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
           if (state.metadata.err) {
             const error = JSON.parse(state.metadata.err) as StreamError;
-            if (error.code === STATUS_CODE_INTERRUPT || !state.data) {
+            if (error.code === HMSH_CODE_INTERRUPT || !state.data) {
               return reject({ ...error, job_id: this.workflowId });
             }
           }
@@ -117,7 +117,7 @@ export class WorkflowHandleService {
       this.hotMesh.sub(topic, async (topic: string, state: JobOutput) => {
         if (state.metadata.err) {
           const error = JSON.parse(state.metadata.err) as StreamError;
-          if (error.code === STATUS_CODE_INTERRUPT || !state.data) {
+          if (error.code === HMSH_CODE_INTERRUPT || !state.data) {
             return await complete(null, state.metadata.err);
           }
         }

package/services/durable/worker.ts

@@ -23,6 +23,7 @@ import {
   StreamData,
   StreamDataResponse,
   StreamStatus } from '../../types/stream';
+import { HMSH_LOGLEVEL } from '../../modules/enums';
 
 export class WorkerService {
   static activityRegistry: Registry = {}; //user's activities
@@ -36,7 +37,7 @@ export class WorkerService {
       return await WorkerService.instances.get(workflowTopic);
     }
     const hotMeshClient = HotMesh.init({
-      logLevel: options?.logLevel as 'debug' ?? 'info',
+      logLevel: options?.logLevel ?? HMSH_LOGLEVEL,
       appId: config.namespace ?? APP_ID,
       engine: { redis: { ...WorkerService.connection } }
     });
@@ -121,7 +122,7 @@ export class WorkerService {
       options: config.connection.options as RedisOptions
     };
     const hotMeshWorker = await HotMesh.init({
-      logLevel: config.options?.logLevel as 'debug' ?? 'info',
+      logLevel: config.options?.logLevel ?? HMSH_LOGLEVEL,
       appId: config.namespace ?? APP_ID,
       engine: { redis: redisConfig },
       workers: [
@@ -172,7 +173,7 @@ export class WorkerService {
       options: config.connection.options as RedisOptions
     };
     const hotMeshWorker = await HotMesh.init({
-      logLevel: config.options?.logLevel as 'debug' ?? 'info',
+      logLevel: config.options?.logLevel ?? HMSH_LOGLEVEL,
       appId: config.namespace ?? APP_ID,
       engine: { redis: redisConfig },
       workers: [{

package/services/engine/index.ts

@@ -1,10 +1,10 @@
 import { KeyType } from '../../modules/key';
 import {
-  OTT_WAIT_TIME,
-  STATUS_CODE_SUCCESS,
-  STATUS_CODE_PENDING,
-  STATUS_CODE_TIMEOUT, 
-  DURABLE_EXPIRE_SECONDS } from '../../modules/enums';
+  HMSH_OTT_WAIT_TIME,
+  HMSH_CODE_SUCCESS,
+  HMSH_CODE_PENDING,
+  HMSH_CODE_TIMEOUT, 
+  HMSH_EXPIRE_JOB_SECONDS } from '../../modules/enums';
 import {
   formatISODate,
   getSubscriptionTopic,
@@ -77,6 +77,7 @@ import {
   StreamError,
   StreamRole,
   StreamStatus } from '../../types/stream';
+import { WorkListTaskType } from '../../types/task';
 
 class EngineService {
   namespace: string;
@@ -438,10 +439,10 @@ class EngineService {
         streamData.code = error.code;
       } else if (emit) {
         streamData.status = StreamStatus.PENDING;
-        streamData.code = STATUS_CODE_PENDING;
+        streamData.code = HMSH_CODE_PENDING;
       } else {
         streamData.status = StreamStatus.SUCCESS;
-        streamData.code = STATUS_CODE_SUCCESS;
+        streamData.code = HMSH_CODE_SUCCESS;
       }
       return (await this.router?.publishMessage(null, streamData)) as string;
     }
@@ -489,7 +490,7 @@ class EngineService {
     };
     return await this.router.publishMessage(null, streamData) as string;
   }
-  async hookTime(jobId: string, gId: string, activityId: string, type?: 'sleep'|'expire'|'interrupt'): Promise<string | void> {
+  async hookTime(jobId: string, gId: string, activityId: string, type?: WorkListTaskType): Promise<string | void> {
     if (type === 'interrupt') {
       return await this.interrupt(
         activityId, //note: 'activityId' is the actually job topic
@@ -499,7 +500,6 @@ class EngineService {
     } else if (type === 'expire') {
       return await this.store.expireJob(jobId, 1);
     }
-    //'sleep': parse the activityId into parts
     const [aid, ...dimensions] = activityId.split(',');
     const dad = `,${dimensions.join(',')}`;
     const streamData: StreamData = {
@@ -575,7 +575,7 @@ class EngineService {
     return await this.subscribe.punsubscribe(KeyType.QUORUM, this.appId, wild);
   }
   //publish and await (returns the job and data (if ready)); throws error with jobid if not
-  async pubsub(topic: string, data: JobData, context?: JobState | null, timeout = OTT_WAIT_TIME): Promise<JobOutput> {
+  async pubsub(topic: string, data: JobData, context?: JobState | null, timeout = HMSH_OTT_WAIT_TIME): Promise<JobOutput> {
     context = {
       metadata: {
         ngn: this.guid,
@@ -597,9 +597,10 @@ class EngineService {
         }
       });
       setTimeout(() => {
+        //note: job is still active (the subscriber timed out)
         this.delistJobCallback(jobId);
         reject({
-          code: STATUS_CODE_TIMEOUT,
+          code: HMSH_CODE_TIMEOUT,
           message: 'timeout',
           job_id: jobId
         } as StreamError);
@@ -685,7 +686,7 @@ class EngineService {
    * it will be expired immediately.
    */
   resolveExpires(context: JobState, options: JobCompletionOptions): number {
-    return options.expire ?? context.metadata.expire ?? DURABLE_EXPIRE_SECONDS;
+    return options.expire ?? context.metadata.expire ?? HMSH_EXPIRE_JOB_SECONDS;
   }
 
 

package/services/hotmesh/index.ts

@@ -16,7 +16,7 @@ import {
 import {
   HotMeshConfig,
   HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback } from '../../types/quorum';
+import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
 import {
   JobStatsInput,
   GetStatsOptions,
@@ -137,6 +137,9 @@ class HotMeshService {
   }
 
   // ************* COMPILER METHODS *************
+  async rollCall(delay?: number): Promise<QuorumProfile[]> {
+    return await this.quorum?.rollCall(delay);
+  }
   async plan(path: string): Promise<HotMeshManifest> {
     return await this.engine?.plan(path);
   }
@@ -147,10 +150,6 @@ class HotMeshService {
     //activation is a quorum operation
     return await this.quorum?.activate(version, delay);
   }
-  async inventory(version: string, delay?: number): Promise<number> {
-    //get count of all peers
-    return await this.quorum?.inventory(delay);
-  }
 
   // ************* REPORTER METHODS *************
   async getStats(topic: string, query: JobStatsInput): Promise<StatsResponse> {

package/services/quorum/index.ts

@@ -14,6 +14,7 @@ import { RedisClientType as IORedisClientType } from '../../types/ioredisclient'
 import {
   QuorumMessage,
   QuorumMessageCallback,
+  QuorumProfile,
   SubscriptionCallback,
   ThrottleMessage
 } from '../../types/quorum';
@@ -26,10 +27,10 @@ const QUORUM_DELAY = 250;
 
 class QuorumService {
   namespace: string;
-  apps: HotMeshApps | null;
   appId: string;
   guid: string;
   engine: EngineService;
+  profiles: QuorumProfile[] = [];
   store: StoreService<RedisClient, RedisMulti> | null;
   subscribe: SubService<RedisClient, RedisMulti> | null;
   logger: ILogger;
@@ -108,9 +109,12 @@ class QuorumService {
       if (message.type === 'activate') {
         self.engine.setCacheMode(message.cache_mode, message.until_version);
       } else if (message.type === 'ping') {
-        this.sayPong(self.appId, self.guid, message.originator);
+        self.sayPong(self.appId, self.guid, message.originator, message.details);
       } else if (message.type === 'pong' && self.guid === message.originator) {
         self.quorum = self.quorum + 1;
+        if (message.profile) {
+          self.profiles.push(message.profile);
+        }
       } else if (message.type === 'throttle') {
         self.engine.throttle(message.throttle);
       } else if (message.type === 'work') {
@@ -127,20 +131,38 @@ class QuorumService {
     };
   }
 
-  async sayPong(appId: string, guid: string, originator: string) {
+  async sayPong(appId: string, guid: string, originator: string, details = false) {
+    let profile: QuorumProfile;
+    if (details) {
+      profile = {
+        engine_id: this.guid,
+        namespace: this.namespace,
+        app_id: this.appId,
+        stream: this.engine.stream.mintKey(KeyType.STREAMS, { appId: this.appId })
+      };
+    }
     this.store.publish(
       KeyType.QUORUM, 
-      { type: 'pong', guid, originator },
+      {
+        type: 'pong',
+        guid, originator,
+        profile,
+      },
       appId,
     );
   }
 
-  async requestQuorum(delay = QUORUM_DELAY): Promise<number> {
+  async requestQuorum(delay = QUORUM_DELAY, details = false): Promise<number> {
     const quorum = this.quorum;
     this.quorum = 0;
+    this.profiles.length = 0;
     await this.store.publish(
       KeyType.QUORUM,
-      { type: 'ping', originator: this.guid },
+      { 
+        type: 'ping',
+        originator: this.guid,
+        details,
+      },
       this.appId,
     );
     await sleepFor(delay);
@@ -166,12 +188,26 @@ class QuorumService {
 
 
   // ************* COMPILER METHODS *************
-  async inventory(delay = QUORUM_DELAY): Promise<number> {
-    await this.requestQuorum(delay);
-    const q1 = await this.requestQuorum(delay);
-    const q2 = await this.requestQuorum(delay);
-    const q3 = await this.requestQuorum(delay);
-    return Math.round((q1 + q2 + q3) / 3);
+  async rollCall(delay = QUORUM_DELAY): Promise<QuorumProfile[]> {
+    await this.requestQuorum(delay, true);
+    const targetStreams = [];
+    const multi = this.store.getMulti();
+    this.profiles.forEach((profile: QuorumProfile) => {
+      if (!targetStreams.includes(profile.stream)) {
+        targetStreams.push(profile.stream);
+        this.store.xlen(profile.stream, multi);
+      }
+    });
+    const stream_depths = await multi.exec() as number[];
+    this.profiles.forEach(async (profile: QuorumProfile) => {
+      const index = targetStreams.indexOf(profile.stream);
+      if (index != -1) {
+        profile.stream_depth = Array.isArray(stream_depths[index]) ?
+          stream_depths[index][1] :
+          stream_depths[index];
+      }
+    });
+    return this.profiles;
   }
   async activate(version: string, delay = QUORUM_DELAY): Promise<boolean> {
     version = version.toString();