@hotmeshio/hotmesh 0.0.48 → 0.0.50

package/services/task/index.ts

@@ -11,12 +11,14 @@ import { KeyType } from '../../types/hotmesh';
 import { JobCompletionOptions, JobState } from '../../types/job';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { WorkListTaskType } from '../../types/task';
+import { VALSEP, WEBSEP } from '../../modules/key';
 
 class TaskService {
   store: StoreService<RedisClient, RedisMulti>;
   logger: ILogger;
   cleanupTimeout: NodeJS.Timeout | null = null;
   isScout: boolean = false;
+  errorCount = 0;
 
   constructor(
     store: StoreService<RedisClient, RedisMulti>,
@@ -29,8 +31,8 @@ class TaskService {
   async processWebHooks(hookEventCallback: HookInterface): Promise<void> {
     const workItemKey = await this.store.getActiveTaskQueue();
     if (workItemKey) {
-      const [topic, sourceKey, scrub, ...sdata] = workItemKey.split('::');
-      const data = JSON.parse(sdata.join('::'));
+      const [topic, sourceKey, scrub, ...sdata] = workItemKey.split(WEBSEP);
+      const data = JSON.parse(sdata.join(WEBSEP));
       const destinationKey = `${sourceKey}:processed`;
       const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
       if (jobId) {
@@ -72,6 +74,7 @@ class TaskService {
     activityId: string,
     type: WorkListTaskType,
     inSeconds = HMSH_FIDELITY_SECONDS,
+    dad: string,
     multi?: RedisMulti,
   ): Promise<void> {
     const fromNow = Date.now() + (inSeconds * 1000);
@@ -83,6 +86,7 @@ class TaskService {
       activityId,
       type,
       awakenTimeSlot,
+      dad,
       multi,
     );
   }
@@ -129,21 +133,29 @@ class TaskService {
             await timeEventCallback(target, gId, activityId, type);
           }
           await sleepFor(0);
+          this.errorCount = 0;
           this.processTimeHooks(timeEventCallback, listKey);
         } else if (workListTask) {
           //a worklist was just emptied; try again immediately
           await sleepFor(0);
+          this.errorCount = 0;
           this.processTimeHooks(timeEventCallback);
         } else {
           //no worklists exist; sleep before checking
           let sleep = XSleepFor(HMSH_FIDELITY_SECONDS * 1000);
           this.cleanupTimeout = sleep.timerId;
           await sleep.promise;
+          this.errorCount = 0;
           this.processTimeHooks(timeEventCallback);
         }
       } catch (err) {
-        //todo: retry connect to redis
-        this.logger.error('task-process-timehooks-error', err);
+        //most common reasons: deleted job not found; container stopping; test stopping
+        //less common: redis/cluster down; retry with fallback (5s max main reassignment)
+        this.logger.warn('task-process-timehooks-error', err);
+        await sleepFor(1_000 * this.errorCount++);
+        if (this.errorCount < 5) {
+          this.processTimeHooks(timeEventCallback);
+        }
       }
     } else {
       //didn't get the scout role; try again in 'one-ish' minutes
@@ -174,10 +186,18 @@ class TaskService {
       const jobId = context.metadata.jid;
       const gId = context.metadata.gid;
       const activityId = hookRule.to;
+      //composite keys are used to fully describe the task target
+      const compositeJobKey = [
+        activityId,
+        dad,
+        gId,
+        jobId
+      ].join(WEBSEP);
+
       const hook: HookSignal = {
         topic,
         resolved,
-        jobId: `${activityId}::${dad}::${gId}::${jobId}`,
+        jobId: compositeJobKey,
       }
       await this.store.setHookSignal(hook, multi);
       return jobId;
@@ -196,17 +216,17 @@ class TaskService {
       const resolved = Pipe.resolve(mapExpression, context);
       const hookSignalId = await this.store.getHookSignal(topic, resolved);
       if (!hookSignalId) {
-        //messages can be double-processed; not an issue; return undefined
-        //users can also provide a bogus topic; not an issue; return undefined
+        //messages can be double-processed; not an issue; return `undefined`
+        //users can also provide a bogus topic; not an issue; return `undefined`
         return undefined;
       }
-      //`aid` is part of composit key, but the hook `topic` is its public interface;
+      //`aid` is part of composite key, but the hook `topic` is its public interface;
       // this means that a new version of the graph can be deployed and the
       // topic can be re-mapped to a different activity id. Outside callers
       // can adhere to the unchanged contract (calling the same topic),
-      // while the internal system can be updated in real time as necessary.
-      const [_aid, dad, gid, ...jid] = hookSignalId.split('::');
-      return [jid.join('::'), hookRule.to, dad, gid];
+      // while the internal system can be updated in real-time as necessary.
+      const [_aid, dad, gid, ...jid] = hookSignalId.split(WEBSEP);
+      return [jid.join(WEBSEP), hookRule.to, dad, gid];
     } else {
       throw new Error('signal-not-found');
     }

package/services/worker/index.ts

@@ -1,5 +1,5 @@
 import { KeyType } from "../../modules/key";
-import { formatISODate, getSystemHealth, identifyRedisType } from "../../modules/utils";
+import { XSleepFor, formatISODate, getSystemHealth, identifyRedisType, sleepFor } from "../../modules/utils";
 import { ConnectorService } from "../connector";
 import { ILogger } from "../logger";
 import { Router } from "../router";
@@ -17,10 +17,12 @@ import { RedisClientType as IORedisClientType } from '../../types/ioredisclient'
 import {
   QuorumMessage,
   QuorumProfile,
+  RollCallMessage,
   SubscriptionCallback } from "../../types/quorum";
 import { RedisClient, RedisMulti } from "../../types/redis";
 import { RedisClientType } from '../../types/redisclient';
-import { StreamRole } from "../../types/stream";
+import { StreamData, StreamRole, StreamDataResponse } from "../../types/stream";
+import { HMSH_QUORUM_ROLLCALL_CYCLES } from "../../modules/enums";
 
 class WorkerService {
   namespace: string;
@@ -28,6 +30,7 @@ class WorkerService {
   guid: string;
   topic: string;
   config: HotMeshConfig;
+  callback: (streamData: StreamData) => Promise<StreamDataResponse|void>;
   store: StoreService<RedisClient, RedisMulti> | null;
   stream: StreamService<RedisClient, RedisMulti> | null;
   subscribe: SubService<RedisClient, RedisMulti> | null;
@@ -35,6 +38,7 @@ class WorkerService {
   logger: ILogger;
   reporting = false;
   inited: string;
+  rollCallInterval: NodeJS.Timeout;
 
   static async init(
     namespace: string,
@@ -58,6 +62,7 @@ class WorkerService {
         service.namespace = namespace;
         service.appId = appId;
         service.guid = guid;
+        service.callback = worker.callback;
         service.topic = worker.topic;
         service.config = config;
         service.logger = logger;
@@ -155,14 +160,51 @@ class WorkerService {
     return async (topic: string, message: QuorumMessage) => {
       self.logger.debug('worker-event-received', { topic, type: message.type });
       if (message.type === 'throttle') {
-        self.throttle(message.throttle);
+        if (message.topic !== null) { //undefined allows passthrough
+          self.throttle(message.throttle);
+        }
       } else if(message.type === 'ping') {
         self.sayPong(self.appId, self.guid, message.originator, message.details);
+      } else if(message.type === 'rollcall') {
+        if (message.topic !== null) { //undefined allows passthrough
+          self.doRollCall(message);
+        }
       }
     };
   }
 
-  async sayPong(appId: string, guid: string, originator: string, details = false) {
+  /**
+   * A quorum-wide command to broadcaset system details.
+   * 
+   */
+  async doRollCall(message: RollCallMessage) {
+    let iteration = 0;
+    let max = !isNaN(message.max) ? message.max : HMSH_QUORUM_ROLLCALL_CYCLES;
+    if (this.rollCallInterval) clearTimeout(this.rollCallInterval);
+    const base = (message.interval / 2);
+    const amount = base + Math.ceil(Math.random() * base);
+    do {
+      await sleepFor(Math.ceil(Math.random() * 1000));
+      await this.sayPong(this.appId, this.guid, null, true, message.signature);
+      if (!message.interval) return;
+      const { promise, timerId } = XSleepFor(amount * 1000);
+      this.rollCallInterval = timerId;
+      await promise;
+    } while (this.rollCallInterval && iteration++ < max - 1);
+  }
+
+  cancelRollCall() {
+    if (this.rollCallInterval) {
+      clearTimeout(this.rollCallInterval);
+      delete this.rollCallInterval;
+    }
+  }
+
+  stop() {
+    this.cancelRollCall();
+  }
+
+  async sayPong(appId: string, guid: string, originator?: string, details = false, signature = false) {
     let profile: QuorumProfile;
     if (details) {
       const params = {
@@ -183,13 +225,15 @@ class WorkerService {
         reclaimDelay: this.router.reclaimDelay,
         reclaimCount: this.router.reclaimCount,
         system: await getSystemHealth(),
+        signature: signature ? this.callback.toString() : undefined,
       };
     }
     this.store.publish(
       KeyType.QUORUM, 
       {
         type: 'pong',
-        guid, originator,
+        guid,
+        originator,
         profile,
       },
       appId,

package/types/activity.ts

@@ -77,6 +77,7 @@ interface AwaitActivity extends BaseActivity {
   type: 'await';
   eventName: string;
   timeout: number;
+  await?: boolean; //if exlicitly false do not await the response
 }
 
 interface WorkerActivity extends BaseActivity {

package/types/durable.ts

@@ -1,5 +1,6 @@
 import { LogLevel } from './logger';
 import { RedisClass, RedisOptions } from './redis';
+import { StringStringType } from './serializer';
 
 type WorkflowConfig = {
   backoffCoefficient?: number; //default 10
@@ -15,6 +16,16 @@ type WorkflowContext = {
    */
   counter: number;
 
+  /**
+   * number as string for the replay cursor
+   */
+  cursor: string;
+
+  /**
+   * the replay hash of name/value pairs representing prior executions
+   */
+  replay: StringStringType;
+
   /**
    * the HotMesh App namespace. `durable` is the default.
    */

package/types/exporter.ts

@@ -26,6 +26,8 @@ export interface JobTimeline {
   dimension: string; //dimensional isolate path
   duplex: 'entry' | 'exit'; //activity entry or exit
   timestamp: string; //actually a number but too many digits for JS
+  created?: string; //actually a number but too many digits for JS
+  updated?: string; //actually a number but too many digits for JS
   actions?: ActivityAction[];
 }
 

package/types/job.ts

@@ -17,6 +17,7 @@ type JobMetadata = {
   pg?: string;  //parent_generational_id (system assigned at trigger inception); pg is the parent job's gid (just in case user created/deleted/created a job with same jid)
   pd?: string;  //parent_dimensional_address
   pa?: string;  //parent_activity_id
+  px?: boolean; //sever the dependency chain if true (startChild/vs/executeChild)
   ngn?: string; //engine guid (one time subscriptions)
   app: string;  //app_id
   vrs: string;  //app version

package/types/quorum.ts

@@ -50,50 +50,65 @@ export interface QuorumProfile {
   reclaimDelay?: number;
   reclaimCount?: number;
   system?: SystemHealth;
+  signature?: string; //stringified function
 }
 
-//used for coordination (like version activation)
-export interface PingMessage {
+interface QuorumMessageBase {
+  guid?: string;
+  topic?: string;
+  type?: string;
+}
+
+// Messages extending QuorumMessageBase
+export interface PingMessage extends QuorumMessageBase {
   type: 'ping';
   originator: string; //guid
   details?: boolean;  //if true, all endpoints will include their profile
 }
 
-export interface WorkMessage {
+export interface WorkMessage extends QuorumMessageBase {
   type: 'work';
   originator: string; //guid
 }
 
-export interface CronMessage {
+export interface CronMessage extends QuorumMessageBase {
   type: 'cron';
   originator: string; //guid
 }
 
-export interface PongMessage {
+export interface PongMessage extends QuorumMessageBase {
   type: 'pong';
   guid: string;            //call initiator
   originator: string;      //clone of originator guid passed in ping
   profile?: QuorumProfile; //contains details about the engine/worker
 }
 
-export interface ActivateMessage {
+export interface ActivateMessage extends QuorumMessageBase {
   type: 'activate';
   cache_mode: 'nocache' | 'cache';
   until_version: string;
 }
 
-export interface JobMessage {
+export interface JobMessage extends QuorumMessageBase {
   type: 'job';
   topic: string; //this comes from the 'publishes' field in the YAML
   job: JobOutput
 }
 
-//delay in ms between fetches from the buffered stream (speed/slow down entire network)
-export interface ThrottleMessage {
+export interface ThrottleMessage extends QuorumMessageBase {
   type: 'throttle';
-  guid?: string; //target the engine quorum
-  topic?: string;  //target a worker quorum
-  throttle: number; //0-n
+  guid?: string; //target engine AND workers with this guid
+  topic?: string;  //target worker(s) matching this topic (pass null to only target the engine, pass undefined to target engine and workers)
+  throttle: number; //0-n; millis
+}
+
+export interface RollCallMessage extends QuorumMessageBase {
+  type: 'rollcall';
+  guid?: string;    //target the engine quorum
+  topic?: string | null;   //target a worker if string; suppress if `null`;
+  interval: number; //every 'n' seconds
+  max?: number;     //max broadcasts
+  signature?: boolean; //include bound worker function in broadcast   
 }
 
 export interface JobMessageCallback {
@@ -114,4 +129,4 @@ export interface QuorumMessageCallback {
  * These messages serve to coordinate the cache invalidation and switch-over
  * to the new version without any downtime and a coordinating parent server.
  */
-export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | RollCallMessage | CronMessage;

package/types/stream.ts

@@ -42,6 +42,7 @@ export interface StreamData {
     trc?: string;   //trace id
     spn?: string;   //span id
     try?: number;   //current try count
+    await?: boolean;  //(waitfor) if explicitly false, do not await; sever the connection
   };
   type?: StreamDataType;
   data: Record<string, unknown>;