@hotmeshio/hotmesh 0.0.49 → 0.0.50

package/services/quorum/index.ts

@@ -1,7 +1,9 @@
 import {
   HMSH_ACTIVATION_MAX_RETRY,
-  HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
+  HMSH_QUORUM_DELAY_MS, 
+  HMSH_QUORUM_ROLLCALL_CYCLES} from '../../modules/enums';
 import {
+  XSleepFor,
   formatISODate,
   getSystemHealth,
   identifyRedisType,
@@ -22,6 +24,7 @@ import {
   QuorumMessage,
   QuorumMessageCallback,
   QuorumProfile,
+  RollCallMessage,
   SubscriptionCallback,
   ThrottleMessage
 } from '../../types/quorum';
@@ -41,6 +44,7 @@ class QuorumService {
   untilVersion: string | null = null;
   quorum: number | null = null;
   callbacks: QuorumMessageCallback[] = [];
+  rollCallInterval: NodeJS.Timeout;
 
   static async init(
     namespace: string,
@@ -136,6 +140,8 @@ class QuorumService {
         self.engine.routeToSubscribers(message.topic, message.job)
       } else if (message.type === 'cron') {
         self.engine.processTimeHooks();
+      } else if (message.type === 'rollcall') {
+        self.doRollCall(message);
       }
       //if there are any callbacks, call them
       if (self.callbacks.length > 0) {
@@ -193,10 +199,40 @@ class QuorumService {
     return quorum;
   }
 
+  /**
+   * A quorum-wide command to broadcaset system details.
+   * 
+   */
+  async doRollCall(message: RollCallMessage) {
+    let iteration = 0;
+    let max = !isNaN(message.max) ? message.max : HMSH_QUORUM_ROLLCALL_CYCLES;
+    if (this.rollCallInterval) clearTimeout(this.rollCallInterval);
+    const base = (message.interval / 2);
+    const amount = base + Math.ceil(Math.random() * base);
+    do {
+      await sleepFor(Math.ceil(Math.random() * 1000));
+      await this.sayPong(this.appId, this.guid, null, true);
+      if (!message.interval) return;
+      const { promise, timerId } = XSleepFor(amount * 1000);
+      this.rollCallInterval = timerId;
+      await promise;
+    } while (this.rollCallInterval && iteration++ < max - 1);
+  }
+
+  cancelRollCall() {
+    if (this.rollCallInterval) {
+      clearTimeout(this.rollCallInterval);
+      delete this.rollCallInterval;
+    }
+  }
+
+  stop() {
+    this.cancelRollCall();
+  }
 
   // ************* PUB/SUB METHODS *************
   //publish a message to the quorum
-  async pub(quorumMessage: ThrottleMessage) {
+  async pub(quorumMessage: QuorumMessage) {
     return await this.store.publish(KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
   }
   //subscribe user to quorum messages

package/services/serializer/index.ts

@@ -19,7 +19,7 @@ export const MDATA_SYMBOLS = {
     KEYS: ['au', 'err', 'l2s']
   },
   JOB: {
-    KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+    KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'px', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
   },
   JOB_UPDATE: {
     KEYS: ['ju', 'err']

package/services/store/index.ts

@@ -2,7 +2,9 @@ import {
   KeyService,
   KeyStoreParams,
   KeyType, 
-  HMNS} from '../../modules/key';
+  HMNS,
+  VALSEP,
+  TYPSEP} from '../../modules/key';
 import { ILogger } from '../logger';
 import { MDATA_SYMBOLS, SerializerService as Serializer } from '../serializer';
 import { Cache } from './cache';
@@ -428,7 +430,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * when `originJobId` is interrupted/expired, the items in the
    * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
    */
-  async registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
+  async registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd = '',  multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
     const dependencyParams = {
       appId: this.appId,
@@ -438,8 +440,13 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       KeyType.JOB_DEPENDENTS,
       dependencyParams,
     );
-    //items listed as job dependencies have different relationships
-    const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
+    const expireTask = [
+      depType,
+      topic,
+      gId,
+      pd,
+      jobId,
+    ].join(VALSEP);
     privateMulti[this.commands.rpush](depKey, expireTask);
     if (!multi) {
       return await privateMulti.exec();
@@ -450,15 +457,20 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * Ensures a `hook signal` is delisted when its parent activity/job
    * is interrupted/expired.
    */
-  async registerSignalDependency(jobId: string, signalKey: string, multi? : U): Promise<any> {
+  async registerSignalDependency(jobId: string, signalKey: string, dad: string, multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
     const dependencyParams = { appId: this.appId, jobId };
     const dependencyKey = this.mintKey(
       KeyType.JOB_DEPENDENTS,
       dependencyParams,
     );
-    //tasks have '4' segments
-    const delistTask = `delist::signal::${jobId}::${signalKey}`;
+    //persiste dependency tasks as multi-segment composite keys
+    const delistTask = [
+      'delist',
+      'signal',
+      jobId,
+      dad,
+      signalKey].join(VALSEP);
     privateMulti[this.commands.rpush](
       dependencyKey,
       delistTask,
@@ -799,11 +811,14 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
   async setHookSignal(hook: HookSignal, multi?: U): Promise<any> {
     const key = this.mintKey(KeyType.SIGNALS, { appId: this.appId });
-    const { topic, resolved, jobId} = hook; //`${activityId}::${dad}::${gId}::${jobId}`
+    //destructure the hook key
+    const { topic, resolved, jobId} = hook;
     const signalKey = `${topic}:${resolved}`;
     const payload = { [signalKey]: jobId };
     await (multi || this.redisClient)[this.commands.hset](key, payload);
-    return await this.registerSignalDependency(jobId.split('::')[3], signalKey, multi);
+    //jobId needs even more destructuring
+    const [_aid, dad, _gid, jid] = jobId.split(VALSEP);
+    return await this.registerSignalDependency(jid, signalKey, dad, multi);
   }
 
   async getHookSignal(topic: string, resolved: string): Promise<string | undefined> {
@@ -875,7 +890,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     const depParams = { appId: this.appId, jobId };
     const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
     const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
-    const depKeyContext = `::${context}::${depKey}`;
+    const depKeyContext = `${TYPSEP}${context}${TYPSEP}${depKey}`;
     const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
     await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
   }
@@ -892,9 +907,15 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * for the given sleep group. Sleep groups are
    * organized into 'n'-second blocks (LISTS))
    */
-  async registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, multi?: U): Promise<void> {
+  async registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void> {
     const listKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
-    const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
+    //construct the composite key (the key has enough info to signal the hook)
+    const timeEvent = [
+      type,
+      activityId,
+      gId,
+      dad,
+      jobId].join(VALSEP);
     const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
     if (multi || len === 1) {
       const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
@@ -909,17 +930,23 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       let [pType, pKey] = this.resolveTaskKeyContext(listKey);
       const timeEvent = await this.redisClient[this.commands.lpop](pKey);
       if (timeEvent) {
-        //there are task types
-        //1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
-        let [type, activityId, gId, ...jobId] = timeEvent.split('::');
+        //deconstruct composite key
+        let [
+          type,
+          activityId,
+          gId,
+          _pd,
+          ...jobId] = timeEvent.split(VALSEP);
+        const jid = jobId.join(VALSEP);
+
         if (type === 'delist') {
           pType = 'delist';
         } else if (type === 'child') {
           pType = 'child';
         } else if (type === 'expire-child') {
-          type = 'expire'; //use the same logic as 'expire'
+          type = 'expire';
         }
-        return [listKey, jobId.join('::'), gId, activityId, pType];
+        return [listKey, jid, gId, activityId, pType];
       }
       await this.redisClient[this.commands.zrem](zsetKey, listKey);
       return true;
@@ -933,13 +960,13 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * do with the work list. (not everything is known in advance,
    * so the ZSET key defines HOW to approach the work in the
    * generic LIST (lists typically contain target job ids)
-   * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+   * @param {string} listKey - composite key
    */
   resolveTaskKeyContext(listKey: string): [WorkListTaskType, string] {
-    if (listKey.startsWith('::INTERRUPT')) {
-      return ['interrupt', listKey.split('::')[2]];
-    } else if (listKey.startsWith('::EXPIRE')) {
-      return ['expire', listKey.split('::')[2]];
+    if (listKey.startsWith(`${TYPSEP}INTERRUPT`)) {
+      return ['interrupt', listKey.split(TYPSEP)[2]];
+    } else if (listKey.startsWith(`${TYPSEP}EXPIRE`)) {
+      return ['expire', listKey.split(TYPSEP)[2]];
     } else {
       return ['sleep', listKey];
     }
@@ -1001,7 +1028,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     await this.redisClient[this.commands.del](jobKey);
   }
 
-  async findJobs(queryString: string = '*', limit: number = 1000, batchSize: number = 1000): Promise<string[]> {
+  async findJobs(queryString: string = '*', limit: number = 1000, batchSize: number = 1000): Promise<[string, string[]]> {
     const matchKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId: queryString });
     let cursor = '0';
     let keys: string[];
@@ -1027,7 +1054,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
         break;
       }
     } while (cursor !== '0');
-    return matchingKeys;
+    return [cursor, matchingKeys];
   }
 
   async findJobFields(jobId: string, fieldMatchPattern: string = '*', limit: number = 1000, batchSize: number = 1000, cursor = '0'): Promise<[string, StringStringType]> {

package/services/task/index.ts

@@ -11,12 +11,14 @@ import { KeyType } from '../../types/hotmesh';
 import { JobCompletionOptions, JobState } from '../../types/job';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { WorkListTaskType } from '../../types/task';
+import { VALSEP, WEBSEP } from '../../modules/key';
 
 class TaskService {
   store: StoreService<RedisClient, RedisMulti>;
   logger: ILogger;
   cleanupTimeout: NodeJS.Timeout | null = null;
   isScout: boolean = false;
+  errorCount = 0;
 
   constructor(
     store: StoreService<RedisClient, RedisMulti>,
@@ -29,8 +31,8 @@ class TaskService {
   async processWebHooks(hookEventCallback: HookInterface): Promise<void> {
     const workItemKey = await this.store.getActiveTaskQueue();
     if (workItemKey) {
-      const [topic, sourceKey, scrub, ...sdata] = workItemKey.split('::');
-      const data = JSON.parse(sdata.join('::'));
+      const [topic, sourceKey, scrub, ...sdata] = workItemKey.split(WEBSEP);
+      const data = JSON.parse(sdata.join(WEBSEP));
       const destinationKey = `${sourceKey}:processed`;
       const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
       if (jobId) {
@@ -72,6 +74,7 @@ class TaskService {
     activityId: string,
     type: WorkListTaskType,
     inSeconds = HMSH_FIDELITY_SECONDS,
+    dad: string,
     multi?: RedisMulti,
   ): Promise<void> {
     const fromNow = Date.now() + (inSeconds * 1000);
@@ -83,6 +86,7 @@ class TaskService {
       activityId,
       type,
       awakenTimeSlot,
+      dad,
       multi,
     );
   }
@@ -129,21 +133,29 @@ class TaskService {
             await timeEventCallback(target, gId, activityId, type);
           }
           await sleepFor(0);
+          this.errorCount = 0;
           this.processTimeHooks(timeEventCallback, listKey);
         } else if (workListTask) {
           //a worklist was just emptied; try again immediately
           await sleepFor(0);
+          this.errorCount = 0;
           this.processTimeHooks(timeEventCallback);
         } else {
           //no worklists exist; sleep before checking
           let sleep = XSleepFor(HMSH_FIDELITY_SECONDS * 1000);
           this.cleanupTimeout = sleep.timerId;
           await sleep.promise;
+          this.errorCount = 0;
           this.processTimeHooks(timeEventCallback);
         }
       } catch (err) {
-        //todo: retry connect to redis
-        this.logger.error('task-process-timehooks-error', err);
+        //most common reasons: deleted job not found; container stopping; test stopping
+        //less common: redis/cluster down; retry with fallback (5s max main reassignment)
+        this.logger.warn('task-process-timehooks-error', err);
+        await sleepFor(1_000 * this.errorCount++);
+        if (this.errorCount < 5) {
+          this.processTimeHooks(timeEventCallback);
+        }
       }
     } else {
       //didn't get the scout role; try again in 'one-ish' minutes
@@ -174,10 +186,18 @@ class TaskService {
       const jobId = context.metadata.jid;
       const gId = context.metadata.gid;
       const activityId = hookRule.to;
+      //composite keys are used to fully describe the task target
+      const compositeJobKey = [
+        activityId,
+        dad,
+        gId,
+        jobId
+      ].join(WEBSEP);
+
       const hook: HookSignal = {
         topic,
         resolved,
-        jobId: `${activityId}::${dad}::${gId}::${jobId}`,
+        jobId: compositeJobKey,
       }
       await this.store.setHookSignal(hook, multi);
       return jobId;
@@ -196,17 +216,17 @@ class TaskService {
       const resolved = Pipe.resolve(mapExpression, context);
       const hookSignalId = await this.store.getHookSignal(topic, resolved);
       if (!hookSignalId) {
-        //messages can be double-processed; not an issue; return undefined
-        //users can also provide a bogus topic; not an issue; return undefined
+        //messages can be double-processed; not an issue; return `undefined`
+        //users can also provide a bogus topic; not an issue; return `undefined`
         return undefined;
       }
-      //`aid` is part of composit key, but the hook `topic` is its public interface;
+      //`aid` is part of composite key, but the hook `topic` is its public interface;
       // this means that a new version of the graph can be deployed and the
       // topic can be re-mapped to a different activity id. Outside callers
       // can adhere to the unchanged contract (calling the same topic),
-      // while the internal system can be updated in real time as necessary.
-      const [_aid, dad, gid, ...jid] = hookSignalId.split('::');
-      return [jid.join('::'), hookRule.to, dad, gid];
+      // while the internal system can be updated in real-time as necessary.
+      const [_aid, dad, gid, ...jid] = hookSignalId.split(WEBSEP);
+      return [jid.join(WEBSEP), hookRule.to, dad, gid];
     } else {
       throw new Error('signal-not-found');
     }

package/services/worker/index.ts

@@ -1,5 +1,5 @@
 import { KeyType } from "../../modules/key";
-import { formatISODate, getSystemHealth, identifyRedisType } from "../../modules/utils";
+import { XSleepFor, formatISODate, getSystemHealth, identifyRedisType, sleepFor } from "../../modules/utils";
 import { ConnectorService } from "../connector";
 import { ILogger } from "../logger";
 import { Router } from "../router";
@@ -17,10 +17,12 @@ import { RedisClientType as IORedisClientType } from '../../types/ioredisclient'
 import {
   QuorumMessage,
   QuorumProfile,
+  RollCallMessage,
   SubscriptionCallback } from "../../types/quorum";
 import { RedisClient, RedisMulti } from "../../types/redis";
 import { RedisClientType } from '../../types/redisclient';
-import { StreamRole } from "../../types/stream";
+import { StreamData, StreamRole, StreamDataResponse } from "../../types/stream";
+import { HMSH_QUORUM_ROLLCALL_CYCLES } from "../../modules/enums";
 
 class WorkerService {
   namespace: string;
@@ -28,6 +30,7 @@ class WorkerService {
   guid: string;
   topic: string;
   config: HotMeshConfig;
+  callback: (streamData: StreamData) => Promise<StreamDataResponse|void>;
   store: StoreService<RedisClient, RedisMulti> | null;
   stream: StreamService<RedisClient, RedisMulti> | null;
   subscribe: SubService<RedisClient, RedisMulti> | null;
@@ -35,6 +38,7 @@ class WorkerService {
   logger: ILogger;
   reporting = false;
   inited: string;
+  rollCallInterval: NodeJS.Timeout;
 
   static async init(
     namespace: string,
@@ -58,6 +62,7 @@ class WorkerService {
         service.namespace = namespace;
         service.appId = appId;
         service.guid = guid;
+        service.callback = worker.callback;
         service.topic = worker.topic;
         service.config = config;
         service.logger = logger;
@@ -155,14 +160,51 @@ class WorkerService {
     return async (topic: string, message: QuorumMessage) => {
       self.logger.debug('worker-event-received', { topic, type: message.type });
       if (message.type === 'throttle') {
-        self.throttle(message.throttle);
+        if (message.topic !== null) { //undefined allows passthrough
+          self.throttle(message.throttle);
+        }
       } else if(message.type === 'ping') {
         self.sayPong(self.appId, self.guid, message.originator, message.details);
+      } else if(message.type === 'rollcall') {
+        if (message.topic !== null) { //undefined allows passthrough
+          self.doRollCall(message);
+        }
       }
     };
   }
 
-  async sayPong(appId: string, guid: string, originator: string, details = false) {
+  /**
+   * A quorum-wide command to broadcaset system details.
+   * 
+   */
+  async doRollCall(message: RollCallMessage) {
+    let iteration = 0;
+    let max = !isNaN(message.max) ? message.max : HMSH_QUORUM_ROLLCALL_CYCLES;
+    if (this.rollCallInterval) clearTimeout(this.rollCallInterval);
+    const base = (message.interval / 2);
+    const amount = base + Math.ceil(Math.random() * base);
+    do {
+      await sleepFor(Math.ceil(Math.random() * 1000));
+      await this.sayPong(this.appId, this.guid, null, true, message.signature);
+      if (!message.interval) return;
+      const { promise, timerId } = XSleepFor(amount * 1000);
+      this.rollCallInterval = timerId;
+      await promise;
+    } while (this.rollCallInterval && iteration++ < max - 1);
+  }
+
+  cancelRollCall() {
+    if (this.rollCallInterval) {
+      clearTimeout(this.rollCallInterval);
+      delete this.rollCallInterval;
+    }
+  }
+
+  stop() {
+    this.cancelRollCall();
+  }
+
+  async sayPong(appId: string, guid: string, originator?: string, details = false, signature = false) {
     let profile: QuorumProfile;
     if (details) {
       const params = {
@@ -183,13 +225,15 @@ class WorkerService {
         reclaimDelay: this.router.reclaimDelay,
         reclaimCount: this.router.reclaimCount,
         system: await getSystemHealth(),
+        signature: signature ? this.callback.toString() : undefined,
       };
     }
     this.store.publish(
       KeyType.QUORUM, 
       {
         type: 'pong',
-        guid, originator,
+        guid,
+        originator,
         profile,
       },
       appId,

package/types/activity.ts

@@ -77,6 +77,7 @@ interface AwaitActivity extends BaseActivity {
   type: 'await';
   eventName: string;
   timeout: number;
+  await?: boolean; //if exlicitly false do not await the response
 }
 
 interface WorkerActivity extends BaseActivity {

package/types/exporter.ts

@@ -26,6 +26,8 @@ export interface JobTimeline {
   dimension: string; //dimensional isolate path
   duplex: 'entry' | 'exit'; //activity entry or exit
   timestamp: string; //actually a number but too many digits for JS
+  created?: string; //actually a number but too many digits for JS
+  updated?: string; //actually a number but too many digits for JS
   actions?: ActivityAction[];
 }
 

package/types/job.ts

@@ -17,6 +17,7 @@ type JobMetadata = {
   pg?: string;  //parent_generational_id (system assigned at trigger inception); pg is the parent job's gid (just in case user created/deleted/created a job with same jid)
   pd?: string;  //parent_dimensional_address
   pa?: string;  //parent_activity_id
+  px?: boolean; //sever the dependency chain if true (startChild/vs/executeChild)
   ngn?: string; //engine guid (one time subscriptions)
   app: string;  //app_id
   vrs: string;  //app version

package/types/quorum.ts

@@ -50,50 +50,65 @@ export interface QuorumProfile {
   reclaimDelay?: number;
   reclaimCount?: number;
   system?: SystemHealth;
+  signature?: string; //stringified function
 }
 
-//used for coordination (like version activation)
-export interface PingMessage {
+interface QuorumMessageBase {
+  guid?: string;
+  topic?: string;
+  type?: string;
+}
+
+// Messages extending QuorumMessageBase
+export interface PingMessage extends QuorumMessageBase {
   type: 'ping';
   originator: string; //guid
   details?: boolean;  //if true, all endpoints will include their profile
 }
 
-export interface WorkMessage {
+export interface WorkMessage extends QuorumMessageBase {
   type: 'work';
   originator: string; //guid
 }
 
-export interface CronMessage {
+export interface CronMessage extends QuorumMessageBase {
   type: 'cron';
   originator: string; //guid
 }
 
-export interface PongMessage {
+export interface PongMessage extends QuorumMessageBase {
   type: 'pong';
   guid: string;            //call initiator
   originator: string;      //clone of originator guid passed in ping
   profile?: QuorumProfile; //contains details about the engine/worker
 }
 
-export interface ActivateMessage {
+export interface ActivateMessage extends QuorumMessageBase {
   type: 'activate';
   cache_mode: 'nocache' | 'cache';
   until_version: string;
 }
 
-export interface JobMessage {
+export interface JobMessage extends QuorumMessageBase {
   type: 'job';
   topic: string; //this comes from the 'publishes' field in the YAML
   job: JobOutput
 }
 
-//delay in ms between fetches from the buffered stream (speed/slow down entire network)
-export interface ThrottleMessage {
+export interface ThrottleMessage extends QuorumMessageBase {
   type: 'throttle';
-  guid?: string; //target the engine quorum
-  topic?: string;  //target a worker quorum
-  throttle: number; //0-n
+  guid?: string; //target engine AND workers with this guid
+  topic?: string;  //target worker(s) matching this topic (pass null to only target the engine, pass undefined to target engine and workers)
+  throttle: number; //0-n; millis
+}
+
+export interface RollCallMessage extends QuorumMessageBase {
+  type: 'rollcall';
+  guid?: string;    //target the engine quorum
+  topic?: string | null;   //target a worker if string; suppress if `null`;
+  interval: number; //every 'n' seconds
+  max?: number;     //max broadcasts
+  signature?: boolean; //include bound worker function in broadcast   
 }
 
 export interface JobMessageCallback {
@@ -114,4 +129,4 @@ export interface QuorumMessageCallback {
  * These messages serve to coordinate the cache invalidation and switch-over
  * to the new version without any downtime and a coordinating parent server.
  */
-export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | RollCallMessage | CronMessage;

package/types/stream.ts

@@ -42,6 +42,7 @@ export interface StreamData {
     trc?: string;   //trace id
     spn?: string;   //span id
     try?: number;   //current try count
+    await?: boolean;  //(waitfor) if explicitly false, do not await; sever the connection
   };
   type?: StreamDataType;
   data: Record<string, unknown>;