@hotmeshio/hotmesh 0.0.48 → 0.0.50

package/build/services/quorum/index.d.ts

@@ -1,10 +1,11 @@
+/// <reference types="node" />
 import { EngineService } from '../engine';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
 import { HotMeshConfig } from '../../types/hotmesh';
-import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
+import { QuorumMessage, QuorumMessageCallback, QuorumProfile, RollCallMessage, SubscriptionCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
@@ -19,6 +20,7 @@ declare class QuorumService {
     untilVersion: string | null;
     quorum: number | null;
     callbacks: QuorumMessageCallback[];
+    rollCallInterval: NodeJS.Timeout;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<QuorumService>;
     verifyQuorumFields(config: HotMeshConfig): void;
     initStoreChannel(store: RedisClient): Promise<void>;
@@ -26,7 +28,14 @@ declare class QuorumService {
     subscriptionHandler(): SubscriptionCallback;
     sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
     requestQuorum(delay?: number, details?: boolean): Promise<number>;
-    pub(quorumMessage: ThrottleMessage): Promise<boolean>;
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    doRollCall(message: RollCallMessage): Promise<void>;
+    cancelRollCall(): void;
+    stop(): void;
+    pub(quorumMessage: QuorumMessage): Promise<boolean>;
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
     rollCall(delay?: number): Promise<QuorumProfile[]>;

package/build/services/quorum/index.js

@@ -90,6 +90,9 @@ class QuorumService {
             else if (message.type === 'cron') {
                 self.engine.processTimeHooks();
             }
+            else if (message.type === 'rollcall') {
+                self.doRollCall(message);
+            }
             //if there are any callbacks, call them
             if (self.callbacks.length > 0) {
                 self.callbacks.forEach(cb => cb(topic, message));
@@ -132,6 +135,36 @@ class QuorumService {
         await (0, utils_1.sleepFor)(delay);
         return quorum;
     }
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    async doRollCall(message) {
+        let iteration = 0;
+        let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
+        if (this.rollCallInterval)
+            clearTimeout(this.rollCallInterval);
+        const base = (message.interval / 2);
+        const amount = base + Math.ceil(Math.random() * base);
+        do {
+            await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
+            await this.sayPong(this.appId, this.guid, null, true);
+            if (!message.interval)
+                return;
+            const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
+            this.rollCallInterval = timerId;
+            await promise;
+        } while (this.rollCallInterval && iteration++ < max - 1);
+    }
+    cancelRollCall() {
+        if (this.rollCallInterval) {
+            clearTimeout(this.rollCallInterval);
+            delete this.rollCallInterval;
+        }
+    }
+    stop() {
+        this.cancelRollCall();
+    }
     // ************* PUB/SUB METHODS *************
     //publish a message to the quorum
     async pub(quorumMessage) {

package/build/services/router/index.d.ts

@@ -22,9 +22,24 @@ declare class Router {
     };
     currentTimerId: NodeJS.Timeout | null;
     shouldConsume: boolean;
+    sleepPromiseResolve: (() => void) | null;
+    innerPromiseResolve: (() => void) | null;
+    isSleeping: boolean;
+    sleepTimout: NodeJS.Timeout | null;
     constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    private resetThrottleState;
     createGroup(stream: string, group: string): Promise<void>;
     publishMessage(topic: string, streamData: StreamData | StreamDataResponse, multi?: RedisMulti): Promise<string | RedisMulti>;
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
+    customSleep(): Promise<void>;
     consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
     isStreamMessage(result: any): boolean;
     consumeOne(stream: string, group: string, id: string, message: string[], callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;

package/build/services/router/index.js

@@ -12,6 +12,10 @@ class Router {
         this.errorCount = 0;
         this.counts = {};
         this.currentTimerId = null;
+        this.sleepPromiseResolve = null;
+        this.innerPromiseResolve = null;
+        this.isSleeping = false;
+        this.sleepTimout = null;
         this.appId = config.appId;
         this.guid = config.guid;
         this.role = config.role;
@@ -21,6 +25,13 @@ class Router {
         this.reclaimDelay = config.reclaimDelay || enums_1.HMSH_XCLAIM_DELAY_MS;
         this.reclaimCount = config.reclaimCount || enums_1.HMSH_XCLAIM_COUNT;
         this.logger = logger;
+        this.resetThrottleState();
+    }
+    resetThrottleState() {
+        this.sleepPromiseResolve = null;
+        this.innerPromiseResolve = null;
+        this.isSleeping = false;
+        this.sleepTimout = null;
     }
     async createGroup(stream, group) {
         try {
@@ -36,6 +47,36 @@ class Router {
         const stream = this.store.mintKey(key_1.KeyType.STREAMS, { appId: this.store.appId, topic });
         return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
     }
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
+    async customSleep() {
+        if (this.throttle === 0)
+            return;
+        if (this.isSleeping)
+            return;
+        this.isSleeping = true;
+        let startTime = Date.now(); //anchor the origin
+        await new Promise(async (outerResolve) => {
+            this.sleepPromiseResolve = outerResolve;
+            let elapsedTime = Date.now() - startTime;
+            while (elapsedTime < this.throttle) {
+                await new Promise((innerResolve) => {
+                    this.innerPromiseResolve = innerResolve;
+                    this.sleepTimout = setTimeout(innerResolve, this.throttle - elapsedTime);
+                });
+                elapsedTime = Date.now() - startTime;
+            }
+            this.resetThrottleState();
+            outerResolve();
+        });
+    }
     async consumeMessages(stream, group, consumer, callback) {
         this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
         Router.instances.add(this);
@@ -43,9 +84,7 @@ class Router {
         await this.createGroup(stream, group);
         let lastCheckedPendingMessagesAt = Date.now();
         async function consume() {
-            let sleep = (0, utils_1.XSleepFor)(this.throttle);
-            this.currentTimerId = sleep.timerId;
-            await sleep.promise;
+            await this.customSleep();
             if (!this.shouldConsume) {
                 this.logger.info(`stream-consumer-stopping`, { group, consumer, stream });
                 return;
@@ -229,17 +268,26 @@ class Router {
         this.cancelThrottle();
     }
     cancelThrottle() {
-        if (this.currentTimerId !== undefined) {
-            clearTimeout(this.currentTimerId);
-            this.currentTimerId = undefined;
+        if (this.sleepTimout) {
+            clearTimeout(this.sleepTimout);
         }
+        this.resetThrottleState();
     }
     setThrottle(delayInMillis) {
         if (!Number.isInteger(delayInMillis) || delayInMillis < 0) {
             throw new Error('Throttle must be a non-negative integer');
         }
+        const wasDecreased = delayInMillis < this.throttle;
         this.throttle = delayInMillis;
-        this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
+        // If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
+        if (wasDecreased) {
+            if (this.sleepTimout) {
+                clearTimeout(this.sleepTimout);
+            }
+            if (this.innerPromiseResolve) {
+                this.innerPromiseResolve();
+            }
+        }
     }
     async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = enums_1.HMSH_XPENDING_COUNT) {
         let pendingMessages = [];

package/build/services/serializer/index.js

@@ -12,7 +12,7 @@ exports.MDATA_SYMBOLS = {
         KEYS: ['au', 'err', 'l2s']
     },
     JOB: {
-        KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+        KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'px', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
     },
     JOB_UPDATE: {
         KEYS: ['ju', 'err']

package/build/services/store/clients/redis.js

@@ -10,6 +10,7 @@ class RedisStoreService extends index_1.StoreService {
             setnx: 'SETNX',
             del: 'DEL',
             expire: 'EXPIRE',
+            hscan: 'HSCAN',
             hset: 'HSET',
             hsetnx: 'HSETNX',
             hincrby: 'HINCRBY',
@@ -29,6 +30,7 @@ class RedisStoreService extends index_1.StoreService {
             lpop: 'LPOP',
             rename: 'RENAME',
             rpush: 'RPUSH',
+            scan: 'SCAN',
             xack: 'XACK',
             xdel: 'XDEL',
             xlen: 'XLEN',

package/build/services/store/index.d.ts

@@ -69,12 +69,12 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * when `originJobId` is interrupted/expired, the items in the
      * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
      */
-    registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
+    registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd?: string, multi?: U): Promise<any>;
     /**
      * Ensures a `hook signal` is delisted when its parent activity/job
      * is interrupted/expired.
      */
-    registerSignalDependency(jobId: string, signalKey: string, multi?: U): Promise<any>;
+    registerSignalDependency(jobId: string, signalKey: string, dad: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
     getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
@@ -133,7 +133,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * for the given sleep group. Sleep groups are
      * organized into 'n'-second blocks (LISTS))
      */
-    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, multi?: U): Promise<void>;
+    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void>;
     getNextTask(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: WorkListTaskType] | boolean>;
     /**
      * when processing time jobs, the target LIST ID returned
@@ -141,7 +141,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * do with the work list. (not everything is known in advance,
      * so the ZSET key defines HOW to approach the work in the
      * generic LIST (lists typically contain target job ids)
-     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     * @param {string} listKey - composite key
      */
     resolveTaskKeyContext(listKey: string): [WorkListTaskType, string];
     /**
@@ -152,5 +152,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      */
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;
+    findJobs(queryString?: string, limit?: number, batchSize?: number): Promise<[string, string[]]>;
+    findJobFields(jobId: string, fieldMatchPattern?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, StringStringType]>;
 }
 export { StoreService };

package/build/services/store/index.js

@@ -38,6 +38,7 @@ class StoreService {
             del: 'del',
             expire: 'expire',
             hset: 'hset',
+            hscan: 'hscan',
             hsetnx: 'hsetnx',
             hincrby: 'hincrby',
             hdel: 'hdel',
@@ -56,6 +57,7 @@ class StoreService {
             lrange: 'lrange',
             rename: 'rename',
             rpush: 'rpush',
+            scan: 'scan',
             xack: 'xack',
             xdel: 'xdel',
         };
@@ -343,15 +345,20 @@ class StoreService {
      * when `originJobId` is interrupted/expired, the items in the
      * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
      */
-    async registerJobDependency(depType, originJobId, topic, jobId, gId, multi) {
+    async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
         const privateMulti = multi || this.getMulti();
         const dependencyParams = {
             appId: this.appId,
             jobId: originJobId,
         };
         const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
-        //items listed as job dependencies have different relationships
-        const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
+        const expireTask = [
+            depType,
+            topic,
+            gId,
+            pd,
+            jobId,
+        ].join(key_1.VALSEP);
         privateMulti[this.commands.rpush](depKey, expireTask);
         if (!multi) {
             return await privateMulti.exec();
@@ -361,12 +368,18 @@ class StoreService {
      * Ensures a `hook signal` is delisted when its parent activity/job
      * is interrupted/expired.
      */
-    async registerSignalDependency(jobId, signalKey, multi) {
+    async registerSignalDependency(jobId, signalKey, dad, multi) {
         const privateMulti = multi || this.getMulti();
         const dependencyParams = { appId: this.appId, jobId };
         const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
-        //tasks have '4' segments
-        const delistTask = `delist::signal::${jobId}::${signalKey}`;
+        //persiste dependency tasks as multi-segment composite keys
+        const delistTask = [
+            'delist',
+            'signal',
+            jobId,
+            dad,
+            signalKey
+        ].join(key_1.VALSEP);
         privateMulti[this.commands.rpush](dependencyKey, delistTask);
         if (!multi) {
             return await privateMulti.exec();
@@ -684,11 +697,14 @@ class StoreService {
     }
     async setHookSignal(hook, multi) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
-        const { topic, resolved, jobId } = hook; //`${activityId}::${dad}::${gId}::${jobId}`
+        //destructure the hook key
+        const { topic, resolved, jobId } = hook;
         const signalKey = `${topic}:${resolved}`;
         const payload = { [signalKey]: jobId };
         await (multi || this.redisClient)[this.commands.hset](key, payload);
-        return await this.registerSignalDependency(jobId.split('::')[3], signalKey, multi);
+        //jobId needs even more destructuring
+        const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
+        return await this.registerSignalDependency(jid, signalKey, dad, multi);
     }
     async getHookSignal(topic, resolved) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
@@ -753,7 +769,7 @@ class StoreService {
         const depParams = { appId: this.appId, jobId };
         const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
         const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
-        const depKeyContext = `::${context}::${depKey}`;
+        const depKeyContext = `${key_1.TYPSEP}${context}${key_1.TYPSEP}${depKey}`;
         const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
         await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
     }
@@ -768,9 +784,16 @@ class StoreService {
      * for the given sleep group. Sleep groups are
      * organized into 'n'-second blocks (LISTS))
      */
-    async registerTimeHook(jobId, gId, activityId, type, deletionTime, multi) {
+    async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
         const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
-        const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
+        //construct the composite key (the key has enough info to signal the hook)
+        const timeEvent = [
+            type,
+            activityId,
+            gId,
+            dad,
+            jobId
+        ].join(key_1.VALSEP);
         const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
         if (multi || len === 1) {
             const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
@@ -784,9 +807,9 @@ class StoreService {
             let [pType, pKey] = this.resolveTaskKeyContext(listKey);
             const timeEvent = await this.redisClient[this.commands.lpop](pKey);
             if (timeEvent) {
-                //there are task types
-                //1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
-                let [type, activityId, gId, ...jobId] = timeEvent.split('::');
+                //deconstruct composite key
+                let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
+                const jid = jobId.join(key_1.VALSEP);
                 if (type === 'delist') {
                     pType = 'delist';
                 }
@@ -794,9 +817,9 @@ class StoreService {
                     pType = 'child';
                 }
                 else if (type === 'expire-child') {
-                    type = 'expire'; //use the same logic as 'expire'
+                    type = 'expire';
                 }
-                return [listKey, jobId.join('::'), gId, activityId, pType];
+                return [listKey, jid, gId, activityId, pType];
             }
             await this.redisClient[this.commands.zrem](zsetKey, listKey);
             return true;
@@ -809,14 +832,14 @@ class StoreService {
      * do with the work list. (not everything is known in advance,
      * so the ZSET key defines HOW to approach the work in the
      * generic LIST (lists typically contain target job ids)
-     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     * @param {string} listKey - composite key
      */
     resolveTaskKeyContext(listKey) {
-        if (listKey.startsWith('::INTERRUPT')) {
-            return ['interrupt', listKey.split('::')[2]];
+        if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
+            return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
         }
-        else if (listKey.startsWith('::EXPIRE')) {
-            return ['expire', listKey.split('::')[2]];
+        else if (listKey.startsWith(`${key_1.TYPSEP}EXPIRE`)) {
+            return ['expire', listKey.split(key_1.TYPSEP)[2]];
         }
         else {
             return ['sleep', listKey];
@@ -876,5 +899,47 @@ class StoreService {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         await this.redisClient[this.commands.del](jobKey);
     }
+    async findJobs(queryString = '*', limit = 1000, batchSize = 1000) {
+        const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId: queryString });
+        let cursor = '0';
+        let keys;
+        const matchingKeys = [];
+        do {
+            const output = await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString());
+            if (Array.isArray(output)) {
+                [cursor, keys] = output;
+                for (let key of [...keys]) {
+                    matchingKeys.push(key);
+                }
+                if (matchingKeys.length >= limit) {
+                    break;
+                }
+            }
+            else {
+                break;
+            }
+        } while (cursor !== '0');
+        return [cursor, matchingKeys];
+    }
+    async findJobFields(jobId, fieldMatchPattern = '*', limit = 1000, batchSize = 1000, cursor = '0') {
+        let fields = [];
+        const matchingFields = {};
+        const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+        let len = 0;
+        do {
+            const output = await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString());
+            if (Array.isArray(output)) {
+                [cursor, fields] = output;
+                for (let i = 0; i < fields.length; i += 2) {
+                    len++;
+                    matchingFields[fields[i]] = fields[i + 1];
+                }
+            }
+            else {
+                break;
+            }
+        } while (cursor !== '0' && len < limit);
+        return [cursor, matchingFields];
+    }
 }
 exports.StoreService = StoreService;

package/build/services/task/index.d.ts

@@ -10,11 +10,12 @@ declare class TaskService {
     logger: ILogger;
     cleanupTimeout: NodeJS.Timeout | null;
     isScout: boolean;
+    errorCount: number;
     constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
     processWebHooks(hookEventCallback: HookInterface): Promise<void>;
     enqueueWorkItems(keys: string[]): Promise<void>;
     registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
-    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds?: number, multi?: RedisMulti): Promise<void>;
+    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
     /**
      * Should this engine instance play the role of 'scout' on behalf
      * of the entire quorum? The scout role is responsible for processing

package/build/services/task/index.js

@@ -5,18 +5,20 @@ const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const pipe_1 = require("../pipe");
 const hotmesh_1 = require("../../types/hotmesh");
+const key_1 = require("../../modules/key");
 class TaskService {
     constructor(store, logger) {
         this.cleanupTimeout = null;
         this.isScout = false;
+        this.errorCount = 0;
         this.logger = logger;
         this.store = store;
     }
     async processWebHooks(hookEventCallback) {
         const workItemKey = await this.store.getActiveTaskQueue();
         if (workItemKey) {
-            const [topic, sourceKey, scrub, ...sdata] = workItemKey.split('::');
-            const data = JSON.parse(sdata.join('::'));
+            const [topic, sourceKey, scrub, ...sdata] = workItemKey.split(key_1.WEBSEP);
+            const data = JSON.parse(sdata.join(key_1.WEBSEP));
             const destinationKey = `${sourceKey}:processed`;
             const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
             if (jobId) {
@@ -41,11 +43,11 @@ class TaskService {
             await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
         }
     }
-    async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, multi) {
+    async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, multi) {
         const fromNow = Date.now() + (inSeconds * 1000);
         const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
         const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
-        await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, multi);
+        await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
     }
     /**
      * Should this engine instance play the role of 'scout' on behalf
@@ -89,11 +91,13 @@ class TaskService {
                         await timeEventCallback(target, gId, activityId, type);
                     }
                     await (0, utils_1.sleepFor)(0);
+                    this.errorCount = 0;
                     this.processTimeHooks(timeEventCallback, listKey);
                 }
                 else if (workListTask) {
                     //a worklist was just emptied; try again immediately
                     await (0, utils_1.sleepFor)(0);
+                    this.errorCount = 0;
                     this.processTimeHooks(timeEventCallback);
                 }
                 else {
@@ -101,12 +105,18 @@ class TaskService {
                     let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
                     this.cleanupTimeout = sleep.timerId;
                     await sleep.promise;
+                    this.errorCount = 0;
                     this.processTimeHooks(timeEventCallback);
                 }
             }
             catch (err) {
-                //todo: retry connect to redis
-                this.logger.error('task-process-timehooks-error', err);
+                //most common reasons: deleted job not found; container stopping; test stopping
+                //less common: redis/cluster down; retry with fallback (5s max main reassignment)
+                this.logger.warn('task-process-timehooks-error', err);
+                await (0, utils_1.sleepFor)(1000 * this.errorCount++);
+                if (this.errorCount < 5) {
+                    this.processTimeHooks(timeEventCallback);
+                }
             }
         }
         else {
@@ -135,10 +145,17 @@ class TaskService {
             const jobId = context.metadata.jid;
             const gId = context.metadata.gid;
             const activityId = hookRule.to;
+            //composite keys are used to fully describe the task target
+            const compositeJobKey = [
+                activityId,
+                dad,
+                gId,
+                jobId
+            ].join(key_1.WEBSEP);
             const hook = {
                 topic,
                 resolved,
-                jobId: `${activityId}::${dad}::${gId}::${jobId}`,
+                jobId: compositeJobKey,
             };
             await this.store.setHookSignal(hook, multi);
             return jobId;
@@ -157,17 +174,17 @@ class TaskService {
             const resolved = pipe_1.Pipe.resolve(mapExpression, context);
             const hookSignalId = await this.store.getHookSignal(topic, resolved);
             if (!hookSignalId) {
-                //messages can be double-processed; not an issue; return undefined
-                //users can also provide a bogus topic; not an issue; return undefined
+                //messages can be double-processed; not an issue; return `undefined`
+                //users can also provide a bogus topic; not an issue; return `undefined`
                 return undefined;
             }
-            //`aid` is part of composit key, but the hook `topic` is its public interface;
+            //`aid` is part of composite key, but the hook `topic` is its public interface;
             // this means that a new version of the graph can be deployed and the
             // topic can be re-mapped to a different activity id. Outside callers
             // can adhere to the unchanged contract (calling the same topic),
-            // while the internal system can be updated in real time as necessary.
-            const [_aid, dad, gid, ...jid] = hookSignalId.split('::');
-            return [jid.join('::'), hookRule.to, dad, gid];
+            // while the internal system can be updated in real-time as necessary.
+            const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
+            return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
         }
         else {
             throw new Error('signal-not-found');

package/build/services/worker/index.d.ts

@@ -1,17 +1,20 @@
+/// <reference types="node" />
 import { ILogger } from "../logger";
 import { Router } from "../router";
 import { StoreService } from '../store';
 import { StreamService } from '../stream';
 import { SubService } from '../sub';
 import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
-import { SubscriptionCallback } from "../../types/quorum";
+import { RollCallMessage, SubscriptionCallback } from "../../types/quorum";
 import { RedisClient, RedisMulti } from "../../types/redis";
+import { StreamData, StreamDataResponse } from "../../types/stream";
 declare class WorkerService {
     namespace: string;
     appId: string;
     guid: string;
     topic: string;
     config: HotMeshConfig;
+    callback: (streamData: StreamData) => Promise<StreamDataResponse | void>;
     store: StoreService<RedisClient, RedisMulti> | null;
     stream: StreamService<RedisClient, RedisMulti> | null;
     subscribe: SubService<RedisClient, RedisMulti> | null;
@@ -19,6 +22,7 @@ declare class WorkerService {
     logger: ILogger;
     reporting: boolean;
     inited: string;
+    rollCallInterval: NodeJS.Timeout;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
     verifyWorkerFields(worker: HotMeshWorker): void;
     initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;
@@ -26,7 +30,14 @@ declare class WorkerService {
     initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
     initRouter(worker: HotMeshWorker, logger: ILogger): Router;
     subscriptionHandler(): SubscriptionCallback;
-    sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    doRollCall(message: RollCallMessage): Promise<void>;
+    cancelRollCall(): void;
+    stop(): void;
+    sayPong(appId: string, guid: string, originator?: string, details?: boolean, signature?: boolean): Promise<void>;
     throttle(delayInMillis: number): Promise<void>;
 }
 export { WorkerService };