@hotmeshio/hotmesh 0.0.49 → 0.0.50

package/build/modules/enums.d.ts

@@ -15,6 +15,7 @@ export declare const HMSH_CODE_DURABLE_MAXED = 597;
 export declare const HMSH_CODE_DURABLE_FATAL = 598;
 export declare const HMSH_CODE_DURABLE_RETRYABLE = 599;
 export declare const HMSH_STATUS_UNKNOWN = "unknown";
+export declare const HMSH_QUORUM_ROLLCALL_CYCLES = 12;
 export declare const HMSH_QUORUM_DELAY_MS = 250;
 export declare const HMSH_ACTIVATION_MAX_RETRY = 3;
 export declare const HMSH_OTT_WAIT_TIME: number;

package/build/modules/enums.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
+exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
 // HOTMESH SYSTEM
 exports.HMSH_LOGLEVEL = process.env.HMSH_LOGLEVEL || 'info';
 // STATUS CODES AND MESSAGES
@@ -20,6 +20,7 @@ exports.HMSH_CODE_DURABLE_FATAL = 598;
 exports.HMSH_CODE_DURABLE_RETRYABLE = 599;
 exports.HMSH_STATUS_UNKNOWN = 'unknown';
 // QUORUM
+exports.HMSH_QUORUM_ROLLCALL_CYCLES = 12; //max iterations
 exports.HMSH_QUORUM_DELAY_MS = 250;
 exports.HMSH_ACTIVATION_MAX_RETRY = 3;
 // ENGINE

package/build/modules/key.d.ts

@@ -27,6 +27,10 @@ import { KeyStoreParams, KeyType } from '../types/hotmesh';
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
 declare const HMNS = "hmsh";
+declare const KEYSEP = ":";
+declare const VALSEP = "::";
+declare const WEBSEP = "::";
+declare const TYPSEP = "::";
 declare class KeyService {
     /**
      * returns a key that can be used to access a value in the key/value store
@@ -41,4 +45,4 @@ declare class KeyService {
      */
     static mintKey(namespace: string, keyType: KeyType, params: KeyStoreParams): string;
 }
-export { KeyService, KeyType, KeyStoreParams, HMNS };
+export { KeyService, KeyType, KeyStoreParams, HMNS, KEYSEP, TYPSEP, WEBSEP, VALSEP };

package/build/modules/key.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HMNS = exports.KeyType = exports.KeyService = void 0;
+exports.VALSEP = exports.WEBSEP = exports.TYPSEP = exports.KEYSEP = exports.HMNS = exports.KeyType = exports.KeyService = void 0;
 const hotmesh_1 = require("../types/hotmesh");
 Object.defineProperty(exports, "KeyType", { enumerable: true, get: function () { return hotmesh_1.KeyType; } });
 /**
@@ -30,8 +30,16 @@ Object.defineProperty(exports, "KeyType", { enumerable: true, get: function () {
  * hmsh:<appid>:sym:keys:<activityid|$subscribes> ->  {hash}    list of symbols based upon schema enums (initially) and adaptively optimized (later) during runtime; if '$subscribes' is used as the activityid, it is a top-level `job` symbol set (for job keys)
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
-const HMNS = "hmsh"; //default
+const HMNS = "hmsh";
 exports.HMNS = HMNS;
+const KEYSEP = ':'; //default delimiter for keys 
+exports.KEYSEP = KEYSEP;
+const VALSEP = '::'; //default delimiter for vals
+exports.VALSEP = VALSEP;
+const WEBSEP = '::'; //default delimiter for webhook vals
+exports.WEBSEP = WEBSEP;
+const TYPSEP = '::'; //delimiter for ZSET task typing (how should a list be used?)
+exports.TYPSEP = TYPSEP;
 class KeyService {
     /**
      * returns a key that can be used to access a value in the key/value store

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.49",
+    "version": "0.0.50",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",
@@ -28,6 +28,7 @@
         "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
         "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
         "test:emit": "NODE_ENV=test jest ./tests/functional/emit/index.test.ts --detectOpenHandles --forceExit --verbose",
+        "test:await": "NODE_ENV=test jest ./tests/functional/awaiter/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:hook": "NODE_ENV=test jest ./tests/functional/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:signal": "NODE_ENV=test jest ./tests/functional/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:interrupt": "NODE_ENV=test jest ./tests/functional/interrupt/index.test.ts --detectOpenHandles --forceExit --verbose",

package/build/services/activities/await.js

@@ -78,6 +78,12 @@ class Await extends activity_1.Activity {
             type: stream_1.StreamDataType.AWAIT,
             data: this.context.data
         };
+        if (this.config.await !== true) {
+            const doAwait = pipe_1.Pipe.resolve(this.config.await, this.context);
+            if (doAwait === false) {
+                streamData.metadata.await = false;
+            }
+        }
         if (this.config.retry) {
             streamData.policies = {
                 retry: this.config.retry

package/build/services/activities/hook.js

@@ -107,7 +107,7 @@ class Hook extends activity_1.Activity {
         }
         else if (this.config.sleep) {
             const duration = pipe_1.Pipe.resolve(this.config.sleep, this.context);
-            await this.engine.taskService.registerTimeHook(this.context.metadata.jid, this.context.metadata.gid, `${this.metadata.aid}${this.metadata.dad || ''}`, 'sleep', duration);
+            await this.engine.taskService.registerTimeHook(this.context.metadata.jid, this.context.metadata.gid, `${this.metadata.aid}${this.metadata.dad || ''}`, 'sleep', duration, this.metadata.dad || '');
             return this.context.metadata.jid;
         }
     }

package/build/services/activities/trigger.d.ts

@@ -8,6 +8,7 @@ declare class Trigger extends Activity {
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
     process(): Promise<string>;
     setStatus(amount: number): Promise<void>;
+    execAdjacentParent(): Promise<void>;
     createInputContext(): Partial<JobState>;
     getState(): Promise<void>;
     bindJobMetadataPaths(): string[];

package/build/services/activities/trigger.js

@@ -31,6 +31,9 @@ class Trigger extends activity_1.Activity {
             await this.setStats(multi);
             await this.registerJobDependency(multi);
             await multi.exec();
+            //if the parent (spawner) chose not to await,
+            // emit the job_id as the data payload { job_id }
+            this.execAdjacentParent();
             telemetry.mapActivityAttributes();
             const jobStatus = Number(this.context.metadata.js);
             telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
@@ -61,6 +64,11 @@ class Trigger extends activity_1.Activity {
     async setStatus(amount) {
         this.context.metadata.js = amount;
     }
+    async execAdjacentParent() {
+        if (this.context.metadata.px) {
+            await this.engine.execAdjacentParent(this.context, { metadata: this.context.metadata, data: { job_id: this.context.metadata.jid } });
+        }
+    }
     createInputContext() {
         const input = {
             [this.metadata.aid]: {
@@ -95,6 +103,7 @@ class Trigger extends activity_1.Activity {
                 pg: this.context.metadata.pg,
                 pd: this.context.metadata.pd,
                 pa: this.context.metadata.pa,
+                px: this.context.metadata.px,
                 app: id,
                 vrs: version,
                 tpc: this.config.subscribes,
@@ -165,10 +174,22 @@ class Trigger extends activity_1.Activity {
         }
         if (resolvedDepKey) {
             const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
-            await this.store.registerJobDependency(isParentOrigin ? 'expire-child' : 'expire', resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
+            let type;
+            if (isParentOrigin) {
+                if (this.context.metadata.px) {
+                    type = 'child';
+                }
+                else {
+                    type = 'expire-child';
+                }
+            }
+            else {
+                type = 'expire';
+            }
+            await this.store.registerJobDependency(type, resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, this.context.metadata.pd, multi);
         }
         if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
-            await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
+            await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, this.context.metadata.pd, multi);
         }
     }
     async setStats(multi) {

package/build/services/durable/exporter.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExporterService = void 0;
 const serializer_1 = require("../serializer");
 const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
  * Splits, Inflates, and Sorts the job data for use in durable contexts
@@ -94,13 +95,27 @@ class ExporterService {
             const activityName = item[1].split('/')[0];
             const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
             const timestamp = item[2];
-            const event = {
+            let event = {
                 activity: activityName,
                 duplex: duplex,
                 dimension: dimensions,
                 timestamp,
+                created: timestamp,
+                updated: timestamp,
             };
-            timeline.push(event);
+            const prior = timeline[timeline.length - 1];
+            if (prior && prior.activity === event.activity && prior.duplex !== event.duplex && prior.dimension === event.dimension) {
+                if (event.duplex === 'exit') {
+                    prior.updated = event.timestamp;
+                }
+                else {
+                    prior.created = event.timestamp;
+                }
+                event = prior;
+            }
+            else {
+                timeline.push(event);
+            }
             if (this.isMainEntry(item[1])) {
                 event.actions = [];
                 this.interleaveActions(actions.main, event.actions);
@@ -163,12 +178,11 @@ class ExporterService {
      * @returns - the organized dependency data
      */
     inflateDependencyData(data, actions) {
-        //console.log('dependency data>', data);
         const hookReg = /([0-9,]+)-(\d+)$/;
         const flowReg = /-(\d+)$/;
         return data.map((dependency, index) => {
-            const [action, topic, gid, ...jid] = dependency.split('::');
-            const jobId = jid.join('::');
+            const [action, topic, gid, _pd, ...jid] = dependency.split(key_1.VALSEP);
+            const jobId = jid.join(key_1.VALSEP);
             const match = jobId.match(hookReg);
             let prefix;
             let type;

package/build/services/engine/index.d.ts

@@ -65,7 +65,7 @@ declare class EngineService {
     resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
     processStreamMessage(streamData: StreamDataResponse): Promise<void>;
     execAdjacentParent(context: JobState, jobOutput: JobOutput, emit?: boolean): Promise<string>;
-    hasParentJob(context: JobState): boolean;
+    hasParentJob(context: JobState, checkSevered?: boolean): boolean;
     resolveError(metadata: JobMetadata): StreamError | undefined;
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<string>;
     scrub(jobId: string): Promise<void>;

package/build/services/engine/index.js

@@ -259,6 +259,7 @@ class EngineService {
                 pg: streamData.metadata.gid,
                 pd: streamData.metadata.dad,
                 pa: streamData.metadata.aid,
+                px: streamData.metadata.await === false,
                 trc: streamData.metadata.trc,
                 spn: streamData.metadata.spn,
             };
@@ -316,7 +317,10 @@ class EngineService {
             return (await this.router?.publishMessage(null, streamData));
         }
     }
-    hasParentJob(context) {
+    hasParentJob(context, checkSevered = false) {
+        if (checkSevered) {
+            return Boolean(context.metadata.pj && context.metadata.pa && !context.metadata.px);
+        }
         return Boolean(context.metadata.pj && context.metadata.pa);
     }
     resolveError(metadata) {
@@ -385,7 +389,12 @@ class EngineService {
             const workItems = await reporter.getWorkItems(resolvedQuery, queryFacets);
             if (workItems.length) {
                 const taskService = new task_1.TaskService(this.store, this.logger);
-                await taskService.enqueueWorkItems(workItems.map(workItem => `${hookTopic}::${workItem}::${keyResolver.scrub || false}::${JSON.stringify(data)}`));
+                await taskService.enqueueWorkItems(workItems.map(workItem => [
+                    hookTopic,
+                    workItem,
+                    keyResolver.scrub || false,
+                    JSON.stringify(data)
+                ].join(key_1.VALSEP)));
                 this.store.publish(key_1.KeyType.QUORUM, { type: 'work', originator: this.guid }, this.appId);
             }
             return workItems;
@@ -504,7 +513,7 @@ class EngineService {
     // ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
     async runJobCompletionTasks(context, options = {}) {
         //'emit' indicates the job is still active
-        const isAwait = this.hasParentJob(context);
+        const isAwait = this.hasParentJob(context, true);
         const isOneTimeSub = this.hasOneTimeSubscription(context);
         const topic = await this.getPublishesTopic(context);
         let msgId;

package/build/services/exporter/index.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExporterService = void 0;
 const serializer_1 = require("../serializer");
 const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
  * Expands process data and includes dependency list
@@ -87,8 +88,8 @@ class ExporterService {
         const hookReg = /([0-9,]+)-(\d+)$/;
         const flowReg = /-(\d+)$/;
         return data.map((dependency, index) => {
-            const [action, topic, gid, ...jid] = dependency.split('::');
-            const jobId = jid.join('::');
+            const [action, topic, gid, _pd, ...jid] = dependency.split(key_1.VALSEP);
+            const jobId = jid.join(key_1.VALSEP);
             const match = jobId.match(hookReg);
             let prefix;
             let type;

package/build/services/hotmesh/index.js

@@ -167,6 +167,10 @@ class HotMeshService {
     }
     stop() {
         this.engine?.taskService.cancelCleanup();
+        this.quorum?.stop();
+        this.workers?.forEach((worker) => {
+            worker.stop();
+        });
     }
     async compress(terms) {
         return await this.engine?.compress(terms);

package/build/services/quorum/index.d.ts

@@ -1,10 +1,11 @@
+/// <reference types="node" />
 import { EngineService } from '../engine';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
 import { HotMeshConfig } from '../../types/hotmesh';
-import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
+import { QuorumMessage, QuorumMessageCallback, QuorumProfile, RollCallMessage, SubscriptionCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
@@ -19,6 +20,7 @@ declare class QuorumService {
     untilVersion: string | null;
     quorum: number | null;
     callbacks: QuorumMessageCallback[];
+    rollCallInterval: NodeJS.Timeout;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<QuorumService>;
     verifyQuorumFields(config: HotMeshConfig): void;
     initStoreChannel(store: RedisClient): Promise<void>;
@@ -26,7 +28,14 @@ declare class QuorumService {
     subscriptionHandler(): SubscriptionCallback;
     sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
     requestQuorum(delay?: number, details?: boolean): Promise<number>;
-    pub(quorumMessage: ThrottleMessage): Promise<boolean>;
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    doRollCall(message: RollCallMessage): Promise<void>;
+    cancelRollCall(): void;
+    stop(): void;
+    pub(quorumMessage: QuorumMessage): Promise<boolean>;
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
     rollCall(delay?: number): Promise<QuorumProfile[]>;

package/build/services/quorum/index.js

@@ -90,6 +90,9 @@ class QuorumService {
             else if (message.type === 'cron') {
                 self.engine.processTimeHooks();
             }
+            else if (message.type === 'rollcall') {
+                self.doRollCall(message);
+            }
             //if there are any callbacks, call them
             if (self.callbacks.length > 0) {
                 self.callbacks.forEach(cb => cb(topic, message));
@@ -132,6 +135,36 @@ class QuorumService {
         await (0, utils_1.sleepFor)(delay);
         return quorum;
     }
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    async doRollCall(message) {
+        let iteration = 0;
+        let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
+        if (this.rollCallInterval)
+            clearTimeout(this.rollCallInterval);
+        const base = (message.interval / 2);
+        const amount = base + Math.ceil(Math.random() * base);
+        do {
+            await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
+            await this.sayPong(this.appId, this.guid, null, true);
+            if (!message.interval)
+                return;
+            const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
+            this.rollCallInterval = timerId;
+            await promise;
+        } while (this.rollCallInterval && iteration++ < max - 1);
+    }
+    cancelRollCall() {
+        if (this.rollCallInterval) {
+            clearTimeout(this.rollCallInterval);
+            delete this.rollCallInterval;
+        }
+    }
+    stop() {
+        this.cancelRollCall();
+    }
     // ************* PUB/SUB METHODS *************
     //publish a message to the quorum
     async pub(quorumMessage) {

package/build/services/serializer/index.js

@@ -12,7 +12,7 @@ exports.MDATA_SYMBOLS = {
         KEYS: ['au', 'err', 'l2s']
     },
     JOB: {
-        KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+        KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'px', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
     },
     JOB_UPDATE: {
         KEYS: ['ju', 'err']

package/build/services/store/index.d.ts

@@ -69,12 +69,12 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * when `originJobId` is interrupted/expired, the items in the
      * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
      */
-    registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
+    registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd?: string, multi?: U): Promise<any>;
     /**
      * Ensures a `hook signal` is delisted when its parent activity/job
      * is interrupted/expired.
      */
-    registerSignalDependency(jobId: string, signalKey: string, multi?: U): Promise<any>;
+    registerSignalDependency(jobId: string, signalKey: string, dad: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
     getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
@@ -133,7 +133,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * for the given sleep group. Sleep groups are
      * organized into 'n'-second blocks (LISTS))
      */
-    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, multi?: U): Promise<void>;
+    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void>;
     getNextTask(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: WorkListTaskType] | boolean>;
     /**
      * when processing time jobs, the target LIST ID returned
@@ -141,7 +141,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * do with the work list. (not everything is known in advance,
      * so the ZSET key defines HOW to approach the work in the
      * generic LIST (lists typically contain target job ids)
-     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     * @param {string} listKey - composite key
      */
     resolveTaskKeyContext(listKey: string): [WorkListTaskType, string];
     /**
@@ -152,7 +152,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      */
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;
-    findJobs(queryString?: string, limit?: number, batchSize?: number): Promise<string[]>;
+    findJobs(queryString?: string, limit?: number, batchSize?: number): Promise<[string, string[]]>;
     findJobFields(jobId: string, fieldMatchPattern?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, StringStringType]>;
 }
 export { StoreService };

package/build/services/store/index.js

@@ -345,15 +345,20 @@ class StoreService {
      * when `originJobId` is interrupted/expired, the items in the
      * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
      */
-    async registerJobDependency(depType, originJobId, topic, jobId, gId, multi) {
+    async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
         const privateMulti = multi || this.getMulti();
         const dependencyParams = {
             appId: this.appId,
             jobId: originJobId,
         };
         const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
-        //items listed as job dependencies have different relationships
-        const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
+        const expireTask = [
+            depType,
+            topic,
+            gId,
+            pd,
+            jobId,
+        ].join(key_1.VALSEP);
         privateMulti[this.commands.rpush](depKey, expireTask);
         if (!multi) {
             return await privateMulti.exec();
@@ -363,12 +368,18 @@ class StoreService {
      * Ensures a `hook signal` is delisted when its parent activity/job
      * is interrupted/expired.
      */
-    async registerSignalDependency(jobId, signalKey, multi) {
+    async registerSignalDependency(jobId, signalKey, dad, multi) {
         const privateMulti = multi || this.getMulti();
         const dependencyParams = { appId: this.appId, jobId };
         const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
-        //tasks have '4' segments
-        const delistTask = `delist::signal::${jobId}::${signalKey}`;
+        //persiste dependency tasks as multi-segment composite keys
+        const delistTask = [
+            'delist',
+            'signal',
+            jobId,
+            dad,
+            signalKey
+        ].join(key_1.VALSEP);
         privateMulti[this.commands.rpush](dependencyKey, delistTask);
         if (!multi) {
             return await privateMulti.exec();
@@ -686,11 +697,14 @@ class StoreService {
     }
     async setHookSignal(hook, multi) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
-        const { topic, resolved, jobId } = hook; //`${activityId}::${dad}::${gId}::${jobId}`
+        //destructure the hook key
+        const { topic, resolved, jobId } = hook;
         const signalKey = `${topic}:${resolved}`;
         const payload = { [signalKey]: jobId };
         await (multi || this.redisClient)[this.commands.hset](key, payload);
-        return await this.registerSignalDependency(jobId.split('::')[3], signalKey, multi);
+        //jobId needs even more destructuring
+        const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
+        return await this.registerSignalDependency(jid, signalKey, dad, multi);
     }
     async getHookSignal(topic, resolved) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
@@ -755,7 +769,7 @@ class StoreService {
         const depParams = { appId: this.appId, jobId };
         const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
         const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
-        const depKeyContext = `::${context}::${depKey}`;
+        const depKeyContext = `${key_1.TYPSEP}${context}${key_1.TYPSEP}${depKey}`;
         const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
         await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
     }
@@ -770,9 +784,16 @@ class StoreService {
      * for the given sleep group. Sleep groups are
      * organized into 'n'-second blocks (LISTS))
      */
-    async registerTimeHook(jobId, gId, activityId, type, deletionTime, multi) {
+    async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
         const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
-        const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
+        //construct the composite key (the key has enough info to signal the hook)
+        const timeEvent = [
+            type,
+            activityId,
+            gId,
+            dad,
+            jobId
+        ].join(key_1.VALSEP);
         const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
         if (multi || len === 1) {
             const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
@@ -786,9 +807,9 @@ class StoreService {
             let [pType, pKey] = this.resolveTaskKeyContext(listKey);
             const timeEvent = await this.redisClient[this.commands.lpop](pKey);
             if (timeEvent) {
-                //there are task types
-                //1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
-                let [type, activityId, gId, ...jobId] = timeEvent.split('::');
+                //deconstruct composite key
+                let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
+                const jid = jobId.join(key_1.VALSEP);
                 if (type === 'delist') {
                     pType = 'delist';
                 }
@@ -796,9 +817,9 @@ class StoreService {
                     pType = 'child';
                 }
                 else if (type === 'expire-child') {
-                    type = 'expire'; //use the same logic as 'expire'
+                    type = 'expire';
                 }
-                return [listKey, jobId.join('::'), gId, activityId, pType];
+                return [listKey, jid, gId, activityId, pType];
             }
             await this.redisClient[this.commands.zrem](zsetKey, listKey);
             return true;
@@ -811,14 +832,14 @@ class StoreService {
      * do with the work list. (not everything is known in advance,
      * so the ZSET key defines HOW to approach the work in the
      * generic LIST (lists typically contain target job ids)
-     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     * @param {string} listKey - composite key
      */
     resolveTaskKeyContext(listKey) {
-        if (listKey.startsWith('::INTERRUPT')) {
-            return ['interrupt', listKey.split('::')[2]];
+        if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
+            return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
         }
-        else if (listKey.startsWith('::EXPIRE')) {
-            return ['expire', listKey.split('::')[2]];
+        else if (listKey.startsWith(`${key_1.TYPSEP}EXPIRE`)) {
+            return ['expire', listKey.split(key_1.TYPSEP)[2]];
         }
         else {
             return ['sleep', listKey];
@@ -898,7 +919,7 @@ class StoreService {
                 break;
             }
         } while (cursor !== '0');
-        return matchingKeys;
+        return [cursor, matchingKeys];
     }
     async findJobFields(jobId, fieldMatchPattern = '*', limit = 1000, batchSize = 1000, cursor = '0') {
         let fields = [];

package/build/services/task/index.d.ts

@@ -10,11 +10,12 @@ declare class TaskService {
     logger: ILogger;
     cleanupTimeout: NodeJS.Timeout | null;
     isScout: boolean;
+    errorCount: number;
     constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
     processWebHooks(hookEventCallback: HookInterface): Promise<void>;
     enqueueWorkItems(keys: string[]): Promise<void>;
     registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
-    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds?: number, multi?: RedisMulti): Promise<void>;
+    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
     /**
      * Should this engine instance play the role of 'scout' on behalf
      * of the entire quorum? The scout role is responsible for processing