@hotmeshio/hotmesh 0.0.36 → 0.0.38

package/build/services/engine/index.js

@@ -10,9 +10,8 @@ const utils_1 = require("../../modules/utils");
 const activities_1 = __importDefault(require("../activities"));
 const compiler_1 = require("../compiler");
 const reporter_1 = require("../reporter");
+const router_1 = require("../router");
 const serializer_1 = require("../serializer");
-const store_1 = require("../signaler/store");
-const stream_1 = require("../signaler/stream");
 const redis_1 = require("../store/clients/redis");
 const ioredis_1 = require("../store/clients/ioredis");
 const redis_2 = require("../stream/clients/redis");
@@ -20,7 +19,7 @@ const ioredis_2 = require("../stream/clients/ioredis");
 const ioredis_3 = require("../sub/clients/ioredis");
 const redis_3 = require("../sub/clients/redis");
 const task_1 = require("../task");
-const stream_2 = require("../../types/stream");
+const stream_1 = require("../../types/stream");
 class EngineService {
     constructor() {
         this.cacheMode = 'cache';
@@ -40,13 +39,10 @@ class EngineService {
             await instance.initStoreChannel(config.engine.store);
             await instance.initSubChannel(config.engine.sub);
             await instance.initStreamChannel(config.engine.stream);
-            instance.streamSignaler = instance.initStreamSignaler(config);
-            instance.streamSignaler.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
-            //the storeSignaler service is used by the engine to create `webhooks`
-            //todo: unify/move to the task service (it manages all `signal` types)
-            instance.storeSignaler = new store_1.StoreSignaler(instance.store, logger);
+            instance.router = instance.initRouter(config);
+            instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
             //the task service is used by the engine to process `webhooks` and `timehooks`
-            instance.task = new task_1.TaskService(instance.store, logger);
+            instance.taskService = new task_1.TaskService(instance.store, logger);
             return instance;
         }
     }
@@ -84,12 +80,12 @@ class EngineService {
         }
         await this.stream.init(this.namespace, this.appId, this.logger);
     }
-    initStreamSignaler(config) {
-        return new stream_1.StreamSignaler({
+    initRouter(config) {
+        return new router_1.Router({
             namespace: this.namespace,
             appId: this.appId,
             guid: this.guid,
-            role: stream_2.StreamRole.ENGINE,
+            role: stream_1.StreamRole.ENGINE,
             reclaimDelay: config.engine.reclaimDelay,
             reclaimCount: config.engine.reclaimCount,
         }, this.stream, this.store, this.logger);
@@ -128,18 +124,17 @@ class EngineService {
         }
     }
     async processWebHooks() {
-        this.task.processWebHooks((this.hook).bind(this));
+        this.taskService.processWebHooks((this.hook).bind(this));
     }
     async processTimeHooks() {
-        this.task.processTimeHooks((this.hookTime).bind(this));
+        this.taskService.processTimeHooks((this.hookTime).bind(this));
     }
     async throttle(delayInMillis) {
-        this.streamSignaler.setThrottle(delayInMillis);
+        this.router.setThrottle(delayInMillis);
     }
     // ************* METADATA/MODEL METHODS *************
     async initActivity(topic, data = {}, context) {
         const [activityId, schema] = await this.getSchema(topic);
-        utils_1.polyfill;
         const ActivityHandler = activities_1.default[utils_1.polyfill.resolveActivityType(schema.type)];
         if (ActivityHandler) {
             const utc = (0, utils_1.formatISODate)(new Date());
@@ -222,37 +217,43 @@ class EngineService {
     async processStreamMessage(streamData) {
         this.logger.debug('engine-process-stream-message', {
             jid: streamData.metadata.jid,
+            gid: streamData.metadata.gid,
             dad: streamData.metadata.dad,
             aid: streamData.metadata.aid,
-            status: streamData.status || stream_2.StreamStatus.SUCCESS,
+            status: streamData.status || stream_1.StreamStatus.SUCCESS,
             code: streamData.code || 200,
             type: streamData.type,
         });
         const context = {
             metadata: {
                 jid: streamData.metadata.jid,
+                gid: streamData.metadata.gid,
                 dad: streamData.metadata.dad,
                 aid: streamData.metadata.aid,
             },
             data: streamData.data,
         };
-        if (streamData.type === stream_2.StreamDataType.TIMEHOOK || streamData.type === stream_2.StreamDataType.WEBHOOK || streamData.type === stream_2.StreamDataType.TRANSITION) {
+        if (streamData.type === stream_1.StreamDataType.TIMEHOOK) {
+            //TIMEHOOK AWAKEN
             const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
-            if (streamData.type === stream_2.StreamDataType.TIMEHOOK) {
-                await activityHandler.processTimeHookEvent(streamData.metadata.jid);
-            }
-            else if (streamData.type === stream_2.StreamDataType.TRANSITION) {
-                await activityHandler.process();
-            }
-            else {
-                //a 202 code keeps the hook alive (hooks are single-use by default)
-                await activityHandler.processWebHookEvent(streamData.status, streamData.code);
-            }
+            await activityHandler.processTimeHookEvent(streamData.metadata.jid);
+        }
+        else if (streamData.type === stream_1.StreamDataType.WEBHOOK) {
+            //WEBHOOK AWAKEN (SIGNAL IN)
+            const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
+            await activityHandler.processWebHookEvent(streamData.status, streamData.code);
+        }
+        else if (streamData.type === stream_1.StreamDataType.TRANSITION) {
+            //TRANSITION (ADJACENT ACTIVITY)
+            const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context); //todo: `as Activity` (type is more generic)
+            await activityHandler.process();
         }
-        else if (streamData.type === stream_2.StreamDataType.AWAIT) {
+        else if (streamData.type === stream_1.StreamDataType.AWAIT) {
+            //TRIGGER JOB
             context.metadata = {
                 ...context.metadata,
                 pj: streamData.metadata.jid,
+                pg: streamData.metadata.gid,
                 pd: streamData.metadata.dad,
                 pa: streamData.metadata.aid,
                 trc: streamData.metadata.trc,
@@ -261,16 +262,19 @@ class EngineService {
             const activityHandler = await this.initActivity(streamData.metadata.topic, streamData.data, context);
             await activityHandler.process();
         }
-        else if (streamData.type === stream_2.StreamDataType.RESULT) {
+        else if (streamData.type === stream_1.StreamDataType.RESULT) {
+            //AWAIT RESULT
             const activityHandler = await this.initActivity(`.${context.metadata.aid}`, streamData.data, context);
             await activityHandler.processEvent(streamData.status, streamData.code);
         }
         else {
+            //WORKER RESULT
             const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, streamData.data, context);
             await activityHandler.processEvent(streamData.status, streamData.code, 'output');
         }
         this.logger.debug('engine-process-stream-message-end', {
             jid: streamData.metadata.jid,
+            gid: streamData.metadata.gid,
             aid: streamData.metadata.aid
         });
     }
@@ -284,28 +288,29 @@ class EngineService {
                 metadata: {
                     guid: (0, utils_1.guid)(),
                     jid: context.metadata.pj,
+                    gid: context.metadata.pg,
                     dad: context.metadata.pd,
                     aid: context.metadata.pa,
                     trc: context.metadata.trc,
                     spn,
                 },
-                type: stream_2.StreamDataType.RESULT,
+                type: stream_1.StreamDataType.RESULT,
                 data: jobOutput.data,
             };
             if (error && error.code) {
-                streamData.status = stream_2.StreamStatus.ERROR;
+                streamData.status = stream_1.StreamStatus.ERROR;
                 streamData.data = error;
                 streamData.code = error.code;
             }
             else if (emit) {
-                streamData.status = stream_2.StreamStatus.PENDING;
+                streamData.status = stream_1.StreamStatus.PENDING;
                 streamData.code = enums_1.STATUS_CODE_PENDING;
             }
             else {
-                streamData.status = stream_2.StreamStatus.SUCCESS;
+                streamData.status = stream_1.StreamStatus.SUCCESS;
                 streamData.code = enums_1.STATUS_CODE_SUCCESS;
             }
-            return (await this.streamSignaler?.publishMessage(null, streamData));
+            return (await this.router?.publishMessage(null, streamData));
         }
     }
     hasParentJob(context) {
@@ -332,11 +337,11 @@ class EngineService {
         await this.store.scrub(jobId);
     }
     // ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
-    async hook(topic, data, status = stream_2.StreamStatus.SUCCESS, code = 200) {
-        const hookRule = await this.storeSignaler.getHookRule(topic);
+    async hook(topic, data, status = stream_1.StreamStatus.SUCCESS, code = 200) {
+        const hookRule = await this.taskService.getHookRule(topic);
         const [aid] = await this.getSchema(`.${hookRule.to}`);
         const streamData = {
-            type: stream_2.StreamDataType.WEBHOOK,
+            type: stream_1.StreamDataType.WEBHOOK,
             status,
             code,
             metadata: {
@@ -346,9 +351,9 @@ class EngineService {
             },
             data,
         };
-        return await this.streamSignaler.publishMessage(null, streamData);
+        return await this.router.publishMessage(null, streamData);
     }
-    async hookTime(jobId, activityId, type) {
+    async hookTime(jobId, gId, activityId, type) {
         if (type === 'interrupt') {
             return await this.interrupt(activityId, //note: 'activityId' is the actually job topic
             jobId, { suppress: true, expire: 1 });
@@ -360,20 +365,21 @@ class EngineService {
         const [aid, ...dimensions] = activityId.split(',');
         const dad = `,${dimensions.join(',')}`;
         const streamData = {
-            type: stream_2.StreamDataType.TIMEHOOK,
+            type: stream_1.StreamDataType.TIMEHOOK,
             metadata: {
                 guid: (0, utils_1.guid)(),
                 jid: jobId,
-                aid,
+                gid: gId,
                 dad,
+                aid,
             },
             data: { timestamp: Date.now() },
         };
-        await this.streamSignaler.publishMessage(null, streamData);
+        await this.router.publishMessage(null, streamData);
     }
     async hookAll(hookTopic, data, keyResolver, queryFacets = []) {
         const config = await this.getVID();
-        const hookRule = await this.storeSignaler.getHookRule(hookTopic);
+        const hookRule = await this.taskService.getHookRule(hookTopic);
         if (hookRule) {
             const subscriptionTopic = await (0, utils_1.getSubscriptionTopic)(hookRule.to, this.store, config);
             const resolvedQuery = await this.resolveQuery(subscriptionTopic, keyResolver);
@@ -485,7 +491,7 @@ class EngineService {
         }
     }
     async add(streamData) {
-        return await this.streamSignaler.publishMessage(null, streamData);
+        return await this.router.publishMessage(null, streamData);
     }
     registerJobCallback(jobId, jobCallback) {
         this.jobCallbacks[jobId] = jobCallback;
@@ -510,7 +516,7 @@ class EngineService {
             this.pubPermSubs(context, jobOutput, options.emit);
         }
         if (!options.emit) {
-            this.task.registerJobForCleanup(context.metadata.jid, this.resolveExpires(context, options), options);
+            this.taskService.registerJobForCleanup(context.metadata.jid, this.resolveExpires(context, options), options);
         }
         return msgId;
     }
@@ -520,7 +526,7 @@ class EngineService {
      * it will be expired immediately.
      */
     resolveExpires(context, options) {
-        return context.metadata.expire ?? options.expire ?? enums_1.DURABLE_EXPIRE_SECONDS;
+        return options.expire ?? context.metadata.expire ?? enums_1.DURABLE_EXPIRE_SECONDS;
     }
     // ****** GET JOB STATE/COLLATION STATUS BY ID *********
     async getStatus(jobId) {

package/build/services/hotmesh/index.d.ts

@@ -4,7 +4,7 @@ import { QuorumService } from '../quorum';
 import { WorkerService } from '../worker';
 import { JobState, JobData, JobOutput, JobStatus, JobInterruptOptions } from '../../types/job';
 import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback } from '../../types/quorum';
+import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
 import { StringAnyType } from '../../types/serializer';
@@ -31,10 +31,10 @@ declare class HotMeshService {
     punsub(wild: string): Promise<void>;
     pubsub(topic: string, data?: JobData, context?: JobState | null, timeout?: number): Promise<JobOutput>;
     add(streamData: StreamData | StreamDataResponse): Promise<string>;
+    rollCall(delay?: number): Promise<QuorumProfile[]>;
     plan(path: string): Promise<HotMeshManifest>;
     deploy(pathOrYAML: string): Promise<HotMeshManifest>;
     activate(version: string, delay?: number): Promise<boolean>;
-    inventory(version: string, delay?: number): Promise<number>;
     getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;

package/build/services/hotmesh/index.js

@@ -7,8 +7,8 @@ const redis_1 = require("../connector/clients/redis");
 const ioredis_1 = require("../connector/clients/ioredis");
 const engine_1 = require("../engine");
 const logger_1 = require("../logger");
-const stream_1 = require("../signaler/stream");
 const quorum_1 = require("../quorum");
+const router_1 = require("../router");
 const worker_1 = require("../worker");
 const connector_1 = require("../connector");
 class HotMeshService {
@@ -90,6 +90,9 @@ class HotMeshService {
         return await this.engine.add(streamData);
     }
     // ************* COMPILER METHODS *************
+    async rollCall(delay) {
+        return await this.quorum?.rollCall(delay);
+    }
     async plan(path) {
         return await this.engine?.plan(path);
     }
@@ -100,10 +103,6 @@ class HotMeshService {
         //activation is a quorum operation
         return await this.quorum?.activate(version, delay);
     }
-    async inventory(version, delay) {
-        //get count of all peers
-        return await this.quorum?.inventory(delay);
-    }
     // ************* REPORTER METHODS *************
     async getStats(topic, query) {
         return await this.engine?.getStats(topic, query);
@@ -141,13 +140,13 @@ class HotMeshService {
     static async stop() {
         if (!this.disconnecting) {
             this.disconnecting = true;
-            await stream_1.StreamSignaler.stopConsuming();
+            await router_1.Router.stopConsuming();
             await redis_1.RedisConnection.disconnectAll();
             await ioredis_1.RedisConnection.disconnectAll();
         }
     }
     stop() {
-        this.engine?.task.cancelCleanup();
+        this.engine?.taskService.cancelCleanup();
     }
     async compress(terms) {
         return await this.engine?.compress(terms);

package/build/services/quorum/index.d.ts

@@ -3,15 +3,15 @@ import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
-import { QuorumMessageCallback, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
-import { HotMeshApps, HotMeshConfig } from '../../types/hotmesh';
+import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
+import { HotMeshConfig } from '../../types/hotmesh';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
-    apps: HotMeshApps | null;
     appId: string;
     guid: string;
     engine: EngineService;
+    profiles: QuorumProfile[];
     store: StoreService<RedisClient, RedisMulti> | null;
     subscribe: SubService<RedisClient, RedisMulti> | null;
     logger: ILogger;
@@ -24,12 +24,12 @@ declare class QuorumService {
     initStoreChannel(store: RedisClient): Promise<void>;
     initSubChannel(sub: RedisClient): Promise<void>;
     subscriptionHandler(): SubscriptionCallback;
-    sayPong(appId: string, guid: string, originator: string): Promise<void>;
-    requestQuorum(delay?: number): Promise<number>;
+    sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
+    requestQuorum(delay?: number, details?: boolean): Promise<number>;
     pub(quorumMessage: ThrottleMessage): Promise<boolean>;
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
-    inventory(delay?: number): Promise<number>;
+    rollCall(delay?: number): Promise<QuorumProfile[]>;
     activate(version: string, delay?: number): Promise<boolean>;
 }
 export { QuorumService };

package/build/services/quorum/index.js

@@ -12,6 +12,7 @@ const redis_2 = require("../sub/clients/redis");
 const QUORUM_DELAY = 250;
 class QuorumService {
     constructor() {
+        this.profiles = [];
         this.cacheMode = 'cache';
         this.untilVersion = null;
         this.quorum = null;
@@ -68,10 +69,13 @@ class QuorumService {
                 self.engine.setCacheMode(message.cache_mode, message.until_version);
             }
             else if (message.type === 'ping') {
-                this.sayPong(self.appId, self.guid, message.originator);
+                self.sayPong(self.appId, self.guid, message.originator, message.details);
             }
             else if (message.type === 'pong' && self.guid === message.originator) {
                 self.quorum = self.quorum + 1;
+                if (message.profile) {
+                    self.profiles.push(message.profile);
+                }
             }
             else if (message.type === 'throttle') {
                 self.engine.throttle(message.throttle);
@@ -91,13 +95,31 @@ class QuorumService {
             }
         };
     }
-    async sayPong(appId, guid, originator) {
-        this.store.publish(key_1.KeyType.QUORUM, { type: 'pong', guid, originator }, appId);
+    async sayPong(appId, guid, originator, details = false) {
+        let profile;
+        if (details) {
+            profile = {
+                engine_id: this.guid,
+                namespace: this.namespace,
+                app_id: this.appId,
+                stream: this.engine.stream.mintKey(key_1.KeyType.STREAMS, { appId: this.appId })
+            };
+        }
+        this.store.publish(key_1.KeyType.QUORUM, {
+            type: 'pong',
+            guid, originator,
+            profile,
+        }, appId);
     }
-    async requestQuorum(delay = QUORUM_DELAY) {
+    async requestQuorum(delay = QUORUM_DELAY, details = false) {
         const quorum = this.quorum;
         this.quorum = 0;
-        await this.store.publish(key_1.KeyType.QUORUM, { type: 'ping', originator: this.guid }, this.appId);
+        this.profiles.length = 0;
+        await this.store.publish(key_1.KeyType.QUORUM, {
+            type: 'ping',
+            originator: this.guid,
+            details,
+        }, this.appId);
         await (0, utils_1.sleepFor)(delay);
         return quorum;
     }
@@ -117,12 +139,26 @@ class QuorumService {
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
     // ************* COMPILER METHODS *************
-    async inventory(delay = QUORUM_DELAY) {
-        await this.requestQuorum(delay);
-        const q1 = await this.requestQuorum(delay);
-        const q2 = await this.requestQuorum(delay);
-        const q3 = await this.requestQuorum(delay);
-        return Math.round((q1 + q2 + q3) / 3);
+    async rollCall(delay = QUORUM_DELAY) {
+        await this.requestQuorum(delay, true);
+        const targetStreams = [];
+        const multi = this.store.getMulti();
+        this.profiles.forEach((profile) => {
+            if (!targetStreams.includes(profile.stream)) {
+                targetStreams.push(profile.stream);
+                this.store.xlen(profile.stream, multi);
+            }
+        });
+        const stream_depths = await multi.exec();
+        this.profiles.forEach(async (profile) => {
+            const index = targetStreams.indexOf(profile.stream);
+            if (index != -1) {
+                profile.stream_depth = Array.isArray(stream_depths[index]) ?
+                    stream_depths[index][1] :
+                    stream_depths[index];
+            }
+        });
+        return this.profiles;
     }
     async activate(version, delay = QUORUM_DELAY) {
         version = version.toString();

package/build/services/{signaler/stream.d.ts → router/index.d.ts}

@@ -4,8 +4,8 @@ import { StoreService } from '../store';
 import { StreamService } from '../stream';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { ReclaimedMessageType, StreamConfig, StreamData, StreamDataResponse, StreamRole } from '../../types/stream';
-declare class StreamSignaler {
-    static signalers: Set<StreamSignaler>;
+declare class Router {
+    static instances: Set<Router>;
     appId: string;
     guid: string;
     role: StreamRole;
@@ -40,4 +40,4 @@ declare class StreamSignaler {
     expireUnacknowledged(reclaimedMessage: ReclaimedMessageType, stream: string, group: string, consumer: string, id: string, count: number): Promise<void>;
     parseStreamData(str: string): [undefined, StreamData] | [Error];
 }
-export { StreamSignaler };
+export { Router };

package/build/services/{signaler/stream.js → router/index.js}

@@ -1,12 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.StreamSignaler = void 0;
+exports.Router = void 0;
 const enums_1 = require("../../modules/enums");
 const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-class StreamSignaler {
+class Router {
     constructor(config, stream, store, logger) {
         this.throttle = 0;
         this.errorCount = 0;
@@ -35,7 +35,7 @@ class StreamSignaler {
     }
     async consumeMessages(stream, group, consumer, callback) {
         this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
-        StreamSignaler.signalers.add(this);
+        Router.instances.add(this);
         this.shouldConsume = true;
         await this.createGroup(stream, group);
         let lastCheckedPendingMessagesAt = Date.now();
@@ -213,7 +213,7 @@ class StreamSignaler {
         };
     }
     static async stopConsuming() {
-        for (const instance of [...StreamSignaler.signalers]) {
+        for (const instance of [...Router.instances]) {
             instance.stopConsuming();
         }
         await (0, utils_1.sleepFor)(enums_1.BLOCK_TIME_MS);
@@ -311,5 +311,5 @@ class StreamSignaler {
         }
     }
 }
-exports.StreamSignaler = StreamSignaler;
-StreamSignaler.signalers = new Set();
+exports.Router = Router;
+Router.instances = new Set();

package/build/services/serializer/index.js

@@ -12,7 +12,7 @@ exports.MDATA_SYMBOLS = {
         KEYS: ['au', 'err', 'l2s']
     },
     JOB: {
-        KEYS: ['ngn', 'tpc', 'pj', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+        KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
     },
     JOB_UPDATE: {
         KEYS: ['ju', 'err']

package/build/services/store/clients/ioredis.d.ts

@@ -24,5 +24,6 @@ declare class IORedisStoreService extends StoreService<RedisClientType, RedisMul
     xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     xack(key: string, group: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
     xdel(key: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
+    xlen(key: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
 }
 export { IORedisStoreService };

package/build/services/store/clients/ioredis.js

@@ -105,5 +105,14 @@ class IORedisStoreService extends index_1.StoreService {
             throw error;
         }
     }
+    async xlen(key, multi) {
+        try {
+            return await (multi || this.redisClient).xlen(key);
+        }
+        catch (error) {
+            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            throw error;
+        }
+    }
 }
 exports.IORedisStoreService = IORedisStoreService;

package/build/services/store/clients/redis.d.ts

@@ -26,5 +26,6 @@ declare class RedisStoreService extends StoreService<RedisClientType, RedisMulti
     xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     xack(key: string, group: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
     xdel(key: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
+    xlen(key: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
 }
 export { RedisStoreService };

package/build/services/store/clients/redis.js

@@ -30,6 +30,7 @@ class RedisStoreService extends index_1.StoreService {
             rpush: 'RPUSH',
             xack: 'XACK',
             xdel: 'XDEL',
+            xlen: 'XLEN',
         };
     }
     getMulti() {
@@ -141,5 +142,20 @@ class RedisStoreService extends index_1.StoreService {
             throw error;
         }
     }
+    async xlen(key, multi) {
+        try {
+            if (multi) {
+                multi.XLEN(key);
+                return multi;
+            }
+            else {
+                return await this.redisClient.XLEN(key);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            throw error;
+        }
+    }
 }
 exports.RedisStoreService = RedisStoreService;

package/build/services/store/index.d.ts

@@ -31,6 +31,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     abstract xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     abstract xack(key: string, group: string, id: string, multi?: U): Promise<number | U>;
     abstract xdel(key: string, id: string, multi?: U): Promise<number | U>;
+    abstract xlen(key: string, multi?: U): Promise<number | U>;
     constructor(redisClient: T);
     init(namespace: string, appId: string, logger: ILogger): Promise<HotMeshApps>;
     isSuccessful(result: any): boolean;
@@ -39,7 +40,12 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null>;
     mintKey(type: KeyType, params: KeyStoreParams): string;
     invalidateCache(): void;
-    reserveEngineId(engineId: string): Promise<boolean>;
+    /**
+     * At any given time only a single engine will
+     * check for and process work items in the
+     * time and signal task queues.
+     */
+    reserveScoutRole(scoutType: 'time' | 'signal', delay?: number): Promise<boolean>;
     getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
     setSettings(manifest: HotMeshSettings): Promise<any>;
     reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;
@@ -61,7 +67,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * list (added via RPUSH) are LPOPed. If origin was expired, then
      * LPOPed items from the list are likewise expired;
      */
-    setDependency(originJobId: string, topic: string, jobId: string, multi?: U): Promise<any>;
+    setDependency(originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
     getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
@@ -107,8 +113,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * for the given sleep group. Sleep groups are
      * organized into 'n'-second blocks (LISTS))
      */
-    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', deletionTime: number, multi?: U): Promise<void>;
-    getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt'] | void>;
+    registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', deletionTime: number, multi?: U): Promise<void>;
+    getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt'] | boolean>;
     /**
      * when processing time jobs, the target LIST ID returned
      * from the ZSET query can be prefixed to denote what to