@hotmeshio/hotmesh 0.0.37 → 0.0.38

package/README.md

@@ -15,7 +15,7 @@ npm install @hotmeshio/hotmesh
 ## Understanding HotMesh
 HotMesh inverts the relationship to Redis: those functions that once used Redis as a cache, are instead *cached and governed* by Redis. Consider the following. It's a typical microservices network, with a tangled mess of services and functions. There's important business logic in there (functions *A*, *B* and *C* are critical!), but they're hard to find and access.
 
-<img src="https://github.com/hotmeshio/sdk-typescript/tree/main/docs/img/operational_data_layer.png" alt="A Tangled Microservices Network with 3 functions buried within" style="max-width:100%;width:600px;">
+<img src="./docs/img/operational_data_layer.png" alt="A Tangled Microservices Network with 3 valuable functions buried within" style="max-width:100%;width:600px;">
 
 HotMesh creates an *ad hoc*, Redis-backed network of functions and organizes them into a unified service mesh. *Any service with access to Redis can join in the network, bypassing the legacy clutter.*
 

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.37",
+    "version": "0.0.38",
     "description": "Unbreakable Workflows",
     "main": "build/index.js",
     "types": "build/index.d.ts",

package/build/services/hotmesh/index.d.ts

@@ -4,7 +4,7 @@ import { QuorumService } from '../quorum';
 import { WorkerService } from '../worker';
 import { JobState, JobData, JobOutput, JobStatus, JobInterruptOptions } from '../../types/job';
 import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback } from '../../types/quorum';
+import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
 import { StringAnyType } from '../../types/serializer';
@@ -31,10 +31,10 @@ declare class HotMeshService {
     punsub(wild: string): Promise<void>;
     pubsub(topic: string, data?: JobData, context?: JobState | null, timeout?: number): Promise<JobOutput>;
     add(streamData: StreamData | StreamDataResponse): Promise<string>;
+    rollCall(delay?: number): Promise<QuorumProfile[]>;
     plan(path: string): Promise<HotMeshManifest>;
     deploy(pathOrYAML: string): Promise<HotMeshManifest>;
     activate(version: string, delay?: number): Promise<boolean>;
-    inventory(version: string, delay?: number): Promise<number>;
     getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;

package/build/services/hotmesh/index.js

@@ -90,6 +90,9 @@ class HotMeshService {
         return await this.engine.add(streamData);
     }
     // ************* COMPILER METHODS *************
+    async rollCall(delay) {
+        return await this.quorum?.rollCall(delay);
+    }
     async plan(path) {
         return await this.engine?.plan(path);
     }
@@ -100,10 +103,6 @@ class HotMeshService {
         //activation is a quorum operation
         return await this.quorum?.activate(version, delay);
     }
-    async inventory(version, delay) {
-        //get count of all peers
-        return await this.quorum?.inventory(delay);
-    }
     // ************* REPORTER METHODS *************
     async getStats(topic, query) {
         return await this.engine?.getStats(topic, query);

package/build/services/quorum/index.d.ts

@@ -3,15 +3,15 @@ import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
-import { QuorumMessageCallback, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
-import { HotMeshApps, HotMeshConfig } from '../../types/hotmesh';
+import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
+import { HotMeshConfig } from '../../types/hotmesh';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
-    apps: HotMeshApps | null;
     appId: string;
     guid: string;
     engine: EngineService;
+    profiles: QuorumProfile[];
     store: StoreService<RedisClient, RedisMulti> | null;
     subscribe: SubService<RedisClient, RedisMulti> | null;
     logger: ILogger;
@@ -24,12 +24,12 @@ declare class QuorumService {
     initStoreChannel(store: RedisClient): Promise<void>;
     initSubChannel(sub: RedisClient): Promise<void>;
     subscriptionHandler(): SubscriptionCallback;
-    sayPong(appId: string, guid: string, originator: string): Promise<void>;
-    requestQuorum(delay?: number): Promise<number>;
+    sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
+    requestQuorum(delay?: number, details?: boolean): Promise<number>;
     pub(quorumMessage: ThrottleMessage): Promise<boolean>;
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
-    inventory(delay?: number): Promise<number>;
+    rollCall(delay?: number): Promise<QuorumProfile[]>;
     activate(version: string, delay?: number): Promise<boolean>;
 }
 export { QuorumService };

package/build/services/quorum/index.js

@@ -12,6 +12,7 @@ const redis_2 = require("../sub/clients/redis");
 const QUORUM_DELAY = 250;
 class QuorumService {
     constructor() {
+        this.profiles = [];
         this.cacheMode = 'cache';
         this.untilVersion = null;
         this.quorum = null;
@@ -68,10 +69,13 @@ class QuorumService {
                 self.engine.setCacheMode(message.cache_mode, message.until_version);
             }
             else if (message.type === 'ping') {
-                this.sayPong(self.appId, self.guid, message.originator);
+                self.sayPong(self.appId, self.guid, message.originator, message.details);
             }
             else if (message.type === 'pong' && self.guid === message.originator) {
                 self.quorum = self.quorum + 1;
+                if (message.profile) {
+                    self.profiles.push(message.profile);
+                }
             }
             else if (message.type === 'throttle') {
                 self.engine.throttle(message.throttle);
@@ -91,13 +95,31 @@ class QuorumService {
             }
         };
     }
-    async sayPong(appId, guid, originator) {
-        this.store.publish(key_1.KeyType.QUORUM, { type: 'pong', guid, originator }, appId);
+    async sayPong(appId, guid, originator, details = false) {
+        let profile;
+        if (details) {
+            profile = {
+                engine_id: this.guid,
+                namespace: this.namespace,
+                app_id: this.appId,
+                stream: this.engine.stream.mintKey(key_1.KeyType.STREAMS, { appId: this.appId })
+            };
+        }
+        this.store.publish(key_1.KeyType.QUORUM, {
+            type: 'pong',
+            guid, originator,
+            profile,
+        }, appId);
     }
-    async requestQuorum(delay = QUORUM_DELAY) {
+    async requestQuorum(delay = QUORUM_DELAY, details = false) {
         const quorum = this.quorum;
         this.quorum = 0;
-        await this.store.publish(key_1.KeyType.QUORUM, { type: 'ping', originator: this.guid }, this.appId);
+        this.profiles.length = 0;
+        await this.store.publish(key_1.KeyType.QUORUM, {
+            type: 'ping',
+            originator: this.guid,
+            details,
+        }, this.appId);
         await (0, utils_1.sleepFor)(delay);
         return quorum;
     }
@@ -117,12 +139,26 @@ class QuorumService {
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
     // ************* COMPILER METHODS *************
-    async inventory(delay = QUORUM_DELAY) {
-        await this.requestQuorum(delay);
-        const q1 = await this.requestQuorum(delay);
-        const q2 = await this.requestQuorum(delay);
-        const q3 = await this.requestQuorum(delay);
-        return Math.round((q1 + q2 + q3) / 3);
+    async rollCall(delay = QUORUM_DELAY) {
+        await this.requestQuorum(delay, true);
+        const targetStreams = [];
+        const multi = this.store.getMulti();
+        this.profiles.forEach((profile) => {
+            if (!targetStreams.includes(profile.stream)) {
+                targetStreams.push(profile.stream);
+                this.store.xlen(profile.stream, multi);
+            }
+        });
+        const stream_depths = await multi.exec();
+        this.profiles.forEach(async (profile) => {
+            const index = targetStreams.indexOf(profile.stream);
+            if (index != -1) {
+                profile.stream_depth = Array.isArray(stream_depths[index]) ?
+                    stream_depths[index][1] :
+                    stream_depths[index];
+            }
+        });
+        return this.profiles;
     }
     async activate(version, delay = QUORUM_DELAY) {
         version = version.toString();

package/build/services/store/clients/ioredis.d.ts

@@ -24,5 +24,6 @@ declare class IORedisStoreService extends StoreService<RedisClientType, RedisMul
     xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     xack(key: string, group: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
     xdel(key: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
+    xlen(key: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
 }
 export { IORedisStoreService };

package/build/services/store/clients/ioredis.js

@@ -105,5 +105,14 @@ class IORedisStoreService extends index_1.StoreService {
             throw error;
         }
     }
+    async xlen(key, multi) {
+        try {
+            return await (multi || this.redisClient).xlen(key);
+        }
+        catch (error) {
+            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            throw error;
+        }
+    }
 }
 exports.IORedisStoreService = IORedisStoreService;

package/build/services/store/clients/redis.d.ts

@@ -26,5 +26,6 @@ declare class RedisStoreService extends StoreService<RedisClientType, RedisMulti
     xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     xack(key: string, group: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
     xdel(key: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
+    xlen(key: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
 }
 export { RedisStoreService };

package/build/services/store/clients/redis.js

@@ -30,6 +30,7 @@ class RedisStoreService extends index_1.StoreService {
             rpush: 'RPUSH',
             xack: 'XACK',
             xdel: 'XDEL',
+            xlen: 'XLEN',
         };
     }
     getMulti() {
@@ -141,5 +142,20 @@ class RedisStoreService extends index_1.StoreService {
             throw error;
         }
     }
+    async xlen(key, multi) {
+        try {
+            if (multi) {
+                multi.XLEN(key);
+                return multi;
+            }
+            else {
+                return await this.redisClient.XLEN(key);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            throw error;
+        }
+    }
 }
 exports.RedisStoreService = RedisStoreService;

package/build/services/store/index.d.ts

@@ -31,6 +31,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     abstract xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     abstract xack(key: string, group: string, id: string, multi?: U): Promise<number | U>;
     abstract xdel(key: string, id: string, multi?: U): Promise<number | U>;
+    abstract xlen(key: string, multi?: U): Promise<number | U>;
     constructor(redisClient: T);
     init(namespace: string, appId: string, logger: ILogger): Promise<HotMeshApps>;
     isSuccessful(result: any): boolean;

package/build/services/stream/clients/ioredis.d.ts

@@ -19,5 +19,6 @@ declare class IORedisStreamService extends StreamService<RedisClientType, RedisM
     xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     xack(key: string, group: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
     xdel(key: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
+    xlen(key: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
 }
 export { IORedisStreamService };

package/build/services/stream/clients/ioredis.js

@@ -25,18 +25,18 @@ class IORedisStreamService extends index_1.StreamService {
             try {
                 return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
             }
-            catch (err) {
+            catch (error) {
                 this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
-                throw err;
+                throw error;
             }
         }
         else {
             try {
                 return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
             }
-            catch (err) {
+            catch (error) {
                 this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
-                throw err;
+                throw error;
             }
         }
     }
@@ -44,9 +44,9 @@ class IORedisStreamService extends index_1.StreamService {
         try {
             return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
         }
-        catch (err) {
-            this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
-            throw err;
+        catch (error) {
+            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+            throw error;
         }
     }
     async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
@@ -56,9 +56,9 @@ class IORedisStreamService extends index_1.StreamService {
             // @ts-ignore
             blockOption, blockTime, streamsOption, streamName, id);
         }
-        catch (err) {
-            this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, err);
-            throw err;
+        catch (error) {
+            this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { error });
+            throw error;
         }
     }
     async xpending(key, group, start, end, count, consumer) {
@@ -75,40 +75,49 @@ class IORedisStreamService extends index_1.StreamService {
             try {
                 return await this.redisClient.call('XPENDING', ...args);
             }
-            catch (err) {
-                this.logger.error('err, args', err, args);
+            catch (error) {
+                this.logger.error('err, args', { error }, args);
             }
         }
-        catch (err) {
-            this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, err);
-            throw err;
+        catch (error) {
+            this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { error });
+            throw error;
         }
     }
     async xclaim(key, group, consumer, minIdleTime, id, ...args) {
         try {
             return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args);
         }
-        catch (err) {
-            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
-            throw err;
+        catch (error) {
+            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+            throw error;
         }
     }
     async xack(key, group, id, multi) {
         try {
             return await (multi || this.redisClient).xack(key, group, id);
         }
-        catch (err) {
-            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
-            throw err;
+        catch (error) {
+            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+            throw error;
         }
     }
     async xdel(key, id, multi) {
         try {
             return await (multi || this.redisClient).xdel(key, id);
         }
-        catch (err) {
-            this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, err);
-            throw err;
+        catch (error) {
+            this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async xlen(key, multi) {
+        try {
+            return await (multi || this.redisClient).xlen(key);
+        }
+        catch (error) {
+            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            throw error;
         }
     }
 }

package/build/services/stream/clients/redis.d.ts

@@ -19,5 +19,6 @@ declare class RedisStreamService extends StreamService<RedisClientType, RedisMul
     xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     xack(key: string, group: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
     xdel(key: string, id: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
+    xlen(key: string, multi?: RedisMultiType): Promise<number | RedisMultiType>;
 }
 export { RedisStreamService };

package/build/services/stream/clients/redis.js

@@ -115,5 +115,20 @@ class RedisStreamService extends index_1.StreamService {
             throw err;
         }
     }
+    async xlen(key, multi) {
+        try {
+            if (multi) {
+                multi.XLEN(key);
+                return multi;
+            }
+            else {
+                return await this.redisClient.XLEN(key);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            throw error;
+        }
+    }
 }
 exports.RedisStreamService = RedisStreamService;

package/build/services/stream/index.d.ts

@@ -17,5 +17,6 @@ declare abstract class StreamService<T, U> {
     abstract xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<ReclaimedMessageType>;
     abstract xack(key: string, group: string, id: string, multi?: U): Promise<number | U>;
     abstract xdel(key: string, id: string, multi?: U): Promise<number | U>;
+    abstract xlen(key: string, multi?: U): Promise<number | U>;
 }
 export { StreamService };

package/build/services/worker/index.d.ts

@@ -25,6 +25,7 @@ declare class WorkerService {
     initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
     initRouter(worker: HotMeshWorker, logger: ILogger): Router;
     subscriptionHandler(): SubscriptionCallback;
+    sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
     throttle(delayInMillis: number): Promise<void>;
 }
 export { WorkerService };

package/build/services/worker/index.js

@@ -96,8 +96,32 @@ class WorkerService {
             if (message.type === 'throttle') {
                 self.throttle(message.throttle);
             }
+            else if (message.type === 'ping') {
+                self.sayPong(self.appId, self.guid, message.originator, message.details);
+            }
         };
     }
+    async sayPong(appId, guid, originator, details = false) {
+        let profile;
+        if (details) {
+            const params = {
+                appId: this.appId,
+                topic: this.topic,
+            };
+            profile = {
+                engine_id: this.guid,
+                namespace: this.namespace,
+                app_id: this.appId,
+                worker_topic: this.topic,
+                stream: this.stream.mintKey(key_1.KeyType.STREAMS, params),
+            };
+        }
+        this.store.publish(key_1.KeyType.QUORUM, {
+            type: 'pong',
+            guid, originator,
+            profile,
+        }, appId);
+    }
     async throttle(delayInMillis) {
         this.router.setThrottle(delayInMillis);
     }

package/build/types/quorum.d.ts

@@ -1,7 +1,16 @@
 import { JobOutput } from "./job";
+export interface QuorumProfile {
+    namespace: string;
+    app_id: string;
+    engine_id: string;
+    worker_topic?: string;
+    stream?: string;
+    stream_depth?: number;
+}
 export interface PingMessage {
     type: 'ping';
     originator: string;
+    details?: boolean;
 }
 export interface WorkMessage {
     type: 'work';
@@ -13,8 +22,9 @@ export interface CronMessage {
 }
 export interface PongMessage {
     type: 'pong';
-    originator: string;
     guid: string;
+    originator: string;
+    profile?: QuorumProfile;
 }
 export interface ActivateMessage {
     type: 'activate';

package/build/types/redisclient.d.ts

@@ -4,6 +4,7 @@ interface RedisMultiType {
     XADD(key: string, id: string, fields: any): this;
     XACK(key: string, group: string, id: string): this;
     XDEL(key: string, id: string): this;
+    XLEN(key: string): this;
     HDEL(key: string, itemId: string): this;
     HGET(key: string, itemId: string): this;
     HGETALL(key: string): this;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.37",
+  "version": "0.0.38",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",

package/services/hotmesh/index.ts

@@ -16,7 +16,7 @@ import {
 import {
   HotMeshConfig,
   HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback } from '../../types/quorum';
+import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
 import {
   JobStatsInput,
   GetStatsOptions,
@@ -137,6 +137,9 @@ class HotMeshService {
   }
 
   // ************* COMPILER METHODS *************
+  async rollCall(delay?: number): Promise<QuorumProfile[]> {
+    return await this.quorum?.rollCall(delay);
+  }
   async plan(path: string): Promise<HotMeshManifest> {
     return await this.engine?.plan(path);
   }
@@ -147,10 +150,6 @@ class HotMeshService {
     //activation is a quorum operation
     return await this.quorum?.activate(version, delay);
   }
-  async inventory(version: string, delay?: number): Promise<number> {
-    //get count of all peers
-    return await this.quorum?.inventory(delay);
-  }
 
   // ************* REPORTER METHODS *************
   async getStats(topic: string, query: JobStatsInput): Promise<StatsResponse> {

package/services/quorum/index.ts

@@ -14,6 +14,7 @@ import { RedisClientType as IORedisClientType } from '../../types/ioredisclient'
 import {
   QuorumMessage,
   QuorumMessageCallback,
+  QuorumProfile,
   SubscriptionCallback,
   ThrottleMessage
 } from '../../types/quorum';
@@ -26,10 +27,10 @@ const QUORUM_DELAY = 250;
 
 class QuorumService {
   namespace: string;
-  apps: HotMeshApps | null;
   appId: string;
   guid: string;
   engine: EngineService;
+  profiles: QuorumProfile[] = [];
   store: StoreService<RedisClient, RedisMulti> | null;
   subscribe: SubService<RedisClient, RedisMulti> | null;
   logger: ILogger;
@@ -108,9 +109,12 @@ class QuorumService {
       if (message.type === 'activate') {
         self.engine.setCacheMode(message.cache_mode, message.until_version);
       } else if (message.type === 'ping') {
-        this.sayPong(self.appId, self.guid, message.originator);
+        self.sayPong(self.appId, self.guid, message.originator, message.details);
       } else if (message.type === 'pong' && self.guid === message.originator) {
         self.quorum = self.quorum + 1;
+        if (message.profile) {
+          self.profiles.push(message.profile);
+        }
       } else if (message.type === 'throttle') {
         self.engine.throttle(message.throttle);
       } else if (message.type === 'work') {
@@ -127,20 +131,38 @@ class QuorumService {
     };
   }
 
-  async sayPong(appId: string, guid: string, originator: string) {
+  async sayPong(appId: string, guid: string, originator: string, details = false) {
+    let profile: QuorumProfile;
+    if (details) {
+      profile = {
+        engine_id: this.guid,
+        namespace: this.namespace,
+        app_id: this.appId,
+        stream: this.engine.stream.mintKey(KeyType.STREAMS, { appId: this.appId })
+      };
+    }
     this.store.publish(
       KeyType.QUORUM, 
-      { type: 'pong', guid, originator },
+      {
+        type: 'pong',
+        guid, originator,
+        profile,
+      },
       appId,
     );
   }
 
-  async requestQuorum(delay = QUORUM_DELAY): Promise<number> {
+  async requestQuorum(delay = QUORUM_DELAY, details = false): Promise<number> {
     const quorum = this.quorum;
     this.quorum = 0;
+    this.profiles.length = 0;
     await this.store.publish(
       KeyType.QUORUM,
-      { type: 'ping', originator: this.guid },
+      { 
+        type: 'ping',
+        originator: this.guid,
+        details,
+      },
       this.appId,
     );
     await sleepFor(delay);
@@ -166,12 +188,26 @@ class QuorumService {
 
 
   // ************* COMPILER METHODS *************
-  async inventory(delay = QUORUM_DELAY): Promise<number> {
-    await this.requestQuorum(delay);
-    const q1 = await this.requestQuorum(delay);
-    const q2 = await this.requestQuorum(delay);
-    const q3 = await this.requestQuorum(delay);
-    return Math.round((q1 + q2 + q3) / 3);
+  async rollCall(delay = QUORUM_DELAY): Promise<QuorumProfile[]> {
+    await this.requestQuorum(delay, true);
+    const targetStreams = [];
+    const multi = this.store.getMulti();
+    this.profiles.forEach((profile: QuorumProfile) => {
+      if (!targetStreams.includes(profile.stream)) {
+        targetStreams.push(profile.stream);
+        this.store.xlen(profile.stream, multi);
+      }
+    });
+    const stream_depths = await multi.exec() as number[];
+    this.profiles.forEach(async (profile: QuorumProfile) => {
+      const index = targetStreams.indexOf(profile.stream);
+      if (index != -1) {
+        profile.stream_depth = Array.isArray(stream_depths[index]) ?
+          stream_depths[index][1] :
+          stream_depths[index];
+      }
+    });
+    return this.profiles;
   }
   async activate(version: string, delay = QUORUM_DELAY): Promise<boolean> {
     version = version.toString();

package/services/store/clients/ioredis.ts

@@ -131,6 +131,15 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
       throw error;
     }
   }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xlen(key);
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
+    }
+  }
 }
 
 export { IORedisStoreService };

package/services/store/clients/redis.ts

@@ -42,6 +42,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       rpush: 'RPUSH',
       xack: 'XACK',
       xdel: 'XDEL',
+      xlen: 'XLEN',
     };
   }
 
@@ -167,6 +168,21 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       throw error;
     }
   }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi.XLEN(key);
+        return multi;
+      } else {
+        return await this.redisClient.XLEN(key);
+      }
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
+    }
+  }
+
 }
 
 export { RedisStoreService };

package/services/store/index.ts

@@ -118,6 +118,10 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     id: string,
     multi?: U
   ): Promise<number|U>;
+  abstract xlen(
+    key: string,
+    multi?: U
+  ): Promise<number|U>;
 
   constructor(redisClient: T) {
     this.redisClient = redisClient;

package/services/stream/clients/ioredis.ts

@@ -33,16 +33,16 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
     if (mkStream === 'MKSTREAM') {
       try {
         return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
-      } catch (err) {
+      } catch (error) {
         this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
-        throw err;
+        throw error;
       }
     } else {
       try {
         return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
-      } catch (err) {
+      } catch (error) {
         this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
-        throw err;
+        throw error;
       }
     }
   }
@@ -50,9 +50,9 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
   async xadd(key: string, id: string, messageId: string, messageValue: string, multi?: RedisMultiType): Promise<string | RedisMultiType> {
     try {
       return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
-    } catch (err) {
-      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+      throw error;
     }
   }
 
@@ -79,9 +79,9 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
         streamName,
         id
       );
-    } catch (err) {
-      this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { error });
+      throw error;
     }
   }
 
@@ -101,12 +101,12 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
       if (consumer) args.push(consumer);
       try {
         return await this.redisClient.call('XPENDING', ...args) as [string, string, number, number][];
-      } catch (err) {
-        this.logger.error('err, args', err, args);
+      } catch (error) {
+        this.logger.error('err, args', { error }, args);
       }
-    } catch (err) {
-      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { error });
+      throw error;
     }
   }
 
@@ -120,27 +120,36 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
   ): Promise<ReclaimedMessageType> {
     try {
       return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args) as unknown as ReclaimedMessageType;
-    } catch (err) {
-      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
   async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
     try {
       return await (multi || this.redisClient).xack(key, group, id);
-    } catch (err) {
-      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
   async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
     try {
       return await (multi || this.redisClient).xdel(key, id);
-    } catch (err) {
-      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { error });
+      throw error;
+    }
+  }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xlen(key);
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
     }
   }
 }

package/services/stream/clients/redis.ts

@@ -139,6 +139,20 @@ class RedisStreamService extends StreamService<RedisClientType, RedisMultiType>
       throw err;
     }
   }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi.XLEN(key);
+        return multi;
+      } else {
+        return await this.redisClient.XLEN(key);
+      }
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
+    }
+  }
 }
 
 export { RedisStreamService };

package/services/stream/index.ts

@@ -52,6 +52,7 @@ abstract class StreamService<T, U> {
     ...args: string[]): Promise<ReclaimedMessageType>;
   abstract xack(key: string, group: string, id: string, multi?: U): Promise<number|U>;
   abstract xdel(key: string, id: string, multi?: U): Promise<number|U>;
+  abstract xlen(key: string, multi?: U): Promise<number|U>;
 }
 
 export { StreamService };

package/services/worker/index.ts

@@ -14,6 +14,7 @@ import { RedisClientType as IORedisClientType } from '../../types/ioredisclient'
 import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
 import {
   QuorumMessage,
+  QuorumProfile,
   SubscriptionCallback } from "../../types/quorum";
 import { RedisClient, RedisMulti } from "../../types/redis";
 import { RedisClientType } from '../../types/redisclient';
@@ -153,10 +154,39 @@ class WorkerService {
       self.logger.debug('worker-event-received', { topic, type: message.type });
       if (message.type === 'throttle') {
         self.throttle(message.throttle);
+      } else if(message.type === 'ping') {
+        self.sayPong(self.appId, self.guid, message.originator, message.details);
       }
     };
   }
 
+  async sayPong(appId: string, guid: string, originator: string, details = false) {
+    let profile: QuorumProfile;
+    if (details) {
+      const params = {
+        appId: this.appId,
+        topic: this.topic,
+      };
+
+      profile = {
+        engine_id: this.guid,
+        namespace: this.namespace,
+        app_id: this.appId,
+        worker_topic: this.topic,
+        stream: this.stream.mintKey(KeyType.STREAMS, params),
+      };
+    }
+    this.store.publish(
+      KeyType.QUORUM, 
+      {
+        type: 'pong',
+        guid, originator,
+        profile,
+      },
+      appId,
+    );
+  }
+
   async throttle(delayInMillis: number) {
     this.router.setThrottle(delayInMillis);
   }

package/types/quorum.ts

@@ -1,9 +1,20 @@
 import { JobOutput } from "./job";
 
-//used for coordination like version activation
+//used for coordination (like version activation)
+
+export interface QuorumProfile {
+  namespace: string;
+  app_id: string;
+  engine_id: string;
+  worker_topic?: string;
+  stream?: string;
+  stream_depth?: number;
+}
+
 export interface PingMessage {
   type: 'ping';
   originator: string; //guid
+  details?: boolean;  //if true, all endpoints will include their profile
 }
 
 export interface WorkMessage {
@@ -16,11 +27,11 @@ export interface CronMessage {
   originator: string; //guid
 }
 
-//used for coordination like version activation
 export interface PongMessage {
   type: 'pong';
-  originator: string; //clone of originator guid passed in ping
-  guid: string;
+  guid: string;            //call initiator
+  originator: string;      //clone of originator guid passed in ping
+  profile?: QuorumProfile; //contains details about the engine/worker
 }
 
 export interface ActivateMessage {

package/types/redisclient.ts

@@ -6,6 +6,7 @@ interface RedisMultiType {
   XADD(key: string, id: string, fields: any): this;
   XACK(key: string, group: string, id: string): this;
   XDEL(key: string, id: string): this;
+  XLEN(key: string): this;
   HDEL(key: string, itemId: string): this;
   HGET(key: string, itemId: string): this;
   HGETALL(key: string): this;