@hotmeshio/hotmesh 0.0.46 → 0.0.48

package/build/modules/utils.d.ts

@@ -4,6 +4,8 @@ import { AppSubscriptions, AppTransitions, AppVID } from "../types/app";
 import { RedisClient, RedisMulti } from "../types/redis";
 import { StringAnyType } from "../types/serializer";
 import { StreamCode, StreamStatus } from "../types/stream";
+import { SystemHealth } from '../types/quorum';
+export declare function getSystemHealth(): Promise<SystemHealth>;
 export declare function sleepFor(ms: number): Promise<unknown>;
 export declare function guid(): string;
 export declare function deterministicRandom(seed: number): number;

package/build/modules/utils.js

@@ -1,7 +1,46 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreHierarchy = exports.getValueByPath = exports.getIndexedHash = exports.getSymVal = exports.getSymKey = exports.formatISODate = exports.getTimeSeries = exports.getSubscriptionTopic = exports.findSubscriptionForTrigger = exports.findTopKey = exports.XSleepFor = exports.matchesStatus = exports.matchesStatusCode = exports.identifyRedisTypeFromClass = exports.polyfill = exports.identifyRedisType = exports.deterministicRandom = exports.guid = exports.sleepFor = void 0;
+exports.restoreHierarchy = exports.getValueByPath = exports.getIndexedHash = exports.getSymVal = exports.getSymKey = exports.formatISODate = exports.getTimeSeries = exports.getSubscriptionTopic = exports.findSubscriptionForTrigger = exports.findTopKey = exports.XSleepFor = exports.matchesStatus = exports.matchesStatusCode = exports.identifyRedisTypeFromClass = exports.polyfill = exports.identifyRedisType = exports.deterministicRandom = exports.guid = exports.sleepFor = exports.getSystemHealth = void 0;
+const os_1 = __importDefault(require("os"));
+const systeminformation_1 = __importDefault(require("systeminformation"));
 const nanoid_1 = require("nanoid");
+async function safeExecute(operation, defaultValue) {
+    try {
+        return await operation;
+    }
+    catch (error) {
+        console.error(`Operation Error: ${error}`);
+        return defaultValue;
+    }
+}
+async function getSystemHealth() {
+    const totalMemory = os_1.default.totalmem();
+    const freeMemory = os_1.default.freemem();
+    const usedMemory = totalMemory - freeMemory;
+    const cpus = os_1.default.cpus();
+    // CPU load calculation remains unchanged
+    const cpuLoad = cpus.map((cpu, i) => {
+        const total = Object.values(cpu.times).reduce((acc, tv) => acc + tv, 0);
+        const idle = cpu.times.idle;
+        const usage = ((total - idle) / total) * 100;
+        return { [`CPU ${i} Usage`]: `${usage.toFixed(2)}%` };
+    });
+    // Wrap each systeminformation call with safeExecute
+    const networkStats = await safeExecute(systeminformation_1.default.networkStats(), []);
+    // Construct the system health object with error handling in mind
+    const systemHealth = {
+        TotalMemoryGB: `${(totalMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+        FreeMemoryGB: `${(freeMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+        UsedMemoryGB: `${(usedMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+        CPULoad: cpuLoad,
+        NetworkStats: networkStats,
+    };
+    return systemHealth;
+}
+exports.getSystemHealth = getSystemHealth;
 async function sleepFor(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.46",
+    "version": "0.0.48",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",
@@ -57,7 +57,14 @@
         "test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
     },
-    "keywords": ["durable workflow", "hotmesh", "service mesh", "workflows", "operational data", "redis"],
+    "keywords": [
+        "durable workflow",
+        "hotmesh",
+        "service mesh",
+        "workflows",
+        "operational data",
+        "redis"
+    ],
     "author": "luke.birdeau@gmail.com",
     "license": "SEE LICENSE IN LICENSE",
     "dependencies": {
@@ -66,6 +73,7 @@
         "js-yaml": "^4.1.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.6",
+        "systeminformation": "^5.22.2",
         "winston": "^3.8.2"
     },
     "devDependencies": {

package/build/services/durable/meshos.d.ts

@@ -77,6 +77,10 @@ export declare class MeshOSService {
      * Creates an FT search index
      */
     static createIndex(): Promise<void>;
+    /**
+     * Lists FT search indexes
+     */
+    static listIndexes(): Promise<string[]>;
     /**
      * stop the workers
      * @returns {Promise<void>}

package/build/services/durable/meshos.js

@@ -39,7 +39,15 @@ class MeshOSService {
     static async createIndex() {
         const my = new this();
         const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
-        search_1.Search.configureSearchIndex(hmClient, my.search);
+        await search_1.Search.configureSearchIndex(hmClient, my.search);
+    }
+    /**
+     * Lists FT search indexes
+     */
+    static async listIndexes() {
+        const my = new this();
+        const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
+        return await search_1.Search.listSearchIndexes(hmClient);
     }
     /**
      * stop the workers

package/build/services/durable/search.d.ts

@@ -8,6 +8,7 @@ export declare class Search {
     searchSessionIndex: number;
     hotMeshClient: HotMesh;
     store: StoreService<RedisClient, RedisMulti> | null;
+    constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string);
     safeKey(key: string): string;
     /**
      * For those deployments with a redis stack backend (with the FT module),
@@ -17,7 +18,13 @@ export declare class Search {
      * in the HASH.
      */
     static configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void>;
-    constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string);
+    /**
+     * For those deployments with a redis stack backend (with the FT module),
+     * this method will list all search indexes.
+     * @param {HotMesh} hotMeshClient - the hotmesh client
+     * @returns {Promise<string[]>} - the list of search indexes
+     */
+    static listSearchIndexes(hotMeshClient: HotMesh): Promise<string[]>;
     /**
      * increments the index to return a unique search session guid when
      * calling any method that produces side effects (changes the value)

package/build/services/durable/search.js

@@ -3,8 +3,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Search = void 0;
 const key_1 = require("../../modules/key");
 class Search {
+    constructor(workflowId, hotMeshClient, searchSessionId) {
+        this.searchSessionIndex = 0;
+        const keyParams = {
+            appId: hotMeshClient.appId,
+            jobId: workflowId
+        };
+        this.jobId = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
+        this.searchSessionId = searchSessionId;
+        this.hotMeshClient = hotMeshClient;
+        this.store = hotMeshClient.engine.store;
+    }
     safeKey(key) {
-        //note: protect the execution namespace with a prefix
         return `_${key}`;
     }
     /**
@@ -40,16 +50,16 @@ class Search {
             }
         }
     }
-    constructor(workflowId, hotMeshClient, searchSessionId) {
-        this.searchSessionIndex = 0;
-        const keyParams = {
-            appId: hotMeshClient.appId,
-            jobId: workflowId
-        };
-        this.jobId = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
-        this.searchSessionId = searchSessionId;
-        this.hotMeshClient = hotMeshClient;
-        this.store = hotMeshClient.engine.store;
+    /**
+     * For those deployments with a redis stack backend (with the FT module),
+     * this method will list all search indexes.
+     * @param {HotMesh} hotMeshClient - the hotmesh client
+     * @returns {Promise<string[]>} - the list of search indexes
+     */
+    static async listSearchIndexes(hotMeshClient) {
+        const store = hotMeshClient.engine.store;
+        const searchIndexes = await store.exec('FT._LIST');
+        return searchIndexes;
     }
     /**
      * increments the index to return a unique search session guid when

package/build/services/engine/index.d.ts

@@ -41,6 +41,7 @@ declare class EngineService {
     jobCallbacks: Record<string, JobMessageCallback>;
     reporting: boolean;
     jobId: number;
+    inited: string;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<EngineService>;
     verifyEngineFields(config: HotMeshConfig): void;
     initStoreChannel(store: RedisClient): Promise<void>;

package/build/services/engine/index.js

@@ -44,6 +44,7 @@ class EngineService {
             instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
             instance.taskService = new task_1.TaskService(instance.store, logger);
             instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
+            instance.inited = (0, utils_1.formatISODate)(new Date());
             return instance;
         }
     }

package/build/services/hotmesh/index.d.ts

@@ -4,7 +4,7 @@ import { QuorumService } from '../quorum';
 import { WorkerService } from '../worker';
 import { JobState, JobData, JobOutput, JobStatus, JobInterruptOptions } from '../../types/job';
 import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
+import { JobMessageCallback, QuorumProfile, ThrottleOptions } from '../../types/quorum';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
 import { StringAnyType, StringStringType } from '../../types/serializer';
@@ -33,6 +33,7 @@ declare class HotMeshService {
     pubsub(topic: string, data?: JobData, context?: JobState | null, timeout?: number): Promise<JobOutput>;
     add(streamData: StreamData | StreamDataResponse): Promise<string>;
     rollCall(delay?: number): Promise<QuorumProfile[]>;
+    throttle(options: ThrottleOptions): Promise<boolean>;
     plan(path: string): Promise<HotMeshManifest>;
     deploy(pathOrYAML: string): Promise<HotMeshManifest>;
     activate(version: string, delay?: number): Promise<boolean>;

package/build/services/hotmesh/index.js

@@ -89,10 +89,24 @@ class HotMeshService {
     async add(streamData) {
         return await this.engine.add(streamData);
     }
-    // ************* COMPILER METHODS *************
+    // ************* QUORUM METHODS *************
     async rollCall(delay) {
         return await this.quorum?.rollCall(delay);
     }
+    async throttle(options) {
+        const throttleMessage = {
+            type: 'throttle',
+            throttle: options.throttle,
+        };
+        if (options.guid) {
+            throttleMessage.guid = options.guid;
+        }
+        else if (options.topic) {
+            throttleMessage.topic = options.topic;
+        }
+        return await this.quorum?.pub(throttleMessage);
+    }
+    // ************* COMPILER METHODS *************
     async plan(path) {
         return await this.engine?.plan(path);
     }

package/build/services/quorum/index.js

@@ -4,8 +4,8 @@ exports.QuorumService = void 0;
 const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const compiler_1 = require("../compiler");
-const redis_1 = require("../store/clients/redis");
 const ioredis_1 = require("../store/clients/ioredis");
+const redis_1 = require("../store/clients/redis");
 const ioredis_2 = require("../sub/clients/ioredis");
 const redis_2 = require("../sub/clients/redis");
 const hotmesh_1 = require("../../types/hotmesh");
@@ -107,6 +107,11 @@ class QuorumService {
                 stream,
                 counts: this.engine.router.counts,
                 timestamp: (0, utils_1.formatISODate)(new Date()),
+                inited: this.engine.inited,
+                throttle: this.engine.router.throttle,
+                reclaimDelay: this.engine.router.reclaimDelay,
+                reclaimCount: this.engine.router.reclaimCount,
+                system: await (0, utils_1.getSystemHealth)(),
             };
         }
         this.store.publish(hotmesh_1.KeyType.QUORUM, {

package/build/services/worker/index.d.ts

@@ -18,6 +18,7 @@ declare class WorkerService {
     router: Router | null;
     logger: ILogger;
     reporting: boolean;
+    inited: string;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
     verifyWorkerFields(worker: HotMeshWorker): void;
     initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;

package/build/services/worker/index.js

@@ -38,6 +38,7 @@ class WorkerService {
                 service.router = service.initRouter(worker, logger);
                 const key = service.stream.mintKey(key_1.KeyType.STREAMS, { appId: service.appId, topic: worker.topic });
                 await service.router.consumeMessages(key, 'WORKER', service.guid, worker.callback);
+                service.inited = (0, utils_1.formatISODate)(new Date());
                 services.push(service);
             }
         }
@@ -116,6 +117,11 @@ class WorkerService {
                 stream: this.stream.mintKey(key_1.KeyType.STREAMS, params),
                 counts: this.router.counts,
                 timestamp: (0, utils_1.formatISODate)(new Date()),
+                inited: this.inited,
+                throttle: this.router.throttle,
+                reclaimDelay: this.router.reclaimDelay,
+                reclaimCount: this.router.reclaimCount,
+                system: await (0, utils_1.getSystemHealth)(),
             };
         }
         this.store.publish(key_1.KeyType.QUORUM, {

package/build/types/index.d.ts

@@ -12,7 +12,7 @@ export { JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, Partial
 export { MappingStatements } from './map';
 export { Pipe, PipeItem, PipeItems } from './pipe';
 export { HotMesh, HotMeshApp, HotMeshApps, HotMeshConfig, HotMeshEngine, RedisConfig, HotMeshGraph, HotMeshManifest, HotMeshSettings, HotMeshWorker, KeyStoreParams, KeyType } from './hotmesh';
-export { ActivateMessage, JobMessage, JobMessageCallback, PingMessage, PongMessage, QuorumMessage, QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage, WorkMessage } from './quorum';
+export { ActivateMessage, JobMessage, JobMessageCallback, PingMessage, PongMessage, QuorumMessage, QuorumMessageCallback, QuorumProfile, SubscriptionCallback, SystemHealth, ThrottleMessage, ThrottleOptions, WorkMessage } from './quorum';
 export { MultiResponseFlags, RedisClient, RedisMulti } from './redis';
 export { RedisClientType, RedisMultiType } from './redisclient';
 export { JSONSchema, StringAnyType, StringScalarType, StringStringType, SymbolMap, SymbolMaps, SymbolRanges, Symbols, SymbolSets } from './serializer';

package/build/types/quorum.d.ts

@@ -1,4 +1,36 @@
 import { JobOutput } from "./job";
+interface CPULoad {
+    [cpu: string]: string;
+}
+interface NetworkStat {
+    iface: string;
+    operstate: string;
+    rx_bytes: number;
+    rx_dropped: number;
+    rx_errors: number;
+    tx_bytes: number;
+    tx_dropped: number;
+    tx_errors: number;
+    rx_sec: number;
+    tx_sec: number;
+    ms: number;
+}
+/** reveals: memory, cpu, network */
+export interface SystemHealth {
+    TotalMemoryGB: string;
+    FreeMemoryGB: string;
+    UsedMemoryGB: string;
+    CPULoad: CPULoad[];
+    NetworkStats: NetworkStat[];
+}
+export type ThrottleOptions = {
+    /** target an engine OR worker by GUID */
+    guid?: string;
+    /** target a worker quorum */
+    topic?: string;
+    /** in milliseconds; default is 0 */
+    throttle: number;
+};
 export interface QuorumProfile {
     namespace: string;
     app_id: string;
@@ -7,7 +39,12 @@ export interface QuorumProfile {
     stream?: string;
     stream_depth?: number;
     counts?: Record<string, number>;
+    inited?: string;
     timestamp?: string;
+    throttle?: number;
+    reclaimDelay?: number;
+    reclaimCount?: number;
+    system?: SystemHealth;
 }
 export interface PingMessage {
     type: 'ping';
@@ -60,3 +97,4 @@ export interface QuorumMessageCallback {
  * to the new version without any downtime and a coordinating parent server.
  */
 export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;
+export {};

package/modules/utils.ts

@@ -1,9 +1,50 @@
+import os from 'os';
+import si from 'systeminformation';
 import { nanoid } from "nanoid";
 import { StoreService } from "../services/store";
 import { AppSubscriptions, AppTransitions, AppVID } from "../types/app";
 import { RedisClient, RedisMulti } from "../types/redis";
 import { StringAnyType } from "../types/serializer";
 import { StreamCode, StreamStatus } from "../types/stream";
+import { SystemHealth } from '../types/quorum';
+
+async function safeExecute<T>(operation: Promise<T>, defaultValue: T): Promise<T> {
+  try {
+    return await operation;
+  } catch (error) {
+    console.error(`Operation Error: ${error}`);
+    return defaultValue;
+  }
+}
+
+export async function getSystemHealth(): Promise<SystemHealth> {
+  const totalMemory = os.totalmem();
+  const freeMemory = os.freemem();
+  const usedMemory = totalMemory - freeMemory;
+  const cpus = os.cpus();
+
+  // CPU load calculation remains unchanged
+  const cpuLoad = cpus.map((cpu, i) => {
+    const total = Object.values(cpu.times).reduce((acc, tv) => acc + tv, 0);
+    const idle = cpu.times.idle;
+    const usage = ((total - idle) / total) * 100;
+    return { [`CPU ${i} Usage`]: `${usage.toFixed(2)}%` };
+  });
+
+  // Wrap each systeminformation call with safeExecute
+  const networkStats = await safeExecute(si.networkStats(), []);
+
+  // Construct the system health object with error handling in mind
+  const systemHealth = {
+    TotalMemoryGB: `${(totalMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+    FreeMemoryGB: `${(freeMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+    UsedMemoryGB: `${(usedMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
+    CPULoad: cpuLoad,
+    NetworkStats: networkStats,
+  };
+
+  return systemHealth as SystemHealth;
+}
 
 export async function sleepFor(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.46",
+  "version": "0.0.48",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -57,7 +57,14 @@
     "test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
   },
-  "keywords": ["durable workflow", "hotmesh", "service mesh", "workflows", "operational data", "redis"],
+  "keywords": [
+    "durable workflow",
+    "hotmesh",
+    "service mesh",
+    "workflows",
+    "operational data",
+    "redis"
+  ],
   "author": "luke.birdeau@gmail.com",
   "license": "SEE LICENSE IN LICENSE",
   "dependencies": {
@@ -66,6 +73,7 @@
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
     "nanoid": "^3.3.6",
+    "systeminformation": "^5.22.2",
     "winston": "^3.8.2"
   },
   "devDependencies": {

package/services/durable/meshos.ts

@@ -125,7 +125,16 @@ export class MeshOSService {
   static async createIndex() {
     const my = new this();
     const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
-    Search.configureSearchIndex(hmClient, my.search)
+    await Search.configureSearchIndex(hmClient, my.search);
+  }
+
+  /**
+   * Lists FT search indexes
+   */
+  static async listIndexes() {
+    const my = new this();
+    const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
+    return await Search.listSearchIndexes(hmClient)
   }
 
   /**

package/services/durable/search.ts

@@ -11,8 +11,18 @@ export class Search {
   hotMeshClient: HotMesh;
   store: StoreService<RedisClient, RedisMulti> | null;
 
+  constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string) {
+    const keyParams = {
+      appId: hotMeshClient.appId,
+      jobId: workflowId
+    }
+    this.jobId = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+    this.searchSessionId = searchSessionId;
+    this.hotMeshClient = hotMeshClient;
+    this.store = hotMeshClient.engine.store as StoreService<RedisClient, RedisMulti>;
+  }
+
   safeKey(key:string): string {
-    //note: protect the execution namespace with a prefix
     return `_${key}`;
   }
 
@@ -49,15 +59,16 @@ export class Search {
     }
   }
 
-  constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string) {
-    const keyParams = {
-      appId: hotMeshClient.appId,
-      jobId: workflowId
-    }
-    this.jobId = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
-    this.searchSessionId = searchSessionId;
-    this.hotMeshClient = hotMeshClient;
-    this.store = hotMeshClient.engine.store as StoreService<RedisClient, RedisMulti>;
+  /**
+   * For those deployments with a redis stack backend (with the FT module),
+   * this method will list all search indexes.
+   * @param {HotMesh} hotMeshClient - the hotmesh client
+   * @returns {Promise<string[]>} - the list of search indexes
+   */
+  static async listSearchIndexes(hotMeshClient: HotMesh): Promise<string[]> {
+    const store = hotMeshClient.engine.store;
+    const searchIndexes = await store.exec('FT._LIST');
+    return searchIndexes as string[];
   }
 
   /**

package/services/durable/worker.ts

@@ -93,7 +93,7 @@ export class WorkerService {
     const worker = new WorkerService();
     worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
     worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
-    Search.configureSearchIndex(worker.workflowRunner, config.search)
+    Search.configureSearchIndex(worker.workflowRunner, config.search);
     await WorkerService.activateWorkflow(worker.workflowRunner);
     return worker;
   }

package/services/engine/index.ts

@@ -98,6 +98,7 @@ class EngineService {
   jobCallbacks: Record<string, JobMessageCallback> = {};
   reporting = false;
   jobId = 1;
+  inited: string;
 
   static async init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<EngineService> {
     if (config.engine) {
@@ -133,6 +134,7 @@ class EngineService {
         instance.store,
         logger,
       );
+      instance.inited = formatISODate(new Date());
       return instance;
     }
   }

package/services/hotmesh/index.ts

@@ -16,7 +16,7 @@ import {
 import {
   HotMeshConfig,
   HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
+import { JobMessageCallback, QuorumProfile, ThrottleMessage, ThrottleOptions } from '../../types/quorum';
 import {
   JobStatsInput,
   GetStatsOptions,
@@ -137,10 +137,25 @@ class HotMeshService {
     return await this.engine.add(streamData) as string;
   }
 
-  // ************* COMPILER METHODS *************
+
+  // ************* QUORUM METHODS *************
   async rollCall(delay?: number): Promise<QuorumProfile[]> {
     return await this.quorum?.rollCall(delay);
   }
+  async throttle(options: ThrottleOptions): Promise<boolean> {
+    const throttleMessage: ThrottleMessage = {
+      type: 'throttle',
+      throttle: options.throttle,
+    };
+    if (options.guid) {
+      throttleMessage.guid = options.guid;
+    } else if (options.topic) {
+      throttleMessage.topic = options.topic;
+    }
+    return await this.quorum?.pub(throttleMessage);
+  }
+
+  // ************* COMPILER METHODS *************
   async plan(path: string): Promise<HotMeshManifest> {
     return await this.engine?.plan(path);
   }

package/services/quorum/index.ts

@@ -1,11 +1,17 @@
-import { HMSH_ACTIVATION_MAX_RETRY, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
-import { formatISODate, identifyRedisType, sleepFor } from '../../modules/utils';
+import {
+  HMSH_ACTIVATION_MAX_RETRY,
+  HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
+import {
+  formatISODate,
+  getSystemHealth,
+  identifyRedisType,
+  sleepFor } from '../../modules/utils';
 import { CompilerService } from '../compiler';
 import { EngineService } from '../engine';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
-import { RedisStoreService as RedisStore } from '../store/clients/redis';
 import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
+import { RedisStoreService as RedisStore } from '../store/clients/redis';
 import { SubService } from '../sub';
 import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
 import { RedisSubService as RedisSub } from '../sub/clients/redis';
@@ -152,6 +158,11 @@ class QuorumService {
         stream,
         counts: this.engine.router.counts,
         timestamp: formatISODate(new Date()),
+        inited: this.engine.inited,
+        throttle: this.engine.router.throttle,
+        reclaimDelay: this.engine.router.reclaimDelay,
+        reclaimCount: this.engine.router.reclaimCount,
+        system: await getSystemHealth(),
       };
     }
     this.store.publish(

package/services/worker/index.ts

@@ -1,5 +1,5 @@
 import { KeyType } from "../../modules/key";
-import { formatISODate, identifyRedisType } from "../../modules/utils";
+import { formatISODate, getSystemHealth, identifyRedisType } from "../../modules/utils";
 import { ConnectorService } from "../connector";
 import { ILogger } from "../logger";
 import { Router } from "../router";
@@ -34,6 +34,7 @@ class WorkerService {
   router: Router | null;
   logger: ILogger;
   reporting = false;
+  inited: string;
 
   static async init(
     namespace: string,
@@ -76,6 +77,7 @@ class WorkerService {
           service.guid,
           worker.callback
         );
+        service.inited = formatISODate(new Date());
         services.push(service);
       }
     }
@@ -176,6 +178,11 @@ class WorkerService {
         stream: this.stream.mintKey(KeyType.STREAMS, params),
         counts: this.router.counts,
         timestamp: formatISODate(new Date()),
+        inited: this.inited,
+        throttle: this.router.throttle,
+        reclaimDelay: this.router.reclaimDelay,
+        reclaimCount: this.router.reclaimCount,
+        system: await getSystemHealth(),
       };
     }
     this.store.publish(

package/types/index.ts

@@ -115,7 +115,9 @@ export {
   QuorumMessageCallback,
   QuorumProfile,
   SubscriptionCallback,
+  SystemHealth,
   ThrottleMessage,
+  ThrottleOptions,
   WorkMessage } from './quorum';
 export {
   MultiResponseFlags,

package/types/quorum.ts

@@ -1,6 +1,40 @@
 import { JobOutput } from "./job";
 
-//used for coordination (like version activation)
+interface CPULoad {
+  [cpu: string]: string;
+}
+
+interface NetworkStat {
+  iface: string;
+  operstate: string;
+  rx_bytes: number;
+  rx_dropped: number;
+  rx_errors: number;
+  tx_bytes: number;
+  tx_dropped: number;
+  tx_errors: number;
+  rx_sec: number;
+  tx_sec: number;
+  ms: number;
+}
+
+/** reveals: memory, cpu, network */
+export interface SystemHealth {
+  TotalMemoryGB: string;
+  FreeMemoryGB: string;
+  UsedMemoryGB: string;
+  CPULoad: CPULoad[];
+  NetworkStats: NetworkStat[];
+}
+
+export type ThrottleOptions = {
+  /** target an engine OR worker by GUID */
+  guid?: string;
+  /** target a worker quorum */
+  topic?: string;
+  /** in milliseconds; default is 0 */
+  throttle: number;
+};
 
 export interface QuorumProfile {
   namespace: string;
@@ -10,9 +44,15 @@ export interface QuorumProfile {
   stream?: string;
   stream_depth?: number;
   counts?: Record<string, number>;
+  inited?: string;
   timestamp?: string;
+  throttle?: number;
+  reclaimDelay?: number;
+  reclaimCount?: number;
+  system?: SystemHealth;
 }
 
+//used for coordination (like version activation)
 export interface PingMessage {
   type: 'ping';
   originator: string; //guid