@hotmeshio/hotmesh 0.0.47 → 0.0.49

package/README.md

@@ -1,7 +1,7 @@
 # HotMesh
 ![alpha release](https://img.shields.io/badge/release-alpha-yellow)
 
-HotMesh elevates Redis from an in-memory data cache to a distributed orchestration engine.
+HotMesh transforms Redis into a distributed orchestration engine.
 
 *Write functions in your own preferred style, and let Redis govern their execution, reliably and durably.*
 

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.47",
+    "version": "0.0.49",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/durable/meshos.js

@@ -187,13 +187,18 @@ class MeshOSService {
         }
         else {
             //limit which hash fields to return
-            if (options.return?.length) {
-                args.push('RETURN');
-                args.push(options.return.length.toString());
-                options.return.forEach(returnField => {
+            args.push('RETURN');
+            args.push(((options.return?.length ?? 0) + 1).toString());
+            args.push('$');
+            options.return?.forEach(returnField => {
+                if (returnField.startsWith('"')) {
+                    //allow literal values to be requested
+                    args.push(returnField.slice(1, -1));
+                }
+                else {
                     args.push(`_${returnField}`);
-                });
-            }
+                }
+            });
             //paginate
             if (options.limit) {
                 args.push('LIMIT', options.limit.start.toString(), options.limit.size.toString());

package/build/services/durable/search.d.ts

@@ -30,10 +30,29 @@ export declare class Search {
      * calling any method that produces side effects (changes the value)
      */
     getSearchSessionGuid(): string;
-    set(...args: string[]): Promise<void>;
+    /**
+     * Sets the fields listed in args. Returns the
+     * count of new fields that were set (does not
+     * count fields that were updated)
+     */
+    set(...args: string[]): Promise<number>;
     get(key: string): Promise<string>;
     mget(...args: string[]): Promise<string[]>;
+    /**
+     * Deletes the fields listed in args. Returns the
+     * count of fields that were deleted.
+     */
     del(...args: string[]): Promise<number | void>;
+    /**
+     * Increments the value of a field by the given amount. Returns the
+     * new value of the field after the increment. Can be
+     * used to decrement the value of a field by specifying a negative.
+     */
     incr(key: string, val: number): Promise<number>;
+    /**
+     * Multiplies the value of a field by the given amount. Returns the
+     * new value of the field after the multiplication. NOTE:
+     * this is exponential multiplication.
+     */
     mult(key: string, val: number): Promise<number>;
 }

package/build/services/durable/search.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Search = void 0;
 const key_1 = require("../../modules/key");
+const storage_1 = require("../../modules/storage");
 class Search {
     constructor(workflowId, hotMeshClient, searchSessionId) {
         this.searchSessionIndex = 0;
@@ -57,9 +58,15 @@ class Search {
      * @returns {Promise<string[]>} - the list of search indexes
      */
     static async listSearchIndexes(hotMeshClient) {
-        const store = hotMeshClient.engine.store;
-        const searchIndexes = await store.exec('FT._LIST');
-        return searchIndexes;
+        try {
+            const store = hotMeshClient.engine.store;
+            const searchIndexes = await store.exec('FT._LIST');
+            return searchIndexes;
+        }
+        catch (err) {
+            hotMeshClient.engine.logger.info('durable-client-search-list-err', { err });
+            return [];
+        }
     }
     /**
      * increments the index to return a unique search session guid when
@@ -69,18 +76,28 @@ class Search {
         //return the search session as it would exist in the search session index
         return `${this.searchSessionId}-${this.searchSessionIndex++}-`;
     }
+    /**
+     * Sets the fields listed in args. Returns the
+     * count of new fields that were set (does not
+     * count fields that were updated)
+     */
     async set(...args) {
         const ssGuid = this.getSearchSessionGuid();
-        const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
-        if (ssGuidValue === 1) {
-            const safeArgs = [];
-            for (let i = 0; i < args.length; i += 2) {
-                const key = this.safeKey(args[i]);
-                const value = args[i + 1].toString();
-                safeArgs.push(key, value);
-            }
-            await this.store.exec('HSET', this.jobId, ...safeArgs);
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Number(replay[ssGuid]);
         }
+        const safeArgs = [];
+        for (let i = 0; i < args.length; i += 2) {
+            const key = this.safeKey(args[i]);
+            const value = args[i + 1].toString();
+            safeArgs.push(key, value);
+        }
+        const fieldCount = await this.store.exec('HSET', this.jobId, ...safeArgs);
+        //no need to wait; set this interim value in the replay
+        this.store.exec('HSET', this.jobId, ssGuid, fieldCount.toString());
+        return Number(fieldCount);
     }
     async get(key) {
         try {
@@ -104,32 +121,63 @@ class Search {
             return [];
         }
     }
+    /**
+     * Deletes the fields listed in args. Returns the
+     * count of fields that were deleted.
+     */
     async del(...args) {
         const ssGuid = this.getSearchSessionGuid();
-        const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
-        if (ssGuidValue === 1) {
-            const safeArgs = [];
-            for (let i = 0; i < args.length; i++) {
-                safeArgs.push(this.safeKey(args[i]));
-            }
-            const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
-            return isNaN(response) ? undefined : Number(response);
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Number(replay[ssGuid]);
+        }
+        const safeArgs = [];
+        for (let i = 0; i < args.length; i++) {
+            safeArgs.push(this.safeKey(args[i]));
         }
+        const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
+        const formattedResponse = isNaN(response) ? 0 : Number(response);
+        //no need to wait; set this interim value in the replay
+        this.store.exec('HSET', this.jobId, ssGuid, formattedResponse.toString());
+        return formattedResponse;
     }
+    /**
+     * Increments the value of a field by the given amount. Returns the
+     * new value of the field after the increment. Can be
+     * used to decrement the value of a field by specifying a negative.
+     */
     async incr(key, val) {
         const ssGuid = this.getSearchSessionGuid();
-        const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
-        if (ssGuidValue === 1) {
-            return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()));
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Number(replay[ssGuid]);
         }
+        const num = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString());
+        //no need to wait; set this interim value in the replay
+        this.store.exec('HSET', this.jobId, ssGuid, num.toString());
+        return Number(num);
     }
+    /**
+     * Multiplies the value of a field by the given amount. Returns the
+     * new value of the field after the multiplication. NOTE:
+     * this is exponential multiplication.
+     */
     async mult(key, val) {
         const ssGuid = this.getSearchSessionGuid();
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            return Math.exp(Number(replay[ssGuid]));
+        }
         const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1'));
         if (ssGuidValue === 1) {
             const log = Math.log(val);
-            const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()));
-            return Math.exp(logTotal);
+            const logTotal = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString());
+            //no need to wait; set this interim value in the replay
+            this.store.exec('HSET', this.jobId, ssGuid, logTotal.toString());
+            return Math.exp(Number(logTotal));
         }
     }
 }

package/build/services/durable/worker.js

@@ -164,16 +164,26 @@ class WorkerService {
                     // garbage collect (expire) this job when originJobId is expired
                     context.set('originJobId', workflowInput.originJobId);
                 }
+                let replayQuery = '';
                 if (workflowInput.workflowDimension) {
                     //every hook function runs in an isolated dimension controlled
                     //by the index assigned when the signal was received; even if the
                     //hook function re-runs, its scope will always remain constant
                     context.set('workflowDimension', workflowInput.workflowDimension);
+                    replayQuery = `-*${workflowInput.workflowDimension}-*`;
+                }
+                else {
+                    //last letter of words like 'hook', 'sleep', 'wait', 'signal', 'search', 'start'
+                    replayQuery = '-*[ehklpt]-*';
                 }
                 context.set('workflowTopic', workflowTopic);
                 context.set('workflowName', workflowTopic.split('-').pop());
                 context.set('workflowTrace', data.metadata.trc);
                 context.set('workflowSpan', data.metadata.spn);
+                const store = this.workflowRunner.engine.store;
+                const [cursor, replay] = await store.findJobFields(workflowInput.workflowId, replayQuery, 50000, 5000);
+                context.set('replay', replay);
+                context.set('cursor', cursor); // if != 0, more remain
                 const workflowResponse = await storage_1.asyncLocalStorage.run(context, async () => {
                     return await workflowFunction.apply(this, workflowInput.arguments);
                 });

package/build/services/durable/workflow.d.ts

@@ -87,6 +87,7 @@ export declare class WorkflowService {
         workflowTopic: any;
         workflowDimension: any;
         counter: any;
+        replay: any;
     };
     /**
      * Executes a function once and caches the result. If the function is called

package/build/services/durable/workflow.js

@@ -76,6 +76,10 @@ class WorkflowService {
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
         const sessionId = `-start${workflowDimension}-${execIndex}-`;
+        const replay = store.get('replay');
+        if (sessionId in replay) {
+            return replay[sessionId];
+        }
         //NOTE: this is the hash prefix; necessary for the search index to locate the entity
         const entityOrEmptyString = options.entity ?? '';
         //If the workflowId is not provided, it is generated from the entity and the workflow name
@@ -172,6 +176,8 @@ class WorkflowService {
     static getContext() {
         const store = storage_1.asyncLocalStorage.getStore();
         const workflowId = store.get('workflowId');
+        const replay = store.get('replay');
+        const cursor = store.get('cursor');
         const workflowDimension = store.get('workflowDimension') ?? '';
         const workflowTopic = store.get('workflowTopic');
         const namespace = store.get('namespace');
@@ -180,7 +186,9 @@ class WorkflowService {
         const COUNTER = store.get('counter');
         return {
             counter: COUNTER.counter,
+            cursor,
             namespace,
+            replay,
             workflowId,
             workflowDimension,
             workflowTopic,
@@ -201,6 +209,10 @@ class WorkflowService {
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
         const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
+        const replay = store.get('replay');
+        if (sessionId in replay) {
+            return false;
+        }
         const keyParams = {
             appId: hotMeshClient.appId,
             jobId: workflowId
@@ -274,6 +286,7 @@ class WorkflowService {
             workflowTopic: store.get('workflowTopic'),
             workflowDimension: store.get('workflowDimension') ?? '',
             counter: store.get('counter'),
+            replay: store.get('replay'),
         };
     }
     /**
@@ -284,9 +297,12 @@ class WorkflowService {
      * @template T - the result type
      */
     static async once(fn, ...args) {
-        const { workflowId, namespace, workflowTopic, workflowDimension, counter: COUNTER, } = WorkflowService.getLocalState();
+        const { workflowId, namespace, workflowTopic, workflowDimension, counter: COUNTER, replay, } = WorkflowService.getLocalState();
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
         const sessionId = `-once${workflowDimension}-${execIndex}-`;
+        if (sessionId in replay) {
+            return JSON.parse(replay[sessionId]);
+        }
         const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
         const keyParams = {
             appId: hotMeshClient.appId,

package/build/services/engine/index.d.ts

@@ -41,6 +41,7 @@ declare class EngineService {
     jobCallbacks: Record<string, JobMessageCallback>;
     reporting: boolean;
     jobId: number;
+    inited: string;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<EngineService>;
     verifyEngineFields(config: HotMeshConfig): void;
     initStoreChannel(store: RedisClient): Promise<void>;

package/build/services/engine/index.js

@@ -44,6 +44,7 @@ class EngineService {
             instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
             instance.taskService = new task_1.TaskService(instance.store, logger);
             instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
+            instance.inited = (0, utils_1.formatISODate)(new Date());
             return instance;
         }
     }

package/build/services/quorum/index.js

@@ -107,6 +107,10 @@ class QuorumService {
                 stream,
                 counts: this.engine.router.counts,
                 timestamp: (0, utils_1.formatISODate)(new Date()),
+                inited: this.engine.inited,
+                throttle: this.engine.router.throttle,
+                reclaimDelay: this.engine.router.reclaimDelay,
+                reclaimCount: this.engine.router.reclaimCount,
                 system: await (0, utils_1.getSystemHealth)(),
             };
         }

package/build/services/router/index.d.ts

@@ -22,9 +22,24 @@ declare class Router {
     };
     currentTimerId: NodeJS.Timeout | null;
     shouldConsume: boolean;
+    sleepPromiseResolve: (() => void) | null;
+    innerPromiseResolve: (() => void) | null;
+    isSleeping: boolean;
+    sleepTimout: NodeJS.Timeout | null;
     constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    private resetThrottleState;
     createGroup(stream: string, group: string): Promise<void>;
     publishMessage(topic: string, streamData: StreamData | StreamDataResponse, multi?: RedisMulti): Promise<string | RedisMulti>;
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
+    customSleep(): Promise<void>;
     consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
     isStreamMessage(result: any): boolean;
     consumeOne(stream: string, group: string, id: string, message: string[], callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;

package/build/services/router/index.js

@@ -12,6 +12,10 @@ class Router {
         this.errorCount = 0;
         this.counts = {};
         this.currentTimerId = null;
+        this.sleepPromiseResolve = null;
+        this.innerPromiseResolve = null;
+        this.isSleeping = false;
+        this.sleepTimout = null;
         this.appId = config.appId;
         this.guid = config.guid;
         this.role = config.role;
@@ -21,6 +25,13 @@ class Router {
         this.reclaimDelay = config.reclaimDelay || enums_1.HMSH_XCLAIM_DELAY_MS;
         this.reclaimCount = config.reclaimCount || enums_1.HMSH_XCLAIM_COUNT;
         this.logger = logger;
+        this.resetThrottleState();
+    }
+    resetThrottleState() {
+        this.sleepPromiseResolve = null;
+        this.innerPromiseResolve = null;
+        this.isSleeping = false;
+        this.sleepTimout = null;
     }
     async createGroup(stream, group) {
         try {
@@ -36,6 +47,36 @@ class Router {
         const stream = this.store.mintKey(key_1.KeyType.STREAMS, { appId: this.store.appId, topic });
         return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
     }
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
+    async customSleep() {
+        if (this.throttle === 0)
+            return;
+        if (this.isSleeping)
+            return;
+        this.isSleeping = true;
+        let startTime = Date.now(); //anchor the origin
+        await new Promise(async (outerResolve) => {
+            this.sleepPromiseResolve = outerResolve;
+            let elapsedTime = Date.now() - startTime;
+            while (elapsedTime < this.throttle) {
+                await new Promise((innerResolve) => {
+                    this.innerPromiseResolve = innerResolve;
+                    this.sleepTimout = setTimeout(innerResolve, this.throttle - elapsedTime);
+                });
+                elapsedTime = Date.now() - startTime;
+            }
+            this.resetThrottleState();
+            outerResolve();
+        });
+    }
     async consumeMessages(stream, group, consumer, callback) {
         this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
         Router.instances.add(this);
@@ -43,9 +84,7 @@ class Router {
         await this.createGroup(stream, group);
         let lastCheckedPendingMessagesAt = Date.now();
         async function consume() {
-            let sleep = (0, utils_1.XSleepFor)(this.throttle);
-            this.currentTimerId = sleep.timerId;
-            await sleep.promise;
+            await this.customSleep();
             if (!this.shouldConsume) {
                 this.logger.info(`stream-consumer-stopping`, { group, consumer, stream });
                 return;
@@ -229,17 +268,26 @@ class Router {
         this.cancelThrottle();
     }
     cancelThrottle() {
-        if (this.currentTimerId !== undefined) {
-            clearTimeout(this.currentTimerId);
-            this.currentTimerId = undefined;
+        if (this.sleepTimout) {
+            clearTimeout(this.sleepTimout);
         }
+        this.resetThrottleState();
     }
     setThrottle(delayInMillis) {
         if (!Number.isInteger(delayInMillis) || delayInMillis < 0) {
             throw new Error('Throttle must be a non-negative integer');
         }
+        const wasDecreased = delayInMillis < this.throttle;
         this.throttle = delayInMillis;
-        this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
+        // If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
+        if (wasDecreased) {
+            if (this.sleepTimout) {
+                clearTimeout(this.sleepTimout);
+            }
+            if (this.innerPromiseResolve) {
+                this.innerPromiseResolve();
+            }
+        }
     }
     async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = enums_1.HMSH_XPENDING_COUNT) {
         let pendingMessages = [];

package/build/services/store/clients/redis.js

@@ -10,6 +10,7 @@ class RedisStoreService extends index_1.StoreService {
             setnx: 'SETNX',
             del: 'DEL',
             expire: 'EXPIRE',
+            hscan: 'HSCAN',
             hset: 'HSET',
             hsetnx: 'HSETNX',
             hincrby: 'HINCRBY',
@@ -29,6 +30,7 @@ class RedisStoreService extends index_1.StoreService {
             lpop: 'LPOP',
             rename: 'RENAME',
             rpush: 'RPUSH',
+            scan: 'SCAN',
             xack: 'XACK',
             xdel: 'XDEL',
             xlen: 'XLEN',

package/build/services/store/index.d.ts

@@ -152,5 +152,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      */
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;
+    findJobs(queryString?: string, limit?: number, batchSize?: number): Promise<string[]>;
+    findJobFields(jobId: string, fieldMatchPattern?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, StringStringType]>;
 }
 export { StoreService };

package/build/services/store/index.js

@@ -38,6 +38,7 @@ class StoreService {
             del: 'del',
             expire: 'expire',
             hset: 'hset',
+            hscan: 'hscan',
             hsetnx: 'hsetnx',
             hincrby: 'hincrby',
             hdel: 'hdel',
@@ -56,6 +57,7 @@ class StoreService {
             lrange: 'lrange',
             rename: 'rename',
             rpush: 'rpush',
+            scan: 'scan',
             xack: 'xack',
             xdel: 'xdel',
         };
@@ -876,5 +878,47 @@ class StoreService {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         await this.redisClient[this.commands.del](jobKey);
     }
+    async findJobs(queryString = '*', limit = 1000, batchSize = 1000) {
+        const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId: queryString });
+        let cursor = '0';
+        let keys;
+        const matchingKeys = [];
+        do {
+            const output = await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString());
+            if (Array.isArray(output)) {
+                [cursor, keys] = output;
+                for (let key of [...keys]) {
+                    matchingKeys.push(key);
+                }
+                if (matchingKeys.length >= limit) {
+                    break;
+                }
+            }
+            else {
+                break;
+            }
+        } while (cursor !== '0');
+        return matchingKeys;
+    }
+    async findJobFields(jobId, fieldMatchPattern = '*', limit = 1000, batchSize = 1000, cursor = '0') {
+        let fields = [];
+        const matchingFields = {};
+        const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+        let len = 0;
+        do {
+            const output = await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString());
+            if (Array.isArray(output)) {
+                [cursor, fields] = output;
+                for (let i = 0; i < fields.length; i += 2) {
+                    len++;
+                    matchingFields[fields[i]] = fields[i + 1];
+                }
+            }
+            else {
+                break;
+            }
+        } while (cursor !== '0' && len < limit);
+        return [cursor, matchingFields];
+    }
 }
 exports.StoreService = StoreService;

package/build/services/worker/index.d.ts

@@ -18,6 +18,7 @@ declare class WorkerService {
     router: Router | null;
     logger: ILogger;
     reporting: boolean;
+    inited: string;
     static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
     verifyWorkerFields(worker: HotMeshWorker): void;
     initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;

package/build/services/worker/index.js

@@ -38,6 +38,7 @@ class WorkerService {
                 service.router = service.initRouter(worker, logger);
                 const key = service.stream.mintKey(key_1.KeyType.STREAMS, { appId: service.appId, topic: worker.topic });
                 await service.router.consumeMessages(key, 'WORKER', service.guid, worker.callback);
+                service.inited = (0, utils_1.formatISODate)(new Date());
                 services.push(service);
             }
         }
@@ -116,6 +117,10 @@ class WorkerService {
                 stream: this.stream.mintKey(key_1.KeyType.STREAMS, params),
                 counts: this.router.counts,
                 timestamp: (0, utils_1.formatISODate)(new Date()),
+                inited: this.inited,
+                throttle: this.router.throttle,
+                reclaimDelay: this.router.reclaimDelay,
+                reclaimCount: this.router.reclaimCount,
                 system: await (0, utils_1.getSystemHealth)(),
             };
         }

package/build/types/durable.d.ts

@@ -1,5 +1,6 @@
 import { LogLevel } from './logger';
 import { RedisClass, RedisOptions } from './redis';
+import { StringStringType } from './serializer';
 type WorkflowConfig = {
     backoffCoefficient?: number;
     maximumAttempts?: number;
@@ -11,6 +12,14 @@ type WorkflowContext = {
      * the reentrant semaphore, incremented in real-time as idempotent statements are re-traversed upon reentry. Indicates the current semaphore count.
      */
     counter: number;
+    /**
+     * number as string for the replay cursor
+     */
+    cursor: string;
+    /**
+     * the replay hash of name/value pairs representing prior executions
+     */
+    replay: StringStringType;
     /**
      * the HotMesh App namespace. `durable` is the default.
      */

package/build/types/quorum.d.ts

@@ -39,7 +39,11 @@ export interface QuorumProfile {
     stream?: string;
     stream_depth?: number;
     counts?: Record<string, number>;
+    inited?: string;
     timestamp?: string;
+    throttle?: number;
+    reclaimDelay?: number;
+    reclaimCount?: number;
     system?: SystemHealth;
 }
 export interface PingMessage {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.47",
+  "version": "0.0.49",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/services/durable/meshos.ts

@@ -288,13 +288,17 @@ export class MeshOSService {
       args.push('LIMIT', '0', '0');
     } else {
       //limit which hash fields to return
-      if (options.return?.length) {
-        args.push('RETURN');
-        args.push(options.return.length.toString());
-        options.return.forEach(returnField => {
+      args.push('RETURN');
+      args.push(((options.return?.length ?? 0) + 1).toString());
+      args.push('$');
+      options.return?.forEach(returnField => {
+        if (returnField.startsWith('"')) {
+          //allow literal values to be requested
+          args.push(returnField.slice(1, -1));
+        } else {
           args.push(`_${returnField}`);
-        });
-      }
+        }
+      });
       //paginate
       if (options.limit) {
         args.push('LIMIT', options.limit.start.toString(), options.limit.size.toString());

package/services/durable/search.ts

@@ -3,6 +3,7 @@ import { RedisClient, RedisMulti } from '../../types/redis';
 import { StoreService } from '../store';
 import { KeyService, KeyType } from '../../modules/key';
 import { WorkflowSearchOptions } from '../../types/durable';
+import { asyncLocalStorage } from '../../modules/storage';
 
 export class Search {
   jobId: string;
@@ -66,9 +67,14 @@ export class Search {
    * @returns {Promise<string[]>} - the list of search indexes
    */
   static async listSearchIndexes(hotMeshClient: HotMesh): Promise<string[]> {
-    const store = hotMeshClient.engine.store;
-    const searchIndexes = await store.exec('FT._LIST');
-    return searchIndexes as string[];
+    try {
+      const store = hotMeshClient.engine.store;
+      const searchIndexes = await store.exec('FT._LIST');
+      return searchIndexes as string[];
+    } catch (err) {
+      hotMeshClient.engine.logger.info('durable-client-search-list-err', { err });
+      return [];
+    }
   }
 
   /**
@@ -80,18 +86,28 @@ export class Search {
     return `${this.searchSessionId}-${this.searchSessionIndex++}-`;
   }
 
-  async set(...args: string[]): Promise<void> {
+  /**
+   * Sets the fields listed in args. Returns the
+   * count of new fields that were set (does not
+   * count fields that were updated)
+   */
+  async set(...args: string[]): Promise<number> {
     const ssGuid = this.getSearchSessionGuid();
-    const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
-    if (ssGuidValue === 1) {
-      const safeArgs: string[] = [];
-      for (let i = 0; i < args.length; i += 2) {
-        const key = this.safeKey(args[i]);
-        const value = args[i+1].toString();
-        safeArgs.push(key, value);
-      }
-      await this.store.exec('HSET', this.jobId, ...safeArgs);
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Number(replay[ssGuid]);
     }
+    const safeArgs: string[] = [];
+    for (let i = 0; i < args.length; i += 2) {
+      const key = this.safeKey(args[i]);
+      const value = args[i+1].toString();
+      safeArgs.push(key, value);
+    }
+    const fieldCount = await this.store.exec('HSET', this.jobId, ...safeArgs);
+    //no need to wait; set this interim value in the replay
+    this.store.exec('HSET', this.jobId, ssGuid, fieldCount.toString());
+    return Number(fieldCount);
   }
 
   async get(key: string): Promise<string> {
@@ -116,34 +132,65 @@ export class Search {
     }
   }
 
+  /**
+   * Deletes the fields listed in args. Returns the
+   * count of fields that were deleted.
+   */
   async del(...args: string[]): Promise<number | void> {
     const ssGuid = this.getSearchSessionGuid();
-    const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
-    if (ssGuidValue === 1) {
-      const safeArgs: string[] = [];
-      for (let i = 0; i < args.length; i++) {
-        safeArgs.push(this.safeKey(args[i]));
-      }
-      const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
-      return isNaN(response as unknown as number) ? undefined : Number(response);
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Number(replay[ssGuid]);
+    }
+    const safeArgs: string[] = [];
+    for (let i = 0; i < args.length; i++) {
+      safeArgs.push(this.safeKey(args[i]));
     }
+    const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
+    const formattedResponse = isNaN(response as unknown as number) ? 0 : Number(response);
+    //no need to wait; set this interim value in the replay
+    this.store.exec('HSET', this.jobId, ssGuid, formattedResponse.toString());
+    return formattedResponse;
   }
 
+  /**
+   * Increments the value of a field by the given amount. Returns the
+   * new value of the field after the increment. Can be
+   * used to decrement the value of a field by specifying a negative.
+   */
   async incr(key: string, val: number): Promise<number> {
     const ssGuid = this.getSearchSessionGuid();
-    const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
-    if (ssGuidValue === 1) {
-      return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string);
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Number(replay[ssGuid]);
     }
+    const num = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string;
+    //no need to wait; set this interim value in the replay
+    this.store.exec('HSET', this.jobId, ssGuid, num.toString());
+    return Number(num);
   }
 
+  /**
+   * Multiplies the value of a field by the given amount. Returns the
+   * new value of the field after the multiplication. NOTE:
+   * this is exponential multiplication.
+   */
   async mult(key: string, val: number): Promise<number> {
     const ssGuid = this.getSearchSessionGuid();
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Math.exp(Number(replay[ssGuid]));
+    }
     const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
     if (ssGuidValue === 1) {
       const log = Math.log(val);
-      const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string);
-      return Math.exp(logTotal);
+      const logTotal = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string;
+      //no need to wait; set this interim value in the replay
+      this.store.exec('HSET', this.jobId, ssGuid, logTotal.toString());
+      return Math.exp(Number(logTotal));
     }
   }
 }

package/services/durable/worker.ts

@@ -211,16 +211,29 @@ export class WorkerService {
           // garbage collect (expire) this job when originJobId is expired
           context.set('originJobId', workflowInput.originJobId);
         }
+        let replayQuery = '';
         if (workflowInput.workflowDimension) {
           //every hook function runs in an isolated dimension controlled
           //by the index assigned when the signal was received; even if the
           //hook function re-runs, its scope will always remain constant
           context.set('workflowDimension', workflowInput.workflowDimension);
+          replayQuery = `-*${workflowInput.workflowDimension}-*`;
+        } else {
+          //last letter of words like 'hook', 'sleep', 'wait', 'signal', 'search', 'start'
+          replayQuery = '-*[ehklpt]-*';
         }
         context.set('workflowTopic', workflowTopic);
         context.set('workflowName', workflowTopic.split('-').pop());
         context.set('workflowTrace', data.metadata.trc);
         context.set('workflowSpan', data.metadata.spn);
+        const store = this.workflowRunner.engine.store;
+        const [cursor, replay] = await store.findJobFields(
+          workflowInput.workflowId,
+          replayQuery,
+          50_000,
+          5_000,);
+        context.set('replay', replay);
+        context.set('cursor', cursor); // if != 0, more remain
         const workflowResponse = await asyncLocalStorage.run(context, async () => {
           return await workflowFunction.apply(this, workflowInput.arguments);
         });
@@ -232,7 +245,6 @@ export class WorkerService {
           data: { response: workflowResponse, done: true }
         };
       } catch (err) {
-
         //not an error...just a trigger to sleep
         if (err instanceof DurableSleepForError) {
           return {

package/services/durable/workflow.ts

@@ -21,6 +21,7 @@ import {
 import { JobInterruptOptions, JobOutput, JobState } from '../../types/job';
 import { StreamStatus } from '../../types/stream';
 import { deterministicRandom } from '../../modules/utils';
+import { StringStringType } from '../../types';
 
 export class WorkflowService {
 
@@ -92,6 +93,10 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const sessionId = `-start${workflowDimension}-${execIndex}-`;
+    const replay = store.get('replay') as StringStringType;
+    if (sessionId in replay) {
+      return replay[sessionId];
+    }
     //NOTE: this is the hash prefix; necessary for the search index to locate the entity
     const entityOrEmptyString = options.entity ?? '';
     //If the workflowId is not provided, it is generated from the entity and the workflow name
@@ -193,6 +198,8 @@ export class WorkflowService {
   static getContext(): WorkflowContext {
     const store = asyncLocalStorage.getStore();
     const workflowId = store.get('workflowId');
+    const replay = store.get('replay');
+    const cursor = store.get('cursor');
     const workflowDimension = store.get('workflowDimension') ?? '';
     const workflowTopic = store.get('workflowTopic');
     const namespace = store.get('namespace');
@@ -201,7 +208,9 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     return {
       counter: COUNTER.counter,
+      cursor,
       namespace,
+      replay,
       workflowId,
       workflowDimension,
       workflowTopic,
@@ -223,6 +232,10 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
+    const replay = store.get('replay') as StringStringType;
+    if (sessionId in replay) {
+      return false;
+    }
     const keyParams = {
       appId: hotMeshClient.appId,
       jobId: workflowId
@@ -300,6 +313,7 @@ export class WorkflowService {
       workflowTopic: store.get('workflowTopic'),
       workflowDimension: store.get('workflowDimension') ?? '',
       counter: store.get('counter'),
+      replay: store.get('replay'),
     }
   }
 
@@ -317,9 +331,13 @@ export class WorkflowService {
       workflowTopic,
       workflowDimension,
       counter: COUNTER,
+      replay,
     } = WorkflowService.getLocalState();
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const sessionId = `-once${workflowDimension}-${execIndex}-`;
+    if (sessionId in replay) {
+      return JSON.parse(replay[sessionId]);
+    }
     const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
     const keyParams = {
       appId: hotMeshClient.appId,

package/services/engine/index.ts

@@ -98,6 +98,7 @@ class EngineService {
   jobCallbacks: Record<string, JobMessageCallback> = {};
   reporting = false;
   jobId = 1;
+  inited: string;
 
   static async init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<EngineService> {
     if (config.engine) {
@@ -133,6 +134,7 @@ class EngineService {
         instance.store,
         logger,
       );
+      instance.inited = formatISODate(new Date());
       return instance;
     }
   }

package/services/quorum/index.ts

@@ -158,6 +158,10 @@ class QuorumService {
         stream,
         counts: this.engine.router.counts,
         timestamp: formatISODate(new Date()),
+        inited: this.engine.inited,
+        throttle: this.engine.router.throttle,
+        reclaimDelay: this.engine.router.reclaimDelay,
+        reclaimCount: this.engine.router.reclaimCount,
         system: await getSystemHealth(),
       };
     }

package/services/router/index.ts

@@ -10,7 +10,7 @@ import {
   HMSH_XCLAIM_DELAY_MS,
   HMSH_XPENDING_COUNT } from '../../modules/enums';
 import { KeyType } from '../../modules/key';
-import { XSleepFor, guid, sleepFor } from '../../modules/utils';
+import { guid, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { StreamService } from '../stream';
@@ -43,6 +43,10 @@ class Router {
   counts: { [key: string]: number } = {};
   currentTimerId: NodeJS.Timeout | null = null;
   shouldConsume: boolean;
+  sleepPromiseResolve: (() => void) | null = null;
+  innerPromiseResolve: (() => void) | null = null;
+  isSleeping: boolean = false;
+  sleepTimout: NodeJS.Timeout | null = null;
 
   constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
     this.appId = config.appId;
@@ -54,6 +58,14 @@ class Router {
     this.reclaimDelay = config.reclaimDelay || HMSH_XCLAIM_DELAY_MS;
     this.reclaimCount = config.reclaimCount || HMSH_XCLAIM_COUNT;
     this.logger = logger;
+    this.resetThrottleState();
+  }
+
+  private resetThrottleState() {
+    this.sleepPromiseResolve = null;
+    this.innerPromiseResolve = null;
+    this.isSleeping = false;
+    this.sleepTimout = null;
   }
 
   async createGroup(stream: string, group: string) {
@@ -72,6 +84,36 @@ class Router {
     return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
   }
 
+  /**
+   * An adjustable throttle that will interrupt a sleeping
+   * router if the throttle is reduced and the sleep time
+   * has elapsed. If the throttle is increased, or if
+   * the sleep time has not elapsed, the router will continue
+   * to sleep until the new termination point. This
+   * allows for dynamic, elastic throttling with smooth
+   * acceleration and deceleration.
+   */
+  public async customSleep(): Promise<void> {
+    if (this.throttle === 0) return;
+    if (this.isSleeping) return;
+    this.isSleeping = true;
+    let startTime = Date.now(); //anchor the origin
+
+    await new Promise<void>(async (outerResolve) => {
+      this.sleepPromiseResolve = outerResolve;
+      let elapsedTime = Date.now() - startTime;
+      while (elapsedTime < this.throttle) {
+        await new Promise<void>((innerResolve) => {
+          this.innerPromiseResolve = innerResolve;
+          this.sleepTimout = setTimeout(innerResolve, this.throttle - elapsedTime);
+        });
+        elapsedTime = Date.now() - startTime;
+      }
+      this.resetThrottleState();
+      outerResolve();
+    });
+  }
+
   async consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse|void>): Promise<void> {
     this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
     Router.instances.add(this);
@@ -80,9 +122,7 @@ class Router {
     let lastCheckedPendingMessagesAt = Date.now();
 
     async function consume() {
-      let sleep = XSleepFor(this.throttle);
-      this.currentTimerId = sleep.timerId;
-      await sleep.promise;
+      await this.customSleep();
       if (!this.shouldConsume) {
         this.logger.info(`stream-consumer-stopping`, { group, consumer, stream });
         return;
@@ -273,18 +313,28 @@ class Router {
   }
 
   cancelThrottle() {
-    if (this.currentTimerId !== undefined) {
-      clearTimeout(this.currentTimerId);
-      this.currentTimerId = undefined;
+    if (this.sleepTimout) {
+      clearTimeout(this.sleepTimout);
     }
+    this.resetThrottleState();
   }
 
-  setThrottle(delayInMillis: number) {
+  public setThrottle(delayInMillis: number): void {
     if (!Number.isInteger(delayInMillis) || delayInMillis < 0) {
       throw new Error('Throttle must be a non-negative integer');
     }
+    const wasDecreased = delayInMillis < this.throttle;
     this.throttle = delayInMillis;
-    this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
+
+    // If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
+    if (wasDecreased) {
+      if (this.sleepTimout)  {
+        clearTimeout(this.sleepTimout);
+      }
+      if (this.innerPromiseResolve) {
+        this.innerPromiseResolve();
+      }
+    }
   }
 
   async claimUnacknowledged(stream: string, group: string, consumer: string, idleTimeMs = this.reclaimDelay, limit = HMSH_XPENDING_COUNT): Promise<[string, [string, string]][]> {

package/services/store/clients/redis.ts

@@ -22,6 +22,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       setnx: 'SETNX',
       del: 'DEL',
       expire: 'EXPIRE',
+      hscan: 'HSCAN',
       hset: 'HSET',
       hsetnx: 'HSETNX',
       hincrby: 'HINCRBY',
@@ -41,6 +42,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       lpop: 'LPOP',
       rename: 'RENAME',
       rpush: 'RPUSH',
+      scan: 'SCAN',
       xack: 'XACK',
       xdel: 'XDEL',
       xlen: 'XLEN',

package/services/store/index.ts

@@ -50,6 +50,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     del: 'del',
     expire: 'expire',
     hset: 'hset',
+    hscan: 'hscan',
     hsetnx: 'hsetnx',
     hincrby: 'hincrby',
     hdel: 'hdel',
@@ -68,6 +69,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     lrange: 'lrange',
     rename: 'rename',
     rpush: 'rpush',
+    scan: 'scan',
     xack: 'xack',
     xdel: 'xdel',
   };
@@ -998,6 +1000,63 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     await this.redisClient[this.commands.del](jobKey);
   }
+
+  async findJobs(queryString: string = '*', limit: number = 1000, batchSize: number = 1000): Promise<string[]> {
+    const matchKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId: queryString });
+    let cursor = '0';
+    let keys: string[];
+    const matchingKeys: string[] = [];
+    do {
+      const output = await this.exec(
+        'SCAN',
+        cursor,
+        'MATCH',
+        matchKey,
+        'COUNT',
+        batchSize.toString(),
+      ) as unknown as [string, string[]];
+      if (Array.isArray(output)) {
+        [cursor, keys] = output;
+        for (let key of [...keys]) {
+          matchingKeys.push(key);
+        }
+        if (matchingKeys.length >= limit) {
+          break;
+        }
+      } else {
+        break;
+      }
+    } while (cursor !== '0');
+    return matchingKeys;
+  }
+
+  async findJobFields(jobId: string, fieldMatchPattern: string = '*', limit: number = 1000, batchSize: number = 1000, cursor = '0'): Promise<[string, StringStringType]> {
+    let fields: string[] = [];
+    const matchingFields: StringStringType = {};
+    const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+    let len = 0;
+    do {
+      const output = await this.exec(
+        'HSCAN',
+        jobKey,
+        cursor,
+        'MATCH',
+        fieldMatchPattern,
+        'COUNT',
+        batchSize.toString(),
+      ) as unknown as [string, string[]];
+      if (Array.isArray(output)) {
+        [cursor, fields] = output;
+        for (let i = 0; i < fields.length; i += 2) {
+          len++;
+          matchingFields[fields[i]] = fields[i + 1];
+        }
+      } else {
+        break;
+      }
+    } while (cursor !== '0' && len < limit);
+    return [cursor, matchingFields];
+  }  
 }
 
 export { StoreService };

package/services/worker/index.ts

@@ -34,6 +34,7 @@ class WorkerService {
   router: Router | null;
   logger: ILogger;
   reporting = false;
+  inited: string;
 
   static async init(
     namespace: string,
@@ -76,6 +77,7 @@ class WorkerService {
           service.guid,
           worker.callback
         );
+        service.inited = formatISODate(new Date());
         services.push(service);
       }
     }
@@ -176,6 +178,10 @@ class WorkerService {
         stream: this.stream.mintKey(KeyType.STREAMS, params),
         counts: this.router.counts,
         timestamp: formatISODate(new Date()),
+        inited: this.inited,
+        throttle: this.router.throttle,
+        reclaimDelay: this.router.reclaimDelay,
+        reclaimCount: this.router.reclaimCount,
         system: await getSystemHealth(),
       };
     }

package/types/durable.ts

@@ -1,5 +1,6 @@
 import { LogLevel } from './logger';
 import { RedisClass, RedisOptions } from './redis';
+import { StringStringType } from './serializer';
 
 type WorkflowConfig = {
   backoffCoefficient?: number; //default 10
@@ -15,6 +16,16 @@ type WorkflowContext = {
    */
   counter: number;
 
+  /**
+   * number as string for the replay cursor
+   */
+  cursor: string;
+
+  /**
+   * the replay hash of name/value pairs representing prior executions
+   */
+  replay: StringStringType;
+
   /**
    * the HotMesh App namespace. `durable` is the default.
    */

package/types/quorum.ts

@@ -44,7 +44,11 @@ export interface QuorumProfile {
   stream?: string;
   stream_depth?: number;
   counts?: Record<string, number>;
+  inited?: string;
   timestamp?: string;
+  throttle?: number;
+  reclaimDelay?: number;
+  reclaimCount?: number;
   system?: SystemHealth;
 }