@hotmeshio/hotmesh 0.0.32 → 0.0.34

package/README.md

@@ -44,7 +44,7 @@ npm install @hotmeshio/hotmesh
       }
     }
     ```
-3. Although you could call your workflow directly (it's just a vanilla function), it's only durable when invoked and orchestrated via HotMesh.
+3. Instance a HotMesh **client** to invoke the workflow.
     ```javascript
     //client.ts
     import { Durable } from '@hotmeshio/hotmesh';
@@ -91,29 +91,77 @@ npm install @hotmeshio/hotmesh
     }
     ```
 
+### Workflow Extensions
 Redis governance delivers more than just reliability. Externalizing state fundamentally changes the execution profile for your functions, allowing you to design long-running, durable workflows. The `Durable.workflow` base class (shown in the examples above) provides additional methods for solving the most common state management challenges.
 
-Include statements like `await Durable.workflow.sleep('1 month')` to pause your workflow function for a month, or `await Durable.workflow.waitForSignal('signal1', 'signal2')` to pause your function until all signals have arrived.
-
- - `waitForSignal` | Pause your function and wait for external event(s) before continuing. The *waitForSignal* method will collate and cache the signals and only awaken your function once all signals have arrived.
- - `signal` | Send a signal (and optional payload) to any paused function.
- - `hook` | Redis governance converts your functions into 're-entrant processes'. Optionally use the *hook* method to spawn parallel execution threads to augment a running workflow.
- - `sleep` | Pause function execution for a ridiculous amount of time (months, years, etc). There's no risk of information loss, as Redis governs function state. When your function awakens, function state is efficiently (and automatically) restored and your function will resume right where it left off.
- - `random` | Generate a deterministic random number that can be used in a reentrant process workflow (replaces `Math.random()`).
- - `executeChild` | Call another durable function and await the response. *Design sophisticated, multi-process solutions by leveraging this command.*
- - `startChild` | Call another durable function, but do not await the response.
- - `search` | Instance a search session (e.g, `const search = MeshOS.search()`)
-    - `set` | Set one or more name/value pairs (e.g, `search.set('name1', 'value1', 'name2', 'value2')`)
-    - `get` | Get a single value by name(e.g, `search.get('name')`)
-    - `mget` | Get multiple values by name (e.g, `search.mget('name1', 'name2')`)
-    - `del` | Delete one or more entries by name and return the number deleted (e.g, `search.del('name1', 'name2')`)
-    - `incr` | Increment (or decrement) a number (e.g, `search.incr('name', -99)`)
-    - `mult` | Multiply a number (e.g, `search.mult('name', 12)`)
- - `find` | Find workflows using the native Redis [FT.*](https://redis.io/commands/ft.search/) search commands
- - `findWhere` | Find workflows using a simplified, JSON-based search syntax that overlays the native Redis FT.SEARCH syntax.
- - `createIndex` | Create a searchable index in Redis using simplified, JSON-based syntax that overlays the native Redis FT.CREATE syntax.
- - `startWorkers` | Start the workers necessary to govern your class (typically called at server startup).
- - `stopWorkers` | Stop all workers (typically called at server shutdown)
+ - `waitForSignal` Pause your function and wait for external event(s) before continuing. The *waitForSignal* method will collate and cache the signals and only awaken your function once all signals have arrived.
+   ```javascript
+    const signals = [a, b] = await Durable.workflow.waitForSignal('sig1', 'sig2')` 
+    ```
+ - `signal` Send a signal (and optional payload) to a paused function awaiting the signal.
+    ```javascript
+      await Durable.workflow.signal('sig1', {payload: 'hi!'});
+    ```
+ - `hook` Redis governance converts your functions into 're-entrant processes'. Optionally use the *hook* method to spawn parallel execution threads to augment a running workflow.
+    ```javascript
+    await Durable.workflow.hook({
+      workflowName: 'newsletter',
+      taskQueue: 'default',
+      args: []
+    });
+    ```
+ - `sleepFor` Pause function execution for a ridiculous amount of time (months, years, etc). There's no risk of information loss, as Redis governs function state. When your function awakens, function state is efficiently (and automatically) restored and your function will resume right where it left off.
+    ```javascript
+    await Durable.workflow.sleepFor('1 month');
+    ```
+ - `random` Generate a deterministic random number that can be used in a reentrant process workflow (replaces `Math.random()`).
+    ```javascript
+    const random = await Durable.workflow.random();
+    ```
+ - `executeChild` Call another durable function and await the response. *Design sophisticated, multi-process solutions by leveraging this command.*
+    ```javascript
+    const jobResponse = await Durable.workflow.executeChild({
+      workflowName: 'newsletter',
+      taskQueue: 'default',
+      args: [{ id, user_id, etc }],
+    });
+    ```
+ - `startChild` Call another durable function, but do not await the response.
+    ```javascript
+    const jobId = await Durable.workflow.startChild({
+      workflowName: 'newsletter',
+      taskQueue: 'default',
+      args: [{ id, user_id, etc }],
+    });
+    ```
+ - `search` Instance a search session
+    ```javascript
+    const search = await Durable.workflow.search();
+    ```
+    - `set` Set one or more name/value pairs
+      ```javascript
+      await search.set('name1', 'value1', 'name2', 'value2');
+      ```
+    - `get` Get a single value by name
+      ```javascript
+      const value = await search.get('name');
+      ```
+    - `mget` Get multiple values by name
+      ```javascript
+      const [val1, val2] = await search.mget('name1', 'name2');
+      ```
+    - `del` Delete one or more entries by name and return the number deleted
+      ```javascript
+      const count = await search.del('name1', 'name2');
+      ```
+    - `incr` Increment (or decrement) a number
+      ```javascript
+      const value = await search.incr('name', 12);
+      ```
+    - `mult` Multiply a number
+      ```javascript
+      const value = await search.mult('name', 12);
+      ```
 
 Refer to the [hotmeshio/samples-typescript](https://github.com/hotmeshio/samples-typescript) repo for usage examples. 
 

package/build/modules/errors.d.ts

@@ -28,6 +28,13 @@ declare class DurableSleepError extends Error {
     dimension: string;
     constructor(message: string, duration: number, index: number, dimension: string);
 }
+declare class DurableSleepForError extends Error {
+    code: number;
+    duration: number;
+    index: number;
+    dimension: string;
+    constructor(message: string, duration: number, index: number, dimension: string);
+}
 declare class DurableTimeoutError extends Error {
     code: number;
     constructor(message: string);
@@ -63,4 +70,4 @@ declare class CollationError extends Error {
     fault: CollationFaultType;
     constructor(status: number, leg: ActivityDuplex, stage: CollationStage, fault?: CollationFaultType);
 }
-export { CollationError, DurableTimeoutError, DurableMaxedError, DurableFatalError, DurableRetryError, DurableWaitForSignalError, DurableIncompleteSignalError, DurableSleepError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };
+export { CollationError, DurableTimeoutError, DurableMaxedError, DurableFatalError, DurableRetryError, DurableWaitForSignalError, DurableIncompleteSignalError, DurableSleepError, DurableSleepForError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };

package/build/modules/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExecActivityError = exports.RegisterTimeoutError = exports.MapDataError = exports.SetStateError = exports.GetStateError = exports.DuplicateJobError = exports.DurableSleepError = exports.DurableIncompleteSignalError = exports.DurableWaitForSignalError = exports.DurableRetryError = exports.DurableFatalError = exports.DurableMaxedError = exports.DurableTimeoutError = exports.CollationError = void 0;
+exports.ExecActivityError = exports.RegisterTimeoutError = exports.MapDataError = exports.SetStateError = exports.GetStateError = exports.DuplicateJobError = exports.DurableSleepForError = exports.DurableSleepError = exports.DurableIncompleteSignalError = exports.DurableWaitForSignalError = exports.DurableRetryError = exports.DurableFatalError = exports.DurableMaxedError = exports.DurableTimeoutError = exports.CollationError = void 0;
 class GetStateError extends Error {
     constructor() {
         super("Error occurred while getting job state");
@@ -31,6 +31,7 @@ class DurableWaitForSignalError extends Error {
     }
 }
 exports.DurableWaitForSignalError = DurableWaitForSignalError;
+/* @deprecated */
 class DurableSleepError extends Error {
     constructor(message, duration, index, dimension) {
         super(message);
@@ -41,6 +42,16 @@ class DurableSleepError extends Error {
     }
 }
 exports.DurableSleepError = DurableSleepError;
+class DurableSleepForError extends Error {
+    constructor(message, duration, index, dimension) {
+        super(message);
+        this.duration = duration;
+        this.index = index;
+        this.dimension = dimension;
+        this.code = 592;
+    }
+}
+exports.DurableSleepForError = DurableSleepForError;
 class DurableTimeoutError extends Error {
     constructor(message) {
         super(message);

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.32",
+    "version": "0.0.34",
     "description": "Unbreakable Workflows",
     "main": "build/index.js",
     "types": "build/index.d.ts",
@@ -23,6 +23,7 @@
         "test:hmsh": "NODE_ENV=test jest ./tests/functional/index.test.ts --detectOpenHandles --verbose",
         "test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
+        "test:trigger": "NODE_ENV=test jest ./tests/unit/services/activities/trigger.test.ts --detectOpenHandles --forceExit --verbose",
         "test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
         "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",

package/build/services/durable/client.js

@@ -93,13 +93,12 @@ class ClientService {
                 };
                 const context = { metadata: { trc, spn }, data: {} };
                 const jobId = await hotMeshClient.pub(`${options.namespace ?? factory_1.APP_ID}.execute`, payload, context);
-                //seed search data if present
+                // Seed search data if present
                 if (jobId && options.search?.data) {
                     const searchSessionId = `-search-0`;
                     const search = new search_1.Search(jobId, hotMeshClient, searchSessionId);
-                    for (const [key, value] of Object.entries(options.search.data)) {
-                        search.set(key, value);
-                    }
+                    const entries = Object.entries(options.search.data).flat();
+                    search.set(...entries);
                 }
                 return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
             },
@@ -179,9 +178,8 @@ class ClientService {
         }
     }
     static async shutdown() {
-        for (const [key, value] of ClientService.instances) {
-            const hotMesh = await value;
-            await hotMesh.stop();
+        for (const [_, hotMeshInstance] of ClientService.instances) {
+            (await hotMeshInstance).stop();
         }
     }
 }

package/build/services/durable/factory.d.ts

@@ -6,7 +6,8 @@
  *
  * ERROR CODES:
  *      594: waitforsignal
- *      595: sleep
+ *      592: sleepFor
+ *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */

package/build/services/durable/factory.js

@@ -9,7 +9,8 @@ exports.DEFAULT_COEFFICIENT = exports.APP_ID = exports.APP_VERSION = exports.get
  *
  * ERROR CODES:
  *      594: waitforsignal
- *      595: sleep
+ *      592: sleepFor
+ *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */
@@ -129,6 +130,20 @@ const getWorkflowYAML = (app, version) => {
               maps:
                 duration: '{$self.output.data.duration}'
                 index: '{$self.output.data.index}'
+            592:
+              schema:
+                type: object
+                properties:
+                  duration:
+                    type: number
+                    description: sleepFor duration in seconds
+                  index:
+                    type: number
+                    description: the current index
+              maps:
+                duration: '{$self.output.data.duration}'
+                index: '{$self.output.data.index}'
+  
           job:
             maps:
               response: '{$self.output.data.response}'
@@ -231,7 +246,20 @@ const getWorkflowYAML = (app, version) => {
               maps:
                 duration: '{$self.output.data.duration}'
                 index: '{$self.output.data.index}'
-
+            592:
+              schema:
+                type: object
+                properties:
+                  duration:
+                    type: number
+                    description: sleepFor duration in seconds
+                  index:
+                    type: number
+                    description: the current index
+              maps:
+                duration: '{$self.output.data.duration}'
+                index: '{$self.output.data.index}'
+  
         siga594:
           title: Signal In - Wait for signals
           type: await
@@ -334,6 +362,19 @@ const getWorkflowYAML = (app, version) => {
             maps:
               duration: '{siga1.output.data.duration}'
 
+        siga592:
+          title: Signal In - Sleep For a duration and then cycle/goto
+          type: hook
+          sleep: '{sigw1.output.data.duration}'
+
+        sigc592:
+          title: Signal In - Goto Activity a1 after sleeping for a duration
+          type: cycle
+          ancestor: siga1
+          input:
+            maps:
+              duration: '{siga1.output.data.duration}'
+    
         siga599:
           title: Signal In - Sleep exponentially longer and retry
           type: hook
@@ -452,6 +493,19 @@ const getWorkflowYAML = (app, version) => {
             maps:
               duration: '{a1.output.data.duration}'
 
+        a592:
+          title: Sleep For a duration and then cycle/goto
+          type: hook
+          sleep: '{w1.output.data.duration}'
+
+        c592:
+          title: Goto Activity a1 after sleeping for a duration
+          type: cycle
+          ancestor: a1
+          input:
+            maps:
+              duration: '{a1.output.data.duration}'
+
         a599:
           title: Sleep exponentially longer before retrying
           type: hook
@@ -655,6 +709,9 @@ const getWorkflowYAML = (app, version) => {
           - to: siga595
             conditions:
               code: 595
+          - to: siga592
+            conditions:
+              code: 592
           - to: siga599
             conditions:
               code: 599
@@ -666,6 +723,8 @@ const getWorkflowYAML = (app, version) => {
           - to: sigc595
             conditions:
               code: 202
+        siga592:
+          - to: sigc592
         siga599:
           - to: sigc599
         a1:
@@ -677,6 +736,9 @@ const getWorkflowYAML = (app, version) => {
           - to: a595
             conditions:
               code: 595
+          - to: a592
+            conditions:
+              code: 592
           - to: a599
             conditions:
               code: 599
@@ -703,6 +765,8 @@ const getWorkflowYAML = (app, version) => {
           - to: c595
             conditions:
               code: 202
+        a592:
+          - to: c592
         a599:
           - to: c599
 

package/build/services/durable/index.d.ts

@@ -12,5 +12,10 @@ export declare const Durable: {
     MeshOS: typeof MeshOSService;
     Worker: typeof WorkerService;
     workflow: typeof WorkflowService;
+    /**
+     * Shutdown everything. All connections, workers, and clients will be closed.
+     * Include in your signal handlers to ensure a clean shutdown.
+     */
+    shutdown(): Promise<void>;
 };
 export type { ContextType };

package/build/services/durable/index.js

@@ -7,6 +7,7 @@ const meshos_1 = require("./meshos");
 const search_1 = require("./search");
 const worker_1 = require("./worker");
 const workflow_1 = require("./workflow");
+const hotmesh_1 = require("../hotmesh");
 exports.Durable = {
     Client: client_1.ClientService,
     Connection: connection_1.ConnectionService,
@@ -14,4 +15,13 @@ exports.Durable = {
     MeshOS: meshos_1.MeshOSService,
     Worker: worker_1.WorkerService,
     workflow: workflow_1.WorkflowService,
+    /**
+     * Shutdown everything. All connections, workers, and clients will be closed.
+     * Include in your signal handlers to ensure a clean shutdown.
+     */
+    async shutdown() {
+        await client_1.ClientService.shutdown();
+        await worker_1.WorkerService.shutdown();
+        await hotmesh_1.HotMeshService.stop();
+    }
 };

package/build/services/durable/meshos.js

@@ -8,7 +8,6 @@ const client_1 = require("./client");
 const search_1 = require("./search");
 const worker_1 = require("./worker");
 const workflow_1 = require("./workflow");
-const stream_1 = require("../signaler/stream");
 /**
  * The base class for running MeshOS workflows.
  * Extend this class, add your Redis config, and add functions to
@@ -47,9 +46,7 @@ class MeshOSService {
      * @returns {Promise<void>}
      */
     static async stopWorkers() {
-        await _1.Durable.Client.shutdown();
-        await _1.Durable.Worker.shutdown();
-        await stream_1.StreamSignaler.stopConsuming();
+        await _1.Durable.shutdown();
     }
     /**
      * Initializes the worker(s). This is a static

package/build/services/durable/worker.js

@@ -178,7 +178,22 @@ class WorkerService {
             }
             catch (err) {
                 //not an error...just a trigger to sleep
-                if (err instanceof errors_1.DurableSleepError) {
+                if (err instanceof errors_1.DurableSleepForError) {
+                    return {
+                        status: stream_1.StreamStatus.SUCCESS,
+                        code: err.code,
+                        metadata: { ...data.metadata },
+                        data: {
+                            code: err.code,
+                            message: JSON.stringify({ duration: err.duration, index: err.index, dimension: err.dimension }),
+                            duration: err.duration,
+                            index: err.index,
+                            dimension: err.dimension
+                        }
+                    };
+                    //deprecated format; not an error...just a trigger to sleep
+                }
+                else if (err instanceof errors_1.DurableSleepError) {
                     return {
                         status: stream_1.StreamStatus.SUCCESS,
                         code: err.code,
@@ -225,9 +240,8 @@ class WorkerService {
         };
     }
     static async shutdown() {
-        for (const [key, value] of WorkerService.instances) {
-            const hotMesh = await value;
-            await hotMesh.stop();
+        for (const [_, hotMeshInstance] of WorkerService.instances) {
+            (await hotMeshInstance).stop();
         }
     }
 }

package/build/services/durable/workflow.d.ts

@@ -81,10 +81,21 @@ export declare class WorkflowService {
      */
     static hook(options: HookOptions): Promise<string>;
     /**
-     * Sleeps for a duration.
+     * Sleeps the workflow for a duration. As the function is reentrant,
+     * upon reentry, the function will traverse prior execution paths up
+     * until the sleep command and then resume execution from that point.
      * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
      * @returns {Promise<number>}
      */
+    static sleepFor(duration: string): Promise<number>;
+    /**
+     * Sleeps the workflow for a duration. As the function is reentrant,
+     * upon reentry, the function will traverse prior execution paths up
+     * until the sleep command and then resume execution from that point.
+     * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
+     * @returns {Promise<number>}
+     * @deprecated - use `sleepFor` instead
+     */
     static sleep(duration: string): Promise<number>;
     /**
      * Waits for a signal to awaken

package/build/services/durable/workflow.js

@@ -262,10 +262,37 @@ class WorkflowService {
         }
     }
     /**
-     * Sleeps for a duration.
+     * Sleeps the workflow for a duration. As the function is reentrant,
+     * upon reentry, the function will traverse prior execution paths up
+     * until the sleep command and then resume execution from that point.
      * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
      * @returns {Promise<number>}
      */
+    static async sleepFor(duration) {
+        const seconds = (0, ms_1.default)(duration) / 1000;
+        const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
+        const namespace = store.get('namespace');
+        const workflowTopic = store.get('workflowTopic');
+        const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
+        if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'sleep')) {
+            const workflowId = store.get('workflowId');
+            const workflowDimension = store.get('workflowDimension') ?? '';
+            const COUNTER = store.get('counter');
+            const execIndex = COUNTER.counter;
+            // spawn a new sleep job if error code 592 is thrown by the worker
+            // NOTE: If this message appears in the stack trace, the `.sleepFor()` method in your workflow code was NOT awaited.
+            throw new errors_1.DurableSleepForError(workflowId, seconds, execIndex, workflowDimension);
+        }
+        return seconds;
+    }
+    /**
+     * Sleeps the workflow for a duration. As the function is reentrant,
+     * upon reentry, the function will traverse prior execution paths up
+     * until the sleep command and then resume execution from that point.
+     * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
+     * @returns {Promise<number>}
+     * @deprecated - use `sleepFor` instead
+     */
     static async sleep(duration) {
         const seconds = (0, ms_1.default)(duration) / 1000;
         const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
@@ -284,7 +311,6 @@ class WorkflowService {
         }
         catch (e) {
             // spawn a new sleep job if error code 595 is thrown by the worker)
-            // NOTE: If this message shows up in your stack trace, you forgot to await `Durable.workflow.sleep()` in your workflow code.
             throw new errors_1.DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
         }
     }

package/build/services/hotmesh/index.d.ts

@@ -16,6 +16,7 @@ declare class HotMeshService {
     quorum: QuorumService | null;
     workers: WorkerService[];
     logger: ILogger;
+    static disconnecting: boolean;
     verifyAndSetNamespace(namespace?: string): void;
     verifyAndSetAppId(appId: string): void;
     static init(config: HotMeshConfig): Promise<HotMeshService>;
@@ -42,7 +43,8 @@ declare class HotMeshService {
     scrub(jobId: string): Promise<void>;
     hook(topic: string, data: JobData, status?: StreamStatus, code?: StreamCode): Promise<string>;
     hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
-    stop(): Promise<void>;
+    static stop(): Promise<void>;
+    stop(): void;
     compress(terms: string[]): Promise<boolean>;
 }
 export { HotMeshService };

package/build/services/hotmesh/index.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HotMeshService = void 0;
 const nanoid_1 = require("nanoid");
 const key_1 = require("../../modules/key");
+const redis_1 = require("../connector/clients/redis");
+const ioredis_1 = require("../connector/clients/ioredis");
 const engine_1 = require("../engine");
 const logger_1 = require("../logger");
 const stream_1 = require("../signaler/stream");
@@ -128,8 +130,15 @@ class HotMeshService {
     async hookAll(hookTopic, data, query, queryFacets = []) {
         return await this.engine?.hookAll(hookTopic, data, query, queryFacets);
     }
-    async stop() {
-        await stream_1.StreamSignaler.stopConsuming();
+    static async stop() {
+        if (!this.disconnecting) {
+            this.disconnecting = true;
+            await stream_1.StreamSignaler.stopConsuming();
+            await redis_1.RedisConnection.disconnectAll();
+            await ioredis_1.RedisConnection.disconnectAll();
+        }
+    }
+    stop() {
         this.engine?.task.cancelCleanup();
     }
     async compress(terms) {
@@ -137,3 +146,4 @@ class HotMeshService {
     }
 }
 exports.HotMeshService = HotMeshService;
+HotMeshService.disconnecting = false;

package/build/services/signaler/stream.js

@@ -220,12 +220,12 @@ class StreamSignaler {
         for (const instance of [...StreamSignaler.signalers]) {
             instance.stopConsuming();
         }
+        await (0, utils_1.sleepFor)(BLOCK_TIME_MS);
     }
     async stopConsuming() {
         this.shouldConsume = false;
         this.logger.info(`stream-consumer-stopping`, this.topic ? { topic: this.topic } : undefined);
         this.cancelThrottle();
-        //await sleepFor(BLOCK_TIME_MS);
     }
     cancelThrottle() {
         if (this.currentTimerId !== undefined) {

package/modules/errors.ts

@@ -33,6 +33,7 @@ class DurableWaitForSignalError extends Error {
   }
 }
 
+/* @deprecated */
 class DurableSleepError extends Error {
   code: number;
   duration: number; //seconds
@@ -46,6 +47,19 @@ class DurableSleepError extends Error {
     this.code = 595;
   }
 }
+class DurableSleepForError extends Error {
+  code: number;
+  duration: number; //seconds
+  index: number;    //execution order in the workflow
+  dimension: string; //hook dimension (e.g., ',0,1,0') (uses empty string for `null`)
+  constructor(message: string, duration: number, index: number, dimension: string) {
+    super(message);
+    this.duration = duration;
+    this.index = index;
+    this.dimension = dimension;
+    this.code = 592;
+  }
+}
 class DurableTimeoutError extends Error {
   code: number;
   constructor(message: string) {
@@ -124,6 +138,7 @@ export {
   DurableWaitForSignalError,
   DurableIncompleteSignalError,
   DurableSleepError,
+  DurableSleepForError,
   DuplicateJobError,
   GetStateError,
   SetStateError,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.32",
+  "version": "0.0.34",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -23,6 +23,7 @@
     "test:hmsh": "NODE_ENV=test jest ./tests/functional/index.test.ts --detectOpenHandles --verbose",
     "test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:trigger": "NODE_ENV=test jest ./tests/unit/services/activities/trigger.test.ts --detectOpenHandles --forceExit --verbose",
     "test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",

package/services/durable/client.ts

@@ -113,13 +113,12 @@ export class ClientService {
         payload,
         context as JobState
       );
-      //seed search data if present
+      // Seed search data if present
       if (jobId && options.search?.data) {
         const searchSessionId = `-search-0`;
         const search = new Search(jobId, hotMeshClient, searchSessionId);
-        for (const [key, value] of Object.entries(options.search.data)) {
-          search.set(key, value);
-        }
+        const entries = Object.entries(options.search.data).flat();
+        search.set(...entries);
       }
       return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
     },
@@ -199,9 +198,8 @@ export class ClientService {
   }
 
   static async shutdown(): Promise<void> {
-    for (const [key, value] of ClientService.instances) {
-      const hotMesh = await value;
-      await hotMesh.stop();
+    for (const [_, hotMeshInstance] of ClientService.instances) {
+      (await hotMeshInstance).stop();
     }
   }
 }

package/services/durable/factory.ts

@@ -6,7 +6,8 @@
  *
  * ERROR CODES:
  *      594: waitforsignal
- *      595: sleep
+ *      592: sleepFor
+ *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */
@@ -126,6 +127,20 @@ const getWorkflowYAML = (app: string, version: string) => {
               maps:
                 duration: '{$self.output.data.duration}'
                 index: '{$self.output.data.index}'
+            592:
+              schema:
+                type: object
+                properties:
+                  duration:
+                    type: number
+                    description: sleepFor duration in seconds
+                  index:
+                    type: number
+                    description: the current index
+              maps:
+                duration: '{$self.output.data.duration}'
+                index: '{$self.output.data.index}'
+  
           job:
             maps:
               response: '{$self.output.data.response}'
@@ -228,7 +243,20 @@ const getWorkflowYAML = (app: string, version: string) => {
               maps:
                 duration: '{$self.output.data.duration}'
                 index: '{$self.output.data.index}'
-
+            592:
+              schema:
+                type: object
+                properties:
+                  duration:
+                    type: number
+                    description: sleepFor duration in seconds
+                  index:
+                    type: number
+                    description: the current index
+              maps:
+                duration: '{$self.output.data.duration}'
+                index: '{$self.output.data.index}'
+  
         siga594:
           title: Signal In - Wait for signals
           type: await
@@ -331,6 +359,19 @@ const getWorkflowYAML = (app: string, version: string) => {
             maps:
               duration: '{siga1.output.data.duration}'
 
+        siga592:
+          title: Signal In - Sleep For a duration and then cycle/goto
+          type: hook
+          sleep: '{sigw1.output.data.duration}'
+
+        sigc592:
+          title: Signal In - Goto Activity a1 after sleeping for a duration
+          type: cycle
+          ancestor: siga1
+          input:
+            maps:
+              duration: '{siga1.output.data.duration}'
+    
         siga599:
           title: Signal In - Sleep exponentially longer and retry
           type: hook
@@ -449,6 +490,19 @@ const getWorkflowYAML = (app: string, version: string) => {
             maps:
               duration: '{a1.output.data.duration}'
 
+        a592:
+          title: Sleep For a duration and then cycle/goto
+          type: hook
+          sleep: '{w1.output.data.duration}'
+
+        c592:
+          title: Goto Activity a1 after sleeping for a duration
+          type: cycle
+          ancestor: a1
+          input:
+            maps:
+              duration: '{a1.output.data.duration}'
+
         a599:
           title: Sleep exponentially longer before retrying
           type: hook
@@ -652,6 +706,9 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: siga595
             conditions:
               code: 595
+          - to: siga592
+            conditions:
+              code: 592
           - to: siga599
             conditions:
               code: 599
@@ -663,6 +720,8 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: sigc595
             conditions:
               code: 202
+        siga592:
+          - to: sigc592
         siga599:
           - to: sigc599
         a1:
@@ -674,6 +733,9 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: a595
             conditions:
               code: 595
+          - to: a592
+            conditions:
+              code: 592
           - to: a599
             conditions:
               code: 599
@@ -700,6 +762,8 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: c595
             conditions:
               code: 202
+        a592:
+          - to: c592
         a599:
           - to: c599
 

package/services/durable/index.ts

@@ -4,6 +4,7 @@ import { MeshOSService } from './meshos';
 import { Search } from './search';
 import { WorkerService } from './worker';
 import { WorkflowService } from './workflow';
+import { HotMeshService } from '../hotmesh';
 import { ContextType } from '../../types/durable';
 
 export const Durable = {
@@ -13,6 +14,16 @@ export const Durable = {
   MeshOS: MeshOSService,
   Worker: WorkerService,
   workflow: WorkflowService,
+
+  /**
+   * Shutdown everything. All connections, workers, and clients will be closed.
+   * Include in your signal handlers to ensure a clean shutdown.
+   */
+  async shutdown(): Promise<void> {
+    await ClientService.shutdown();
+    await WorkerService.shutdown();
+    await HotMeshService.stop();
+  }
 };
 
 export type { ContextType };

package/services/durable/meshos.ts

@@ -7,7 +7,6 @@ import { WorkflowHandleService } from './handle';
 import { Search } from './search';
 import { WorkerService as Worker } from './worker';
 import { WorkflowService } from './workflow';
-import { StreamSignaler } from '../signaler/stream';
 import {
   FindOptions,
   FindWhereOptions,
@@ -135,9 +134,7 @@ export class MeshOSService {
    * @returns {Promise<void>}
    */
   static async stopWorkers(): Promise<void> {
-    await Durable.Client.shutdown();
-    await Durable.Worker.shutdown();
-    await StreamSignaler.stopConsuming();
+    await Durable.shutdown();
   }
 
   /**

package/services/durable/worker.ts

@@ -4,6 +4,7 @@ import {
   DurableMaxedError,
   DurableRetryError,
   DurableSleepError,
+  DurableSleepForError,
   DurableTimeoutError, 
   DurableWaitForSignalError} from '../../modules/errors';
 import { asyncLocalStorage } from './asyncLocalStorage';
@@ -226,7 +227,22 @@ export class WorkerService {
       } catch (err) {
 
         //not an error...just a trigger to sleep
-        if (err instanceof DurableSleepError) {
+        if (err instanceof DurableSleepForError) {
+          return {
+            status: StreamStatus.SUCCESS,
+            code: err.code,
+            metadata: { ...data.metadata },
+            data: {
+              code: err.code,
+              message: JSON.stringify({ duration: err.duration, index: err.index, dimension: err.dimension }),
+              duration: err.duration,
+              index: err.index,
+              dimension: err.dimension
+            }
+          } as StreamDataResponse;
+
+        //deprecated format; not an error...just a trigger to sleep
+        } else if (err instanceof DurableSleepError) {
           return {
             status: StreamStatus.SUCCESS,
             code: err.code,
@@ -275,9 +291,8 @@ export class WorkerService {
   }
 
   static async shutdown(): Promise<void> {
-    for (const [key, value] of WorkerService.instances) {
-      const hotMesh = await value;
-      await hotMesh.stop();
+    for (const [_, hotMeshInstance] of WorkerService.instances) {
+      (await hotMeshInstance).stop();
     }
   }
 }

package/services/durable/workflow.ts

@@ -3,6 +3,7 @@ import ms from 'ms';
 import {
   DurableIncompleteSignalError,
   DurableSleepError,
+  DurableSleepForError,
   DurableWaitForSignalError } from '../../modules/errors';
 import { KeyService, KeyType } from '../../modules/key';
 import { asyncLocalStorage } from './asyncLocalStorage';
@@ -289,10 +290,38 @@ export class WorkflowService {
   }
 
   /**
-   * Sleeps for a duration.
+   * Sleeps the workflow for a duration. As the function is reentrant, 
+   * upon reentry, the function will traverse prior execution paths up
+   * until the sleep command and then resume execution from that point.
    * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
    * @returns {Promise<number>}
    */
+  static async sleepFor(duration: string): Promise<number> {
+    const seconds = ms(duration) / 1000;
+    const store = asyncLocalStorage.getStore();
+    const namespace = store.get('namespace');
+    const workflowTopic = store.get('workflowTopic');
+    const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
+    if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'sleep')) {
+      const workflowId = store.get('workflowId');
+      const workflowDimension = store.get('workflowDimension') ?? '';
+      const COUNTER = store.get('counter');
+      const execIndex = COUNTER.counter;
+      // spawn a new sleep job if error code 592 is thrown by the worker
+      // NOTE: If this message appears in the stack trace, the `.sleepFor()` method in your workflow code was NOT awaited.
+      throw new DurableSleepForError(workflowId, seconds, execIndex, workflowDimension);
+    }
+    return seconds;
+  }
+
+  /**
+   * Sleeps the workflow for a duration. As the function is reentrant, 
+   * upon reentry, the function will traverse prior execution paths up
+   * until the sleep command and then resume execution from that point.
+   * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
+   * @returns {Promise<number>}
+   * @deprecated - use `sleepFor` instead
+   */
   static async sleep(duration: string): Promise<number> {
     const seconds = ms(duration) / 1000;
 
@@ -312,7 +341,6 @@ export class WorkflowService {
       return seconds;
     } catch (e) {
       // spawn a new sleep job if error code 595 is thrown by the worker)
-      // NOTE: If this message shows up in your stack trace, you forgot to await `Durable.workflow.sleep()` in your workflow code.
       throw new DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
     }
   }

package/services/hotmesh/index.ts

@@ -1,5 +1,7 @@
 import { nanoid } from 'nanoid';
 import { HMNS } from '../../modules/key';
+import { RedisConnection } from '../connector/clients/redis';
+import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
 import { EngineService } from '../engine';
 import { LoggerService, ILogger } from '../logger';
 import { StreamSignaler } from '../signaler/stream';
@@ -32,6 +34,8 @@ class HotMeshService {
   workers: WorkerService[] = [];
   logger: ILogger;
 
+  static disconnecting = false;
+
   verifyAndSetNamespace(namespace?: string) {
     if (!namespace) {
       this.namespace = HMNS;
@@ -176,8 +180,16 @@ class HotMeshService {
     return await this.engine?.hookAll(hookTopic, data, query, queryFacets);
   }
 
-  async stop() {
-    await StreamSignaler.stopConsuming();
+  static async stop() {
+    if (!this.disconnecting) {
+      this.disconnecting = true;
+      await StreamSignaler.stopConsuming();
+      await RedisConnection.disconnectAll();
+      await IORedisConnection.disconnectAll();
+    }
+  }
+
+  stop() {
     this.engine?.task.cancelCleanup();
   }
 

package/services/signaler/stream.ts

@@ -254,13 +254,13 @@ class StreamSignaler {
     for (const instance of [...StreamSignaler.signalers]) {
       instance.stopConsuming();
     }
+    await sleepFor(BLOCK_TIME_MS);
   }
   
   async stopConsuming() {
     this.shouldConsume = false;
     this.logger.info(`stream-consumer-stopping`, this.topic ? { topic: this.topic } : undefined);
     this.cancelThrottle();
-    //await sleepFor(BLOCK_TIME_MS);
   }
 
   cancelThrottle() {