@hotmeshio/hotmesh 0.0.19 → 0.0.20

package/README.md

@@ -1,7 +1,7 @@
 # HotMesh
 ![alpha release](https://img.shields.io/badge/release-alpha-yellow)
 
-Elevate Redis from an in-memory data store to a game-changing [service mesh](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/faq.md#what-is-hotmesh). Turn your unpredictable functions into unbreakable workflows.
+Elevate Redis from an in-memory data cache to a game-changing [service mesh](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/faq.md#what-is-hotmesh). Turn your unpredictable functions into unbreakable workflows.
 
 ## Install
 [![npm version](https://badge.fury.io/js/%40hotmeshio%2Fhotmesh.svg)](https://badge.fury.io/js/%40hotmeshio%2Fhotmesh)
@@ -13,7 +13,7 @@ npm install @hotmeshio/hotmesh
 ## Design
 The HotMesh SDK is designed to keep your code front-and-center. Write functions as you normally would, then use HotMesh to make them durable.
 
-1. Start by defining **activities**. Activities are those functions that will be invoked by your workflow. They are commonly used to read and write to databases and invoke external services. They can be written in any style, using any framework, and can even be legacy functions you've already written. The only requirement is that they return a Promise. *Note how the `saludar` example throws an error 50% of the time. It doesn't matter how unpredictable your functions are, HotMesh will retry as necessary until they succeed.*
+1. Start by defining **activities**. Activities can be written in any style, using any framework, and can even be legacy functions you've already written. The only requirement is that they return a Promise. *Note how the `saludar` example throws an error 50% of the time. It doesn't matter how unpredictable your functions are, HotMesh will retry as necessary until they succeed.*
     ```javascript
     //activities.ts
 
@@ -47,7 +47,7 @@ The HotMesh SDK is designed to keep your code front-and-center. Write functions
     }
     ```
 
-3. Although you could call your workflow directly (it's just a vanilla function), it's only durable when invoked and orchestrated via HotMesh. By using a HotMesh **client** to trigger the function, it's guaranteed to return a result.
+3. Although you could call your workflow directly (it's just a vanilla function), it's only durable when invoked and orchestrated via HotMesh.
     ```javascript
     //client.ts
 
@@ -75,7 +75,7 @@ The HotMesh SDK is designed to keep your code front-and-center. Write functions
     }
     ```
 
-4. The last step is to create a **worker** and link it to your workflow function. Workers listen for tasks on their assigned channel, executing the targeted workflow function until it succeeds.
+4. Finally, create a **worker** and link your workflow function. Workers listen for tasks on their assigned Redis stream and invoke your workflow function each time they receive an event.
     ```javascript
     //worker.ts
 

package/build/modules/errors.d.ts

@@ -25,7 +25,8 @@ declare class DurableSleepError extends Error {
     code: number;
     duration: number;
     index: number;
-    constructor(message: string, duration: number, index: number);
+    dimension: string;
+    constructor(message: string, duration: number, index: number, dimension: string);
 }
 declare class DurableTimeoutError extends Error {
     code: number;

package/build/modules/errors.js

@@ -32,10 +32,11 @@ class DurableWaitForSignalError extends Error {
 }
 exports.DurableWaitForSignalError = DurableWaitForSignalError;
 class DurableSleepError extends Error {
-    constructor(message, duration, index) {
+    constructor(message, duration, index, dimension) {
         super(message);
         this.duration = duration;
         this.index = index;
+        this.dimension = dimension;
         this.code = 595;
     }
 }

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.19",
+    "version": "0.0.20",
     "description": "Unbreakable Workflows",
     "main": "build/index.js",
     "types": "build/index.d.ts",
@@ -45,6 +45,7 @@
         "test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
+        "test:durable:hook": "NODE_ENV=test jest ./tests/durable/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:durable:fatal": "NODE_ENV=test jest ./tests/durable/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:durable:sleep": "NODE_ENV=test jest ./tests/durable/sleep/index.test.ts --detectOpenHandles --forceExit --verbose",

package/build/services/activities/activity.d.ts

@@ -26,7 +26,7 @@ declare class Activity {
     adjacentIndex: number;
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
     setLeg(leg: ActivityLeg): void;
-    processEvent(status?: StreamStatus, code?: StreamCode, type?: 'hook' | 'output', jobId?: string): Promise<void>;
+    processEvent(status?: StreamStatus, code?: StreamCode, type?: 'hook' | 'output'): Promise<void>;
     processPending(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags>;
     processSuccess(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags>;
     processError(telemetry: TelemetryService, type: string): Promise<MultiResponseFlags>;
@@ -49,7 +49,7 @@ declare class Activity {
     bindActivityState(state: StringAnyType): void;
     bindJobMetadataPaths(): string[];
     bindActivityMetadataPaths(): string[];
-    getState(jobId?: string): Promise<void>;
+    getState(): Promise<void>;
     initDimensionalAddress(dad: string): void;
     initSelf(context: StringAnyType): JobState;
     initPolicies(context: JobState): void;

package/build/services/activities/activity.js

@@ -30,16 +30,20 @@ class Activity {
         this.leg = leg;
     }
     //********  DUPLEX RE-ENTRY POINT  ********//
-    async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200, type = 'output', jobId) {
+    async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200, type = 'output') {
         this.setLeg(2);
-        const jid = this.context.metadata.jid || jobId;
+        const jid = this.context.metadata.jid;
+        if (!jid) {
+            this.logger.error('activity-process-event-error', { message: 'job id is undefined' });
+            return;
+        }
         const aid = this.metadata.aid;
         this.status = status;
         this.code = code;
         this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
         let telemetry;
         try {
-            await this.getState(jobId);
+            await this.getState();
             const aState = await collator_1.CollatorService.notarizeReentry(this);
             this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(aState);
             telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
@@ -67,7 +71,6 @@ class Activity {
                 this.logger.info('process-event-inactive-error', { error });
                 return;
             }
-            console.error(error);
             this.logger.error('activity-process-event-error', { error });
             telemetry && telemetry.setActivityError(error.message);
             throw error;
@@ -251,7 +254,7 @@ class Activity {
         const keys_to_save = this.leg === 1 ? 'ACTIVITY' : 'ACTIVITY_UPDATE';
         return serializer_1.MDATA_SYMBOLS[keys_to_save].KEYS.map((key) => `output/metadata/${key}`);
     }
-    async getState(jobId) {
+    async getState() {
         //assemble list of paths necessary to create 'job state' from the 'symbol hash'
         const jobSymbolHashName = `$${this.config.subscribes}`;
         const consumes = {
@@ -277,10 +280,9 @@ class Activity {
         }
         telemetry_1.TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
         let { dad, jid } = this.context.metadata;
-        jobId = jobId || jid;
-        const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad);
+        const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
         //`state` is a flat hash; context is a tree
-        const [state, status] = await this.store.getState(jobId, consumes, dIds);
+        const [state, status] = await this.store.getState(jid, consumes, dIds);
         this.context = (0, utils_1.restoreHierarchy)(state);
         this.initDimensionalAddress(dad);
         this.initSelf(this.context);

package/build/services/activities/hook.d.ts

@@ -4,6 +4,7 @@ import { ActivityData, ActivityMetadata, ActivityType, HookActivity } from '../.
 import { HookRule } from '../../types/hook';
 import { JobState, JobStatus } from '../../types/job';
 import { RedisMulti } from '../../types/redis';
+import { StreamCode, StreamStatus } from '../../types/stream';
 /**
  * Listens for `webhook`, `timehook`, and `cycle` (repeat) signals
  */
@@ -14,7 +15,7 @@ declare class Hook extends Activity {
     doesHook(): boolean;
     getHookRule(topic: string): Promise<HookRule | undefined>;
     registerHook(multi?: RedisMulti): Promise<string | void>;
-    processWebHookEvent(): Promise<JobStatus | void>;
+    processWebHookEvent(status?: StreamStatus, code?: StreamCode): Promise<JobStatus | void>;
     processTimeHookEvent(jobId: string): Promise<JobStatus | void>;
 }
 export { Hook };

package/build/services/activities/hook.js

@@ -85,7 +85,7 @@ class Hook extends activity_1.Activity {
     async registerHook(multi) {
         if (this.config.hook?.topic) {
             const signaler = new store_1.StoreSignaler(this.store, this.logger);
-            return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
+            return await signaler.registerWebHook(this.config.hook.topic, this.context, this.resolveDad(), multi);
         }
         else if (this.config.sleep) {
             const durationInSeconds = pipe_1.Pipe.resolve(this.config.sleep, this.context);
@@ -96,19 +96,22 @@ class Hook extends activity_1.Activity {
             return jobId;
         }
     }
-    async processWebHookEvent() {
+    async processWebHookEvent(status = stream_1.StreamStatus.SUCCESS, code = 200) {
         this.logger.debug('hook-process-web-hook-event', {
             topic: this.config.hook.topic,
-            aid: this.metadata.aid
+            aid: this.metadata.aid,
+            status,
+            code,
         });
         const signaler = new store_1.StoreSignaler(this.store, this.logger);
         const data = { ...this.data };
-        const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
-        if (jobId) {
-            //if a webhook signal is sent that includes 'keep_alive' the hook will remain open
-            const code = data.keep_alive ? 202 : 200;
-            await this.processEvent(stream_1.StreamStatus.SUCCESS, code, 'hook', jobId);
-            if (code === 200) {
+        const signal = await signaler.processWebHookSignal(this.config.hook.topic, data);
+        if (signal) {
+            const [jobId, aid, dad] = signal;
+            this.context.metadata.jid = jobId;
+            this.context.metadata.dad = dad;
+            await this.processEvent(status, code, 'hook');
+            if (code === 200) { //otherwise 202 for pending/keepalive
                 await signaler.deleteWebHookSignal(this.config.hook.topic, data);
             }
         } //else => already resolved

package/build/services/activities/signal.d.ts

@@ -8,6 +8,10 @@ declare class Signal extends Activity {
     process(): Promise<string>;
     mapSignalData(): Record<string, any>;
     mapResolverData(): Record<string, any>;
+    /**
+     * The signal activity will hook one
+     */
+    hookOne(): Promise<string>;
     /**
      * The signal activity will hook all paused jobs that share the same job key.
      */

package/build/services/activities/signal.js

@@ -31,8 +31,12 @@ class Signal extends activity_1.Activity {
             await collator_1.CollatorService.notarizeEarlyCompletion(this, multi);
             await this.setStatus(this.adjacencyList.length - 1, multi);
             const multiResponse = await multi.exec();
-            //signal to awaken all paused jobs that share the targeted job key
-            await this.hookAll();
+            if (this.config.subtype === 'all') {
+                await this.hookAll();
+            }
+            else {
+                await this.hookOne();
+            }
             //transition to adjacent activities
             const jobStatus = this.resolveStatus(multiResponse);
             const attrs = { 'app.job.jss': jobStatus };
@@ -71,6 +75,16 @@ class Signal extends activity_1.Activity {
             return mapper.mapRules();
         }
     }
+    /**
+     * The signal activity will hook one
+     */
+    async hookOne() {
+        const topic = pipe_1.Pipe.resolve(this.config.topic, this.context);
+        const signalInputData = this.mapSignalData();
+        const status = pipe_1.Pipe.resolve(this.config.status, this.context);
+        const code = pipe_1.Pipe.resolve(this.config.code, this.context);
+        return await this.engine.hook(topic, signalInputData, status, code);
+    }
     /**
      * The signal activity will hook all paused jobs that share the same job key.
      */

package/build/services/durable/client.d.ts

@@ -1,12 +1,12 @@
 import { WorkflowHandleService } from './handle';
 import { HotMeshService as HotMesh } from '../hotmesh';
-import { ClientConfig, Connection, WorkflowOptions, WorkflowSearchOptions } from '../../types/durable';
+import { ClientConfig, Connection, HookOptions, WorkflowOptions, WorkflowSearchOptions } from '../../types/durable';
 export declare class ClientService {
     connection: Connection;
     options: WorkflowOptions;
     static instances: Map<string, HotMesh | Promise<HotMesh>>;
     constructor(config: ClientConfig);
-    getHotMeshClient: (worflowTopic: string) => Promise<HotMesh>;
+    getHotMeshClient: (worflowTopic: string, namespace?: string) => Promise<HotMesh>;
     /**
      * For those deployments with a redis stack backend (with the FT module),
      * this method will configure the search index for the workflow.
@@ -15,9 +15,19 @@ export declare class ClientService {
     search: (hotMeshClient: HotMesh, index: string, query: string[]) => Promise<string[]>;
     workflow: {
         start: (options: WorkflowOptions) => Promise<WorkflowHandleService>;
-        signal: (signalId: string, data: Record<any, any>) => Promise<string>;
-        getHandle: (taskQueue: string, workflowName: string, workflowId: string) => Promise<WorkflowHandleService>;
-        search: (taskQueue: string, workflowName: string, index: string, ...query: string[]) => Promise<string[]>;
+        /**
+         * send a message to a running workflow that is paused and awaiting the signal
+         */
+        signal: (signalId: string, data: Record<any, any>, namespace?: string) => Promise<string>;
+        /**
+         * send a message to spawn an parallel in-process thread of execution
+         * with the same job state as the main thread but bound to a different
+         * handler function. All job state will be journaled to the same hash
+         * as is used by the main thread.
+         */
+        hook: (options: HookOptions) => Promise<string>;
+        getHandle: (taskQueue: string, workflowName: string, workflowId: string, namespace?: string) => Promise<WorkflowHandleService>;
+        search: (taskQueue: string, workflowName: string, namespace: null | string, index: string, ...query: string[]) => Promise<string[]>;
     };
     activateWorkflow(hotMesh: HotMesh, appId?: string, version?: string): Promise<void>;
     static shutdown(): Promise<void>;

package/build/services/durable/client.js

@@ -7,15 +7,16 @@ const handle_1 = require("./handle");
 const hotmesh_1 = require("../hotmesh");
 const key_1 = require("../../modules/key");
 const search_1 = require("./search");
+const types_1 = require("../../types");
 class ClientService {
     constructor(config) {
-        this.getHotMeshClient = async (worflowTopic) => {
+        this.getHotMeshClient = async (worflowTopic, namespace) => {
             //NOTE: every unique topic inits a new engine
             if (ClientService.instances.has(worflowTopic)) {
                 return await ClientService.instances.get(worflowTopic);
             }
             const hotMeshClient = hotmesh_1.HotMeshService.init({
-                appId: factory_1.APP_ID,
+                appId: namespace ?? factory_1.APP_ID,
                 engine: {
                     redis: {
                         class: this.connection.class,
@@ -26,7 +27,7 @@ class ClientService {
             ClientService.instances.set(worflowTopic, hotMeshClient);
             //since the YAML topic is dynamic, it MUST be manually created before use
             const store = (await hotMeshClient).engine.store;
-            const params = { appId: factory_1.APP_ID, topic: worflowTopic };
+            const params = { appId: namespace ?? factory_1.APP_ID, topic: worflowTopic };
             const streamKey = store.mintKey(key_1.KeyType.STREAMS, params);
             try {
                 await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
@@ -34,7 +35,7 @@ class ClientService {
             catch (err) {
                 //ignore if already exists
             }
-            await this.activateWorkflow(await hotMeshClient);
+            await this.activateWorkflow(await hotMeshClient, namespace ?? factory_1.APP_ID);
             return hotMeshClient;
         };
         /**
@@ -46,7 +47,7 @@ class ClientService {
                 const store = hotMeshClient.engine.store;
                 const schema = [];
                 for (const [key, value] of Object.entries(search.schema)) {
-                    //prefix with a comma (avoids collisions with hotmesh reserved words)
+                    //prefix with an underscore (avoids collisions with hotmesh reserved symbols)
                     schema.push(`_${key}`);
                     schema.push(value.type);
                     if (value.sortable) {
@@ -79,7 +80,7 @@ class ClientService {
                 const spn = options.workflowSpan;
                 //topic is concat of taskQueue and workflowName
                 const workflowTopic = `${taskQueueName}-${workflowName}`;
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+                const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
                 this.configureSearchIndex(hotMeshClient, options.search);
                 const payload = {
                     arguments: [...options.args],
@@ -89,27 +90,49 @@ class ClientService {
                     backoffCoefficient: options.config?.backoffCoefficient || factory_1.DEFAULT_COEFFICIENT,
                 };
                 const context = { metadata: { trc, spn }, data: {} };
-                const jobId = await hotMeshClient.pub(factory_1.SUBSCRIBES_TOPIC, payload, context);
+                const jobId = await hotMeshClient.pub(`${options.namespace ?? factory_1.APP_ID}.execute`, payload, context);
                 if (jobId && options.search?.data) {
                     //job successfully kicked off; there is default job data to persist
-                    const search = new search_1.Search(jobId, hotMeshClient);
+                    const searchSessionId = `-search-0`;
+                    const search = new search_1.Search(jobId, hotMeshClient, searchSessionId);
                     for (const [key, value] of Object.entries(options.search.data)) {
                         search.set(key, value);
                     }
                 }
                 return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
             },
-            signal: async (signalId, data) => {
-                return await (await this.getHotMeshClient('durable.wfs.signal')).hook('durable.wfs.signal', { id: signalId, data });
+            /**
+             * send a message to a running workflow that is paused and awaiting the signal
+             */
+            signal: async (signalId, data, namespace) => {
+                const topic = `${namespace ?? factory_1.APP_ID}.wfs.signal`;
+                return await (await this.getHotMeshClient(topic, namespace)).hook(topic, { id: signalId, data });
             },
-            getHandle: async (taskQueue, workflowName, workflowId) => {
+            /**
+             * send a message to spawn an parallel in-process thread of execution
+             * with the same job state as the main thread but bound to a different
+             * handler function. All job state will be journaled to the same hash
+             * as is used by the main thread.
+             */
+            hook: async (options) => {
+                const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
+                const payload = {
+                    arguments: [...options.args],
+                    id: options.workflowId,
+                    workflowTopic,
+                    backoffCoefficient: options.config?.backoffCoefficient || factory_1.DEFAULT_COEFFICIENT,
+                };
+                const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
+                return await hotMeshClient.hook(`${hotMeshClient.appId}.flow.signal`, payload, types_1.StreamStatus.PENDING, 202);
+            },
+            getHandle: async (taskQueue, workflowName, workflowId, namespace) => {
                 const workflowTopic = `${taskQueue}-${workflowName}`;
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+                const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
                 return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
             },
-            search: async (taskQueue, workflowName, index, ...query) => {
+            search: async (taskQueue, workflowName, namespace, index, ...query) => {
                 const workflowTopic = `${taskQueue}-${workflowName}`;
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+                const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
                 try {
                     return await this.search(hotMeshClient, index, query);
                 }

package/build/services/durable/factory.d.ts

@@ -2,7 +2,7 @@
  * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
  *       workflows will be retried on the following schedule (8 times in 27 hours):
  *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
- * 593:
+ *
  * 594: waitforsignal
  * 595: sleep
  * 596, 597, 598: fatal
@@ -11,19 +11,5 @@
 declare const getWorkflowYAML: (app: string, version: string) => string;
 declare const APP_VERSION = "1";
 declare const APP_ID = "durable";
-declare const ACTIVITY_SUBSCRIBES_TOPIC = "durable.activity.execute";
-declare const ACTIVITY_PUBLISHES_TOPIC = "durable.activity.executed";
-declare const SLEEP_SUBSCRIBES_TOPIC = "durable.sleep.execute";
-declare const SLEEP_PUBLISHES_TOPIC = "durable.sleep.executed";
-declare const WFS_SUBSCRIBES_TOPIC = "durable.wfs.execute";
-declare const WFS_PUBLISHES_TOPIC = "durable.wfs.executed";
-declare const WFSC_SUBSCRIBES_TOPIC = "durable.wfsc.execute";
-declare const WFSC_PUBLISHES_TOPIC = "durable.wfsc.executed";
-declare const SUBSCRIBES_TOPIC = "durable.execute";
-declare const PUBLISHES_TOPIC = "durable.executed";
-declare const HOOK_ID = "durable.awaken";
-declare const ACTIVITY_HOOK_ID = "durable.activity.awaken";
-declare const SLEEP_HOOK_ID = "durable.sleep.awaken";
-declare const WFS_HOOK_ID = "durable.wfs.awaken";
 declare const DEFAULT_COEFFICIENT = 10;
-export { getWorkflowYAML, APP_VERSION, APP_ID, ACTIVITY_SUBSCRIBES_TOPIC, ACTIVITY_PUBLISHES_TOPIC, SLEEP_SUBSCRIBES_TOPIC, SLEEP_PUBLISHES_TOPIC, SUBSCRIBES_TOPIC, PUBLISHES_TOPIC, HOOK_ID, ACTIVITY_HOOK_ID, SLEEP_HOOK_ID, DEFAULT_COEFFICIENT, WFS_SUBSCRIBES_TOPIC, WFS_PUBLISHES_TOPIC, WFSC_SUBSCRIBES_TOPIC, WFSC_PUBLISHES_TOPIC, WFS_HOOK_ID, };
+export { getWorkflowYAML, APP_VERSION, APP_ID, DEFAULT_COEFFICIENT, };