@hotmeshio/hotmesh 0.0.19 → 0.0.20

package/build/types/durable.d.ts

@@ -15,6 +15,7 @@ type WorkflowSearchOptions = {
     data?: Record<string, string>;
 };
 type WorkflowOptions = {
+    namespace?: string;
     taskQueue: string;
     args: any[];
     workflowId: string;
@@ -25,6 +26,15 @@ type WorkflowOptions = {
     search?: WorkflowSearchOptions;
     config?: WorkflowConfig;
 };
+type HookOptions = {
+    namespace?: string;
+    taskQueue: string;
+    args: any[];
+    workflowId: string;
+    workflowName?: string;
+    search?: WorkflowSearchOptions;
+    config?: WorkflowConfig;
+};
 type SignalOptions = {
     taskQueue: string;
     data: Record<string, any>;
@@ -63,6 +73,7 @@ type WorkerConfig = {
     search?: WorkflowSearchOptions;
 };
 type WorkerOptions = {
+    logLevel?: string;
     maxSystemRetries?: number;
     backoffCoefficient?: number;
 };
@@ -84,4 +95,4 @@ type ActivityConfig = {
         maximumInterval: string;
     };
 };
-export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, SignalOptions, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, SignalOptions, HookOptions, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };

package/build/types/hook.d.ts

@@ -12,7 +12,6 @@ interface HookConditions {
 }
 interface HookRule {
     to: string;
-    keep_alive?: boolean;
     conditions: HookConditions;
 }
 interface HookRules {

package/build/types/index.d.ts

@@ -3,7 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
-export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, HookOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/modules/errors.ts

@@ -36,11 +36,13 @@ class DurableWaitForSignalError extends Error {
 class DurableSleepError extends Error {
   code: number;
   duration: number; //seconds
-  index: number;    //execution order in the workflow 
-  constructor(message: string, duration: number, index: number) {
+  index: number;    //execution order in the workflow
+  dimension: string; //hook dimension (e.g., ',0,1,0') (uses empty string for `null`)
+  constructor(message: string, duration: number, index: number, dimension: string) {
     super(message);
     this.duration = duration;
     this.index = index;
+    this.dimension = dimension;
     this.code = 595;
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.19",
+  "version": "0.0.20",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -45,6 +45,7 @@
     "test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:hook": "NODE_ENV=test jest ./tests/durable/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:fatal": "NODE_ENV=test jest ./tests/durable/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:sleep": "NODE_ENV=test jest ./tests/durable/sleep/index.test.ts --detectOpenHandles --forceExit --verbose",

package/services/activities/activity.ts

@@ -70,16 +70,20 @@ class Activity {
   }
 
   //********  DUPLEX RE-ENTRY POINT  ********//
-  async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output', jobId?: string): Promise<void> {
+  async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output'): Promise<void> {
     this.setLeg(2);
-    const jid = this.context.metadata.jid || jobId;
+    const jid = this.context.metadata.jid;
+    if (!jid) {
+      this.logger.error('activity-process-event-error', { message: 'job id is undefined' });
+      return;
+    }
     const aid = this.metadata.aid;
     this.status = status;
     this.code = code;
     this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
     let telemetry: TelemetryService;
     try {
-      await this.getState(jobId);
+      await this.getState();
       const aState = await CollatorService.notarizeReentry(this);
       this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
 
@@ -107,7 +111,6 @@ class Activity {
         this.logger.info('process-event-inactive-error', { error });
         return;
       }
-      console.error(error);
       this.logger.error('activity-process-event-error', { error });
       telemetry && telemetry.setActivityError(error.message);
       throw error;
@@ -323,7 +326,7 @@ class Activity {
     return MDATA_SYMBOLS[keys_to_save].KEYS.map((key) => `output/metadata/${key}`);
   }
 
-  async getState(jobId?: string) {
+  async getState() {
     //assemble list of paths necessary to create 'job state' from the 'symbol hash'
     const jobSymbolHashName = `$${this.config.subscribes}`;
     const consumes: Consumes = {
@@ -348,10 +351,9 @@ class Activity {
     }
     TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
     let { dad, jid } = this.context.metadata;
-    jobId = jobId || jid;
-    const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad);
+    const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
     //`state` is a flat hash; context is a tree
-    const [state, status] = await this.store.getState(jobId, consumes, dIds);
+    const [state, status] = await this.store.getState(jid, consumes, dIds);
     this.context = restoreHierarchy(state) as JobState;
     this.initDimensionalAddress(dad);
     this.initSelf(this.context);

package/services/activities/hook.ts

@@ -16,7 +16,7 @@ import {
   MultiResponseFlags,
   RedisMulti } from '../../types/redis';
 import { StringScalarType } from '../../types/serializer';
-import { StreamStatus } from '../../types/stream';
+import { StreamCode, StreamStatus } from '../../types/stream';
 
 /**
  * Listens for `webhook`, `timehook`, and `cycle` (repeat) signals
@@ -108,7 +108,7 @@ class Hook extends Activity {
   async registerHook(multi?: RedisMulti): Promise<string | void> {
     if (this.config.hook?.topic) {
       const signaler = new StoreSignaler(this.store, this.logger);
-      return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
+      return await signaler.registerWebHook(this.config.hook.topic, this.context, this.resolveDad(), multi);
     } else if (this.config.sleep) {
       const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
       const jobId = this.context.metadata.jid;
@@ -119,19 +119,22 @@ class Hook extends Activity {
     }
   }
 
-  async processWebHookEvent(): Promise<JobStatus | void> {
+  async processWebHookEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<JobStatus | void> {
     this.logger.debug('hook-process-web-hook-event', {
       topic: this.config.hook.topic,
-      aid: this.metadata.aid
+      aid: this.metadata.aid,
+      status,
+      code,
     });
     const signaler = new StoreSignaler(this.store, this.logger);
     const data = { ...this.data };
-    const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
-    if (jobId) {
-      //if a webhook signal is sent that includes 'keep_alive' the hook will remain open
-      const code = data.keep_alive ? 202 : 200;
-      await this.processEvent(StreamStatus.SUCCESS, code, 'hook', jobId);
-      if (code === 200) {
+    const signal = await signaler.processWebHookSignal(this.config.hook.topic, data);
+    if (signal) {
+      const [jobId, aid, dad] = signal;
+      this.context.metadata.jid = jobId;
+      this.context.metadata.dad = dad;
+      await this.processEvent(status, code, 'hook');
+      if (code === 200) { //otherwise 202 for pending/keepalive
         await signaler.deleteWebHookSignal(this.config.hook.topic, data);
       }
     } //else => already resolved

package/services/activities/signal.ts

@@ -10,7 +10,7 @@ import {
   ActivityMetadata,
   SignalActivity } from '../../types/activity';
 import { JobState } from '../../types/job';
-import { MultiResponseFlags, RedisMulti } from '../../types/redis';
+import { MultiResponseFlags } from '../../types/redis';
 import { StringScalarType } from '../../types/serializer';
 import { JobStatsInput } from '../../types/stats';
 
@@ -50,8 +50,11 @@ class Signal extends Activity {
       await this.setStatus(this.adjacencyList.length - 1, multi);
       const multiResponse = await multi.exec() as MultiResponseFlags;
 
-      //signal to awaken all paused jobs that share the targeted job key
-      await this.hookAll();
+      if (this.config.subtype === 'all') {
+        await this.hookAll();
+      } else {
+        await this.hookOne();
+      }
 
       //transition to adjacent activities
       const jobStatus = this.resolveStatus(multiResponse);
@@ -92,6 +95,17 @@ class Signal extends Activity {
     }
   }
 
+  /**
+   * The signal activity will hook one
+   */
+  async hookOne(): Promise<string> {
+    const topic = Pipe.resolve(this.config.topic, this.context);
+    const signalInputData = this.mapSignalData();
+    const status = Pipe.resolve(this.config.status, this.context);
+    const code = Pipe.resolve(this.config.code, this.context);
+    return await this.engine.hook(topic, signalInputData, status, code);
+  }
+
   /**
    * The signal activity will hook all paused jobs that share the same job key.
    */

package/services/durable/client.ts

@@ -1,15 +1,17 @@
 import { nanoid } from 'nanoid';
-import { APP_ID, APP_VERSION, DEFAULT_COEFFICIENT, SUBSCRIBES_TOPIC, getWorkflowYAML } from './factory';
+import { APP_ID, APP_VERSION, DEFAULT_COEFFICIENT, getWorkflowYAML } from './factory';
 import { WorkflowHandleService } from './handle';
 import { HotMeshService as HotMesh } from '../hotmesh';
 import {
   ClientConfig,
   Connection,
+  HookOptions,
   WorkflowOptions, 
   WorkflowSearchOptions} from '../../types/durable';
 import { JobState } from '../../types/job';
 import { KeyService, KeyType } from '../../modules/key';
 import { Search } from './search';
+import { StreamStatus } from '../../types';
 
 export class ClientService {
 
@@ -21,14 +23,14 @@ export class ClientService {
     this.connection = config.connection;
   }
 
-  getHotMeshClient = async (worflowTopic: string) => {
+  getHotMeshClient = async (worflowTopic: string, namespace?: string) => {
     //NOTE: every unique topic inits a new engine
     if (ClientService.instances.has(worflowTopic)) {
       return await ClientService.instances.get(worflowTopic);
     }
 
     const hotMeshClient = HotMesh.init({
-      appId: APP_ID,
+      appId: namespace ?? APP_ID,
       engine: {
         redis: {
           class: this.connection.class,
@@ -40,14 +42,14 @@ export class ClientService {
 
     //since the YAML topic is dynamic, it MUST be manually created before use
     const store = (await hotMeshClient).engine.store;
-    const params = { appId: APP_ID, topic: worflowTopic };
+    const params = { appId: namespace ?? APP_ID, topic: worflowTopic };
     const streamKey = store.mintKey(KeyType.STREAMS, params);
     try {
       await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
     } catch (err) {
       //ignore if already exists
     }
-    await this.activateWorkflow(await hotMeshClient);
+    await this.activateWorkflow(await hotMeshClient, namespace ?? APP_ID);
     return hotMeshClient;
   }
 
@@ -60,7 +62,7 @@ export class ClientService {
       const store = hotMeshClient.engine.store;
       const schema: string[] = [];
       for (const [key, value] of Object.entries(search.schema)) {
-        //prefix with a comma (avoids collisions with hotmesh reserved words)
+        //prefix with an underscore (avoids collisions with hotmesh reserved symbols)
         schema.push(`_${key}`);
         schema.push(value.type);
         if (value.sortable) {
@@ -94,7 +96,7 @@ export class ClientService {
       const spn = options.workflowSpan;
       //topic is concat of taskQueue and workflowName
       const workflowTopic = `${taskQueueName}-${workflowName}`;
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
       this.configureSearchIndex(hotMeshClient, options.search)
       const payload = {
         arguments: [...options.args],
@@ -105,12 +107,13 @@ export class ClientService {
       }
       const context = { metadata: { trc, spn }, data: {}};
       const jobId = await hotMeshClient.pub(
-        SUBSCRIBES_TOPIC,
+        `${options.namespace ?? APP_ID}.execute`,
         payload,
         context as JobState);
        if (jobId && options.search?.data) {
         //job successfully kicked off; there is default job data to persist
-        const search = new Search(jobId, hotMeshClient);
+        const searchSessionId = `-search-0`;
+        const search = new Search(jobId, hotMeshClient, searchSessionId);
         for (const [key, value] of Object.entries(options.search.data)) {
           search.set(key, value);
         }
@@ -118,19 +121,41 @@ export class ClientService {
       return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
     },
 
-    signal: async (signalId: string, data: Record<any, any>): Promise<string> => {
-      return await (await this.getHotMeshClient('durable.wfs.signal')).hook('durable.wfs.signal', { id: signalId, data });
+    /**
+     * send a message to a running workflow that is paused and awaiting the signal
+     */
+    signal: async (signalId: string, data: Record<any, any>, namespace?: string): Promise<string> => {
+      const topic = `${namespace ?? APP_ID}.wfs.signal`;
+      return await (await this.getHotMeshClient(topic, namespace)).hook(topic, { id: signalId, data });
     },
 
-    getHandle: async (taskQueue: string, workflowName: string, workflowId: string): Promise<WorkflowHandleService> => {
+    /**
+     * send a message to spawn an parallel in-process thread of execution
+     * with the same job state as the main thread but bound to a different
+     * handler function. All job state will be journaled to the same hash
+     * as is used by the main thread.
+     */
+    hook: async (options: HookOptions): Promise<string> => {
+      const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
+      const payload = {
+        arguments: [...options.args],
+        id: options.workflowId,
+        workflowTopic,
+        backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
+      }
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
+      return await hotMeshClient.hook(`${hotMeshClient.appId}.flow.signal`, payload, StreamStatus.PENDING, 202);
+    },
+
+    getHandle: async (taskQueue: string, workflowName: string, workflowId: string, namespace?: string): Promise<WorkflowHandleService> => {
       const workflowTopic = `${taskQueue}-${workflowName}`;
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
       return new WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
     },
 
-    search: async (taskQueue: string, workflowName: string, index: string, ...query: string[]): Promise<string[]> => {
+    search: async (taskQueue: string, workflowName: string, namespace: null | string, index: string, ...query: string[]): Promise<string[]> => {
       const workflowTopic = `${taskQueue}-${workflowName}`;
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic);
+      const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
       try {
         return await this.search(hotMeshClient, index, query);
       } catch (err) {