@hotmeshio/hotmesh 0.0.22 → 0.0.24

package/services/durable/meshos.ts

@@ -0,0 +1,344 @@
+import { nanoid } from 'nanoid';
+
+import { ClientService as Client } from './client';
+import { WorkflowHandleService } from './handle';
+import { Search } from './search';
+import { WorkerService as Worker } from './worker';
+import { FindOptions, MeshOSActivityOptions, MeshOSOptions, WorkflowSearchOptions } from '../../types/durable';
+import { RedisOptions, RedisClass } from '../../types/redis';
+import { StringAnyType } from '../../types';
+import { Durable } from '.';
+import { asyncLocalStorage } from './asyncLocalStorage';
+import { WorkflowService } from './workflow';
+
+/**
+ * The base class for running MeshOS workflows.
+ * Extend and register subclass methods by name to
+ * execute as durable workflows, backed by Redis.
+ */
+
+export class MeshOSService {
+
+  /**
+   * The top-level Redis isolation. All workflow data is
+   * isolated within this namespace. Values should be
+   * lower-case with no spaces (e.g, 'staging', 'prod', 'test',
+   * 'routing-stagig', 'reporting-prod', etc.). 
+   *  1) only url-safe values are allowed;
+   *  2) the 'a' symbol is reserved by HotMesh for indexing apps
+   */
+  namespace = 'durable';
+
+  /**
+   * Data is routed to workers that specify this task queue.
+   * Setting the task queue when the worker is created will
+   * ensure that the worker only receives messages destined
+   * for the queue. Callers can specify the taskQue to when
+   * starting a job to call those workers.
+   */
+  taskQueue = 'default';
+
+  /**
+   * These methods run as durable workflows
+   */
+  workflowFunctions: Array<MeshOSOptions | string> = [];
+
+  /**
+   * These methods run as hooks (hook into a running workflow)
+   */
+  hookFunctions: Array<MeshOSOptions | string> = [];
+
+  /**
+   * These methods run as proxied activities (and are safely memoized)
+   */
+  proxyFunctions: Array<MeshOSActivityOptions | string> = [];
+
+  /**
+   * The workflow GUID. Workflows will be persisted to
+   * Redis using the pattern hmsh:<namespace>:j:<id>.
+   */
+  id: string;
+
+  /**
+   * The Redis connection options. NOTE: Redis and IORedis
+   * use different formats for their connection config.
+   */
+  redisOptions: RedisOptions = {
+    host: 'localhost',
+    port: 6379,
+    password: '',
+    db: 0,
+  };
+
+  /**
+   * The Redis connection class.
+   * 
+   * @example
+   * import Redis from 'ioredis';
+   * import * as Redis from 'redis';
+   */
+  redisClass: RedisClass;
+
+  /**
+   * Optional model declaration (custom workflow state)
+   */
+  model: StringAnyType;
+
+  /**
+   * Optional configuration for Redis FT search
+   */
+  search: WorkflowSearchOptions;
+
+  static MeshOS = WorkflowService
+
+  static async getHotMeshClient (redisClass: RedisClass, redisOptions: RedisOptions, namespace: string, taskQueue: string) {
+    const client = new Client({
+      connection: {
+        class: redisClass,
+        options: redisOptions,
+      }
+    });
+    return await client.getHotMeshClient(taskQueue, namespace);
+  }
+
+  /**
+   * mints a workflow ID, using the search prefix.
+   * NOTE: The prefix is necesary when indexing
+   *       HASHes when FT search is enabled.
+   * @returns {string}
+   */
+  static mintGuid(): string {
+    const my = new this();
+    return `${my.search?.prefix?.[0]}${nanoid()}}`;
+  }
+
+  /**
+   * Creates an FT search index
+   */
+  static async createIndex() {
+    const my = new this();
+    const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
+    Search.configureSearchIndex(hmClient, my.search)
+  }
+
+  /**
+   * Initialize the worker(s) for the entity. This is a static
+   * method that allows for optional task Queue targeting.
+   * NOTE: Allow List may be optionally used to only wrap
+   *       specific methods in this class. 
+   * @param {string} taskQueue 
+   * @param {string[]} allowList 
+   */
+  static async startWorkers(taskQueue?: string, allowList: Array<MeshOSOptions | string> = []) {
+    const my = new this();
+
+    //helper functions
+    const resolveFunctionNames = (arr: any[]) => arr.map(item => typeof item === 'string' ? item : item.name);
+    const belongsTo = (name: string, target: Array<MeshOSOptions | MeshOSActivityOptions | string>): boolean => {
+      const isWorkflow = target.find((item: MeshOSOptions | string) => {
+        return typeof item === 'string' ? item === name : item.name === name;
+      });
+      return isWorkflow !== undefined;
+    };
+
+    // proxy registered activities
+    const proxyFunctionNames = resolveFunctionNames([...my.proxyFunctions]);
+    if (proxyFunctionNames.length) {
+      const proxyActivities = proxyFunctionNames.reduce((acc, funcName) => {
+        let originalMethod = my[funcName];
+        if (typeof originalMethod === 'function') {
+          acc[funcName] = async (...args: any[]) => {
+            return await originalMethod.apply(my, args);
+          }
+        }
+        return acc;
+      }, {});
+      const proxiedActivities = Durable.workflow.proxyActivities({
+        activities: proxyActivities
+      });
+      //WATCH!: unsure if this will pollute the scope; don't think
+      //      so as activity functions are terminal in the chain.
+      Object.assign(my, proxiedActivities);
+    }
+
+    const functionsToIterate = allowList.length ? resolveFunctionNames(allowList) : resolveFunctionNames([...my.workflowFunctions, ...my.hookFunctions]);
+
+    // Iterating through the functions sequentially
+    for (const funcName of functionsToIterate) {
+      const originalMethod = my[funcName];
+      if (typeof originalMethod === 'function') {
+
+        //wrap the function to return
+        const wrappedFunction = {
+          [funcName]: async (...args: any[]) => {
+            const store = asyncLocalStorage.getStore();
+            const workflowId = store.get('workflowId');
+
+            //use a Proxy to wrap hook methods
+            const context = new Proxy(my, {
+              get: (target, prop, receiver) => {
+                if (prop === 'id') {
+                  return workflowId;
+                } else if (typeof target[prop] === 'function') {
+                  return (...args: any[]) => {
+                    return new Promise(async (resolve, reject) => {
+                      if (belongsTo(prop as string, my.hookFunctions)) {
+                        return WorkflowService.hook({
+                          namespace: my.namespace,
+                          taskQueue: my.taskQueue,
+                          workflowName: prop as string,
+                          workflowId,
+                          args,
+                        }).then(resolve).catch(reject);
+                      }
+                      //otherwise, call the method as a standard instance method.
+                      target[prop].apply(this, args).then(resolve).catch(reject);
+                    });
+                  }
+                }
+                return Reflect.get(target, prop, receiver);
+              },
+            });
+            return await originalMethod.apply(context, args);
+          }
+        };
+
+        //start the worker
+        await Worker.create({
+          namespace: my.namespace,
+          connection: {
+            class: my.redisClass,
+            options: my.redisOptions,
+          },
+          taskQueue: taskQueue ?? my.taskQueue,
+          workflow: wrappedFunction,
+        });
+      }
+    }
+  }
+
+  /**
+   * executes the redis FT search query
+   * @example '@_quantity:[89 89]'
+   * @param {any[]} args
+   * @returns {string}
+   */
+  static async find(options: FindOptions, ...args: string[]): Promise<string[] | [number]> {
+    const my = new this();
+    const client = new Client({ connection: {
+      class: my.redisClass,
+      options: my.redisOptions 
+    }});
+    //workflow name is the function name driving the workflow
+    let workflowName: string;
+    if (options?.workflowName) {
+      workflowName = options?.workflowName
+    } else if(my.workflowFunctions?.length) {
+      let target = my.workflowFunctions[0];
+      if (typeof target === 'string') {
+        workflowName = target;
+      } else {
+        workflowName = target.name;
+      }
+    }
+    return await client.workflow.search(
+      options?.taskQueue ?? my.taskQueue,
+      workflowName,
+      my.namespace,
+      my.search.index,
+      ...args,
+    ); //[count, [id, fields[], id, fields[], id, fields[], ...]]
+  }
+
+  /**
+   * returns the workflow handle. The handle can then be
+   * used to query for status, state, custom state, etc.
+   * @param {string} id 
+   * @returns {Promise<WorkflowHandleService>}
+   */
+  static async get(id: string): Promise<WorkflowHandleService> {
+    const my = new this();
+    const client = new Client({ connection: {
+      class: my.redisClass,
+      options: my.redisOptions 
+    }});
+    let workflowName: string;
+    let target = my.workflowFunctions[0];
+    if (typeof target === 'string') {
+      workflowName = target;
+    } else {
+      workflowName = target.name;
+    }
+    return await client.workflow.getHandle(
+      my.taskQueue,
+      workflowName,
+      id,
+      my.namespace,
+    );
+  }
+
+  /**
+   * Optionally include a target taskQueue to exec the
+   * workflow's call on a specific worker queue.
+   */
+  constructor(id?: string, options?: Record<string, any>) {
+    this.id = id;
+    if (options?.taskQueue) {
+      this.taskQueue = options.taskQueue;
+    } else if (!id && !options?.taskQueue) {
+      return this;
+    }
+
+    function belongsTo(name: string, target: Array<MeshOSOptions | MeshOSActivityOptions | string>): boolean {
+      const isWorkflow = target.find((item: MeshOSOptions | string) => {
+        return typeof item === 'string' ? item === name : item.name === name;
+      });
+      return isWorkflow !== undefined;
+    }
+
+    return new Proxy(this, {
+      get: (target, prop, receiver) => {
+        if (typeof target[prop] === 'function') {
+          return (...args: any[]) => {
+            return new Promise(async (resolve, reject) => {
+              const client = new Client({ connection: {
+                class: this.redisClass,
+                options: this.redisOptions 
+              }});
+              if (belongsTo(prop as string, this.workflowFunctions)) {
+                //start a new workflow
+                const handle = await client.workflow.start({
+                  namespace: this.namespace,
+                  args,
+                  taskQueue: this.taskQueue,
+                  workflowName: prop as string,
+                  workflowId: this.id,
+                });
+                if (options?.await) {
+                  //wait for the workflow to complete
+                  const result = await handle.result();
+                  return resolve(result);
+                } else {
+                  //return the workflow handle
+                  return resolve(handle);
+                }
+              } else if (belongsTo(prop as string, this.hookFunctions)) {
+                //hook into a running workflow
+                return client.workflow.hook({
+                  namespace: this.namespace,
+                  taskQueue: this.taskQueue,
+                  workflowName: prop as string,
+                  workflowId: this.id,
+                  args,
+                }).then(resolve).catch(reject);
+              }
+              //otherwise, call the method as a standard instance method.
+              target[prop].apply(this, args).then(resolve).catch(reject);
+            });
+          };
+        }
+        return Reflect.get(target, prop, receiver);
+      },
+    });
+  }
+}

package/services/durable/search.ts

@@ -2,6 +2,7 @@ import { HotMeshService as HotMesh } from '../hotmesh'
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { StoreService } from '../store';
 import { KeyService, KeyType } from '../../modules/key';
+import { WorkflowSearchOptions } from '../../types/durable';
 
 export class Search {
   jobId: string;
@@ -11,11 +12,43 @@ export class Search {
   store: StoreService<RedisClient, RedisMulti> | null;
 
   safeKey(key:string): string {
-    //note: protect the execution namespace with a prefix,
-    //so its design never conflicts with the hotmesh keyspace
+    //note: protect the execution namespace with a prefix
     return `_${key}`;
   }
 
+  /**
+   * For those deployments with a redis stack backend (with the FT module),
+   * this method will configure the search index for the workflow. For all
+   * others, this method will exit/fail gracefully and not index
+   * the fields in the HASH. However, all values are still available
+   * in the HASH.
+   */
+  static async configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> {
+    if (search?.schema) {
+      const store = hotMeshClient.engine.store;
+      const schema: string[] = [];
+      for (const [key, value] of Object.entries(search.schema)) {
+        //prefix with a comma (avoids collisions with hotmesh reserved words)
+        schema.push(`_${key}`);
+        schema.push(value.type);
+        if (value.sortable) {
+          schema.push('SORTABLE');
+        }
+      }
+      try {
+        const keyParams = {
+          appId: hotMeshClient.appId,
+          jobId: ''
+        }
+        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
+        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length.toString(), ...prefixes, 'SCHEMA', ...schema);
+      } catch (err) {
+        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
+      }
+    }
+  }
+
   constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string) {
     const keyParams = {
       appId: hotMeshClient.appId,

package/services/durable/worker.ts

@@ -6,7 +6,6 @@ import {
   DurableSleepError,
   DurableTimeoutError, 
   DurableWaitForSignalError} from '../../modules/errors';
-import { KeyService, KeyType } from '../../modules/key';
 import { asyncLocalStorage } from './asyncLocalStorage';
 import { APP_ID, APP_VERSION, getWorkflowYAML } from './factory';
 import { HotMeshService as HotMesh } from '../hotmesh';
@@ -16,9 +15,9 @@ import {
   Registry,
   WorkerConfig,
   WorkerOptions,
-  WorkflowDataType, 
-  WorkflowSearchOptions} from '../../types/durable';
+  WorkflowDataType } from '../../types/durable';
 import { RedisClass, RedisOptions } from '../../types/redis';
+import { Search } from './search';
 import {
   StreamData,
   StreamDataResponse,
@@ -73,46 +72,15 @@ export class WorkerService {
       Object.keys(activities).forEach(key => {
         if (activities[key].name && typeof WorkerService.activityRegistry[activities[key].name] !== 'function') {
           WorkerService.activityRegistry[activities[key].name] = (activities as any)[key] as Function;
+        } else if (typeof (activities as any)[key] === 'function') {
+          WorkerService.activityRegistry[key] = (activities as any)[key] as Function;
         }
       });
     }
     return WorkerService.activityRegistry;
   }
 
-  /**
-   * For those deployments with a redis stack backend (with the FT module),
-   * this method will configure the search index for the workflow. For all
-   * others, this method will fail gracefully. In all cases, the values
-   * will be stored in the workflow's central HASH data structure, allowing
-   * for manual traversal and inspection as well.
-   */
-  static async configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> {
-    if (search?.schema) {
-      const store = hotMeshClient.engine.store;
-      const schema: string[] = [];
-      for (const [key, value] of Object.entries(search.schema)) {
-        //prefix with a comma (avoids collisions with hotmesh reserved words)
-        schema.push(`_${key}`);
-        schema.push(value.type);
-        if (value.sortable) {
-          schema.push('SORTABLE');
-        }
-      }
-      try {
-        const keyParams = {
-          appId: hotMeshClient.appId,
-          jobId: ''
-        }
-        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
-        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
-        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
-      } catch (err) {
-        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
-      }
-    }
-  }
-
-  static async create(config: WorkerConfig) {
+  static async create(config: WorkerConfig): Promise<WorkerService> {
     WorkerService.connection = config.connection;
     const workflow = config.workflow;
     const [workflowFunctionName, workflowFunction] = WorkerService.resolveWorkflowTarget(workflow);
@@ -124,21 +92,22 @@ export class WorkerService {
     const worker = new WorkerService();
     worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
     worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
-    WorkerService.configureSearchIndex(worker.workflowRunner, config.search)
+    Search.configureSearchIndex(worker.workflowRunner, config.search)
     await WorkerService.activateWorkflow(worker.workflowRunner);
     return worker;
   }
 
-  static resolveWorkflowTarget(workflow: object | Function): [string, Function] {
+  static resolveWorkflowTarget(workflow: object | Function, name?: string): [string, Function] {
     let workflowFunction: Function;
     if (typeof workflow === 'function') {
       workflowFunction = workflow;
+      return [workflowFunction.name ?? name, workflowFunction];
     } else {
       const workflowFunctionNames = Object.keys(workflow);
-      workflowFunction = workflow[workflowFunctionNames[workflowFunctionNames.length - 1]];
-      return WorkerService.resolveWorkflowTarget(workflowFunction);
+      const lastFunctionName = workflowFunctionNames[workflowFunctionNames.length - 1];
+      workflowFunction = workflow[lastFunctionName];
+      return WorkerService.resolveWorkflowTarget(workflowFunction, lastFunctionName);
     }
-    return [workflowFunction.name, workflowFunction];
   }
 
   async run() {

package/services/durable/workflow.ts

@@ -37,7 +37,7 @@ export class WorkflowService {
     //this is risky but MUST be allowed. Users MAY set the workflowId,
     //but if there is a naming collision, the data from the target entity will be used
     //as there is know way of knowing if the item was generated via a prior run of the workflow
-    const childJobId = options.workflowId ?? `${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
+    const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
     const parentWorkflowId = `${workflowId}-f`;
 
     const client = new Client({
@@ -80,7 +80,7 @@ export class WorkflowService {
     const workflowSpan = store.get('workflowSpan');
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    const childJobId = options.workflowId ?? `${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
+    const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
     const parentWorkflowId = `${workflowId}-f`;
     const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
 
@@ -106,9 +106,12 @@ export class WorkflowService {
     }
   }
 
+  /**
+   * wrap all activities in a proxy that will durably run them
+   */
   static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT> {
     if (options.activities) {
-      WorkerService.registerActivities(options.activities)
+      WorkerService.registerActivities(options.activities);
     }
 
     const proxy: any = {};
@@ -122,6 +125,9 @@ export class WorkflowService {
     return proxy;
   }
 
+  /**
+   * return a search session for use when reading/writing to the workflow HASH
+   */
   static async search(): Promise<Search> {
     const store = asyncLocalStorage.getStore();
     const workflowId = store.get('workflowId');
@@ -130,15 +136,26 @@ export class WorkflowService {
     const namespace = store.get('namespace');
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
+    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const searchSessionId = `-search${workflowDimension}-${execIndex}`;
     return new Search(workflowId, hotMeshClient, searchSessionId);
   }
 
+  /**
+   * return a handle to the hotmesh client currently running the workflow
+   */
+  static async getHotMesh(): Promise<HotMesh> {
+    const store = asyncLocalStorage.getStore();
+    const workflowTopic = store.get('workflowTopic');
+    const namespace = store.get('namespace');
+    return await WorkerService.getHotMesh(workflowTopic, { namespace });
+  }
+
   /**
    * those methods that may only be called once must be protected by flagging
-   * their execution with a unique key (the key is stored in the workflow state)
+   * their execution with a unique key (the key is stored in the HASH alongside
+   * process state and job state)
    */
   static async isSideEffectAllowed(hotMeshClient: HotMesh, prefix:string): Promise<boolean> {
     const store = asyncLocalStorage.getStore();
@@ -146,15 +163,12 @@ export class WorkflowService {
     const workflowDimension = store.get('workflowDimension') ?? '';
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
-    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const keyParams = {
       appId: hotMeshClient.appId,
-      jobId: ''
+      jobId: workflowId
     }
-    const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
-    const workflowGuid = `${hotMeshPrefix}${workflowId}`;
+    const workflowGuid = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
     const guidValue = Number(await hotMeshClient.engine.store.exec('HINCRBYFLOAT', workflowGuid, sessionId, '1') as string);
     return guidValue === 1;
   }
@@ -165,8 +179,11 @@ export class WorkflowService {
    */
   static async signal(signalId: string, data: Record<any, any>): Promise<string> {
     const store = asyncLocalStorage.getStore();
+    const workflowTopic = store.get('workflowTopic');
     const namespace = store.get('namespace');
-    const hotMeshClient = await WorkerService.getHotMesh(`${namespace}.wfs.signal`, { namespace });
+    const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
+    //todo: this particular one is better patterned as a get/set,
+    //since the receipt is a meaningful string (the stream id)
     if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'signal')) {
       return await hotMeshClient.hook(`${namespace}.wfs.signal`, { id: signalId, data });
     }
@@ -179,8 +196,9 @@ export class WorkflowService {
    */
   static async hook(options: HookOptions): Promise<string> {
     const store = asyncLocalStorage.getStore();
+    const workflowTopic = store.get('workflowTopic');
     const namespace = store.get('namespace');
-    const hotMeshClient = await WorkerService.getHotMesh(`${namespace}.flow.signal`, { namespace });
+    const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
     if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'hook')) {
       const store = asyncLocalStorage.getStore();
       const workflowId = options.workflowId ?? store.get('workflowId');
@@ -208,7 +226,7 @@ export class WorkflowService {
     const workflowTopic = store.get('workflowTopic');
     const workflowDimension = store.get('workflowDimension') ?? '';
     const namespace = store.get('namespace');
-    const sleepJobId = `${workflowId}-$sleep${workflowDimension}-${execIndex}`;
+    const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
 
     try {
       const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
@@ -216,9 +234,8 @@ export class WorkflowService {
       //if no error is thrown, we've already slept, return the delay
       return seconds;
     } catch (e) {
-      //if an error, the sleep job was not found...rethrow error; sleep job
-      // will be automatically created according to the DAG rules (they
       // spawn a new sleep job if error code 595 is thrown by the worker)
+      // NOTE: If this message shows up in your stack trace, you forgot to await `Durable.workflow.sleep()` in your workflow code.
       throw new DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
     }
   }
@@ -238,7 +255,7 @@ export class WorkflowService {
     const signalResults: any[] = [];
     for (const signal of signals) {
       const execIndex = COUNTER.counter = COUNTER.counter + 1;
-      const wfsJobId = `${workflowId}-$wfs${workflowDimension}-${execIndex}`;
+      const wfsJobId = `-${workflowId}-$wfs${workflowDimension}-${execIndex}`;
       try {
         if (allAreComplete) {
           const state = await hotMeshClient.getState(`${hotMeshClient.appId}.wfs.execute`, wfsJobId);
@@ -285,7 +302,7 @@ export class WorkflowService {
       const spn = store.get('workflowSpan');
       const namespace = store.get('namespace');
       const activityTopic = `${workflowTopic}-activity`;
-      const activityJobId = `${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
+      const activityJobId = `-${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
 
       let activityState: JobOutput
       try {

package/services/engine/index.ts

@@ -53,6 +53,7 @@ import {
   SubscriptionCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { RedisClientType } from '../../types/redisclient';
+import { StringAnyType, StringStringType } from '../../types/serializer';
 import {
   GetStatsOptions,
   IdsResponse,
@@ -67,7 +68,6 @@ import {
   StreamError,
   StreamRole,
   StreamStatus } from '../../types/stream';
-import { StringStringType } from '../../types';
 
 //wait time to see if a job is complete
 const OTT_WAIT_TIME = 1000;
@@ -629,6 +629,9 @@ class EngineService {
     }
     return stateTree;
   }
+  async getQueryState(jobId: string, fields: string[]): Promise<StringAnyType> {
+    return await this.store.getQueryState(jobId, fields);
+  }
 
   async compress(terms: string[]): Promise<boolean> {
     const existingSymbols = await this.store.getSymbolValues();