@hotmeshio/hotmesh 0.0.21 → 0.0.23

package/services/durable/meshdb.ts

@@ -0,0 +1,254 @@
+import { nanoid } from 'nanoid';
+
+import { ClientService as Client } from './client';
+import { WorkflowHandleService } from './handle';
+import { Search } from './search';
+import { WorkerService as Worker } from './worker';
+import { WorkflowSearchOptions } from '../../types/durable';
+import { RedisOptions, RedisClass } from '../../types/redis';
+
+/**
+ * A base class for configuration and setup of
+ * a reentrant process database. Entities modeled as
+ * subclasses of this class will execute as reentrant
+ * processes with a 'main' execution thread and 'n'
+ * parallel hook threads.
+ * 
+ * @example
+ * //RUN (start a workflow)
+ * const myInstance = new MeshDB('someIdempotentGuid');
+ * const handle = await myInstance.create(100);
+ * await handle.result(); //100
+ *
+ * //UPDATE (update a workflow)
+ * const result = await myInstance.decrement(11);
+ */
+
+export class MeshDBService {
+
+  /**
+   * The name of the main method. When this method
+   * is invoked/proxied, it is assumed that a new
+   * workflow instance is being created. In all other
+   * cases, the call is assumed to be a hook/update
+   */
+  main = 'create';
+
+  /**
+   * The GUID for the workflow (assigned when created). This
+   * value should be idempotent and will be rejected if an
+   * instance is already running with the same id.
+   */
+  id: string;
+
+  /**
+   * test value
+   */
+  value: number;
+
+  /**
+   * The top-level Redis isolation. All workflow data is
+   * isolated within this namespace. Values should be
+   * lower-case with no spaces (e.g, 'staging', 'prod', 'test',
+   * 'routing-stagig', 'reporting-prod', etc.). 
+   *  1) only url-safe values are allowed;
+   *  2) the 'a' symbol is reserved by HotMesh for indexing apps
+   */
+  namespace = 'durable';
+
+  /**
+   * The second-level isolation. Data is routed to workers
+   * that specify this task queue. Setting the task queue
+   * when the worker is created will ensure that the worker
+   * only receives messages destined for the queue. This
+   * allows callers to specify specific workers/containers
+   * for specific tasks. Only url-safe values are allowed.
+   */
+  taskQueue = 'default';
+
+  /**
+   * The Redis connection options. NOTE: Redis and IORedis
+   * use different formats for their connection config.
+   */
+  redisOptions: RedisOptions = {
+    host: 'localhost',
+    port: 6379,
+    password: '',
+    db: 0,
+  };
+
+  /**
+   * The Redis connection class. Import as follows
+   * within the base subclass as follows:
+   * 
+   * @example
+   * import Redis from 'ioredis';
+   * import * as Redis from 'redis';
+   */
+  redisClass: RedisClass | null = null;
+
+  /**
+   * Configuration for the the Redis FT search index.
+   */
+  search: WorkflowSearchOptions;
+
+  static async getHotMeshClient (redisClass: RedisClass, redisOptions: RedisOptions, namespace: string, taskQueue: string) {
+    const client = new Client({
+      connection: {
+        class: redisClass,
+        options: redisOptions,
+      }
+    });
+    return await client.getHotMeshClient(taskQueue, namespace);
+  }
+
+  /**
+   * mints a new key, using the provided search prefix, ensuring
+   * new workflows are properly indexed
+   * @returns {string}
+   */
+  static mintGuid(): string {
+    const my = new this();
+    return `${my.search?.prefix?.[0]}${nanoid()}}`;
+  }
+
+  /**
+   * Creates an FT search index
+   */
+  static async createIndex() {
+    const my = new this();
+    const hmClient = await MeshDBService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
+    Search.configureSearchIndex(hmClient, my.search)
+  }
+
+  /**
+   * Initialize the worker(s) for the entity. This is a static
+   * method that allows for optional task Queue targeting.
+   * NOTE: Allow List may be optionally used 
+   * @param {string} taskQueue 
+   * @param {string[]} allowList 
+   */
+  static async doWork(taskQueue?: string, allowList?: string[]) {
+    const my = new this();
+    let prototype = Object.getPrototypeOf(my);
+    const durablePromises = [];
+    const found = [];
+
+    while (prototype !== null && !Object.getOwnPropertyNames(prototype).includes('__proto__')) {
+      const promises = Object.getOwnPropertyNames(prototype).map((prop) => {
+        if (found.includes(prop) || ['constructor'].includes(prop) || (allowList && !allowList.includes(prop))) {
+          return;
+        }
+        const originalMethod = my[prop];
+        if (typeof originalMethod === 'function') {
+          found.push(prop);
+          return Worker.create({
+            namespace: my.namespace,
+            connection: {
+              class: my.redisClass,
+              options: my.redisOptions,
+            },
+            taskQueue: taskQueue ?? my.taskQueue,
+            workflow: originalMethod,
+          });
+        }
+      }).filter(p => p !== undefined); // filter out undefined values
+      durablePromises.push(...promises);
+      prototype = Object.getPrototypeOf(prototype);
+    }
+    await Promise.all(durablePromises);
+  }
+
+  /**
+   * executes the redis FT search query
+   * @example '@_quantity:[89 89]'
+   * @param {any[]} args
+   * @returns {string}
+   */
+  static async find(...args: string[]): Promise<string[] | [number]> {
+    const my = new this();
+    const client = new Client({ connection: {
+      class: my.redisClass,
+      options: my.redisOptions 
+    }});
+    return await client.workflow.search(
+      my.taskQueue,
+      my.main,
+      my.namespace,
+      my.search.index,
+      ...args,
+    );
+    //[count, [id, fields[], id, fields[], id, fields[], ...]]
+  }
+
+  /**
+   * returns the workflow handle (use the handle to call:
+   * `state`, `status`, `queryStatus`, and `result`)
+   * @param {string} id 
+   * @returns {Promise<WorkflowHandleService>}
+   */
+  static async get(id: string): Promise<WorkflowHandleService> {
+    const my = new this();
+    const client = new Client({ connection: {
+      class: my.redisClass,
+      options: my.redisOptions 
+    }});
+    return await client.workflow.getHandle(
+      my.taskQueue,
+      my.main,
+      id,
+      my.namespace,
+    );
+  }
+
+  /**
+   * Initialize with an idempotent workflow identifier.
+   * Optionally include a target taskQueue to send
+   * events to a specific worker.
+   */
+  constructor(id?: string, taskQueue?: string) {
+    this.id = id;
+    if (taskQueue) {
+      this.taskQueue = taskQueue;
+    } else if (!id && !taskQueue) {
+      return this;
+    }
+
+    return new Proxy(this, {
+      get: (target, prop, receiver) => {
+        if (typeof target[prop] === 'function') {
+          return (...args: any[]) => {
+
+            return new Promise(async (resolve, reject) => {
+              const client = new Client({ connection: {
+                class: this.redisClass,
+                options: this.redisOptions 
+              }});
+              if (prop === this.main) {
+                //start a new workflow (main method was called)
+                return client.workflow.start({
+                  namespace: this.namespace,
+                  args,
+                  taskQueue: this.taskQueue,
+                  workflowName: prop,
+                  workflowId: this.id,
+                }).then(resolve).catch(reject);
+              } else if (prop !== 'constructor') {
+                //update an existing workflow (hook/signal-in)
+                return client.workflow.hook({
+                  namespace: this.namespace,
+                  taskQueue: this.taskQueue,
+                  workflowName: prop as string,
+                  workflowId: this.id,
+                  args,
+                }).then(resolve).catch(reject);
+              }
+              target[prop].apply(this, args).then(resolve).catch(reject);
+            });
+          };
+        }
+        return Reflect.get(target, prop, receiver);
+      },
+    });
+  }
+}

package/services/durable/search.ts

@@ -2,6 +2,7 @@ import { HotMeshService as HotMesh } from '../hotmesh'
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { StoreService } from '../store';
 import { KeyService, KeyType } from '../../modules/key';
+import { WorkflowSearchOptions } from '../../types/durable';
 
 export class Search {
   jobId: string;
@@ -11,11 +12,44 @@ export class Search {
   store: StoreService<RedisClient, RedisMulti> | null;
 
   safeKey(key:string): string {
-    //note: protect the execution namespace with a prefix,
-    //so its design never conflicts with the hotmesh keyspace
+    //note: protect the execution namespace with a prefix
     return `_${key}`;
   }
 
+  /**
+   * For those deployments with a redis stack backend (with the FT module),
+   * this method will configure the search index for the workflow. For all
+   * others, this method will exit/fail gracefully and not index
+   * the fields in the HASH. However, all values are still available
+   * in the HASH.
+   */
+  static async configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> {
+    if (search?.schema) {
+      const store = hotMeshClient.engine.store;
+      const schema: string[] = [];
+      for (const [key, value] of Object.entries(search.schema)) {
+        //prefix with a comma (avoids collisions with hotmesh reserved words)
+        schema.push(`_${key}`);
+        schema.push(value.type);
+        if (value.sortable) {
+          schema.push('SORTABLE');
+        }
+      }
+      try {
+        const keyParams = {
+          appId: hotMeshClient.appId,
+          jobId: ''
+        }
+        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
+        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
+        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length.toString(), ...prefixes, 'SCHEMA', ...schema);
+      } catch (err) {
+        console.error(err);
+        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
+      }
+    }
+  }
+
   constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string) {
     const keyParams = {
       appId: hotMeshClient.appId,

package/services/durable/worker.ts

@@ -6,7 +6,6 @@ import {
   DurableSleepError,
   DurableTimeoutError, 
   DurableWaitForSignalError} from '../../modules/errors';
-import { KeyService, KeyType } from '../../modules/key';
 import { asyncLocalStorage } from './asyncLocalStorage';
 import { APP_ID, APP_VERSION, getWorkflowYAML } from './factory';
 import { HotMeshService as HotMesh } from '../hotmesh';
@@ -16,9 +15,9 @@ import {
   Registry,
   WorkerConfig,
   WorkerOptions,
-  WorkflowDataType, 
-  WorkflowSearchOptions} from '../../types/durable';
+  WorkflowDataType } from '../../types/durable';
 import { RedisClass, RedisOptions } from '../../types/redis';
+import { Search } from './search';
 import {
   StreamData,
   StreamDataResponse,
@@ -79,39 +78,6 @@ export class WorkerService {
     return WorkerService.activityRegistry;
   }
 
-  /**
-   * For those deployments with a redis stack backend (with the FT module),
-   * this method will configure the search index for the workflow. For all
-   * others, this method will fail gracefully. In all cases, the values
-   * will be stored in the workflow's central HASH data structure, allowing
-   * for manual traversal and inspection as well.
-   */
-  static async configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> {
-    if (search?.schema) {
-      const store = hotMeshClient.engine.store;
-      const schema: string[] = [];
-      for (const [key, value] of Object.entries(search.schema)) {
-        //prefix with a comma (avoids collisions with hotmesh reserved words)
-        schema.push(`_${key}`);
-        schema.push(value.type);
-        if (value.sortable) {
-          schema.push('SORTABLE');
-        }
-      }
-      try {
-        const keyParams = {
-          appId: hotMeshClient.appId,
-          jobId: ''
-        }
-        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
-        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
-        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
-      } catch (err) {
-        hotMeshClient.engine.logger.info('durable-client-search-err', { err });
-      }
-    }
-  }
-
   static async create(config: WorkerConfig) {
     WorkerService.connection = config.connection;
     const workflow = config.workflow;
@@ -124,7 +90,7 @@ export class WorkerService {
     const worker = new WorkerService();
     worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
     worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
-    WorkerService.configureSearchIndex(worker.workflowRunner, config.search)
+    Search.configureSearchIndex(worker.workflowRunner, config.search)
     await WorkerService.activateWorkflow(worker.workflowRunner);
     return worker;
   }

package/services/durable/workflow.ts

@@ -106,6 +106,9 @@ export class WorkflowService {
     }
   }
 
+  /**
+   * wrap all activities in a proxy that will durably run them
+   */
   static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT> {
     if (options.activities) {
       WorkerService.registerActivities(options.activities)
@@ -122,6 +125,9 @@ export class WorkflowService {
     return proxy;
   }
 
+  /**
+   * return a search session for use when reading/writing to the workflow HASH
+   */
   static async search(): Promise<Search> {
     const store = asyncLocalStorage.getStore();
     const workflowId = store.get('workflowId');
@@ -130,15 +136,16 @@ export class WorkflowService {
     const namespace = store.get('namespace');
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
+    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const searchSessionId = `-search${workflowDimension}-${execIndex}`;
     return new Search(workflowId, hotMeshClient, searchSessionId);
   }
 
   /**
    * those methods that may only be called once must be protected by flagging
-   * their execution with a unique key (the key is stored in the workflow state)
+   * their execution with a unique key (the key is stored in the HASH alongside
+   * process state and job state)
    */
   static async isSideEffectAllowed(hotMeshClient: HotMesh, prefix:string): Promise<boolean> {
     const store = asyncLocalStorage.getStore();
@@ -146,15 +153,12 @@ export class WorkflowService {
     const workflowDimension = store.get('workflowDimension') ?? '';
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
-    //this ID is used as a item key with a hash (dash prefix ensures no collision)
     const keyParams = {
       appId: hotMeshClient.appId,
-      jobId: ''
+      jobId: workflowId
     }
-    const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
-    const workflowGuid = `${hotMeshPrefix}${workflowId}`;
+    const workflowGuid = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
     const guidValue = Number(await hotMeshClient.engine.store.exec('HINCRBYFLOAT', workflowGuid, sessionId, '1') as string);
     return guidValue === 1;
   }
@@ -183,15 +187,11 @@ export class WorkflowService {
     const hotMeshClient = await WorkerService.getHotMesh(`${namespace}.flow.signal`, { namespace });
     if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'hook')) {
       const store = asyncLocalStorage.getStore();
-      let workflowId: string;
-      let workflowTopic: string;
-      if (options.workflowId && options.taskQueue && options.workflowName) {
-        workflowId = options.workflowId;
+      const workflowId = options.workflowId ?? store.get('workflowId');
+      let workflowTopic = store.get('workflowTopic');
+      if (options.taskQueue && options.workflowName) {
         workflowTopic = `${options.taskQueue}-${options.workflowName}`;
-      } else {
-        workflowId = store.get('workflowId');
-        workflowTopic = store.get('workflowTopic');
-      }
+      } //else this is essentially recursion as the function calls itself
       const payload = {
         arguments: [...options.args],
         id: workflowId,

package/services/engine/index.ts

@@ -53,6 +53,7 @@ import {
   SubscriptionCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { RedisClientType } from '../../types/redisclient';
+import { StringAnyType, StringStringType } from '../../types/serializer';
 import {
   GetStatsOptions,
   IdsResponse,
@@ -67,7 +68,6 @@ import {
   StreamError,
   StreamRole,
   StreamStatus } from '../../types/stream';
-import { StringStringType } from '../../types';
 
 //wait time to see if a job is complete
 const OTT_WAIT_TIME = 1000;
@@ -629,6 +629,9 @@ class EngineService {
     }
     return stateTree;
   }
+  async getQueryState(jobId: string, fields: string[]): Promise<StringAnyType> {
+    return await this.store.getQueryState(jobId, fields);
+  }
 
   async compress(terms: string[]): Promise<boolean> {
     const existingSymbols = await this.store.getSymbolValues();

package/services/hotmesh/index.ts

@@ -21,6 +21,7 @@ import {
   StatsResponse } from '../../types/stats';
 import { ConnectorService } from '../connector';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
+import { StringAnyType } from '../../types/serializer';
 
 class HotMeshService {
   namespace: string;
@@ -59,7 +60,7 @@ class HotMeshService {
     instance.logger = new LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
     await instance.initEngine(config, instance.logger);
     await instance.initQuorum(config, instance.engine, instance.logger);
-    await instance.initWorkers(config, instance.logger);
+    await instance.doWork(config, instance.logger);
     return instance;
   }
 
@@ -97,7 +98,7 @@ class HotMeshService {
     }
   }
 
-  async initWorkers(config: HotMeshConfig, logger: ILogger) {
+  async doWork(config: HotMeshConfig, logger: ILogger) {
     this.workers = await WorkerService.init(
       this.namespace,
       this.appId,
@@ -152,6 +153,9 @@ class HotMeshService {
   async getState(topic: string, jobId: string): Promise<JobOutput> {
     return this.engine?.getState(topic, jobId);
   }
+  async getQueryState(jobId: string, fields: string[]): Promise<StringAnyType> {
+    return await this.engine?.getQueryState(jobId, fields);
+  }
   async getIds(topic: string, query: JobStatsInput, queryFacets = []): Promise<IdsResponse> {
     return await this.engine?.getIds(topic, query, queryFacets);
   }

package/services/store/index.ts

@@ -476,6 +476,21 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return jobId;
   }
 
+  /**
+   * returns custom search fields and values. The fields param
+   * should not prefix items with an underscore.
+   */
+  async getQueryState(jobId: string, fields: string[]): Promise<StringAnyType> {
+    const key = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+    const _fields = fields.map(field => `_${field}`);
+    const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
+    const jobData: StringAnyType = {};
+    fields.forEach((field, index) => {
+      jobData[field] = jobDataArray[index];
+    });
+    return jobData;
+  }
+
   async getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined> {
     //get abbreviated field list (the symbols for the paths)
     const key = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });

package/types/durable.ts

@@ -29,9 +29,9 @@ type WorkflowOptions = {
 
 type HookOptions = {
   namespace?: string;   //'durable' is the default namespace if not provided; similar to setting `appid` in the YAML
-  taskQueue: string;
+  taskQueue?: string;
   args: any[];          //input arguments to pass into the hook
-  workflowId: string;   //execution id (the job id to hook into)
+  workflowId?: string;   //execution id (the job id to hook into)
   workflowName?: string; //the name of the user's hook function
   search?: WorkflowSearchOptions //bind additional search terms immediately before hook reentry
   config?: WorkflowConfig; //hook function constraints (backoffCoefficient, maximumAttempts, maximumInterval, initialInterval)
@@ -57,12 +57,27 @@ type WorkflowDataType = {
   workflowTopic: string;
 }
 
+type MeshDBClassConfig = {
+  namespace: string;
+  taskQueue: string;
+  redisOptions: RedisOptions;
+  redisClass: RedisClass;
+}
+
+type MeshDBConfig = {
+  taskQueue?: string;
+  index?: {
+    index: string;
+    prefix: string[];
+    schema: Record<string, {type: 'TEXT' | 'NUMERIC' | 'TAG', sortable: boolean}>;
+  };
+}
+
 type ConnectionConfig = {
   class: RedisClass;
   options: RedisOptions;
 }
 type Connection =  ConnectionConfig;
-type NativeConnection =  ConnectionConfig;
 
 type ClientConfig = {
   connection: Connection;
@@ -115,11 +130,12 @@ export {
   ContextType,
   ConnectionConfig,
   Connection,
-  NativeConnection,
   ProxyType,
   Registry,
   SignalOptions,
   HookOptions,
+  MeshDBClassConfig,
+  MeshDBConfig,
   WorkerConfig,
   WorkflowConfig,
   WorkerOptions,

package/types/index.ts

@@ -34,10 +34,11 @@ export {
     ContextType,
     ConnectionConfig,
     Connection,
-    NativeConnection,
     ProxyType,
     Registry,
     HookOptions,
+    MeshDBClassConfig,
+    MeshDBConfig,
     WorkflowConfig,
     WorkerConfig,
     WorkerOptions,

package/build/services/durable/native.d.ts

@@ -1,4 +0,0 @@
-import { Connection, ConnectionConfig } from "../../types/durable";
-export declare class NativeConnectionService {
-    static connect(config: ConnectionConfig): Promise<Connection>;
-}

package/build/services/durable/native.js

@@ -1,46 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NativeConnectionService = void 0;
-/*
-
-Here is an example of how the methods in this file are used:
-
-./worker.ts
-
-import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
-import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
-
-import * as workflows from './workflows';
-
-async function run() {
-  const connection = await NativeConnection.connect({
-    class: Redis,
-    options: {
-      host: 'localhost',
-      port: 6379,
-    },
-  });
-  const worker = await Worker.create({
-    connection,
-    taskQueue: 'hello-world',
-    workflow: workflows.example,
-    activities,
-  });
-  await worker.run();
-}
-
-run().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});
-
-*/
-class NativeConnectionService {
-    static async connect(config) {
-        return {
-            class: config.class,
-            options: { ...config.options },
-        };
-    }
-}
-exports.NativeConnectionService = NativeConnectionService;