@hotmeshio/hotmesh 0.0.22 → 0.0.23

package/build/services/durable/worker.js

@@ -3,10 +3,10 @@ var _a;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerService = void 0;
 const errors_1 = require("../../modules/errors");
-const key_1 = require("../../modules/key");
 const asyncLocalStorage_1 = require("./asyncLocalStorage");
 const factory_1 = require("./factory");
 const hotmesh_1 = require("../hotmesh");
+const search_1 = require("./search");
 const stream_1 = require("../../types/stream");
 class WorkerService {
     static async activateWorkflow(hotMesh) {
@@ -45,39 +45,6 @@ class WorkerService {
         }
         return WorkerService.activityRegistry;
     }
-    /**
-     * For those deployments with a redis stack backend (with the FT module),
-     * this method will configure the search index for the workflow. For all
-     * others, this method will fail gracefully. In all cases, the values
-     * will be stored in the workflow's central HASH data structure, allowing
-     * for manual traversal and inspection as well.
-     */
-    static async configureSearchIndex(hotMeshClient, search) {
-        if (search?.schema) {
-            const store = hotMeshClient.engine.store;
-            const schema = [];
-            for (const [key, value] of Object.entries(search.schema)) {
-                //prefix with a comma (avoids collisions with hotmesh reserved words)
-                schema.push(`_${key}`);
-                schema.push(value.type);
-                if (value.sortable) {
-                    schema.push('SORTABLE');
-                }
-            }
-            try {
-                const keyParams = {
-                    appId: hotMeshClient.appId,
-                    jobId: ''
-                };
-                const hotMeshPrefix = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
-                const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
-                await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
-            }
-            catch (err) {
-                hotMeshClient.engine.logger.info('durable-client-search-err', { err });
-            }
-        }
-    }
     static async create(config) {
         WorkerService.connection = config.connection;
         const workflow = config.workflow;
@@ -89,7 +56,7 @@ class WorkerService {
         const worker = new WorkerService();
         worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
         worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
-        WorkerService.configureSearchIndex(worker.workflowRunner, config.search);
+        search_1.Search.configureSearchIndex(worker.workflowRunner, config.search);
         await WorkerService.activateWorkflow(worker.workflowRunner);
         return worker;
     }

package/build/services/durable/workflow.d.ts

@@ -10,11 +10,18 @@ export declare class WorkflowService {
      * spawn a child workflow. return the childJobId.
      */
     static startChild<T>(options: WorkflowOptions): Promise<string>;
+    /**
+     * wrap all activities in a proxy that will durably run them
+     */
     static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT>;
+    /**
+     * return a search session for use when reading/writing to the workflow HASH
+     */
     static search(): Promise<Search>;
     /**
      * those methods that may only be called once must be protected by flagging
-     * their execution with a unique key (the key is stored in the workflow state)
+     * their execution with a unique key (the key is stored in the HASH alongside
+     * process state and job state)
      */
     static isSideEffectAllowed(hotMeshClient: HotMesh, prefix: string): Promise<boolean>;
     /**

package/build/services/durable/workflow.js

@@ -89,6 +89,9 @@ class WorkflowService {
             return childJobId;
         }
     }
+    /**
+     * wrap all activities in a proxy that will durably run them
+     */
     static proxyActivities(options) {
         if (options.activities) {
             worker_1.WorkerService.registerActivities(options.activities);
@@ -103,6 +106,9 @@ class WorkflowService {
         }
         return proxy;
     }
+    /**
+     * return a search session for use when reading/writing to the workflow HASH
+     */
     static async search() {
         const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
         const workflowId = store.get('workflowId');
@@ -111,14 +117,15 @@ class WorkflowService {
         const namespace = store.get('namespace');
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
-        //this ID is used as a item key with a hash (dash prefix ensures no collision)
         const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
+        //this ID is used as a item key with a hash (dash prefix ensures no collision)
         const searchSessionId = `-search${workflowDimension}-${execIndex}`;
         return new search_1.Search(workflowId, hotMeshClient, searchSessionId);
     }
     /**
      * those methods that may only be called once must be protected by flagging
-     * their execution with a unique key (the key is stored in the workflow state)
+     * their execution with a unique key (the key is stored in the HASH alongside
+     * process state and job state)
      */
     static async isSideEffectAllowed(hotMeshClient, prefix) {
         const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
@@ -126,15 +133,12 @@ class WorkflowService {
         const workflowDimension = store.get('workflowDimension') ?? '';
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
-        //this ID is used as a item key with a hash (dash prefix ensures no collision)
         const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
-        //this ID is used as a item key with a hash (dash prefix ensures no collision)
         const keyParams = {
             appId: hotMeshClient.appId,
-            jobId: ''
+            jobId: workflowId
         };
-        const hotMeshPrefix = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
-        const workflowGuid = `${hotMeshPrefix}${workflowId}`;
+        const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
         const guidValue = Number(await hotMeshClient.engine.store.exec('HINCRBYFLOAT', workflowGuid, sessionId, '1'));
         return guidValue === 1;
     }

package/build/services/engine/index.d.ts

@@ -18,6 +18,7 @@ import { JobState, JobData, JobMetadata, JobOutput, JobStatus } from '../../type
 import { HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshSettings } from '../../types/hotmesh';
 import { JobMessageCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
+import { StringAnyType } from '../../types/serializer';
 import { GetStatsOptions, IdsResponse, JobStatsInput, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamError, StreamStatus } from '../../types/stream';
 declare class EngineService {
@@ -82,6 +83,7 @@ declare class EngineService {
     runJobCompletionTasks(context: JobState, emit?: boolean): Promise<void>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;
+    getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     compress(terms: string[]): Promise<boolean>;
 }
 export { EngineService };

package/build/services/engine/index.js

@@ -523,6 +523,9 @@ class EngineService {
         }
         return stateTree;
     }
+    async getQueryState(jobId, fields) {
+        return await this.store.getQueryState(jobId, fields);
+    }
     async compress(terms) {
         const existingSymbols = await this.store.getSymbolValues();
         const startIndex = Object.keys(existingSymbols).length;

package/build/services/hotmesh/index.d.ts

@@ -7,6 +7,7 @@ import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
 import { JobMessageCallback } from '../../types/quorum';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
+import { StringAnyType } from '../../types/serializer';
 declare class HotMeshService {
     namespace: string;
     appId: string;
@@ -21,7 +22,7 @@ declare class HotMeshService {
     static guid(): string;
     initEngine(config: HotMeshConfig, logger: ILogger): Promise<void>;
     initQuorum(config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<void>;
-    initWorkers(config: HotMeshConfig, logger: ILogger): Promise<void>;
+    doWork(config: HotMeshConfig, logger: ILogger): Promise<void>;
     pub(topic: string, data?: JobData, context?: JobState): Promise<string>;
     sub(topic: string, callback: JobMessageCallback): Promise<void>;
     unsub(topic: string): Promise<void>;
@@ -35,6 +36,7 @@ declare class HotMeshService {
     getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;
+    getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getIds(topic: string, query: JobStatsInput, queryFacets?: any[]): Promise<IdsResponse>;
     resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
     scrub(jobId: string): Promise<void>;

package/build/services/hotmesh/index.js

@@ -45,7 +45,7 @@ class HotMeshService {
         instance.logger = new logger_1.LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
         await instance.initEngine(config, instance.logger);
         await instance.initQuorum(config, instance.engine, instance.logger);
-        await instance.initWorkers(config, instance.logger);
+        await instance.doWork(config, instance.logger);
         return instance;
     }
     static guid() {
@@ -62,7 +62,7 @@ class HotMeshService {
             this.quorum = await quorum_1.QuorumService.init(this.namespace, this.appId, this.guid, config, engine, logger);
         }
     }
-    async initWorkers(config, logger) {
+    async doWork(config, logger) {
         this.workers = await worker_1.WorkerService.init(this.namespace, this.appId, this.guid, config, logger);
     }
     // ************* PUB/SUB METHODS *************
@@ -108,6 +108,9 @@ class HotMeshService {
     async getState(topic, jobId) {
         return this.engine?.getState(topic, jobId);
     }
+    async getQueryState(jobId, fields) {
+        return await this.engine?.getQueryState(jobId, fields);
+    }
     async getIds(topic, query, queryFacets = []) {
         return await this.engine?.getIds(topic, query, queryFacets);
     }

package/build/services/store/index.d.ts

@@ -61,6 +61,11 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setStatus(collationKeyStatus: number, jobId: string, appId: string, multi?: U): Promise<any>;
     getStatus(jobId: string, appId: string): Promise<number>;
     setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
+    /**
+     * returns custom search fields and values. The fields param
+     * should not prefix items with an underscore.
+     */
+    getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
     collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
     setStateNX(jobId: string, appId: string): Promise<boolean>;

package/build/services/store/index.js

@@ -394,6 +394,20 @@ class StoreService {
         await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
         return jobId;
     }
+    /**
+     * returns custom search fields and values. The fields param
+     * should not prefix items with an underscore.
+     */
+    async getQueryState(jobId, fields) {
+        const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+        const _fields = fields.map(field => `_${field}`);
+        const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
+        const jobData = {};
+        fields.forEach((field, index) => {
+            jobData[field] = jobDataArray[index];
+        });
+        return jobData;
+    }
     async getState(jobId, consumes, dIds) {
         //get abbreviated field list (the symbols for the paths)
         const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });

package/build/types/durable.d.ts

@@ -52,12 +52,28 @@ type WorkflowDataType = {
     workflowId: string;
     workflowTopic: string;
 };
+type MeshDBClassConfig = {
+    namespace: string;
+    taskQueue: string;
+    redisOptions: RedisOptions;
+    redisClass: RedisClass;
+};
+type MeshDBConfig = {
+    taskQueue?: string;
+    index?: {
+        index: string;
+        prefix: string[];
+        schema: Record<string, {
+            type: 'TEXT' | 'NUMERIC' | 'TAG';
+            sortable: boolean;
+        }>;
+    };
+};
 type ConnectionConfig = {
     class: RedisClass;
     options: RedisOptions;
 };
 type Connection = ConnectionConfig;
-type NativeConnection = ConnectionConfig;
 type ClientConfig = {
     connection: Connection;
 };
@@ -95,4 +111,4 @@ type ActivityConfig = {
         maximumInterval: string;
     };
 };
-export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, SignalOptions, HookOptions, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, HookOptions, MeshDBClassConfig, MeshDBConfig, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };

package/build/types/index.d.ts

@@ -3,7 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
-export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, HookOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, HookOptions, MeshDBClassConfig, MeshDBConfig, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.22",
+  "version": "0.0.23",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -43,6 +43,7 @@
     "test:sub:redis": "NODE_ENV=test jest ./tests/functional/sub/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:sub:ioredis": "NODE_ENV=test jest ./tests/functional/sub/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:meshdb": "NODE_ENV=test jest ./tests/durable/meshdb/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hook": "NODE_ENV=test jest ./tests/durable/hook/index.test.ts --detectOpenHandles --forceExit --verbose",

package/services/durable/connection.ts

@@ -1,45 +1,5 @@
 import { Connection, ConnectionConfig } from "../../types/durable";
 
-/*
-Here is an example of how the methods in this file are used:
-
-./client.ts
-
-import { Durable } from '@hotmeshio/hotmesh';
-import Redis from 'ioredis';
-import { nanoid } from 'nanoid';
-
-async function run() {
-  const connection = await Durable.Connection.connect({
-    class: Redis,
-    options: {
-      host: 'localhost',
-      port: 6379,
-    },
-  });
-
-  const client = new Durable.Client({
-    connection,
-  });
-
-  const handle = await client.workflow.start(example, {
-    taskQueue: 'hello-world',
-    args: ['HotMesh'],
-    workflowName: 'example',
-    workflowId: nanoid(),
-  });
-
-  console.log(`Started workflow ${handle.workflowId}`);
-  console.log(await handle.result());
-}
-
-run().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});
-
-*/
-
 export class ConnectionService {
   static async connect(config: ConnectionConfig): Promise<Connection> {
     return {

package/services/durable/factory.ts

@@ -2,11 +2,13 @@
  * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
  *       workflows will be retried on the following schedule (8 times in 27 hours):
  *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ * TODO: Max Interval, Min Interval, Initial Interval
  *
- * 594: waitforsignal
- * 595: sleep
- * 596, 597, 598: fatal
- * 599: retry
+ * ERROR CODES:
+ *      594: waitforsignal
+ *      595: sleep
+ *      596, 597, 598: fatal
+ *      599: retry
  */
 const getWorkflowYAML = (app: string, version: string) => {
   return `app:

package/services/durable/handle.ts

@@ -16,6 +16,22 @@ export class WorkflowHandleService {
     await this.hotMesh.hook(`${this.hotMesh.appId}.wfs.signal`, { id: signalId, data });
   }
 
+  async state(metadata = false): Promise<Record<string, any>> {
+    const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
+    if (!state.data && state.metadata.err) {
+      throw new Error(JSON.parse(state.metadata.err));
+    }
+    return metadata ? state : state.data;
+  }
+
+  async queryState(fields: string[]): Promise<Record<string, any>> {
+    return await this.hotMesh.getQueryState(this.workflowId, fields);
+  }
+
+  async status(): Promise<number> {
+    return await this.hotMesh.getStatus(this.workflowId);
+  }
+
   async result(loadState?: boolean): Promise<any> {
     if (loadState) {
       const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
@@ -26,8 +42,7 @@ export class WorkflowHandleService {
         //child flows are never technically 'done' as they have an open hook
         //that is tied to the parent flow's completion. so, we need to check
         //the 'done' flag on the child flow's payload (not the 'js' metadata field
-        //which is typically used); the loadState parameter ensures this
-        //check happens early
+        //which is typically used); the `loadState` parameter triggers this
         return state.data.response;
       }
     }

package/services/durable/index.ts

@@ -1,58 +1,14 @@
 import { ClientService } from './client';
 import { ConnectionService } from './connection';
-import { NativeConnectionService } from './native';
+import { MeshDBService } from './meshdb';
 import { WorkerService } from './worker';
 import { WorkflowService } from './workflow';
 import { ContextType } from '../../types/durable';
 
-/**
- * As a durable integration platform, HotMesh
- * can model and emulate other durable systems
- * (like Temporal). As you review the code in
- * this file, note the following:
- * 
- * 1) There is no central governing server.
- * HotMesh is a client-side SDK that connects to Redis
- * using CQRS principles to implicitly drive
- * orchestrations using a headless quorum. Stream
- * semantics guarantee that all events are
- * processed by the quorum of connected clients.
- * 
- * 2) Every developer-defined `workflow` function
- * is assigned a HotMesh workflow (which runs in
- * the background) to support it.
- * 
- * If the HotMesh workflow is not yet defined,
- * it will be deployed and activated on-the-fly.
- * (The generated DAG will have one Trigger Activity
- * and one Worker Activity.) The Worker Activity
- * is configured to catch execution errors and
- * return them, using a 'pending' status to
- * indicate that the workflow is still running.
- * It is possible for workflow activities to throw
- * errors that will force the entire workflow to
- * fail. This is not the case here. The workflow will
- * continue to run until it is completed or cancelled.
- * 
- * 2) Every developer-defined `activity` function
- * is assigned a HotMesh workflow (which runs in
- * the background) to support it.
- * 
- * (The generated DAG will have one Trigger Activity and one
- * Worker Activity.) The JOB ID for Worker Activity executions
- * is derived from the containing JOB ID,
- * allowing Activity state to be 'replayed' when the workflow
- * is run again. The Activity Function Runner (activity proxy)
- * is configured similar to how the workflow worker is and will
- * catch execution errors and return them to the caller, using a
- * 'pending' status to indicate that the activity is still running.
- * This allows the activity to be retried until it succeeds.
- */
-
 export const Durable = {
   Client: ClientService,
   Connection: ConnectionService,
-  NativeConnection: NativeConnectionService,
+  MeshDB: MeshDBService,
   Worker: WorkerService,
   workflow: WorkflowService,
 };