@hotmeshio/hotmesh 0.0.22 → 0.0.24

package/build/services/hotmesh/index.d.ts

@@ -7,6 +7,7 @@ import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
 import { JobMessageCallback } from '../../types/quorum';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
+import { StringAnyType } from '../../types/serializer';
 declare class HotMeshService {
     namespace: string;
     appId: string;
@@ -21,7 +22,7 @@ declare class HotMeshService {
     static guid(): string;
     initEngine(config: HotMeshConfig, logger: ILogger): Promise<void>;
     initQuorum(config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<void>;
-    initWorkers(config: HotMeshConfig, logger: ILogger): Promise<void>;
+    doWork(config: HotMeshConfig, logger: ILogger): Promise<void>;
     pub(topic: string, data?: JobData, context?: JobState): Promise<string>;
     sub(topic: string, callback: JobMessageCallback): Promise<void>;
     unsub(topic: string): Promise<void>;
@@ -35,6 +36,7 @@ declare class HotMeshService {
     getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;
+    getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getIds(topic: string, query: JobStatsInput, queryFacets?: any[]): Promise<IdsResponse>;
     resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
     scrub(jobId: string): Promise<void>;

package/build/services/hotmesh/index.js

@@ -45,7 +45,7 @@ class HotMeshService {
         instance.logger = new logger_1.LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
         await instance.initEngine(config, instance.logger);
         await instance.initQuorum(config, instance.engine, instance.logger);
-        await instance.initWorkers(config, instance.logger);
+        await instance.doWork(config, instance.logger);
         return instance;
     }
     static guid() {
@@ -62,7 +62,7 @@ class HotMeshService {
             this.quorum = await quorum_1.QuorumService.init(this.namespace, this.appId, this.guid, config, engine, logger);
         }
     }
-    async initWorkers(config, logger) {
+    async doWork(config, logger) {
         this.workers = await worker_1.WorkerService.init(this.namespace, this.appId, this.guid, config, logger);
     }
     // ************* PUB/SUB METHODS *************
@@ -108,6 +108,9 @@ class HotMeshService {
     async getState(topic, jobId) {
         return this.engine?.getState(topic, jobId);
     }
+    async getQueryState(jobId, fields) {
+        return await this.engine?.getQueryState(jobId, fields);
+    }
     async getIds(topic, query, queryFacets = []) {
         return await this.engine?.getIds(topic, query, queryFacets);
     }

package/build/services/signaler/stream.js

@@ -37,7 +37,7 @@ class StreamSignaler {
             await this.store.xgroup('CREATE', stream, group, '$', 'MKSTREAM');
         }
         catch (err) {
-            this.logger.info('consumer-group-exists', { stream, group });
+            this.logger.debug('consumer-group-exists', { stream, group });
         }
     }
     async publishMessage(topic, streamData, multi) {
@@ -121,7 +121,6 @@ class StreamSignaler {
             output = await callback(input);
         }
         catch (error) {
-            console.error(error);
             this.logger.error(`stream-call-function-error`, { error });
             output = this.structureUnhandledError(input, error);
         }

package/build/services/store/clients/ioredis.js

@@ -45,7 +45,7 @@ class IORedisStoreService extends index_1.StoreService {
                 return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
             }
             catch (err) {
-                this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
+                this.logger.debug(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
                 throw err;
             }
         }
@@ -54,7 +54,7 @@ class IORedisStoreService extends index_1.StoreService {
                 return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
             }
             catch (err) {
-                this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
+                this.logger.debug(`Consumer group not created for key: ${key} and group: ${groupName}`);
                 throw err;
             }
         }

package/build/services/store/clients/redis.js

@@ -68,7 +68,7 @@ class RedisStoreService extends index_1.StoreService {
         }
         catch (error) {
             const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
-            this.logger.info(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { error });
+            this.logger.debug(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { error });
             throw error;
         }
     }

package/build/services/store/index.d.ts

@@ -61,6 +61,11 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setStatus(collationKeyStatus: number, jobId: string, appId: string, multi?: U): Promise<any>;
     getStatus(jobId: string, appId: string): Promise<number>;
     setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
+    /**
+     * returns custom search fields and values. The fields param
+     * should not prefix items with an underscore.
+     */
+    getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
     collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
     setStateNX(jobId: string, appId: string): Promise<boolean>;

package/build/services/store/index.js

@@ -394,6 +394,20 @@ class StoreService {
         await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
         return jobId;
     }
+    /**
+     * returns custom search fields and values. The fields param
+     * should not prefix items with an underscore.
+     */
+    async getQueryState(jobId, fields) {
+        const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+        const _fields = fields.map(field => `_${field}`);
+        const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
+        const jobData = {};
+        fields.forEach((field, index) => {
+            jobData[field] = jobDataArray[index];
+        });
+        return jobData;
+    }
     async getState(jobId, consumes, dIds) {
         //get abbreviated field list (the symbols for the paths)
         const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });

package/build/types/durable.d.ts

@@ -10,7 +10,7 @@ type WorkflowSearchOptions = {
     prefix?: string[];
     schema?: Record<string, {
         type: 'TEXT' | 'NUMERIC' | 'TAG';
-        sortable: boolean;
+        sortable?: boolean;
     }>;
     data?: Record<string, string>;
 };
@@ -52,12 +52,28 @@ type WorkflowDataType = {
     workflowId: string;
     workflowTopic: string;
 };
+type MeshOSClassConfig = {
+    namespace: string;
+    taskQueue: string;
+    redisOptions: RedisOptions;
+    redisClass: RedisClass;
+};
+type MeshOSConfig = {
+    taskQueue?: string;
+    index?: {
+        index: string;
+        prefix: string[];
+        schema: Record<string, {
+            type: 'TEXT' | 'NUMERIC' | 'TAG';
+            sortable: boolean;
+        }>;
+    };
+};
 type ConnectionConfig = {
     class: RedisClass;
     options: RedisOptions;
 };
 type Connection = ConnectionConfig;
-type NativeConnection = ConnectionConfig;
 type ClientConfig = {
     connection: Connection;
 };
@@ -68,10 +84,24 @@ type WorkerConfig = {
     connection: Connection;
     namespace?: string;
     taskQueue: string;
-    workflow: Function;
+    workflow: Function | Record<string | symbol, Function>;
     options?: WorkerOptions;
     search?: WorkflowSearchOptions;
 };
+type FindOptions = {
+    workflowName?: string;
+    taskQueue?: string;
+    namespace?: string;
+    index?: string;
+};
+type MeshOSOptions = {
+    name: string;
+    options: WorkerOptions;
+};
+type MeshOSActivityOptions = {
+    name: string;
+    options: ActivityConfig;
+};
 type WorkerOptions = {
     logLevel?: string;
     maxSystemRetries?: number;
@@ -95,4 +125,4 @@ type ActivityConfig = {
         maximumInterval: string;
     };
 };
-export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, SignalOptions, HookOptions, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, FindOptions, HookOptions, MeshOSActivityOptions, MeshOSClassConfig, MeshOSConfig, MeshOSOptions, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };

package/build/types/index.d.ts

@@ -3,7 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';
 export { CollationFaultType, CollationStage } from './collator';
-export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, HookOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
+export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, FindOptions, HookOptions, MeshOSActivityOptions, MeshOSClassConfig, MeshOSConfig, MeshOSOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
 export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
 export { ILogger } from './logger';

package/index.ts

@@ -1,5 +1,6 @@
 import { Durable } from './services/durable';
+import { MeshOSService as MeshOS } from './services/durable/meshos';
 import { HotMeshService as HotMesh } from './services/hotmesh';
 import { HotMeshConfig } from './types/hotmesh';
 
-export { Durable, HotMesh, HotMeshConfig };
+export { Durable, HotMesh, HotMeshConfig, MeshOS };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.22",
+  "version": "0.0.24",
   "description": "Unbreakable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -43,6 +43,7 @@
     "test:sub:redis": "NODE_ENV=test jest ./tests/functional/sub/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:sub:ioredis": "NODE_ENV=test jest ./tests/functional/sub/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:meshos": "NODE_ENV=test jest ./tests/durable/meshos/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:durable:hook": "NODE_ENV=test jest ./tests/durable/hook/index.test.ts --detectOpenHandles --forceExit --verbose",

package/services/durable/connection.ts

@@ -1,45 +1,5 @@
 import { Connection, ConnectionConfig } from "../../types/durable";
 
-/*
-Here is an example of how the methods in this file are used:
-
-./client.ts
-
-import { Durable } from '@hotmeshio/hotmesh';
-import Redis from 'ioredis';
-import { nanoid } from 'nanoid';
-
-async function run() {
-  const connection = await Durable.Connection.connect({
-    class: Redis,
-    options: {
-      host: 'localhost',
-      port: 6379,
-    },
-  });
-
-  const client = new Durable.Client({
-    connection,
-  });
-
-  const handle = await client.workflow.start(example, {
-    taskQueue: 'hello-world',
-    args: ['HotMesh'],
-    workflowName: 'example',
-    workflowId: nanoid(),
-  });
-
-  console.log(`Started workflow ${handle.workflowId}`);
-  console.log(await handle.result());
-}
-
-run().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});
-
-*/
-
 export class ConnectionService {
   static async connect(config: ConnectionConfig): Promise<Connection> {
     return {

package/services/durable/factory.ts

@@ -2,11 +2,13 @@
  * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
  *       workflows will be retried on the following schedule (8 times in 27 hours):
  *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ * TODO: Max Interval, Min Interval, Initial Interval
  *
- * 594: waitforsignal
- * 595: sleep
- * 596, 597, 598: fatal
- * 599: retry
+ * ERROR CODES:
+ *      594: waitforsignal
+ *      595: sleep
+ *      596, 597, 598: fatal
+ *      599: retry
  */
 const getWorkflowYAML = (app: string, version: string) => {
   return `app:
@@ -262,11 +264,11 @@ const getWorkflowYAML = (app: string, version: string) => {
                   - ['{@string.concat}']
               cycleWorkflowId:
                 '@pipe':
-                  - ['{$job.metadata.jid}', '-$wfc', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
+                  - ['-', '{$job.metadata.jid}', '-$wfc', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
                   - ['{@string.concat}']
               baseWorkflowId:
                 '@pipe':
-                  - ['{$job.metadata.jid}', '-$wfs', '{sig.output.metadata.dad}', '-']
+                  - ['-', '{$job.metadata.jid}', '-$wfs', '{sig.output.metadata.dad}', '-']
                   - ['{@string.concat}']
           output:
             schema:
@@ -310,7 +312,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                   - ['{@string.concat}']
               workflowId:
                 '@pipe':
-                  - ['{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
+                  - ['-', '{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
                   - ['{@string.concat}']
           output:
             schema:
@@ -380,11 +382,11 @@ const getWorkflowYAML = (app: string, version: string) => {
                   - ['{@string.concat}']
               cycleWorkflowId:
                 '@pipe':
-                  - ['{$job.metadata.jid}', '-$wfc-', '{w1.output.data.index}']
+                  - ['-', '{$job.metadata.jid}', '-$wfc-', '{w1.output.data.index}']
                   - ['{@string.concat}']
               baseWorkflowId:
                 '@pipe':
-                  - ['{$job.metadata.jid}', '-$wfs-']
+                  - ['-', '{$job.metadata.jid}', '-$wfs-']
                   - ['{@string.concat}']
           output:
             schema:
@@ -428,7 +430,7 @@ const getWorkflowYAML = (app: string, version: string) => {
                   - ['{@string.concat}']
               workflowId:
                 '@pipe':
-                  - ['{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
+                  - ['-', '{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
                   - ['{@string.concat}']
           output:
             schema:

package/services/durable/handle.ts

@@ -16,6 +16,22 @@ export class WorkflowHandleService {
     await this.hotMesh.hook(`${this.hotMesh.appId}.wfs.signal`, { id: signalId, data });
   }
 
+  async state(metadata = false): Promise<Record<string, any>> {
+    const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
+    if (!state.data && state.metadata.err) {
+      throw new Error(JSON.parse(state.metadata.err));
+    }
+    return metadata ? state : state.data;
+  }
+
+  async queryState(fields: string[]): Promise<Record<string, any>> {
+    return await this.hotMesh.getQueryState(this.workflowId, fields);
+  }
+
+  async status(): Promise<number> {
+    return await this.hotMesh.getStatus(this.workflowId);
+  }
+
   async result(loadState?: boolean): Promise<any> {
     if (loadState) {
       const state = await this.hotMesh.getState(`${this.hotMesh.appId}.execute`, this.workflowId);
@@ -23,11 +39,8 @@ export class WorkflowHandleService {
         throw new Error(JSON.parse(state.metadata.err));
       }
       if (state?.data?.done) {
-        //child flows are never technically 'done' as they have an open hook
-        //that is tied to the parent flow's completion. so, we need to check
-        //the 'done' flag on the child flow's payload (not the 'js' metadata field
-        //which is typically used); the loadState parameter ensures this
-        //check happens early
+        //child flows are never 'done'; they use a hook
+        //that only closes upon parent flow completion.
         return state.data.response;
       }
     }

package/services/durable/index.ts

@@ -1,58 +1,14 @@
 import { ClientService } from './client';
 import { ConnectionService } from './connection';
-import { NativeConnectionService } from './native';
+import { MeshOSService } from './meshos';
 import { WorkerService } from './worker';
 import { WorkflowService } from './workflow';
 import { ContextType } from '../../types/durable';
 
-/**
- * As a durable integration platform, HotMesh
- * can model and emulate other durable systems
- * (like Temporal). As you review the code in
- * this file, note the following:
- * 
- * 1) There is no central governing server.
- * HotMesh is a client-side SDK that connects to Redis
- * using CQRS principles to implicitly drive
- * orchestrations using a headless quorum. Stream
- * semantics guarantee that all events are
- * processed by the quorum of connected clients.
- * 
- * 2) Every developer-defined `workflow` function
- * is assigned a HotMesh workflow (which runs in
- * the background) to support it.
- * 
- * If the HotMesh workflow is not yet defined,
- * it will be deployed and activated on-the-fly.
- * (The generated DAG will have one Trigger Activity
- * and one Worker Activity.) The Worker Activity
- * is configured to catch execution errors and
- * return them, using a 'pending' status to
- * indicate that the workflow is still running.
- * It is possible for workflow activities to throw
- * errors that will force the entire workflow to
- * fail. This is not the case here. The workflow will
- * continue to run until it is completed or cancelled.
- * 
- * 2) Every developer-defined `activity` function
- * is assigned a HotMesh workflow (which runs in
- * the background) to support it.
- * 
- * (The generated DAG will have one Trigger Activity and one
- * Worker Activity.) The JOB ID for Worker Activity executions
- * is derived from the containing JOB ID,
- * allowing Activity state to be 'replayed' when the workflow
- * is run again. The Activity Function Runner (activity proxy)
- * is configured similar to how the workflow worker is and will
- * catch execution errors and return them to the caller, using a
- * 'pending' status to indicate that the activity is still running.
- * This allows the activity to be retried until it succeeds.
- */
-
 export const Durable = {
   Client: ClientService,
   Connection: ConnectionService,
-  NativeConnection: NativeConnectionService,
+  MeshOS: MeshOSService,
   Worker: WorkerService,
   workflow: WorkflowService,
 };