@hotmeshio/hotmesh 0.0.12 → 0.0.14

package/build/services/activities/worker.js

@@ -5,6 +5,7 @@ const errors_1 = require("../../modules/errors");
 const activity_1 = require("./activity");
 const collator_1 = require("../collator");
 const telemetry_1 = require("../telemetry");
+const pipe_1 = require("../pipe");
 class Worker extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -14,21 +15,24 @@ class Worker extends activity_1.Activity {
         this.logger.debug('worker-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
         let telemetry;
         try {
+            //confirm entry is allowed and restore state
             this.setLeg(1);
             await collator_1.CollatorService.notarizeEntry(this);
             await this.getState();
             telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
             telemetry.startActivitySpan(this.leg);
             this.mapInputData();
+            //save state and authorize reentry
             const multi = this.store.getMulti();
             //todo: await this.registerTimeout();
+            const messageId = await this.execActivity(multi);
             await collator_1.CollatorService.authorizeReentry(this, multi);
             await this.setState(multi);
             await this.setStatus(0, multi);
             const multiResponse = await multi.exec();
+            //telemetry
             telemetry.mapActivityAttributes();
             const jobStatus = this.resolveStatus(multiResponse);
-            const messageId = await this.execActivity();
             telemetry.setActivityAttributes({
                 'app.activity.mid': messageId,
                 'app.job.jss': jobStatus
@@ -37,10 +41,10 @@ class Worker extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.GetStateError) {
-                this.logger.error('worker-get-state-error', error);
+                this.logger.error('worker-get-state-error', { error });
             }
             else {
-                this.logger.error('worker-process-error', error);
+                this.logger.error('worker-process-error', { error });
             }
             telemetry.setActivityError(error.message);
             throw error;
@@ -50,13 +54,14 @@ class Worker extends activity_1.Activity {
             this.logger.debug('worker-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
         }
     }
-    async execActivity() {
+    async execActivity(multi) {
+        const topic = pipe_1.Pipe.resolve(this.config.subtype, this.context);
         const streamData = {
             metadata: {
                 jid: this.context.metadata.jid,
                 dad: this.metadata.dad,
                 aid: this.metadata.aid,
-                topic: this.config.subtype,
+                topic,
                 spn: this.context['$self'].output.metadata.l1s,
                 trc: this.context.metadata.trc,
             },
@@ -67,7 +72,7 @@ class Worker extends activity_1.Activity {
                 retry: this.config.retry
             };
         }
-        return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData));
+        return (await this.engine.streamSignaler?.publishMessage(topic, streamData, multi));
     }
 }
 exports.Worker = Worker;

package/build/services/compiler/deployer.js

@@ -5,6 +5,7 @@ const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const collator_1 = require("../collator");
 const serializer_1 = require("../serializer");
+const pipe_1 = require("../pipe");
 const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
 const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
 const DEFAULT_RANGE_SIZE = DEFAULT_METADATA_RANGE_SIZE + DEFAULT_DATA_RANGE_SIZE;
@@ -405,7 +406,8 @@ class Deployer {
             const activities = graph.activities;
             for (const activityKey in activities) {
                 const activity = activities[activityKey];
-                if (activity.type === 'worker') {
+                //only precreate if the topic is concrete and not `mappable`
+                if (activity.type === 'worker' && pipe_1.Pipe.resolve(activity.subtype, {}) === activity.subtype) {
                     params.topic = activity.subtype;
                     const key = this.store.mintKey(key_1.KeyType.STREAMS, params);
                     //create one worker group per unique activity subtype (the topic)

package/build/services/durable/client.d.ts

@@ -6,10 +6,11 @@ export declare class ClientService {
     options: WorkflowOptions;
     static instances: Map<string, HotMesh | Promise<HotMesh>>;
     constructor(config: ClientConfig);
-    getHotMesh: (worflowTopic: string) => Promise<HotMesh>;
+    getHotMeshClient: (worflowTopic: string) => Promise<HotMesh>;
     workflow: {
         start: (options: WorkflowOptions) => Promise<WorkflowHandleService>;
+        signal: (signalId: string, data: Record<any, any>) => Promise<string>;
     };
-    activateWorkflow(hotMesh: HotMesh, workflowTopic: string): Promise<void>;
+    activateWorkflow(hotMesh: HotMesh, appId?: string, version?: string): Promise<void>;
     static shutdown(): Promise<void>;
 }

package/build/services/durable/client.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ClientService = void 0;
+const nanoid_1 = require("nanoid");
+const factory_1 = require("./factory");
 const handle_1 = require("./handle");
 const hotmesh_1 = require("../hotmesh");
-const factory_1 = require("./factory");
+const key_1 = require("../../modules/key");
 /*
 Here is an example of how the methods in this file are used:
 
@@ -46,12 +48,13 @@ run().catch((err) => {
 */
 class ClientService {
     constructor(config) {
-        this.getHotMesh = async (worflowTopic) => {
+        this.getHotMeshClient = async (worflowTopic) => {
+            //NOTE: every unique topic inits a new engine
             if (ClientService.instances.has(worflowTopic)) {
                 return await ClientService.instances.get(worflowTopic);
             }
-            const hotMesh = hotmesh_1.HotMeshService.init({
-                appId: worflowTopic,
+            const hotMeshClient = hotmesh_1.HotMeshService.init({
+                appId: factory_1.APP_ID,
                 engine: {
                     redis: {
                         class: this.connection.class,
@@ -59,9 +62,19 @@ class ClientService {
                     }
                 }
             });
-            ClientService.instances.set(worflowTopic, hotMesh);
-            await this.activateWorkflow(await hotMesh, worflowTopic);
-            return hotMesh;
+            ClientService.instances.set(worflowTopic, hotMeshClient);
+            //since the YAML topic is dynamic, it MUST be manually created before use
+            const store = (await hotMeshClient).engine.store;
+            const params = { appId: factory_1.APP_ID, topic: worflowTopic };
+            const streamKey = store.mintKey(key_1.KeyType.STREAMS, params);
+            try {
+                await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
+            }
+            catch (err) {
+                //ignore if already exists
+            }
+            await this.activateWorkflow(await hotMeshClient);
+            return hotMeshClient;
         };
         this.workflow = {
             start: async (options) => {
@@ -69,40 +82,45 @@ class ClientService {
                 const workflowName = options.workflowName;
                 const trc = options.workflowTrace;
                 const spn = options.workflowSpan;
+                //topic is concat of taskQueue and workflowName
                 const workflowTopic = `${taskQueueName}-${workflowName}`;
-                const hotMesh = await this.getHotMesh(workflowTopic);
+                const hotMeshClient = await this.getHotMeshClient(workflowTopic);
                 const payload = {
                     arguments: [...options.args],
-                    workflowId: options.workflowId,
+                    workflowId: options.workflowId || (0, nanoid_1.nanoid)(),
+                    workflowTopic: workflowTopic,
+                    backoffCoefficient: options.config?.backoffCoefficient || factory_1.DEFAULT_COEFFICIENT,
                 };
                 const context = { metadata: { trc, spn }, data: {} };
-                const jobId = await hotMesh.pub(workflowTopic, payload, context);
-                return new handle_1.WorkflowHandleService(hotMesh, workflowTopic, jobId);
+                const jobId = await hotMeshClient.pub(factory_1.SUBSCRIBES_TOPIC, payload, context);
+                return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
             },
+            signal: async (signalId, data) => {
+                return await (await this.getHotMeshClient('durable.wfs.signal')).hook('durable.wfs.signal', { id: signalId, data });
+            }
         };
         this.connection = config.connection;
     }
-    async activateWorkflow(hotMesh, workflowTopic) {
-        const version = '1';
-        const app = await hotMesh.engine.store.getApp(workflowTopic);
+    async activateWorkflow(hotMesh, appId = factory_1.APP_ID, version = factory_1.APP_VERSION) {
+        const app = await hotMesh.engine.store.getApp(appId);
         const appVersion = app?.version;
         if (isNaN(appVersion)) {
             try {
-                await hotMesh.deploy((0, factory_1.getWorkflowYAML)(workflowTopic, version));
+                await hotMesh.deploy((0, factory_1.getWorkflowYAML)(appId, version));
                 await hotMesh.activate(version);
             }
-            catch (err) {
-                hotMesh.engine.logger.error('durable-client-deploy-activate-err', err);
-                throw err;
+            catch (error) {
+                hotMesh.engine.logger.error('durable-client-deploy-activate-err', { error });
+                throw error;
             }
         }
         else if (app && !app.active) {
             try {
                 await hotMesh.activate(version);
             }
-            catch (err) {
-                hotMesh.engine.logger.error('durable-client-activate-err', err);
-                throw err;
+            catch (error) {
+                hotMesh.engine.logger.error('durable-client-activate-err', { error });
+                throw error;
             }
         }
     }

package/build/services/durable/factory.d.ts

@@ -1,20 +1,24 @@
 /**
- * 1) `maxSystemRetries` | can be 0 to 3 and represents milliseconds;
- * if there is an error, the workflow will retry up to `maxSystemRetries` times
- * delaying by 10, 100, and 1000ms; this is a system level retry
- * and is not configurable. It exists to handle intermittent network
- * errors. (NOTE: each retry spawns a new transition stream)
- *
- * 2) `backoffExponent` | can be any number and represents `seconds` when applied;
- * retries will happen indefinitely and adhere to the
- * exponential backoff algorithm by multiplying by `backoffExponent`.
- * For example, if `backoffExponent` is 10, the workflow will retry
- * in 10s, 100s, 1000s, 10000s, etc.
- *
- * EXAMPLE | Using `maxSystemRetries = 3` and `backoffExponent = 10`, errant
- * workflows will be retried on the following schedule (8 times in 27 hours):
- * => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
+ * NOTE: Using `maxSystemRetries = 3` and `backoffCoefficient = 10`, errant
+ *       workflows will be retried on the following schedule (8 times in 27 hours):
+ *       => 10ms, 100ms, 1000ms, 10s, 100s, 1_000s, 10_000s, 100_000s
  */
-declare const getWorkflowYAML: (topic: string, version?: string, maxSystemRetries?: number, backoffExponent?: number) => string;
-declare const getActivityYAML: (topic: string, version?: string) => string;
-export { getActivityYAML, getWorkflowYAML };
+declare const getWorkflowYAML: (app: string, version: string) => string;
+declare const APP_VERSION = "1";
+declare const APP_ID = "durable";
+declare const ACTIVITY_SUBSCRIBES_TOPIC = "durable.activity.execute";
+declare const ACTIVITY_PUBLISHES_TOPIC = "durable.activity.executed";
+declare const SLEEP_SUBSCRIBES_TOPIC = "durable.sleep.execute";
+declare const SLEEP_PUBLISHES_TOPIC = "durable.sleep.executed";
+declare const WFS_SUBSCRIBES_TOPIC = "durable.wfs.execute";
+declare const WFS_PUBLISHES_TOPIC = "durable.wfs.executed";
+declare const WFSC_SUBSCRIBES_TOPIC = "durable.wfsc.execute";
+declare const WFSC_PUBLISHES_TOPIC = "durable.wfsc.executed";
+declare const SUBSCRIBES_TOPIC = "durable.execute";
+declare const PUBLISHES_TOPIC = "durable.executed";
+declare const HOOK_ID = "durable.awaken";
+declare const ACTIVITY_HOOK_ID = "durable.activity.awaken";
+declare const SLEEP_HOOK_ID = "durable.sleep.awaken";
+declare const WFS_HOOK_ID = "durable.wfs.awaken";
+declare const DEFAULT_COEFFICIENT = 10;
+export { getWorkflowYAML, APP_VERSION, APP_ID, ACTIVITY_SUBSCRIBES_TOPIC, ACTIVITY_PUBLISHES_TOPIC, SLEEP_SUBSCRIBES_TOPIC, SLEEP_PUBLISHES_TOPIC, SUBSCRIBES_TOPIC, PUBLISHES_TOPIC, HOOK_ID, ACTIVITY_HOOK_ID, SLEEP_HOOK_ID, DEFAULT_COEFFICIENT, WFS_SUBSCRIBES_TOPIC, WFS_PUBLISHES_TOPIC, WFSC_SUBSCRIBES_TOPIC, WFSC_PUBLISHES_TOPIC, WFS_HOOK_ID, };