@hotmeshio/hotmesh 0.0.4 → 0.0.5

package/build/esm/services/durable/factory.js

@@ -0,0 +1,119 @@
+const getWorkflowYAML = (topic, version = '1') => {
+    return `app:
+  id: ${topic}
+  version: '${version}'
+  graphs:
+    - subscribes: ${topic}
+      publishes: ${topic}
+      expire: 120
+      input:
+        schema:
+          type: object
+          properties:
+            workflowId:
+              type: string
+            arguments:
+              type: array
+      output:
+        schema:
+          type: object
+          properties:
+            response:
+              type: any
+
+      activities:
+        t1:
+          type: trigger
+          stats:
+            id: '{$self.input.data.workflowId}'
+        a1:
+          type: worker
+          topic: ${topic}
+          input:
+            schema:
+              type: object
+              properties:
+                workflowId:
+                  type: string
+                arguments:
+                  type: array
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+              arguments: '{t1.output.data.arguments}'
+          output:
+            schema:
+              type: object
+              properties:
+                response:
+                  type: any
+          job:
+            maps:
+              response: '{$self.output.data.response}'
+      transitions:
+        t1:
+          - to: a1`;
+};
+const getActivityYAML = (topic, version = '1') => {
+    return `app:
+  id: ${topic}
+  version: '${version}'
+  graphs:
+    - subscribes: ${topic}
+      input:
+        schema:
+          type: object
+          properties:
+            workflowId:
+              type: string
+            workflowTopic:
+              type: string
+            activityName:
+              type: array
+            arguments:
+              type: array
+      output:
+        schema:
+          type: object
+          properties:
+            response:
+              type: any
+
+      activities:
+        t1:
+          type: trigger
+          stats:
+            id: '{$self.input.data.workflowId}'
+        a1:
+          type: worker
+          topic: ${topic}
+          input:
+            schema:
+              type: object
+              properties:
+                workflowId:
+                  type: string
+                workflowTopic:
+                  type: string
+                activityName:
+                  type: array
+                arguments:
+                  type: array
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+              workflowTopic: '{t1.output.data.workflowTopic}'
+              activityName: '{t1.output.data.activityName}'
+              arguments: '{t1.output.data.arguments}'
+          output:
+            schema:
+              type: object
+              properties:
+                response:
+                  type: any
+          job:
+            maps:
+              response: '{$self.output.data.response}'
+      transitions:
+        t1:
+          - to: a1`;
+};
+export { getActivityYAML, getWorkflowYAML };

package/build/esm/services/durable/handle.d.ts

@@ -0,0 +1,8 @@
+import { HotMeshService as HotMesh } from '../hotmesh';
+export declare class WorkflowHandleService {
+    hotMesh: HotMesh;
+    workflowTopic: string;
+    workflowId: string;
+    constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string);
+    result(): Promise<any>;
+}

package/build/esm/services/durable/handle.js

@@ -0,0 +1,34 @@
+export class WorkflowHandleService {
+    constructor(hotMesh, workflowTopic, workflowId) {
+        this.workflowTopic = workflowTopic;
+        this.workflowId = workflowId;
+        this.hotMesh = hotMesh;
+    }
+    async result() {
+        let status = await this.hotMesh.getStatus(this.workflowId);
+        const topic = `${this.workflowTopic}.${this.workflowId}`;
+        if (status == 0) {
+            return (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response;
+        }
+        return new Promise((resolve, reject) => {
+            let isResolved = false;
+            //common fulfill/unsubscribe
+            const complete = async (response) => {
+                if (isResolved)
+                    return;
+                isResolved = true;
+                this.hotMesh.unsub(topic);
+                resolve(response || (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response);
+            };
+            this.hotMesh.sub(topic, async (topic, message) => {
+                await complete(message.data?.response);
+            });
+            setTimeout(async () => {
+                status = await this.hotMesh.getStatus(this.workflowId);
+                if (status == 0) {
+                    await complete();
+                }
+            }, 0);
+        });
+    }
+}

package/build/esm/services/durable/index.d.ts

@@ -0,0 +1,57 @@
+import { ClientService } from './client';
+import { ConnectionService } from './connection';
+import { NativeConnectionService } from './native';
+import { WorkerService } from './worker';
+import { WorkflowService } from './workflow';
+import { ContextType } from '../../types/durable';
+/**
+ * As a durable integration platform, HotMesh
+ * can model and emulate other durable systems
+ * (like Temporal). As you review the code in
+ * this file, note the following:
+ *
+ * 1) There is no central governing server.
+ * HotMesh is a client-side SDK that connects to Redis
+ * using CQRS principles to implicitly drive
+ * orchestrations using a headless quorum. Stream
+ * semantics guarantee that all events are
+ * processed by the quorum of connected clients.
+ *
+ * 2) Every developer-defined `workflow` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * If the HotMesh workflow is not yet defined,
+ * it will be deployed and activated on-the-fly.
+ * (The generated DAG will have one Trigger Activity
+ * and one Worker Activity.) The Worker Activity
+ * is configured to catch execution errors and
+ * return them, using a 'pending' status to
+ * indicate that the workflow is still running.
+ * It is possible for workflow activities to throw
+ * errors that will force the entire workflow to
+ * fail. This is not the case here. The workflow will
+ * continue to run until it is completed or cancelled.
+ *
+ * 2) Every developer-defined `activity` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * (The generated DAG will have one Trigger Activity and one
+ * Worker Activity.) The JOB ID for Worker Activity executions
+ * is derived from the containing JOB ID,
+ * allowing Activity state to be 'replayed' when the workflow
+ * is run again. The Activity Function Runner (activity proxy)
+ * is configured similar to how the workflow worker is and will
+ * catch execution errors and return them to the caller, using a
+ * 'pending' status to indicate that the activity is still running.
+ * This allows the activity to be retried until it succeeds.
+ */
+export declare const Durable: {
+    Client: typeof ClientService;
+    Connection: typeof ConnectionService;
+    NativeConnection: typeof NativeConnectionService;
+    Worker: typeof WorkerService;
+    workflow: typeof WorkflowService;
+};
+export type { ContextType };

package/build/esm/services/durable/index.js

@@ -0,0 +1,55 @@
+import { ClientService } from './client';
+import { ConnectionService } from './connection';
+import { NativeConnectionService } from './native';
+import { WorkerService } from './worker';
+import { WorkflowService } from './workflow';
+/**
+ * As a durable integration platform, HotMesh
+ * can model and emulate other durable systems
+ * (like Temporal). As you review the code in
+ * this file, note the following:
+ *
+ * 1) There is no central governing server.
+ * HotMesh is a client-side SDK that connects to Redis
+ * using CQRS principles to implicitly drive
+ * orchestrations using a headless quorum. Stream
+ * semantics guarantee that all events are
+ * processed by the quorum of connected clients.
+ *
+ * 2) Every developer-defined `workflow` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * If the HotMesh workflow is not yet defined,
+ * it will be deployed and activated on-the-fly.
+ * (The generated DAG will have one Trigger Activity
+ * and one Worker Activity.) The Worker Activity
+ * is configured to catch execution errors and
+ * return them, using a 'pending' status to
+ * indicate that the workflow is still running.
+ * It is possible for workflow activities to throw
+ * errors that will force the entire workflow to
+ * fail. This is not the case here. The workflow will
+ * continue to run until it is completed or cancelled.
+ *
+ * 2) Every developer-defined `activity` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * (The generated DAG will have one Trigger Activity and one
+ * Worker Activity.) The JOB ID for Worker Activity executions
+ * is derived from the containing JOB ID,
+ * allowing Activity state to be 'replayed' when the workflow
+ * is run again. The Activity Function Runner (activity proxy)
+ * is configured similar to how the workflow worker is and will
+ * catch execution errors and return them to the caller, using a
+ * 'pending' status to indicate that the activity is still running.
+ * This allows the activity to be retried until it succeeds.
+ */
+export const Durable = {
+    Client: ClientService,
+    Connection: ConnectionService,
+    NativeConnection: NativeConnectionService,
+    Worker: WorkerService,
+    workflow: WorkflowService,
+};

package/build/esm/services/durable/native.d.ts

@@ -0,0 +1,4 @@
+import { Connection, ConnectionConfig } from "../../types/durable";
+export declare class NativeConnectionService {
+    static connect(config: ConnectionConfig): Promise<Connection>;
+}

package/build/esm/services/durable/native.js

@@ -0,0 +1,43 @@
+/*
+
+Here is an example of how the methods in this file are used:
+
+./worker.ts
+
+import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
+
+import * as activities from './activities';
+
+async function run() {
+  const connection = await NativeConnection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+  const worker = await Worker.create({
+    connection,
+    namespace: 'default',
+    taskQueue: 'hello-world',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+  });
+  await worker.run();
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+
+*/
+export class NativeConnectionService {
+    static async connect(config) {
+        return {
+            class: config.class,
+            options: { ...config.options },
+        };
+    }
+}

package/build/esm/services/durable/worker.d.ts

@@ -0,0 +1,36 @@
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { Connection, Registry, WorkerConfig } from "../../types/durable";
+export declare class WorkerService {
+    static activityRegistry: Registry;
+    static connection: Connection;
+    static instances: Map<string, HotMesh | Promise<HotMesh>>;
+    workflowRunner: HotMesh;
+    static getHotMesh: (worflowTopic: string) => Promise<HotMesh>;
+    static activateWorkflow(hotMesh: HotMesh, topic: string, factory: Function): Promise<void>;
+    /**
+     * The `worker` calls `registerActivities` immediately BEFORE
+     * dynamically importing the user's workflow module. That file
+     * contains a call, `proxyActivities`, which needs this info.
+     *
+     * NOTE: The `worker` and `client` both call `proxyActivities`,
+     * as a natural result of importing worflows.ts. However,
+     * because the worker imports the workflows dynamically AFTER
+     * the activities are loaded, there will be items in the registry,
+     * allowing proxyActivities to succeed.
+     */
+    static registerActivities<ACT>(activities: ACT): Registry;
+    static create(config: WorkerConfig): Promise<WorkerService>;
+    run(): Promise<void>;
+    initActivityWorkflow(config: WorkerConfig, activityTopic: string): Promise<HotMesh>;
+    wrapActivityFunctions(): Function;
+    activateActivityWorkflow(hotMesh: HotMesh, activityTopic: string): Promise<void>;
+    initWorkerWorkflow(config: WorkerConfig, workflowTopic: string, workflowFunction: Function): Promise<HotMesh>;
+    static Context: {
+        info: () => {
+            workflowId: string;
+            workflowTopic: string;
+        };
+    };
+    wrapWorkflowFunction(workflowFunction: Function, workflowTopic: string): Function;
+    static shutdown(): Promise<void>;
+}

package/build/esm/services/durable/worker.js

@@ -0,0 +1,258 @@
+var _a;
+import { asyncLocalStorage } from './asyncLocalStorage';
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { StreamStatus } from '../../types/stream';
+import { getWorkflowYAML, getActivityYAML } from './factory';
+/*
+Here is an example of how the methods in this file are used:
+
+./worker.ts
+
+import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
+
+import * as activities from './activities';
+
+async function run() {
+  const connection = await NativeConnection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+  const worker = await Worker.create({
+    connection,
+    namespace: 'default',
+    taskQueue: 'hello-world',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+  });
+  await worker.run();
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+*/
+class WorkerService {
+    static async activateWorkflow(hotMesh, topic, factory) {
+        const version = '1';
+        const app = await hotMesh.engine.store.getApp(topic);
+        const appVersion = app?.version;
+        if (!appVersion) {
+            try {
+                await hotMesh.deploy(factory(topic, version));
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-worker-deploy-activate-err', err);
+                throw err;
+            }
+        }
+        else if (app && !app.active) {
+            try {
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-worker-activate-err', err);
+                throw err;
+            }
+        }
+    }
+    /**
+     * The `worker` calls `registerActivities` immediately BEFORE
+     * dynamically importing the user's workflow module. That file
+     * contains a call, `proxyActivities`, which needs this info.
+     *
+     * NOTE: The `worker` and `client` both call `proxyActivities`,
+     * as a natural result of importing worflows.ts. However,
+     * because the worker imports the workflows dynamically AFTER
+     * the activities are loaded, there will be items in the registry,
+     * allowing proxyActivities to succeed.
+     */
+    static registerActivities(activities) {
+        Object.keys(activities).forEach(key => {
+            WorkerService.activityRegistry[key] = activities[key];
+        });
+        return WorkerService.activityRegistry;
+    }
+    static async create(config) {
+        WorkerService.connection = config.connection;
+        //pre-cache user activity functions
+        WorkerService.registerActivities(config.activities);
+        //import the user's workflow file (triggers activity functions to be wrapped)
+        const workflow = await import(config.workflowsPath);
+        const workflowFunctionNames = Object.keys(workflow);
+        const workflowFunctionName = workflowFunctionNames[workflowFunctionNames.length - 1];
+        const workflowFunction = workflow[workflowFunctionName];
+        const baseTopic = `${config.taskQueue}-${workflowFunctionName}`;
+        const activityTopic = `${baseTopic}-activity`;
+        const workflowTopic = `${baseTopic}`;
+        //initialize supporting workflows
+        const worker = new WorkerService();
+        const activityRunner = await worker.initActivityWorkflow(config, activityTopic);
+        await WorkerService.activateWorkflow(activityRunner, activityTopic, getActivityYAML);
+        worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
+        await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML);
+        return worker;
+    }
+    async run() {
+        if (this.workflowRunner) {
+            this.workflowRunner.engine.logger.info('WorkerService is running');
+        }
+        else {
+            console.log('WorkerService is running');
+        }
+    }
+    async initActivityWorkflow(config, activityTopic) {
+        const redisConfig = {
+            class: config.connection.class,
+            options: config.connection.options
+        };
+        const hmshInstance = await HotMesh.init({
+            appId: activityTopic,
+            engine: { redis: redisConfig },
+            workers: [
+                { topic: activityTopic,
+                    redis: redisConfig,
+                    callback: this.wrapActivityFunctions().bind(this)
+                }
+            ]
+        });
+        WorkerService.instances.set(activityTopic, hmshInstance);
+        return hmshInstance;
+    }
+    wrapActivityFunctions() {
+        return async (data) => {
+            try {
+                //always run the activity function when instructed; return the response
+                const activityInput = data.data;
+                const activityName = activityInput.activityName;
+                const activityFunction = WorkerService.activityRegistry[activityName];
+                const pojoResponse = await activityFunction.apply(this, activityInput.arguments);
+                return {
+                    status: StreamStatus.SUCCESS,
+                    metadata: { ...data.metadata },
+                    data: { response: pojoResponse }
+                };
+            }
+            catch (err) {
+                console.error(err);
+                //todo (make retry configurable)
+                return {
+                    status: StreamStatus.PENDING,
+                    metadata: { ...data.metadata },
+                    data: { error: err }
+                };
+            }
+        };
+    }
+    async activateActivityWorkflow(hotMesh, activityTopic) {
+        const version = '1';
+        const app = await hotMesh.engine.store.getApp(activityTopic);
+        const appVersion = app?.version;
+        if (isNaN(appVersion)) {
+            try {
+                await hotMesh.deploy(getActivityYAML(activityTopic, version));
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                console.log('durable-worker-activity-deploy-activate-error', err);
+                throw err;
+            }
+        }
+        else if (app && !app.active) {
+            try {
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-worker-activity-activate-err', err);
+                throw err;
+            }
+        }
+    }
+    async initWorkerWorkflow(config, workflowTopic, workflowFunction) {
+        const redisConfig = {
+            class: config.connection.class,
+            options: config.connection.options
+        };
+        const hmshInstance = await HotMesh.init({
+            appId: workflowTopic,
+            engine: { redis: redisConfig },
+            workers: [
+                { topic: workflowTopic,
+                    redis: redisConfig,
+                    callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic).bind(this)
+                }
+            ]
+        });
+        WorkerService.instances.set(workflowTopic, hmshInstance);
+        return hmshInstance;
+    }
+    wrapWorkflowFunction(workflowFunction, workflowTopic) {
+        return async (data) => {
+            try {
+                //incoming data payload has arguments and workflowId
+                const workflowInput = data.data;
+                const context = new Map();
+                const counter = { counter: 0 };
+                context.set('counter', counter);
+                context.set('workflowId', workflowInput.workflowId);
+                context.set('workflowTopic', workflowTopic);
+                context.set('workflowName', workflowTopic.split('-').pop());
+                context.set('workflowTrace', data.metadata.trc);
+                context.set('workflowSpan', data.metadata.spn);
+                const workflowResponse = await asyncLocalStorage.run(context, async () => {
+                    return await workflowFunction.apply(this, workflowInput.arguments);
+                });
+                return {
+                    code: 200,
+                    status: StreamStatus.SUCCESS,
+                    metadata: { ...data.metadata },
+                    data: { response: workflowResponse }
+                };
+            }
+            catch (err) {
+                //todo: (retryable error types)
+                return {
+                    code: 500,
+                    status: StreamStatus.PENDING,
+                    metadata: { ...data.metadata },
+                    data: { error: err }
+                };
+            }
+        };
+    }
+    static async shutdown() {
+        for (const [key, value] of WorkerService.instances) {
+            const hotMesh = await value;
+            await hotMesh.stop();
+        }
+    }
+}
+_a = WorkerService;
+WorkerService.activityRegistry = {}; //user's activities
+WorkerService.instances = new Map();
+WorkerService.getHotMesh = async (worflowTopic) => {
+    if (WorkerService.instances.has(worflowTopic)) {
+        return await WorkerService.instances.get(worflowTopic);
+    }
+    const hotMesh = HotMesh.init({
+        appId: worflowTopic,
+        engine: { redis: { ...WorkerService.connection } }
+    });
+    WorkerService.instances.set(worflowTopic, hotMesh);
+    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML);
+    return hotMesh;
+};
+WorkerService.Context = {
+    info: () => {
+        return {
+            workflowId: '',
+            workflowTopic: '',
+        };
+    },
+};
+export { WorkerService };

package/build/esm/services/durable/workflow.d.ts

@@ -0,0 +1,6 @@
+import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable";
+export declare class WorkflowService {
+    static executeChild<T>(options: WorkflowOptions): Promise<T>;
+    static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT>;
+    static wrapActivity<T>(activityName: string, options?: ActivityConfig): T;
+}