@hotmeshio/hotmesh 0.0.4 → 0.0.6

package/build/esm/services/durable/handle.d.ts

@@ -0,0 +1,8 @@
+import { HotMeshService as HotMesh } from '../hotmesh';
+export declare class WorkflowHandleService {
+    hotMesh: HotMesh;
+    workflowTopic: string;
+    workflowId: string;
+    constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string);
+    result(): Promise<any>;
+}

package/build/esm/services/durable/handle.js

@@ -0,0 +1,34 @@
+export class WorkflowHandleService {
+    constructor(hotMesh, workflowTopic, workflowId) {
+        this.workflowTopic = workflowTopic;
+        this.workflowId = workflowId;
+        this.hotMesh = hotMesh;
+    }
+    async result() {
+        let status = await this.hotMesh.getStatus(this.workflowId);
+        const topic = `${this.workflowTopic}.${this.workflowId}`;
+        if (status == 0) {
+            return (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response;
+        }
+        return new Promise((resolve, reject) => {
+            let isResolved = false;
+            //common fulfill/unsubscribe
+            const complete = async (response) => {
+                if (isResolved)
+                    return;
+                isResolved = true;
+                this.hotMesh.unsub(topic);
+                resolve(response || (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response);
+            };
+            this.hotMesh.sub(topic, async (topic, message) => {
+                await complete(message.data?.response);
+            });
+            setTimeout(async () => {
+                status = await this.hotMesh.getStatus(this.workflowId);
+                if (status == 0) {
+                    await complete();
+                }
+            }, 0);
+        });
+    }
+}

package/build/esm/services/durable/index.d.ts

@@ -0,0 +1,57 @@
+import { ClientService } from './client';
+import { ConnectionService } from './connection';
+import { NativeConnectionService } from './native';
+import { WorkerService } from './worker';
+import { WorkflowService } from './workflow';
+import { ContextType } from '../../types/durable';
+/**
+ * As a durable integration platform, HotMesh
+ * can model and emulate other durable systems
+ * (like Temporal). As you review the code in
+ * this file, note the following:
+ *
+ * 1) There is no central governing server.
+ * HotMesh is a client-side SDK that connects to Redis
+ * using CQRS principles to implicitly drive
+ * orchestrations using a headless quorum. Stream
+ * semantics guarantee that all events are
+ * processed by the quorum of connected clients.
+ *
+ * 2) Every developer-defined `workflow` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * If the HotMesh workflow is not yet defined,
+ * it will be deployed and activated on-the-fly.
+ * (The generated DAG will have one Trigger Activity
+ * and one Worker Activity.) The Worker Activity
+ * is configured to catch execution errors and
+ * return them, using a 'pending' status to
+ * indicate that the workflow is still running.
+ * It is possible for workflow activities to throw
+ * errors that will force the entire workflow to
+ * fail. This is not the case here. The workflow will
+ * continue to run until it is completed or cancelled.
+ *
+ * 2) Every developer-defined `activity` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * (The generated DAG will have one Trigger Activity and one
+ * Worker Activity.) The JOB ID for Worker Activity executions
+ * is derived from the containing JOB ID,
+ * allowing Activity state to be 'replayed' when the workflow
+ * is run again. The Activity Function Runner (activity proxy)
+ * is configured similar to how the workflow worker is and will
+ * catch execution errors and return them to the caller, using a
+ * 'pending' status to indicate that the activity is still running.
+ * This allows the activity to be retried until it succeeds.
+ */
+export declare const Durable: {
+    Client: typeof ClientService;
+    Connection: typeof ConnectionService;
+    NativeConnection: typeof NativeConnectionService;
+    Worker: typeof WorkerService;
+    workflow: typeof WorkflowService;
+};
+export type { ContextType };

package/build/esm/services/durable/index.js

@@ -0,0 +1,55 @@
+import { ClientService } from './client';
+import { ConnectionService } from './connection';
+import { NativeConnectionService } from './native';
+import { WorkerService } from './worker';
+import { WorkflowService } from './workflow';
+/**
+ * As a durable integration platform, HotMesh
+ * can model and emulate other durable systems
+ * (like Temporal). As you review the code in
+ * this file, note the following:
+ *
+ * 1) There is no central governing server.
+ * HotMesh is a client-side SDK that connects to Redis
+ * using CQRS principles to implicitly drive
+ * orchestrations using a headless quorum. Stream
+ * semantics guarantee that all events are
+ * processed by the quorum of connected clients.
+ *
+ * 2) Every developer-defined `workflow` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * If the HotMesh workflow is not yet defined,
+ * it will be deployed and activated on-the-fly.
+ * (The generated DAG will have one Trigger Activity
+ * and one Worker Activity.) The Worker Activity
+ * is configured to catch execution errors and
+ * return them, using a 'pending' status to
+ * indicate that the workflow is still running.
+ * It is possible for workflow activities to throw
+ * errors that will force the entire workflow to
+ * fail. This is not the case here. The workflow will
+ * continue to run until it is completed or cancelled.
+ *
+ * 2) Every developer-defined `activity` function
+ * is assigned a HotMesh workflow (which runs in
+ * the background) to support it.
+ *
+ * (The generated DAG will have one Trigger Activity and one
+ * Worker Activity.) The JOB ID for Worker Activity executions
+ * is derived from the containing JOB ID,
+ * allowing Activity state to be 'replayed' when the workflow
+ * is run again. The Activity Function Runner (activity proxy)
+ * is configured similar to how the workflow worker is and will
+ * catch execution errors and return them to the caller, using a
+ * 'pending' status to indicate that the activity is still running.
+ * This allows the activity to be retried until it succeeds.
+ */
+export const Durable = {
+    Client: ClientService,
+    Connection: ConnectionService,
+    NativeConnection: NativeConnectionService,
+    Worker: WorkerService,
+    workflow: WorkflowService,
+};

package/build/esm/services/durable/native.d.ts

@@ -0,0 +1,4 @@
+import { Connection, ConnectionConfig } from "../../types/durable";
+export declare class NativeConnectionService {
+    static connect(config: ConnectionConfig): Promise<Connection>;
+}

package/build/esm/services/durable/native.js

@@ -0,0 +1,43 @@
+/*
+
+Here is an example of how the methods in this file are used:
+
+./worker.ts
+
+import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
+
+import * as activities from './activities';
+
+async function run() {
+  const connection = await NativeConnection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+  const worker = await Worker.create({
+    connection,
+    namespace: 'default',
+    taskQueue: 'hello-world',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+  });
+  await worker.run();
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+
+*/
+export class NativeConnectionService {
+    static async connect(config) {
+        return {
+            class: config.class,
+            options: { ...config.options },
+        };
+    }
+}

package/build/esm/services/durable/worker.d.ts

@@ -0,0 +1,37 @@
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { Connection, Registry, WorkerConfig } from "../../types/durable";
+export declare class WorkerService {
+    static activityRegistry: Registry;
+    static connection: Connection;
+    static instances: Map<string, HotMesh | Promise<HotMesh>>;
+    workflowRunner: HotMesh;
+    static getHotMesh: (worflowTopic: string) => Promise<HotMesh>;
+    static activateWorkflow(hotMesh: HotMesh, topic: string, factory: Function): Promise<void>;
+    /**
+     * The `worker` calls `registerActivities` immediately BEFORE
+     * dynamically importing the user's workflow module. That file
+     * contains a call, `proxyActivities`, which needs this info.
+     *
+     * NOTE: The `worker` and `client` both call `proxyActivities`,
+     * as a natural result of importing worflows.ts. However,
+     * because the worker imports the workflows dynamically AFTER
+     * the activities are loaded, there will be items in the registry,
+     * allowing proxyActivities to succeed.
+     */
+    static registerActivities<ACT>(activities: ACT): Registry;
+    static create(config: WorkerConfig): Promise<WorkerService>;
+    static resolveWorkflowTarget(workflow: object | Function): [string, Function];
+    run(): Promise<void>;
+    initActivityWorkflow(config: WorkerConfig, activityTopic: string): Promise<HotMesh>;
+    wrapActivityFunctions(): Function;
+    activateActivityWorkflow(hotMesh: HotMesh, activityTopic: string): Promise<void>;
+    initWorkerWorkflow(config: WorkerConfig, workflowTopic: string, workflowFunction: Function): Promise<HotMesh>;
+    static Context: {
+        info: () => {
+            workflowId: string;
+            workflowTopic: string;
+        };
+    };
+    wrapWorkflowFunction(workflowFunction: Function, workflowTopic: string): Function;
+    static shutdown(): Promise<void>;
+}

package/build/esm/services/durable/worker.js

@@ -0,0 +1,268 @@
+var _a;
+import { asyncLocalStorage } from './asyncLocalStorage';
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { StreamStatus } from '../../types/stream';
+import { getWorkflowYAML, getActivityYAML } from './factory';
+/*
+Here is an example of how the methods in this file are used:
+
+./worker.ts
+
+import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
+
+import * as activities from './activities';
+
+async function run() {
+  const connection = await NativeConnection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+  const worker = await Worker.create({
+    connection,
+    namespace: 'default',
+    taskQueue: 'hello-world',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+  });
+  await worker.run();
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+*/
+class WorkerService {
+    static async activateWorkflow(hotMesh, topic, factory) {
+        const version = '1';
+        const app = await hotMesh.engine.store.getApp(topic);
+        const appVersion = app?.version;
+        if (!appVersion) {
+            try {
+                await hotMesh.deploy(factory(topic, version));
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-worker-deploy-activate-err', err);
+                throw err;
+            }
+        }
+        else if (app && !app.active) {
+            try {
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-worker-activate-err', err);
+                throw err;
+            }
+        }
+    }
+    /**
+     * The `worker` calls `registerActivities` immediately BEFORE
+     * dynamically importing the user's workflow module. That file
+     * contains a call, `proxyActivities`, which needs this info.
+     *
+     * NOTE: The `worker` and `client` both call `proxyActivities`,
+     * as a natural result of importing worflows.ts. However,
+     * because the worker imports the workflows dynamically AFTER
+     * the activities are loaded, there will be items in the registry,
+     * allowing proxyActivities to succeed.
+     */
+    static registerActivities(activities) {
+        Object.keys(activities).forEach(key => {
+            WorkerService.activityRegistry[key] = activities[key];
+        });
+        return WorkerService.activityRegistry;
+    }
+    static async create(config) {
+        WorkerService.connection = config.connection;
+        //pre-cache user activity functions
+        WorkerService.registerActivities(config.activities);
+        //import the user's workflow file (triggers activity functions to be wrapped)
+        const workflow = await import(config.workflowsPath);
+        const [workflowFunctionName, workflowFunction] = WorkerService.resolveWorkflowTarget(workflow);
+        const baseTopic = `${config.taskQueue}-${workflowFunctionName}`;
+        const activityTopic = `${baseTopic}-activity`;
+        const workflowTopic = `${baseTopic}`;
+        //initialize supporting workflows
+        const worker = new WorkerService();
+        const activityRunner = await worker.initActivityWorkflow(config, activityTopic);
+        await WorkerService.activateWorkflow(activityRunner, activityTopic, getActivityYAML);
+        worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
+        await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML);
+        return worker;
+    }
+    static resolveWorkflowTarget(workflow) {
+        let workflowFunction;
+        if (typeof workflow === 'function') {
+            workflowFunction = workflow;
+        }
+        else {
+            const workflowFunctionNames = Object.keys(workflow);
+            workflowFunction = workflow[workflowFunctionNames[workflowFunctionNames.length - 1]];
+            return WorkerService.resolveWorkflowTarget(workflowFunction);
+        }
+        return [workflowFunction.name, workflowFunction];
+    }
+    async run() {
+        if (this.workflowRunner) {
+            this.workflowRunner.engine.logger.info('WorkerService is running');
+        }
+        else {
+            console.log('WorkerService is running');
+        }
+    }
+    async initActivityWorkflow(config, activityTopic) {
+        const redisConfig = {
+            class: config.connection.class,
+            options: config.connection.options
+        };
+        const hmshInstance = await HotMesh.init({
+            appId: activityTopic,
+            engine: { redis: redisConfig },
+            workers: [
+                { topic: activityTopic,
+                    redis: redisConfig,
+                    callback: this.wrapActivityFunctions().bind(this)
+                }
+            ]
+        });
+        WorkerService.instances.set(activityTopic, hmshInstance);
+        return hmshInstance;
+    }
+    wrapActivityFunctions() {
+        return async (data) => {
+            try {
+                //always run the activity function when instructed; return the response
+                const activityInput = data.data;
+                const activityName = activityInput.activityName;
+                const activityFunction = WorkerService.activityRegistry[activityName];
+                const pojoResponse = await activityFunction.apply(this, activityInput.arguments);
+                return {
+                    status: StreamStatus.SUCCESS,
+                    metadata: { ...data.metadata },
+                    data: { response: pojoResponse }
+                };
+            }
+            catch (err) {
+                console.error(err);
+                //todo (make retry configurable)
+                return {
+                    status: StreamStatus.PENDING,
+                    metadata: { ...data.metadata },
+                    data: { error: err }
+                };
+            }
+        };
+    }
+    async activateActivityWorkflow(hotMesh, activityTopic) {
+        const version = '1';
+        const app = await hotMesh.engine.store.getApp(activityTopic);
+        const appVersion = app?.version;
+        if (isNaN(appVersion)) {
+            try {
+                await hotMesh.deploy(getActivityYAML(activityTopic, version));
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                console.log('durable-worker-activity-deploy-activate-error', err);
+                throw err;
+            }
+        }
+        else if (app && !app.active) {
+            try {
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-worker-activity-activate-err', err);
+                throw err;
+            }
+        }
+    }
+    async initWorkerWorkflow(config, workflowTopic, workflowFunction) {
+        const redisConfig = {
+            class: config.connection.class,
+            options: config.connection.options
+        };
+        const hmshInstance = await HotMesh.init({
+            appId: workflowTopic,
+            engine: { redis: redisConfig },
+            workers: [
+                { topic: workflowTopic,
+                    redis: redisConfig,
+                    callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic).bind(this)
+                }
+            ]
+        });
+        WorkerService.instances.set(workflowTopic, hmshInstance);
+        return hmshInstance;
+    }
+    wrapWorkflowFunction(workflowFunction, workflowTopic) {
+        return async (data) => {
+            try {
+                //incoming data payload has arguments and workflowId
+                const workflowInput = data.data;
+                const context = new Map();
+                const counter = { counter: 0 };
+                context.set('counter', counter);
+                context.set('workflowId', workflowInput.workflowId);
+                context.set('workflowTopic', workflowTopic);
+                context.set('workflowName', workflowTopic.split('-').pop());
+                context.set('workflowTrace', data.metadata.trc);
+                context.set('workflowSpan', data.metadata.spn);
+                const workflowResponse = await asyncLocalStorage.run(context, async () => {
+                    return await workflowFunction.apply(this, workflowInput.arguments);
+                });
+                return {
+                    code: 200,
+                    status: StreamStatus.SUCCESS,
+                    metadata: { ...data.metadata },
+                    data: { response: workflowResponse }
+                };
+            }
+            catch (err) {
+                //todo: (retryable error types)
+                return {
+                    code: 500,
+                    status: StreamStatus.PENDING,
+                    metadata: { ...data.metadata },
+                    data: { error: err }
+                };
+            }
+        };
+    }
+    static async shutdown() {
+        for (const [key, value] of WorkerService.instances) {
+            const hotMesh = await value;
+            await hotMesh.stop();
+        }
+    }
+}
+_a = WorkerService;
+WorkerService.activityRegistry = {}; //user's activities
+WorkerService.instances = new Map();
+WorkerService.getHotMesh = async (worflowTopic) => {
+    if (WorkerService.instances.has(worflowTopic)) {
+        return await WorkerService.instances.get(worflowTopic);
+    }
+    const hotMesh = HotMesh.init({
+        appId: worflowTopic,
+        engine: { redis: { ...WorkerService.connection } }
+    });
+    WorkerService.instances.set(worflowTopic, hotMesh);
+    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML);
+    return hotMesh;
+};
+WorkerService.Context = {
+    info: () => {
+        return {
+            workflowId: '',
+            workflowTopic: '',
+        };
+    },
+};
+export { WorkerService };

package/build/esm/services/durable/workflow.d.ts

@@ -0,0 +1,6 @@
+import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable";
+export declare class WorkflowService {
+    static executeChild<T>(options: WorkflowOptions): Promise<T>;
+    static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT>;
+    static wrapActivity<T>(activityName: string, options?: ActivityConfig): T;
+}

package/build/esm/services/durable/workflow.js

@@ -0,0 +1,128 @@
+import ms from 'ms';
+import { asyncLocalStorage } from './asyncLocalStorage';
+import { WorkerService } from './worker';
+import { ClientService as Client } from './client';
+import { ConnectionService as Connection } from './connection';
+/*
+`proxyActivities` returns a wrapped instance of the
+target activity, so that when the workflow calls a
+proxied activity, it is actually calling the proxy
+function, which in turn calls the activity function.
+
+`proxyActivities` must be called AFTER the activities
+have been registered in order to work properly.
+If the activities are not already registered,
+`proxyActivities` will throw an error. This is OK.
+
+The `client` (client.ts) is equivalent to the
+HotMesh `engine`. The jobs it creates will be
+put in the taskQueue. When the `worker` (worker.ts)
+is eventually initialized (if it happens to be inited later),
+it will see the items in the queue and process them. If it happens
+to already be inited, the jobs will immediately be dequeued and
+processed. In either case, the jobs will be processed.
+
+Here is an example of how the methods in this file are used:
+
+./workflows.ts
+
+import { Durable } from '@hotmeshio/hotmesh';
+import type * as activities from './activities';
+const { greet } = Durable.workflow.proxyActivities<typeof activities>({
+  startToCloseTimeout: '1 minute',
+  retryPolicy: {
+    initialInterval: '5 seconds',  // Initial delay between retries
+    maximumAttempts: 3,            // Max number of retry attempts
+    backoffCoefficient: 2.0,       // Backoff factor for delay between retries: delay = initialInterval * (backoffCoefficient ^ retry_attempt)
+    maximumInterval: '30 seconds', // Max delay between retries
+  },
+});
+
+export async function example(name: string): Promise<string> {
+  return await greet(name);
+}
+*/
+export class WorkflowService {
+    static async executeChild(options) {
+        const store = asyncLocalStorage.getStore();
+        if (!store) {
+            throw new Error('durable-store-not-found');
+        }
+        const workflowId = store.get('workflowId');
+        const workflowTrace = store.get('workflowTrace');
+        const workflowSpan = store.get('workflowSpan');
+        const client = new Client({
+            connection: await Connection.connect(WorkerService.connection),
+        });
+        //todo: should I allow-cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
+        const handle = await client.workflow.start({
+            ...options,
+            workflowId: `${workflowId}${options.workflowId}`,
+            workflowTrace,
+            workflowSpan,
+        });
+        const result = await handle.result();
+        return result;
+    }
+    static proxyActivities(options) {
+        const proxy = {};
+        const keys = Object.keys(WorkerService.activityRegistry);
+        if (keys.length) {
+            keys.forEach((key) => {
+                const activityFunction = WorkerService.activityRegistry[key];
+                proxy[key] = WorkflowService.wrapActivity(key, options);
+            });
+        }
+        return proxy;
+    }
+    static wrapActivity(activityName, options) {
+        return async function () {
+            const store = asyncLocalStorage.getStore();
+            if (!store) {
+                throw new Error('durable-store-not-found');
+            }
+            const COUNTER = store.get('counter');
+            //increment by state (not value) to avoid race conditions
+            const execIndex = COUNTER.counter = COUNTER.counter + 1;
+            const workflowId = store.get('workflowId');
+            const workflowTopic = store.get('workflowTopic');
+            const trc = store.get('workflowTrace');
+            const spn = store.get('workflowSpan');
+            const activityTopic = `${workflowTopic}-activity`;
+            const activityJobId = `${workflowId}-${activityName}-${execIndex}`;
+            let activityState;
+            try {
+                const hmshInstance = await WorkerService.getHotMesh(activityTopic);
+                activityState = await hmshInstance.getState(activityTopic, activityJobId);
+                if (activityState.metadata.js == 1) {
+                    //return immediately
+                    return activityState.data?.response;
+                }
+                //one time subscription
+                return await new Promise((resolve, reject) => {
+                    hmshInstance.sub(activityTopic, async (topic, message) => {
+                        const response = message.data?.response;
+                        hmshInstance.unsub(activityTopic);
+                        // Resolve the Promise when the callback is triggered with a message
+                        resolve(response);
+                    });
+                });
+            }
+            catch (e) {
+                //expected; thrown by `getState` when the job cannot be found
+                const duration = ms(options?.startToCloseTimeout || '1 minute');
+                const payload = {
+                    arguments: Array.from(arguments),
+                    workflowId: activityJobId,
+                    workflowTopic,
+                    activityName,
+                };
+                //start the job
+                const hmshInstance = await WorkerService.getHotMesh(activityTopic);
+                const context = { metadata: { trc, spn }, data: {} };
+                const jobOutput = await hmshInstance.pubsub(activityTopic, payload, context, duration);
+                return jobOutput.data.response;
+            }
+        };
+    }
+}