@hotmeshio/hotmesh 0.0.1

package/services/durable/native.ts

@@ -0,0 +1,46 @@
+import { Connection, ConnectionConfig } from "../../types/durable";
+
+/*
+
+Here is an example of how the methods in this file are used:
+
+./worker.ts
+
+import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
+
+import * as activities from './activities';
+
+async function run() {
+  const connection = await NativeConnection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+  const worker = await Worker.create({
+    connection,
+    namespace: 'default',
+    taskQueue: 'hello-world',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+  });
+  await worker.run();
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+
+*/
+
+export class NativeConnectionService {
+  static async connect(config: ConnectionConfig): Promise<Connection> {
+    return {
+      class: config.class,
+      options: { ...config.options },
+     } as Connection;
+  }
+}

package/services/durable/worker.ts

@@ -0,0 +1,254 @@
+import { asyncLocalStorage } from './asyncLocalStorage';
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { RedisClass, RedisOptions } from '../../types/redis';
+import { StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
+import { ActivityDataType, Connection, Registry, WorkerConfig, WorkflowDataType } from "../../types/durable";
+import { getWorkflowYAML, getActivityYAML } from './factory';
+
+/*
+Here is an example of how the methods in this file are used:
+
+./worker.ts
+
+import { Durable: { NativeConnection, Worker } } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis'; //OR `import * as Redis from 'redis';`
+
+import * as activities from './activities';
+
+async function run() {
+  const connection = await NativeConnection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+  const worker = await Worker.create({
+    connection,
+    namespace: 'default',
+    taskQueue: 'hello-world',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+  });
+  await worker.run();
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+*/
+
+export class WorkerService {
+  static activityRegistry: Registry = {}; //user's activities
+  static connection: Connection;
+  static instances = new Map<string, HotMesh | Promise<HotMesh>>();
+  workflowRunner: HotMesh;
+
+  static getHotMesh = async (worflowTopic: string) => {
+    if (WorkerService.instances.has(worflowTopic)) {
+      return await WorkerService.instances.get(worflowTopic);
+    }
+    const hotMesh = HotMesh.init({
+      appId: worflowTopic,
+      engine: { redis: { ...WorkerService.connection } }
+    });
+    WorkerService.instances.set(worflowTopic, hotMesh);
+    await WorkerService.activateWorkflow(await hotMesh, worflowTopic, getWorkflowYAML);
+    return hotMesh;
+  }
+
+  static async activateWorkflow(hotMesh: HotMesh, topic: string, factory: Function) {
+    const version = '1';
+    const app = await hotMesh.engine.store.getApp(topic);
+    const appVersion = app?.version;
+    if(!appVersion) {
+      try {
+        await hotMesh.deploy(factory(topic, version));
+        await hotMesh.activate(version);
+      } catch (err) {
+        hotMesh.engine.logger.error('durable-worker-workflow-activation-error', err);
+        throw err;
+      }
+    }
+  }
+
+  /**
+   * The `worker` calls `registerActivities` immediately BEFORE
+   * dynamically importing the user's workflow module. That file
+   * contains a call, `proxyActivities`, which needs this info.
+   * 
+   * NOTE: The `worker` and `client` both call `proxyActivities`,
+   * as a natural result of importing worflows.ts. However,
+   * because the worker imports the workflows dynamically AFTER
+   * the activities are loaded, there will be items in the registry,
+   * allowing proxyActivities to succeed.
+   */
+  static registerActivities<ACT>(activities: ACT): Registry  {
+    Object.keys(activities).forEach(key => {
+      WorkerService.activityRegistry[key] = (activities as any)[key];
+    });
+    return WorkerService.activityRegistry;
+  }
+
+  static async create(config: WorkerConfig) {
+    WorkerService.connection = config.connection;
+    //pre-cache user activity functions
+    WorkerService.registerActivities<typeof config.activities>(config.activities);
+    //import the user's workflow file (triggers activity functions to be wrapped)
+    const workflow = await import(config.workflowsPath);
+    const workflowFunctionNames = Object.keys(workflow);
+    const workflowFunctionName = workflowFunctionNames[workflowFunctionNames.length - 1];
+    const workflowFunction = workflow[workflowFunctionName];
+    const baseTopic = `${config.taskQueue}-${workflowFunctionName}`;
+    const activityTopic = `${baseTopic}-activity`;
+    const workflowTopic = `${baseTopic}`;
+
+    //initialize supporting workflows
+    const worker = new WorkerService();
+
+    const activityRunner = await worker.initActivityWorkflow(config, activityTopic);
+    await WorkerService.activateWorkflow(activityRunner, activityTopic, getActivityYAML);
+    worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
+    await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, getWorkflowYAML);
+    return worker;
+  }
+
+  async run() {
+    if (this.workflowRunner) {
+      this.workflowRunner.engine.logger.info('WorkerService is running');
+    } else {
+      console.log('WorkerService is running');
+    }
+  }
+
+  async initActivityWorkflow(config: WorkerConfig, activityTopic: string): Promise<HotMesh> {
+    const redisConfig = {
+      class: config.connection.class as RedisClass,
+      options: config.connection.options as RedisOptions
+    };
+    const hmshInstance = await HotMesh.init({
+      appId: activityTopic,
+      engine: { redis: redisConfig },
+      workers: [
+        { topic: activityTopic,
+          redis: redisConfig,
+          callback: this.wrapActivityFunctions().bind(this)
+        }
+      ]
+    });
+    WorkerService.instances.set(activityTopic, hmshInstance);
+    return hmshInstance;
+  }
+
+  wrapActivityFunctions(): Function {
+    return async (data: StreamData): Promise<StreamDataResponse> => {
+      try {
+        //always run the activity function when instructed; return the response
+        const activityInput = data.data as unknown as ActivityDataType;
+        const activityName = activityInput.activityName;
+        const activityFunction = WorkerService.activityRegistry[activityName];
+        const pojoResponse = await activityFunction.apply(this, activityInput.arguments);
+
+        return {
+          status: StreamStatus.SUCCESS,
+          metadata: { ...data.metadata },
+          data: { response: pojoResponse }
+        };
+      } catch (err) {
+        console.error(err);
+        //todo (make retry configurable)
+        return {
+          status: StreamStatus.PENDING,
+          metadata: { ...data.metadata },
+          data: { error: err }
+        } as StreamDataResponse;
+      }
+    }
+  }
+
+  async activateActivityWorkflow(hotMesh: HotMesh, activityTopic: string) {
+    const version = '1';
+    const app = await hotMesh.engine.store.getApp(activityTopic);
+    const appVersion = app?.version as unknown as number;
+    if(isNaN(appVersion)) {
+      try {
+        await hotMesh.deploy(getActivityYAML(activityTopic, version));
+        await hotMesh.activate(version);
+      } catch (err) {
+        console.log('durable-worker-activity-workflow-activation-error', err);
+        throw err;
+      }
+    }
+  }
+
+  async initWorkerWorkflow(config: WorkerConfig, workflowTopic: string, workflowFunction: Function): Promise<HotMesh> {
+    const redisConfig = {
+      class: config.connection.class as RedisClass,
+      options: config.connection.options as RedisOptions
+    };
+    const hmshInstance = await HotMesh.init({
+      appId: workflowTopic,
+      engine: { redis: redisConfig },
+      workers: [
+        { topic: workflowTopic,
+          redis: redisConfig,
+          callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic).bind(this)
+        }
+      ]
+    });
+    WorkerService.instances.set(workflowTopic, hmshInstance);
+    return hmshInstance;
+  }
+
+  static Context = {
+    info: () => {
+      return {
+        workflowId: '',
+        workflowTopic: '',
+      }
+    },
+  };
+
+  wrapWorkflowFunction(workflowFunction: Function, workflowTopic: string): Function {
+    return async (data: StreamData): Promise<StreamDataResponse> => {
+      try {
+        //incoming data payload has arguments and workflowId
+        const workflowInput = data.data as unknown as WorkflowDataType;
+        const context = new Map();
+        const counter = { counter: 0 };
+        context.set('counter', counter);
+        context.set('workflowId', workflowInput.workflowId);
+        context.set('workflowTopic', workflowTopic);
+        context.set('workflowName', workflowTopic.split('-').pop());
+        context.set('workflowTrace', data.metadata.trc);
+        context.set('workflowSpan', data.metadata.spn);
+        const workflowResponse = await asyncLocalStorage.run(context, async () => {
+          return await workflowFunction.apply(this, workflowInput.arguments);
+        });
+
+        return {
+          code: 200,
+          status: StreamStatus.SUCCESS,
+          metadata: { ...data.metadata },
+          data: { response: workflowResponse }
+        };
+      } catch (err) {
+        //todo: (retryable error types)
+        return {
+          code: 500,
+          status: StreamStatus.PENDING,
+          metadata: { ...data.metadata },
+          data: { error: err }
+        } as StreamDataResponse;
+      }
+    }
+  }
+
+  static async shutdown(): Promise<void> {
+    for (const [key, value] of WorkerService.instances) {
+      const hotMesh = await value;
+      await hotMesh.stop();
+    }
+  }
+}

package/services/durable/workflow.ts

@@ -0,0 +1,136 @@
+import ms from 'ms';
+
+import { asyncLocalStorage } from './asyncLocalStorage';
+import { WorkerService } from './worker';
+import { ClientService as Client } from './client';
+import { ConnectionService as Connection } from './connection';
+import { ActivityConfig, ProxyType, WorkflowOptions } from "../../types/durable";
+import { JobOutput, JobState } from '../../types';
+
+/*
+`proxyActivities` returns a wrapped instance of the 
+target activity, so that when the workflow calls a
+proxied activity, it is actually calling the proxy
+function, which in turn calls the activity function.
+
+`proxyActivities` must be called AFTER the activities
+have been registered in order to work properly.
+If the activities are not already registered,
+`proxyActivities` will throw an error. This is OK.
+
+The `client` (client.ts) is equivalent to the 
+HotMesh `engine`. The jobs it creates will be
+put in the taskQueue. When the `worker` (worker.ts)
+is eventually initialized (if it happens to be inited later),
+it will see the items in the queue and process them. If it happens
+to already be inited, the jobs will immediately be dequeued and
+processed. In either case, the jobs will be processed.
+
+Here is an example of how the methods in this file are used:
+
+./workflows.ts
+
+import { Durable } from '@hotmeshio/hotmesh';
+import type * as activities from './activities';
+const { greet } = Durable.workflow.proxyActivities<typeof activities>({
+  startToCloseTimeout: '1 minute',
+  retryPolicy: {
+    initialInterval: '5 seconds',  // Initial delay between retries
+    maximumAttempts: 3,            // Max number of retry attempts
+    backoffCoefficient: 2.0,       // Backoff factor for delay between retries: delay = initialInterval * (backoffCoefficient ^ retry_attempt)
+    maximumInterval: '30 seconds', // Max delay between retries
+  },
+});
+
+export async function example(name: string): Promise<string> {
+  return await greet(name);
+}
+*/
+
+export class WorkflowService {
+  static async executeChild<T>(options: WorkflowOptions): Promise<T> {
+    const store = asyncLocalStorage.getStore();
+    if (!store) {
+      throw new Error('durable-store-not-found');
+    }
+    const workflowId = store.get('workflowId');
+    const workflowTrace = store.get('workflowTrace');
+    const workflowSpan = store.get('workflowSpan');
+
+    const client = new Client({
+      connection: await Connection.connect(WorkerService.connection),
+    });
+    //todo: should I allow-cross/app callback (pj:'@DURABLE@hello-world@<pjid>'/pa: <paid>/pd: <pdad>)
+    const handle = await client.workflow.start({
+      ...options,
+      workflowId: `${workflowId}${options.workflowId}`, //concat
+      workflowTrace,
+      workflowSpan,
+    });
+    const result = await handle.result();
+    return result as T;
+  }
+
+  static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT> {
+    const proxy: any = {};
+    const keys = Object.keys(WorkerService.activityRegistry);
+    if (keys.length) {
+      keys.forEach((key: string) => {
+        const activityFunction = WorkerService.activityRegistry[key];
+        proxy[key] = WorkflowService.wrapActivity<typeof activityFunction>(key, options);
+      });
+    }
+    return proxy;
+  }
+
+  static wrapActivity<T>(activityName: string, options?: ActivityConfig): T {
+    return async function() {
+      const store = asyncLocalStorage.getStore();
+      if (!store) {
+        throw new Error('durable-store-not-found');
+      }
+      const COUNTER = store.get('counter');
+      //increment by state (not value) to avoid race conditions
+      const execIndex = COUNTER.counter = COUNTER.counter + 1;
+      const workflowId = store.get('workflowId');
+      const workflowTopic = store.get('workflowTopic');
+      const trc = store.get('workflowTrace');
+      const spn = store.get('workflowSpan');
+      const activityTopic = `${workflowTopic}-activity`;
+      const activityJobId = `${workflowId}-${activityName}-${execIndex}`;
+
+      let activityState: JobOutput
+      try {
+        const hmshInstance = await WorkerService.getHotMesh(activityTopic);
+        activityState = await hmshInstance.getState(activityTopic, activityJobId);
+        if (activityState.metadata.js == 1) {
+          //return immediately
+          return activityState.data?.response as T;
+        }
+        //one time subscription
+        return await new Promise((resolve, reject) => {
+          hmshInstance.sub(activityTopic, async (topic, message) => {
+            const response = message.data?.response;
+            hmshInstance.unsub(activityTopic);
+            // Resolve the Promise when the callback is triggered with a message
+            resolve(response);
+          });
+        });
+      } catch (e) {
+        //expected; thrown by `getState` when the job cannot be found
+        const duration = ms(options?.startToCloseTimeout || '1 minute');
+        const payload = {
+          arguments: Array.from(arguments),
+          workflowId: activityJobId,
+          workflowTopic,
+          activityName,
+        };
+        //start the job
+        const hmshInstance = await WorkerService.getHotMesh(activityTopic);
+        const context = { metadata: { trc, spn }, data: {}};
+        const jobOutput = await hmshInstance.pubsub(activityTopic, payload, context as JobState, duration);
+        return jobOutput.data.response as T;
+      }
+    } as T;
+  }
+}