@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/meshcall/index.js

@@ -0,0 +1,452 @@
+"use strict";
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MeshCall = void 0;
+const hotmesh_1 = require("../hotmesh");
+const enums_1 = require("../../modules/enums");
+const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
+const cron_1 = require("../pipe/functions/cron");
+const factory_1 = require("./schemas/factory");
+/**
+ * MeshCall connects any function as an idempotent endpoint.
+ * Call functions from anywhere on the network connected to the
+ * target backend (Postgres, Redis/ValKey, NATS, etc). Function
+ * responses are cacheable and invocations can be scheduled to
+ * run as idempotent cron jobs (this one runs nightly at midnight
+ * and uses Postgres as the backend provider).
+ *
+ * @example
+ * ```typescript
+ * import { Client as Postgres } from 'pg';
+ * import { MeshCall } from '@hotmesh/meshcall';
+ *
+ * MeshCall.cron({
+ *   topic: 'my.cron.function',
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   callback: async () => {
+ *     //your code here...anything goes
+ *   },
+ *   options: { id: 'myDailyCron123', interval: '0 0 * * *' }
+ * });
+ * ```
+ */
+class MeshCall {
+    /**
+     * @private
+     */
+    constructor() { }
+    /**
+     * iterates cached worker/engine instances to locate the first match
+     * with the provided namespace and connection options
+     * @private
+     */
+    static async findFirstMatching(targets, namespace = key_1.HMNS, config, options = {}) {
+        for (const [id, hotMeshInstance] of targets) {
+            const hotMesh = await hotMeshInstance;
+            const appId = hotMesh.engine.appId;
+            if (appId === namespace) {
+                if (id.startsWith(MeshCall.hashOptions(config))) {
+                    if (Boolean(options.readonly) == Boolean(hotMesh.engine.router.readonly)) {
+                        return hotMeshInstance;
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * @private
+     */
+    static async verifyWorkflowActive(hotMesh, appId = key_1.HMNS, count = 0) {
+        const app = await hotMesh.engine.store.getApp(appId);
+        const appVersion = app?.version;
+        if (isNaN(appVersion)) {
+            if (count > 10) {
+                throw new Error('Workflow failed to activate');
+            }
+            await (0, utils_1.sleepFor)(enums_1.HMSH_QUORUM_DELAY_MS * 2);
+            return await MeshCall.verifyWorkflowActive(hotMesh, appId, count + 1);
+        }
+        return true;
+    }
+    /**
+     * @private
+     */
+    static async activateWorkflow(hotMesh, appId = key_1.HMNS, version = factory_1.VERSION) {
+        const app = await hotMesh.engine.store.getApp(appId);
+        const appVersion = app?.version;
+        if (appVersion === version && !app.active) {
+            try {
+                await hotMesh.activate(version);
+            }
+            catch (error) {
+                hotMesh.engine.logger.error('meshcall-client-activate-err', {
+                    error,
+                });
+                throw error;
+            }
+        }
+        else if (isNaN(Number(appVersion)) || appVersion < version) {
+            try {
+                await hotMesh.deploy((0, factory_1.getWorkflowYAML)(appId));
+                await hotMesh.activate(version);
+            }
+            catch (error) {
+                hotMesh.engine.logger.error('meshcall-client-deploy-activate-err', {
+                    error,
+                });
+                throw error;
+            }
+        }
+    }
+    /**
+     * Returns a cached worker instance or creates a new one
+     * @private
+     */
+    static async getInstance(namespace, providerConfig, options = {}) {
+        let hotMeshInstance;
+        if (!options.readonly) {
+            hotMeshInstance = await MeshCall.findFirstMatching(MeshCall.workers, namespace, providerConfig, options);
+        }
+        if (!hotMeshInstance) {
+            hotMeshInstance = await MeshCall.findFirstMatching(MeshCall.engines, namespace, providerConfig, options);
+            if (!hotMeshInstance) {
+                hotMeshInstance = (await MeshCall.getHotMeshClient(namespace, providerConfig, options));
+            }
+        }
+        return hotMeshInstance;
+    }
+    /**
+     * connection re-use is important when making repeated calls, but
+     * only if the connection options are an exact match. this method
+     * hashes the connection options to ensure that the same connection
+     */
+    static hashOptions(connection) {
+        if ('options' in connection) {
+            //shorthand format
+            return (0, utils_1.hashOptions)(connection.options);
+        }
+        else {
+            //longhand format (sub, store, stream, pub, search)
+            const response = [];
+            for (const p in connection) {
+                if (connection[p].options) {
+                    response.push((0, utils_1.hashOptions)(connection[p].options));
+                }
+            }
+            return response.join('');
+        }
+    }
+    /**
+     * Connects and links a worker function to the mesh
+     * @example
+     * ```typescript
+     * import { Client as Postgres } from 'pg';
+     * import { MeshCall } from '@hotmesh/meshcall';
+     *
+     * MeshCall.connect({
+     *   topic: 'my.function',
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   callback: async (arg1: any) => {
+     *     //your code here...
+     *   }
+     * });
+     * ```
+     */
+    static async connect(params) {
+        const targetNamespace = params.namespace ?? key_1.HMNS;
+        const optionsHash = MeshCall.hashOptions(utils_1.polyfill.providerConfig(params));
+        const targetTopic = `${optionsHash}.${targetNamespace}.${params.topic}`;
+        const connection = utils_1.polyfill.providerConfig(params);
+        const hotMeshWorker = await hotmesh_1.HotMesh.init({
+            guid: params.guid,
+            logLevel: params.logLevel ?? enums_1.HMSH_LOGLEVEL,
+            appId: params.namespace ?? key_1.HMNS,
+            engine: { connection },
+            workers: [
+                {
+                    topic: params.topic,
+                    connection,
+                    callback: async function (input) {
+                        const response = await params.callback.apply(this, input.data.args);
+                        return {
+                            metadata: { ...input.metadata },
+                            data: { response },
+                        };
+                    },
+                },
+            ],
+        });
+        MeshCall.workers.set(targetTopic, hotMeshWorker);
+        await MeshCall.activateWorkflow(hotMeshWorker, targetNamespace);
+        return hotMeshWorker;
+    }
+    /**
+     * Calls a function and returns the response.
+     *
+     * @template U - the return type of the linked worker function
+     *
+     * @example
+     * ```typescript
+     * const response = await MeshCall.exec({
+     *   topic: 'my.function',
+     *   args: [{ my: 'args' }],
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   }
+     * });
+     * ```
+     */
+    static async exec(params) {
+        const TOPIC = `${params.namespace ?? key_1.HMNS}.call`;
+        const hotMeshInstance = await MeshCall.getInstance(params.namespace, utils_1.polyfill.providerConfig(params));
+        let id = params.options?.id;
+        if (id) {
+            if (params.options?.flush) {
+                await hotMeshInstance.scrub(id);
+            }
+            else if (params.options?.ttl) {
+                //check cache
+                try {
+                    const cached = await hotMeshInstance.getState(TOPIC, id);
+                    if (cached) {
+                        //todo: check if present; await if not (subscribe)
+                        return cached.data.response;
+                    }
+                }
+                catch (error) {
+                    //just swallow error; it means the cache is empty (no doc by that id)
+                }
+            }
+        }
+        else {
+            id = hotmesh_1.HotMesh.guid();
+        }
+        let expire = 1;
+        if (params.options?.ttl) {
+            expire = (0, utils_1.s)(params.options.ttl);
+        }
+        const jobOutput = await hotMeshInstance.pubsub(TOPIC, { id, expire, topic: params.topic, args: params.args }, null, 30000);
+        return jobOutput?.data?.response;
+    }
+    /**
+     * Clears a cached function response.
+     *
+     * @example
+     * ```typescript
+     * import { Client as Postgres } from 'pg';
+     * import { MeshCall } from '@hotmesh/meshcall';
+     *
+     * MeshCall.flush({
+     *   topic: 'my.function',
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   options: { id: 'myCachedExecFunctionId' }
+     * });
+     * ```
+     */
+    static async flush(params) {
+        const hotMeshInstance = await MeshCall.getInstance(params.namespace, utils_1.polyfill.providerConfig(params));
+        await hotMeshInstance.scrub(params.id ?? params?.options?.id);
+    }
+    /**
+     * Schedules a cron job to run at a specified interval
+     * with optional args. Provided arguments are passed to the
+     * callback function each time the cron job runs. The `id`
+     * option is used to uniquely identify the cron job, allowing
+     * it to be interrupted at any time.
+     *
+     * @example
+     * ```typescript
+     * import { Client as Postgres } from 'pg';
+     * import { MeshCall } from '@hotmesh/meshcall';
+     *
+     * MeshCall.cron({
+     *   topic: 'my.cron.function',
+     *   args: ['arg1', 'arg2'], //optionally pass args
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   callback: async (arg1: any, arg2: any) => {
+     *     //your code here...
+     *   },
+     *   options: { id: 'myDailyCron123', interval: '0 0 * * *' }
+     * });
+     * ```
+     */
+    static async cron(params) {
+        let hotMeshInstance;
+        let readonly = true;
+        if (params.callback) {
+            //always connect cron worker if provided
+            hotMeshInstance = await MeshCall.connect({
+                logLevel: params.logLevel,
+                guid: params.guid,
+                topic: params.topic,
+                connection: utils_1.polyfill.providerConfig(params),
+                callback: params.callback,
+                namespace: params.namespace,
+            });
+            readonly = false;
+        }
+        else {
+            //this is a readonly cron connection which means
+            //it is only being created to connect as a readonly member
+            //of the mesh network that the cron is running on. it
+            //can start a job, but it cannot run the job itself in RO mode.
+        }
+        //configure job inputs
+        const TOPIC = `${params.namespace ?? key_1.HMNS}.cron`;
+        const maxCycles = params.options.maxCycles ?? 100000;
+        let interval = enums_1.HMSH_FIDELITY_SECONDS;
+        let delay;
+        let cron;
+        if ((0, utils_1.isValidCron)(params.options.interval)) {
+            //cron syntax
+            cron = params.options.interval;
+            const nextDelay = new cron_1.CronHandler().nextDelay(cron);
+            delay = nextDelay > 0 ? nextDelay : undefined;
+        }
+        else {
+            const seconds = (0, utils_1.s)(params.options.interval);
+            interval = Math.max(seconds, enums_1.HMSH_FIDELITY_SECONDS);
+            delay = params.options.delay ? (0, utils_1.s)(params.options.delay) : undefined;
+        }
+        try {
+            if (!hotMeshInstance) {
+                //get or create a read-only engine instance to start the cron
+                hotMeshInstance = await MeshCall.getInstance(params.namespace, utils_1.polyfill.providerConfig(params), { readonly, guid: params.guid });
+                await MeshCall.createStream(hotMeshInstance, params.topic, params.namespace);
+            }
+            //spawn the job (ok if it's a duplicate)
+            await hotMeshInstance.pub(TOPIC, {
+                id: params.options.id,
+                topic: params.topic,
+                args: params.args,
+                interval,
+                cron,
+                maxCycles,
+                delay,
+            });
+            return true;
+        }
+        catch (error) {
+            if (error.message.includes(`Duplicate job: ${params.options.id}`)) {
+                return false;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Interrupts a running cron job. Returns `true` if the job
+     * was successfully interrupted, or `false` if the job was not
+     * found.
+     *
+     * @example
+     * ```typescript
+     * import { Client as Postgres } from 'pg';
+     * import { MeshCall } from '@hotmesh/meshcall';
+     *
+     * MeshCall.interrupt({
+     *   topic: 'my.cron.function',
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   options: { id: 'myDailyCron123' }
+     * });
+     * ```
+     */
+    static async interrupt(params) {
+        const hotMeshInstance = await MeshCall.getInstance(params.namespace, utils_1.polyfill.providerConfig(params));
+        try {
+            await hotMeshInstance.interrupt(`${params.namespace ?? key_1.HMNS}.cron`, params.options.id, { throw: false, expire: 1 });
+        }
+        catch (error) {
+            //job doesn't exist; is already stopped
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Shuts down all meshcall instances. Call this method
+     * from the SIGTERM handler in your application.
+     */
+    static async shutdown() {
+        for (const [_, hotMeshInstance] of MeshCall.workers) {
+            (await hotMeshInstance).stop();
+        }
+        for (const [_, hotMeshInstance] of MeshCall.engines) {
+            (await hotMeshInstance).stop();
+        }
+        await hotmesh_1.HotMesh.stop();
+    }
+}
+exports.MeshCall = MeshCall;
+_a = MeshCall;
+/**
+ * @private
+ */
+MeshCall.workers = new Map();
+/**
+ * @private
+ */
+MeshCall.engines = new Map();
+/**
+ * @private
+ */
+MeshCall.connections = new Map();
+/**
+ * @private
+ */
+MeshCall.getHotMeshClient = async (namespace, connection, options = {}) => {
+    //namespace isolation requires the connection options to be hashed
+    //as multiple intersecting databases can be used by the same service
+    const optionsHash = MeshCall.hashOptions(connection);
+    const targetNS = namespace ?? key_1.HMNS;
+    const connectionNS = `${optionsHash}.${targetNS}`;
+    if (MeshCall.engines.has(connectionNS)) {
+        const hotMeshClient = await MeshCall.engines.get(connectionNS);
+        await _a.verifyWorkflowActive(hotMeshClient, targetNS);
+        return hotMeshClient;
+    }
+    //create and cache an instance
+    const hotMeshClient = hotmesh_1.HotMesh.init({
+        guid: options.guid,
+        appId: targetNS,
+        logLevel: enums_1.HMSH_LOGLEVEL,
+        engine: {
+            connection,
+            readonly: options.readonly,
+        },
+    });
+    MeshCall.engines.set(connectionNS, hotMeshClient);
+    await _a.activateWorkflow(await hotMeshClient, targetNS);
+    return hotMeshClient;
+};
+/**
+ * Creates a stream where messages can be published to ensure there is a
+ * channel in place when the message arrives (a race condition for those
+ * platforms without implicit topic setup).
+ * @private
+ */
+MeshCall.createStream = async (hotMeshClient, workflowTopic, namespace) => {
+    const params = { appId: namespace ?? key_1.HMNS, topic: workflowTopic };
+    const streamKey = hotMeshClient.engine.store.mintKey(key_1.KeyType.STREAMS, params);
+    try {
+        await hotMeshClient.engine.stream.createConsumerGroup(streamKey, 'WORKER');
+    }
+    catch (err) {
+        //ignore if already exists
+    }
+};

package/build/services/meshcall/schemas/factory.d.ts

@@ -0,0 +1,9 @@
+export declare const VERSION = "2";
+/**
+ * Provides the YAML necessary to create a workflow for the `MeshCall` service.
+ * The `appId` parameter is optional and defaults to the HMNS constant ('hmsh').
+ *
+ * The version is a string as it supports semantic versioning. It is also optional
+ * and defaults to '1'.
+ */
+export declare const getWorkflowYAML: (appId?: string, version?: string) => string;

package/build/services/meshcall/schemas/factory.js

@@ -0,0 +1,189 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getWorkflowYAML = exports.VERSION = void 0;
+const key_1 = require("../../../modules/key");
+exports.VERSION = '2';
+/**
+ * Provides the YAML necessary to create a workflow for the `MeshCall` service.
+ * The `appId` parameter is optional and defaults to the HMNS constant ('hmsh').
+ *
+ * The version is a string as it supports semantic versioning. It is also optional
+ * and defaults to '1'.
+ */
+const getWorkflowYAML = (appId = key_1.HMNS, version = exports.VERSION) => {
+    return `app:
+  id: ${appId}
+  version: '${version}'
+  graphs:
+    - subscribes: ${appId}.call
+
+      expire: '{trigger.output.data.expire}'
+
+      input:
+        schema:
+          type: object
+          properties:
+            id:
+              type: string
+              description: unique id for this workflow
+            expire:
+              type: number
+              description: time in seconds to expire the completed job and remove it from the cache
+            topic:
+              type: string
+              description: topic assigned to locate the worker
+            args:
+              type: array
+              description: arguments to pass to the worker function
+              items:
+                type: any
+
+      output:
+        schema:
+          type: object
+          properties:
+            response:
+              type: any
+
+      activities:
+        trigger:
+          type: trigger
+
+          stats:
+            id: '{$self.input.data.id}'
+
+        worker:
+          type: worker
+          topic: '{trigger.output.data.topic}'
+          input:
+            schema:
+              type: object
+              properties:
+                args:
+                  type: any
+            maps:
+              args: '{trigger.output.data.args}'
+          output:
+            schema:
+              type: object
+              properties:
+                response:
+                  type: any
+          job:
+            maps:
+              response: '{$self.output.data.response}'
+
+      transitions:
+        trigger:
+          - to: worker
+
+    - subscribes: ${appId}.cron
+
+      input:
+        schema:
+          type: object
+          properties:
+            id:
+              type: string
+              description: unique id for this workflow
+            delay:
+              type: number
+              description: time in seconds to sleep before invoking the first cycle
+            interval:
+              type: number
+              description: time in seconds to sleep before the next cycle (also min interval in seconds if cron is provided)
+            cron:
+              type: string
+              description: cron expression to determine the next cycle (takes precedence over interval)
+            topic:
+              type: string
+              description: topic assigned to locate the worker
+            args:
+              type: array
+              description: arguments to pass to the worker function
+              items:
+                type: any
+            maxCycles:
+              type: number
+              description: maximum number of cycles to run before stopping
+
+      output:
+        schema:
+          type: object
+          properties:
+            response:
+              type: any
+
+      activities:
+        trigger_cron:
+          type: trigger
+          stats:
+            id: '{$self.input.data.id}'
+        
+        cycle_hook_cron:
+          type: hook
+          output:
+            schema:
+              type: object
+              properties:
+                sleepSeconds:
+                  type: number
+                iterationCount:
+                  type: number
+            maps:
+              sleepSeconds:
+                '@pipe':
+                  - ['{trigger_cron.output.data.delay}', '{@symbol.undefined}']
+                  - ['{@conditional.nullish}']
+              iterationCount: 1
+
+        sleep_cron:
+          type: hook
+          sleep: '{cycle_hook_cron.output.data.sleepSeconds}'
+        
+        worker_cron:
+          type: worker
+          topic: '{trigger_cron.output.data.topic}'
+          input:
+            schema:
+              type: object
+              properties:
+                args:
+                  type: any
+            maps:
+              args: '{trigger_cron.output.data.args}'
+
+        cycle_cron:
+          type: cycle
+          ancestor: cycle_hook_cron
+          input:
+            maps:
+              sleepSeconds:
+                '@pipe':
+                  - ['{trigger_cron.output.data.cron}']
+                  - ['{@cron.nextDelay}', '{trigger_cron.output.data.interval}']
+                  - ['{@math.max}']
+              iterationCount:
+                '@pipe':
+                  - ['{cycle_hook_cron.output.data.iterationCount}', 1]
+                  - ['{@math.add}']
+
+      transitions:
+        trigger_cron:
+          - to: cycle_hook_cron
+        cycle_hook_cron:
+          - to: sleep_cron
+        sleep_cron:
+          - to: worker_cron
+        worker_cron:
+          - to: cycle_cron
+            conditions:
+              match:
+                - expected: true
+                  actual: 
+                    '@pipe':
+                      - ['{cycle_hook_cron.output.data.iterationCount}', '{trigger_cron.output.data.maxCycles}']
+                      - ['{@conditional.less_than}']
+`;
+};
+exports.getWorkflowYAML = getWorkflowYAML;