@hotmeshio/hotmesh 0.0.4 → 0.0.6

package/build/esm/services/compiler/validator.js

@@ -0,0 +1,131 @@
+import { Pipe } from "../pipe";
+class Validator {
+    constructor(manifest) {
+        this.manifest = null;
+        this.activityIds = [];
+        this.mappingStatements = {};
+        this.store = null;
+        this.manifest = manifest;
+    }
+    /**
+     * validate the manifest file
+     */
+    async validate(store) {
+        this.store = store;
+        this.getMappingStatements();
+        this.validateActivityIds();
+        this.validateReferencedActivityIds();
+        this.validateMappingStatements();
+        this.validateTransitions();
+        this.validateTransitionConditions();
+        this.validateStats();
+        this.validateSchemas();
+        this.validateUniqueHandledTopics();
+        this.validateGraphPublishSubscribe();
+        this.validateHooks();
+        this.validateConditionalStatements();
+    }
+    // 1.1) Validate the manifest file activity ids are unique (no duplicates)
+    validateActivityIds() {
+        const activityIdsSet = new Set();
+        this.manifest.app.graphs.forEach((graph) => {
+            const ids = Object.keys(graph.activities);
+            // Check for duplicates and add ids to the set
+            ids.forEach((id) => {
+                if (activityIdsSet.has(id)) {
+                    throw new Error(`Duplicate activity id found: ${id}`);
+                }
+                else {
+                    activityIdsSet.add(id);
+                }
+            });
+        });
+        this.activityIds = Array.from(activityIdsSet);
+    }
+    isMappingStatement(value) {
+        return typeof value === 'string' && value.startsWith('{') && value.endsWith('}');
+    }
+    extractMappingStatements(obj, result, currentActivityId) {
+        for (const key in obj) {
+            if (typeof obj[key] === 'object' && obj[key] !== null) {
+                this.extractMappingStatements(obj[key], result, currentActivityId);
+            }
+            else if (this.isMappingStatement(obj[key])) {
+                if (!result[currentActivityId]) {
+                    result[currentActivityId] = [];
+                }
+                result[currentActivityId].push(obj[key]);
+            }
+        }
+    }
+    getMappingStatements() {
+        const mappingStatements = {};
+        this.manifest.app.graphs.forEach((graph) => {
+            const activities = graph.activities;
+            for (const activityId in activities) {
+                const activity = activities[activityId];
+                this.extractMappingStatements(activity, mappingStatements, activityId);
+            }
+        });
+        this.mappingStatements = mappingStatements;
+    }
+    // 1.2) Validate no activity ids are referenced that don't exist
+    validateReferencedActivityIds() {
+        // get list of all mapping statements and validate
+        const mappingStatements = this.mappingStatements;
+        const activityIds = this.activityIds;
+        for (const activity in mappingStatements) {
+            const statements = mappingStatements[activity];
+            statements.forEach((statement) => {
+                if (statement.startsWith('{') && statement.endsWith('}')) {
+                    const statementParts = statement.slice(1, -1).split('.');
+                    const referencedActivityId = statementParts[0];
+                    if (!(Validator.SYS_VARS.includes(referencedActivityId) || activityIds.includes(referencedActivityId) || this.isFunction(statement))) {
+                        throw new Error(`Mapping statement references non-existent activity: ${statement}`);
+                    }
+                }
+            });
+        }
+    }
+    isFunction(value) {
+        return value.startsWith('{@') && Pipe.resolveFunction(value);
+    }
+    // 1.3) Validate the mapping/@pipe statements are valid
+    validateMappingStatements() {
+        // Implement the method content
+    }
+    // 1.4) Validate the transitions are valid
+    validateTransitions() {
+        // Implement the method content
+    }
+    // 1.5) Validate the transition conditions are valid
+    validateTransitionConditions() {
+        // Implement the method content
+    }
+    // 1.6) Validate the stats
+    validateStats() {
+        // Implement the method content
+    }
+    // 1.7) Validate the schemas
+    validateSchemas() {
+        // Implement the method content
+    }
+    // 1.8) Validate the topics are unique and handled
+    validateUniqueHandledTopics() {
+        // Implement the method content
+    }
+    // 1.9) Validate that every graph has publishes and subscribes
+    validateGraphPublishSubscribe() {
+        // Implement the method content
+    }
+    // 1.10) Validate hooks, including mapping statements
+    validateHooks() {
+        // Implement the method content
+    }
+    // 1.11) Validate conditional statements
+    validateConditionalStatements() {
+        // Implement the method content
+    }
+}
+Validator.SYS_VARS = ['$app', '$self', '$graph', '$job'];
+export { Validator };

package/build/esm/services/connector/clients/ioredis.d.ts

@@ -0,0 +1,13 @@
+import { RedisClientOptions, RedisClassType, RedisClientType } from '../../../types/ioredisclient';
+declare class RedisConnection {
+    private connection;
+    private static instances;
+    private id;
+    private static clientOptions;
+    private createConnection;
+    getClient(): RedisClientType;
+    disconnect(): Promise<void>;
+    static connect(id: string, Redis: RedisClassType, options?: RedisClientOptions): Promise<RedisConnection>;
+    static disconnectAll(): Promise<void>;
+}
+export { RedisConnection };

package/build/esm/services/connector/clients/ioredis.js

@@ -0,0 +1,47 @@
+class RedisConnection {
+    constructor() {
+        this.connection = null;
+        this.id = null;
+    }
+    async createConnection(Redis, options) {
+        return new Redis(options);
+    }
+    getClient() {
+        if (!this.connection) {
+            throw new Error('Redis client is not connected');
+        }
+        return this.connection;
+    }
+    async disconnect() {
+        if (this.connection) {
+            await this.connection.quit();
+            this.connection = null;
+        }
+        if (this.id) {
+            RedisConnection.instances.delete(this.id);
+        }
+    }
+    static async connect(id, Redis, options) {
+        if (this.instances.has(id)) {
+            return this.instances.get(id);
+        }
+        const instance = new RedisConnection();
+        const opts = options ? { ...options } : { ...this.clientOptions };
+        instance.connection = await instance.createConnection(Redis, opts);
+        instance.id = id;
+        this.instances.set(id, instance);
+        return instance;
+    }
+    static async disconnectAll() {
+        await Promise.all(Array.from(this.instances.values()).map((instance) => instance.disconnect()));
+        this.instances.clear();
+    }
+}
+RedisConnection.instances = new Map();
+RedisConnection.clientOptions = {
+    host: 'localhost',
+    port: 6379,
+    //password: config.REDIS_PASSWORD,
+    //db: config.REDIS_DATABASE,
+};
+export { RedisConnection };

package/build/esm/services/connector/clients/redis.d.ts

@@ -0,0 +1,13 @@
+import { RedisClientType, RedisClientOptions, RedisClassType } from '../../../types/redisclient';
+declare class RedisConnection {
+    private connection;
+    private static instances;
+    private id;
+    private static clientOptions;
+    private createConnection;
+    getClient(): RedisClientType;
+    disconnect(): Promise<void>;
+    static connect(id: string, Redis: RedisClassType, options?: RedisClientOptions): Promise<RedisConnection>;
+    static disconnectAll(): Promise<void>;
+}
+export { RedisConnection, RedisClientType };

package/build/esm/services/connector/clients/redis.js

@@ -0,0 +1,59 @@
+class RedisConnection {
+    constructor() {
+        this.connection = null;
+        this.id = null;
+    }
+    async createConnection(Redis, options) {
+        return new Promise((resolve, reject) => {
+            const client = Redis.createClient(options);
+            client.on('error', (error) => {
+                reject(error);
+            });
+            client.on('ready', () => {
+                resolve(client);
+            });
+            client.connect();
+        });
+    }
+    getClient() {
+        if (!this.connection) {
+            throw new Error('Redis client is not connected');
+        }
+        return this.connection;
+    }
+    async disconnect() {
+        if (this.connection) {
+            await this.connection.quit();
+            this.connection = null;
+        }
+        if (this.id) {
+            RedisConnection.instances.delete(this.id);
+        }
+    }
+    static async connect(id, Redis, options) {
+        if (this.instances.has(id)) {
+            return this.instances.get(id);
+        }
+        const instance = new RedisConnection();
+        const opts = options ? { ...options } : { ...this.clientOptions };
+        instance.connection = await instance.createConnection(Redis, opts);
+        instance.id = id;
+        this.instances.set(id, instance);
+        return instance;
+    }
+    static async disconnectAll() {
+        await Promise.all(Array.from(this.instances.values()).map((instance) => instance.disconnect()));
+        this.instances.clear();
+    }
+}
+RedisConnection.instances = new Map();
+RedisConnection.clientOptions = {
+    socket: {
+        host: 'localhost',
+        port: 6379,
+        tls: false,
+    },
+    //password: config.REDIS_PASSWORD,
+    //database: config.REDIS_DATABASE,
+};
+export { RedisConnection };

package/build/esm/services/connector/index.d.ts

@@ -0,0 +1,5 @@
+import { HotMeshEngine, HotMeshWorker } from '../../types/hotmesh';
+import { RedisClass, RedisOptions } from '../../types/redis';
+export declare class ConnectorService {
+    static initRedisClients(Redis: RedisClass, options: RedisOptions, target: HotMeshEngine | HotMeshWorker): Promise<void>;
+}

package/build/esm/services/connector/index.js

@@ -0,0 +1,27 @@
+import { v4 as uuidv4 } from 'uuid';
+import { identifyRedisTypeFromClass } from '../../modules/utils';
+import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
+import { RedisConnection } from '../connector/clients/redis';
+export class ConnectorService {
+    //1) Initialize `store`, `stream`, and `subscription` Redis clients.
+    //2) Bind to the target if not already present
+    static async initRedisClients(Redis, options, target) {
+        if (!target.store || !target.stream || !target.sub) {
+            const instances = [];
+            if (identifyRedisTypeFromClass(Redis) === 'redis') {
+                for (let i = 1; i <= 3; i++) {
+                    instances.push(RedisConnection.connect(uuidv4(), Redis, options));
+                }
+            }
+            else {
+                for (let i = 1; i <= 3; i++) {
+                    instances.push(IORedisConnection.connect(uuidv4(), Redis, options));
+                }
+            }
+            const [store, stream, sub] = await Promise.all(instances);
+            target.store = target.store || store.getClient();
+            target.stream = target.stream || stream.getClient();
+            target.sub = target.sub || sub.getClient();
+        }
+    }
+}

package/build/esm/services/dimension/index.d.ts

@@ -0,0 +1,29 @@
+import { HotMeshGraph } from '../../types/hotmesh';
+declare class DimensionService {
+    static targetLength: number;
+    /**
+     * entry point for compiler-type activities. This is called by the compiler
+     * to bind the sorted activity IDs to the trigger activity. These are then used
+     * at runtime by the activities to track job/activity status.
+     * @param graphs
+     */
+    static compile(graphs: HotMeshGraph[]): void;
+    /**
+     * All activities exist on a dimensional plane. Zero
+     * is the default and is implied if no dimension is
+     * present in the hash item key. EVERY value in the
+     * job ledger is dimensionalized even if the dimension
+     * is not present. The key, `AaA`, might not contain
+     * a dimensional index, but it is still implicitly
+     * dimensionalized as `AaA,0` (assuming a trigger).
+     * A value of `AxY,0,0,0,0,1,0,0` would reflect that
+     * an ancestor activity was dimensionalized beyond
+     * the default. The dimensional string must
+     * be included if not zero. There is likely a preceding
+     * sibling dimension, so it would not need to include
+     * the suffix, so these addresses are equivalent:
+     * `AxY,0,0,0,0,0,0,0` == `AxY` for said sibling.
+     */
+    static getSeed(index?: number): string;
+}
+export { DimensionService };

package/build/esm/services/dimension/index.js

@@ -0,0 +1,32 @@
+class DimensionService {
+    /**
+     * entry point for compiler-type activities. This is called by the compiler
+     * to bind the sorted activity IDs to the trigger activity. These are then used
+     * at runtime by the activities to track job/activity status.
+     * @param graphs
+     */
+    static compile(graphs) {
+    }
+    /**
+     * All activities exist on a dimensional plane. Zero
+     * is the default and is implied if no dimension is
+     * present in the hash item key. EVERY value in the
+     * job ledger is dimensionalized even if the dimension
+     * is not present. The key, `AaA`, might not contain
+     * a dimensional index, but it is still implicitly
+     * dimensionalized as `AaA,0` (assuming a trigger).
+     * A value of `AxY,0,0,0,0,1,0,0` would reflect that
+     * an ancestor activity was dimensionalized beyond
+     * the default. The dimensional string must
+     * be included if not zero. There is likely a preceding
+     * sibling dimension, so it would not need to include
+     * the suffix, so these addresses are equivalent:
+     * `AxY,0,0,0,0,0,0,0` == `AxY` for said sibling.
+     */
+    static getSeed(index = 0) {
+        return `,${index}`;
+    }
+}
+//max int digit count that supports `hincrby`
+DimensionService.targetLength = 15;
+export { DimensionService };

package/build/esm/services/durable/asyncLocalStorage.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="node" />
+import { AsyncLocalStorage } from 'async_hooks';
+export declare const asyncLocalStorage: AsyncLocalStorage<Map<string, any>>;

package/build/esm/services/durable/asyncLocalStorage.js

@@ -0,0 +1,2 @@
+import { AsyncLocalStorage } from 'async_hooks';
+export const asyncLocalStorage = new AsyncLocalStorage();

package/build/esm/services/durable/client.d.ts

@@ -0,0 +1,15 @@
+import { WorkflowHandleService } from './handle';
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { ClientConfig, Connection, WorkflowOptions } from '../../types/durable';
+export declare class ClientService {
+    connection: Connection;
+    options: WorkflowOptions;
+    static instances: Map<string, HotMesh | Promise<HotMesh>>;
+    constructor(config: ClientConfig);
+    getHotMesh: (worflowTopic: string) => Promise<HotMesh>;
+    workflow: {
+        start: (options: WorkflowOptions) => Promise<WorkflowHandleService>;
+    };
+    activateWorkflow(hotMesh: HotMesh, workflowTopic: string): Promise<void>;
+    static shutdown(): Promise<void>;
+}

package/build/esm/services/durable/client.js

@@ -0,0 +1,114 @@
+import { WorkflowHandleService } from './handle';
+import { HotMeshService as HotMesh } from '../hotmesh';
+import { getWorkflowYAML } from './factory';
+/*
+Here is an example of how the methods in this file are used:
+
+./client.ts
+
+import { Durable } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis';
+import { example } from './workflows';
+import { v4 as uuidv4 } from 'uuid';
+
+async function run() {
+  const connection = await Durable.Connection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+
+  const client = new Durable.Client({
+    connection,
+  });
+
+  const handle = await client.workflow.start({
+    args: ['HotMesh'],
+    taskQueue: 'hello-world',
+    workflowName: 'example',
+    workflowId: 'workflow-' + uuidv4(),
+  });
+
+  console.log(`Started workflow ${handle.workflowId}`);
+  console.log(await handle.result());
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+
+*/
+class ClientService {
+    constructor(config) {
+        this.getHotMesh = async (worflowTopic) => {
+            if (ClientService.instances.has(worflowTopic)) {
+                return await ClientService.instances.get(worflowTopic);
+            }
+            const hotMesh = HotMesh.init({
+                appId: worflowTopic,
+                engine: {
+                    redis: {
+                        class: this.connection.class,
+                        options: this.connection.options,
+                    }
+                }
+            });
+            ClientService.instances.set(worflowTopic, hotMesh);
+            await this.activateWorkflow(await hotMesh, worflowTopic);
+            return hotMesh;
+        };
+        this.workflow = {
+            start: async (options) => {
+                const taskQueueName = options.taskQueue;
+                const workflowName = options.workflowName;
+                const trc = options.workflowTrace;
+                const spn = options.workflowSpan;
+                const workflowTopic = `${taskQueueName}-${workflowName}`;
+                const hotMesh = await this.getHotMesh(workflowTopic);
+                const payload = {
+                    arguments: [...options.args],
+                    workflowId: options.workflowId,
+                };
+                const context = { metadata: { trc, spn }, data: {} };
+                const jobId = await hotMesh.pub(workflowTopic, payload, context);
+                return new WorkflowHandleService(hotMesh, workflowTopic, jobId);
+            },
+        };
+        this.connection = config.connection;
+    }
+    async activateWorkflow(hotMesh, workflowTopic) {
+        const version = '1';
+        const app = await hotMesh.engine.store.getApp(workflowTopic);
+        const appVersion = app?.version;
+        if (isNaN(appVersion)) {
+            try {
+                await hotMesh.deploy(getWorkflowYAML(workflowTopic, version));
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-client-deploy-activate-err', err);
+                throw err;
+            }
+        }
+        else if (app && !app.active) {
+            try {
+                await hotMesh.activate(version);
+            }
+            catch (err) {
+                hotMesh.engine.logger.error('durable-client-activate-err', err);
+                throw err;
+            }
+        }
+    }
+    static async shutdown() {
+        for (const [key, value] of ClientService.instances) {
+            const hotMesh = await value;
+            await hotMesh.stop();
+        }
+    }
+}
+ClientService.instances = new Map();
+export { ClientService };

package/build/esm/services/durable/connection.d.ts

@@ -0,0 +1,4 @@
+import { Connection, ConnectionConfig } from "../../types/durable";
+export declare class ConnectionService {
+    static connect(config: ConnectionConfig): Promise<Connection>;
+}

package/build/esm/services/durable/connection.js

@@ -0,0 +1,47 @@
+/*
+Here is an example of how the methods in this file are used:
+
+./client.ts
+
+import { Durable } from '@hotmeshio/hotmesh';
+import Redis from 'ioredis';
+import { v4 as uuidv4 } from 'uuid';
+
+async function run() {
+  const connection = await Durable.Connection.connect({
+    class: Redis,
+    options: {
+      host: 'localhost',
+      port: 6379,
+    },
+  });
+
+  const client = new Durable.Client({
+    connection,
+  });
+
+  const handle = await client.workflow.start(example, {
+    taskQueue: 'hello-world',
+    args: ['HotMesh'],
+    workflowName: 'example',
+    workflowId: uuidv4(),
+  });
+
+  console.log(`Started workflow ${handle.workflowId}`);
+  console.log(await handle.result());
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+
+*/
+export class ConnectionService {
+    static async connect(config) {
+        return {
+            class: config.class,
+            options: { ...config.options },
+        };
+    }
+}

package/build/esm/services/durable/factory.d.ts

@@ -0,0 +1,3 @@
+declare const getWorkflowYAML: (topic: string, version?: string) => string;
+declare const getActivityYAML: (topic: string, version?: string) => string;
+export { getActivityYAML, getWorkflowYAML };

package/build/esm/services/durable/factory.js

@@ -0,0 +1,119 @@
+const getWorkflowYAML = (topic, version = '1') => {
+    return `app:
+  id: ${topic}
+  version: '${version}'
+  graphs:
+    - subscribes: ${topic}
+      publishes: ${topic}
+      expire: 120
+      input:
+        schema:
+          type: object
+          properties:
+            workflowId:
+              type: string
+            arguments:
+              type: array
+      output:
+        schema:
+          type: object
+          properties:
+            response:
+              type: any
+
+      activities:
+        t1:
+          type: trigger
+          stats:
+            id: '{$self.input.data.workflowId}'
+        a1:
+          type: worker
+          topic: ${topic}
+          input:
+            schema:
+              type: object
+              properties:
+                workflowId:
+                  type: string
+                arguments:
+                  type: array
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+              arguments: '{t1.output.data.arguments}'
+          output:
+            schema:
+              type: object
+              properties:
+                response:
+                  type: any
+          job:
+            maps:
+              response: '{$self.output.data.response}'
+      transitions:
+        t1:
+          - to: a1`;
+};
+const getActivityYAML = (topic, version = '1') => {
+    return `app:
+  id: ${topic}
+  version: '${version}'
+  graphs:
+    - subscribes: ${topic}
+      input:
+        schema:
+          type: object
+          properties:
+            workflowId:
+              type: string
+            workflowTopic:
+              type: string
+            activityName:
+              type: array
+            arguments:
+              type: array
+      output:
+        schema:
+          type: object
+          properties:
+            response:
+              type: any
+
+      activities:
+        t1:
+          type: trigger
+          stats:
+            id: '{$self.input.data.workflowId}'
+        a1:
+          type: worker
+          topic: ${topic}
+          input:
+            schema:
+              type: object
+              properties:
+                workflowId:
+                  type: string
+                workflowTopic:
+                  type: string
+                activityName:
+                  type: array
+                arguments:
+                  type: array
+            maps:
+              workflowId: '{t1.output.data.workflowId}'
+              workflowTopic: '{t1.output.data.workflowTopic}'
+              activityName: '{t1.output.data.activityName}'
+              arguments: '{t1.output.data.arguments}'
+          output:
+            schema:
+              type: object
+              properties:
+                response:
+                  type: any
+          job:
+            maps:
+              response: '{$self.output.data.response}'
+      transitions:
+        t1:
+          - to: a1`;
+};
+export { getActivityYAML, getWorkflowYAML };