@hotmeshio/hotmesh 0.0.4 → 0.0.6

package/build/esm/services/compiler/deployer.js

@@ -0,0 +1,423 @@
+import { KeyType } from '../../modules/key';
+import { getSymKey } from '../../modules/utils';
+import { CollatorService } from '../collator';
+import { SerializerService } from '../serializer';
+const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
+const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
+const DEFAULT_RANGE_SIZE = DEFAULT_METADATA_RANGE_SIZE + DEFAULT_DATA_RANGE_SIZE;
+class Deployer {
+    constructor(manifest) {
+        this.manifest = null;
+        this.manifest = manifest;
+    }
+    async deploy(store) {
+        this.store = store;
+        CollatorService.compile(this.manifest.app.graphs);
+        this.convertTopicsToTypes();
+        this.copyJobSchemas();
+        this.bindBackRefs();
+        this.bindParents();
+        this.resolveMappingDependencies();
+        this.resolveJobMapsPaths();
+        await this.generateSymKeys();
+        await this.generateSymVals();
+        await this.deployHookPatterns();
+        await this.deployActivitySchemas();
+        await this.deploySubscriptions();
+        await this.deployTransitions();
+        await this.deployConsumerGroups();
+    }
+    getVID() {
+        return {
+            id: this.manifest.app.id,
+            version: this.manifest.app.version,
+        };
+    }
+    async generateSymKeys() {
+        //note: symbol ranges are additive (per version); path assignments are immutable
+        const appId = this.manifest.app.id;
+        for (const graph of this.manifest.app.graphs) {
+            //generate JOB symbols
+            const [, trigger] = this.findTrigger(graph);
+            const topic = trigger.subscribes;
+            const [lower, upper, symbols] = await this.store.reserveSymbolRange(`$${topic}`, DEFAULT_RANGE_SIZE, 'JOB');
+            const prefix = ''; //job meta/data is NOT namespaced
+            const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, trigger.PRODUCES);
+            if (Object.keys(newSymbols).length) {
+                await this.store.addSymbols(`$${topic}`, newSymbols);
+            }
+            //generate ACTIVITY symbols
+            for (const [activityId, activity] of Object.entries(graph.activities)) {
+                const [lower, upper, symbols] = await this.store.reserveSymbolRange(activityId, DEFAULT_RANGE_SIZE, 'ACTIVITY');
+                const prefix = `${activityId}/`; //activity meta/data is namespaced
+                this.bindSelf(activity.consumes, activity.produces, activityId);
+                const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, activity.produces);
+                if (Object.keys(newSymbols).length) {
+                    await this.store.addSymbols(activityId, newSymbols);
+                }
+            }
+        }
+    }
+    bindSelf(consumes, produces, activityId) {
+        //bind self-referential mappings
+        for (const selfId of [activityId, '$self']) {
+            const selfConsumes = consumes[selfId];
+            if (selfConsumes) {
+                for (const path of selfConsumes) {
+                    if (!produces.includes(path)) {
+                        produces.push(path);
+                    }
+                }
+            }
+        }
+    }
+    bindSymbols(startIndex, maxIndex, existingSymbols, prefix, produces) {
+        let newSymbols = {};
+        let currentSymbols = { ...existingSymbols };
+        for (let path of produces) {
+            const fullPath = `${prefix}${path}`;
+            if (!currentSymbols[fullPath]) {
+                if (startIndex > maxIndex) {
+                    throw new Error('Symbol index out of bounds');
+                }
+                const symbol = getSymKey(startIndex);
+                startIndex++;
+                newSymbols[fullPath] = symbol;
+                currentSymbols[fullPath] = symbol; // update the currentSymbols to include this new symbol
+            }
+        }
+        return newSymbols;
+    }
+    copyJobSchemas() {
+        const graphs = this.manifest.app.graphs;
+        for (const graph of graphs) {
+            const jobSchema = graph.output?.schema;
+            const outputSchema = graph.input?.schema;
+            if (!jobSchema && !outputSchema)
+                continue;
+            const activities = graph.activities;
+            // Find the trigger activity and bind the job schema to it
+            // at execution time, the trigger is a standin for the job
+            for (const activityKey in activities) {
+                if (activities[activityKey].type === 'trigger') {
+                    const trigger = activities[activityKey];
+                    if (jobSchema) {
+                        //possible for trigger to have job mappings
+                        if (!trigger.job) {
+                            trigger.job = {};
+                        }
+                        trigger.job.schema = jobSchema;
+                    }
+                    if (outputSchema) {
+                        //impossible for trigger to have output mappings.
+                        trigger.output = { schema: outputSchema };
+                    }
+                }
+            }
+        }
+    }
+    bindBackRefs() {
+        for (const graph of this.manifest.app.graphs) {
+            const activities = graph.activities;
+            const triggerId = this.findTrigger(graph)[0];
+            for (const activityKey in activities) {
+                activities[activityKey].trigger = triggerId;
+                activities[activityKey].subscribes = graph.subscribes;
+                if (graph.publishes) {
+                    activities[activityKey].publishes = graph.publishes;
+                }
+                if (graph.expire) {
+                    activities[activityKey].expire = graph.expire;
+                }
+            }
+        }
+    }
+    //more intuitive for SDK users to use 'topic',
+    //but the compiler is desiged to be generic and uses 'subtypes'
+    convertTopicsToTypes() {
+        for (const graph of this.manifest.app.graphs) {
+            const activities = graph.activities;
+            for (const activityKey in activities) {
+                const activity = activities[activityKey];
+                if (['worker', 'await'].includes(activity.type) && activity.topic && !activity.subtype) {
+                    activity.subtype = activity.topic;
+                }
+            }
+        }
+    }
+    async bindParents() {
+        const graphs = this.manifest.app.graphs;
+        for (const graph of graphs) {
+            if (graph.transitions) {
+                for (const fromActivity in graph.transitions) {
+                    const toTransitions = graph.transitions[fromActivity];
+                    for (const transition of toTransitions) {
+                        const to = transition.to;
+                        //DAGs have one parent; easy to optimize for
+                        graph.activities[to].parent = fromActivity;
+                    }
+                }
+            }
+        }
+    }
+    collectValues(schema, values) {
+        for (const [key, value] of Object.entries(schema)) {
+            if (key === 'enum' || key === 'examples' || key === 'default') {
+                if (Array.isArray(value)) {
+                    for (const v of value) {
+                        if (typeof v === 'string' && v.length > 5) {
+                            values.add(v);
+                        }
+                    }
+                }
+                else if (typeof value === 'string' && value.length > 5) {
+                    values.add(value);
+                }
+            }
+            else if (typeof value === 'object') {
+                this.collectValues(value, values);
+            }
+        }
+    }
+    traverse(obj, values) {
+        for (const value of Object.values(obj)) {
+            if (typeof value === 'object') {
+                if ('schema' in value) {
+                    this.collectValues(value.schema, values);
+                }
+                else {
+                    this.traverse(value, values);
+                }
+            }
+        }
+    }
+    async generateSymVals() {
+        const uniqueStrings = new Set();
+        for (const graph of this.manifest.app.graphs) {
+            this.traverse(graph, uniqueStrings);
+        }
+        const existingSymbols = await this.store.getSymbolValues();
+        const startIndex = Object.keys(existingSymbols).length;
+        const maxIndex = Math.pow(52, 2) - 1;
+        const newSymbols = SerializerService.filterSymVals(startIndex, maxIndex, existingSymbols, uniqueStrings);
+        await this.store.addSymbolValues(newSymbols);
+    }
+    resolveJobMapsPaths() {
+        function parsePaths(obj) {
+            let result = [];
+            function traverse(obj, path = []) {
+                for (let key in obj) {
+                    if (typeof obj[key] === 'object' && obj[key] !== null && !('@pipe' in obj[key])) {
+                        let newPath = [...path, key];
+                        traverse(obj[key], newPath);
+                    }
+                    else {
+                        const finalPath = `data/${[...path, key].join('/')}`;
+                        if (!result.includes(finalPath)) {
+                            result.push(finalPath);
+                        }
+                    }
+                }
+            }
+            if (obj) {
+                traverse(obj);
+            }
+            return result;
+        }
+        for (const graph of this.manifest.app.graphs) {
+            let results = [];
+            const [, trigger] = this.findTrigger(graph);
+            for (const activityKey in graph.activities) {
+                const activity = graph.activities[activityKey];
+                results = results.concat(parsePaths(activity.job?.maps));
+            }
+            trigger.PRODUCES = results;
+        }
+    }
+    resolveMappingDependencies() {
+        const dynamicMappingRules = [];
+        //recursive function to descend into the object and find all dynamic mapping rules
+        function traverse(obj, consumes) {
+            for (const key in obj) {
+                if (typeof obj[key] === 'string') {
+                    const stringValue = obj[key];
+                    const dynamicMappingRuleMatch = stringValue.match(/^\{[^@].*}$/);
+                    if (dynamicMappingRuleMatch) {
+                        if (stringValue.split('.')[1] !== 'input') {
+                            dynamicMappingRules.push(stringValue);
+                            consumes.push(stringValue);
+                        }
+                    }
+                }
+                else if (typeof obj[key] === 'object' && obj[key] !== null) {
+                    traverse(obj[key], consumes);
+                }
+            }
+        }
+        const graphs = this.manifest.app.graphs;
+        for (const graph of graphs) {
+            const activities = graph.activities;
+            for (const activityId in activities) {
+                const activity = activities[activityId];
+                activity.consumes = [];
+                traverse(activity, activity.consumes);
+                activity.consumes = this.groupMappingRules(activity.consumes);
+            }
+        }
+        const groupedRules = this.groupMappingRules(dynamicMappingRules);
+        // Iterate through the graph and add 'produces' field to each activity
+        for (const graph of graphs) {
+            const activities = graph.activities;
+            for (const activityId in activities) {
+                const activity = activities[activityId];
+                activity.produces = groupedRules[`${activityId}`] || [];
+            }
+        }
+    }
+    groupMappingRules(rules) {
+        rules = Array.from(new Set(rules)).sort();
+        // Group by the first symbol before the period (this is the activity name)
+        const groupedRules = {};
+        for (const rule of rules) {
+            const [group, resolved] = this.resolveMappableValue(rule);
+            if (!groupedRules[group]) {
+                groupedRules[group] = [];
+            }
+            groupedRules[group].push(resolved);
+        }
+        return groupedRules;
+    }
+    resolveMappableValue(mappable) {
+        mappable = mappable.substring(1, mappable.length - 1);
+        const parts = mappable.split('.');
+        if (parts[0] === '$job') {
+            const [group, ...path] = parts;
+            return [group, path.join('/')];
+        }
+        else {
+            //normalize paths to be relative to the activity
+            const [group, type, subtype, ...path] = parts;
+            const prefix = {
+                hook: 'hook/data',
+                input: 'input/data',
+                output: subtype === 'data' ? 'output/data' : 'output/metadata'
+            }[type];
+            return [group, `${prefix}/${path.join('/')}`];
+        }
+    }
+    async deployActivitySchemas() {
+        const graphs = this.manifest.app.graphs;
+        const activitySchemas = {};
+        for (const graph of graphs) {
+            const activities = graph.activities;
+            for (const activityKey in activities) {
+                activitySchemas[activityKey] = activities[activityKey];
+            }
+        }
+        await this.store.setSchemas(activitySchemas, this.getVID());
+    }
+    async deploySubscriptions() {
+        const graphs = this.manifest.app.graphs;
+        const publicSubscriptions = {};
+        for (const graph of graphs) {
+            const activities = graph.activities;
+            const subscribesTopic = graph.subscribes;
+            // Find the activity ID associated with the subscribes topic
+            for (const activityKey in activities) {
+                if (activities[activityKey].type === 'trigger') {
+                    publicSubscriptions[subscribesTopic] = activityKey;
+                    break;
+                }
+            }
+        }
+        await this.store.setSubscriptions(publicSubscriptions, this.getVID());
+    }
+    findTrigger(graph) {
+        for (const activityKey in graph.activities) {
+            const activity = graph.activities[activityKey];
+            if (activity.type === 'trigger') {
+                return [activityKey, activity];
+            }
+        }
+        return null;
+    }
+    async deployTransitions() {
+        const graphs = this.manifest.app.graphs;
+        const privateSubscriptions = {};
+        for (const graph of graphs) {
+            if (graph.subscribes && graph.subscribes.startsWith('.')) {
+                const [triggerId] = this.findTrigger(graph);
+                if (triggerId) {
+                    privateSubscriptions[graph.subscribes] = { [triggerId]: true };
+                }
+            }
+            if (graph.transitions) {
+                for (const fromActivity in graph.transitions) {
+                    const toTransitions = graph.transitions[fromActivity];
+                    const toValues = {};
+                    for (const transition of toTransitions) {
+                        const to = transition.to;
+                        if (transition.conditions) {
+                            toValues[to] = transition.conditions;
+                        }
+                        else {
+                            toValues[to] = true;
+                        }
+                    }
+                    if (Object.keys(toValues).length > 0) {
+                        privateSubscriptions['.' + fromActivity] = toValues;
+                    }
+                }
+            }
+        }
+        await this.store.setTransitions(privateSubscriptions, this.getVID());
+    }
+    async deployHookPatterns() {
+        const graphs = this.manifest.app.graphs;
+        const hookRules = {};
+        for (const graph of graphs) {
+            if (graph.hooks) {
+                for (const topic in graph.hooks) {
+                    hookRules[topic] = graph.hooks[topic];
+                    const activityId = graph.hooks[topic][0].to;
+                    const targetActivity = graph.activities[activityId];
+                    if (targetActivity) {
+                        if (!targetActivity.hook) {
+                            targetActivity.hook = {};
+                        }
+                        //create back-reference to the hook topic
+                        targetActivity.hook.topic = topic;
+                    }
+                }
+            }
+        }
+        await this.store.setHookRules(hookRules);
+    }
+    async deployConsumerGroups() {
+        //create one engine group
+        const params = { appId: this.manifest.app.id };
+        const key = this.store.mintKey(KeyType.STREAMS, params);
+        await this.deployConsumerGroup(key, 'ENGINE');
+        for (const graph of this.manifest.app.graphs) {
+            const activities = graph.activities;
+            for (const activityKey in activities) {
+                const activity = activities[activityKey];
+                if (activity.type === 'worker') {
+                    params.topic = activity.subtype;
+                    const key = this.store.mintKey(KeyType.STREAMS, params);
+                    //create one worker group per unique activity subtype (the topic)
+                    await this.deployConsumerGroup(key, 'WORKER');
+                }
+            }
+        }
+    }
+    async deployConsumerGroup(stream, group) {
+        try {
+            await this.store.xgroup('CREATE', stream, group, '$', 'MKSTREAM');
+        }
+        catch (err) {
+            this.store.logger.info('consumer-group-exists', { stream, group });
+        }
+    }
+}
+export { Deployer };

package/build/esm/services/compiler/index.d.ts

@@ -0,0 +1,30 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { HotMeshManifest } from '../../types/hotmesh';
+import { RedisClient, RedisMulti } from '../../types/redis';
+/**
+ * The compiler service converts a graph into a executable program.
+ */
+declare class CompilerService {
+    store: StoreService<RedisClient, RedisMulti> | null;
+    logger: ILogger;
+    constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    /**
+     * verifies and plans the deployment of an app to Redis; the app is not deployed yet
+     * @param path
+     */
+    plan(mySchemaOrPath: string): Promise<HotMeshManifest>;
+    isPath(input: string): boolean;
+    /**
+     * deploys an app to Redis but does NOT activate it.
+     */
+    deploy(mySchemaOrPath: string): Promise<HotMeshManifest>;
+    /**
+     * activates a deployed version of an app;
+     * @param appId
+     * @param appVersion
+     */
+    activate(appId: string, appVersion: string): Promise<boolean>;
+    saveAsJSON(originalPath: string, schema: HotMeshManifest): Promise<void>;
+}
+export { CompilerService };

package/build/esm/services/compiler/index.js

@@ -0,0 +1,82 @@
+import $RefParser from '@apidevtools/json-schema-ref-parser';
+import yaml from 'js-yaml';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { Deployer } from './deployer';
+import { Validator } from './validator';
+/**
+ * The compiler service converts a graph into a executable program.
+ */
+class CompilerService {
+    constructor(store, logger) {
+        this.store = store;
+        this.logger = logger;
+    }
+    /**
+     * verifies and plans the deployment of an app to Redis; the app is not deployed yet
+     * @param path
+     */
+    async plan(mySchemaOrPath) {
+        try {
+            let schema;
+            if (this.isPath(mySchemaOrPath)) {
+                schema = await $RefParser.dereference(mySchemaOrPath);
+            }
+            else {
+                schema = yaml.load(mySchemaOrPath);
+            }
+            // 1) validate the manifest file
+            const validator = new Validator(schema);
+            validator.validate(this.store);
+            // 2) todo: add a PlannerService module that will plan the deployment (what might break, drift, etc)
+            return schema;
+        }
+        catch (err) {
+            this.logger.error('compiler-plan-error', err);
+        }
+    }
+    isPath(input) {
+        return !input.trim().startsWith('app:');
+    }
+    /**
+     * deploys an app to Redis but does NOT activate it.
+     */
+    async deploy(mySchemaOrPath) {
+        try {
+            let schema;
+            if (this.isPath(mySchemaOrPath)) {
+                schema = await $RefParser.dereference(mySchemaOrPath);
+                await this.saveAsJSON(mySchemaOrPath, schema);
+            }
+            else {
+                schema = yaml.load(mySchemaOrPath);
+            }
+            // 2) validate the manifest file (synchronous operation...no callbacks)
+            const validator = new Validator(schema);
+            validator.validate(this.store);
+            // 3) deploy the schema (segment, optimize, etc; save to Redis)
+            const deployer = new Deployer(schema);
+            await deployer.deploy(this.store);
+            // 4) save the app version to Redis (so it can be activated later)
+            await this.store.setApp(schema.app.id, schema.app.version);
+            return schema;
+        }
+        catch (err) {
+            this.logger.error('compiler-deploy-error', err);
+        }
+    }
+    /**
+     * activates a deployed version of an app;
+     * @param appId
+     * @param appVersion
+     */
+    async activate(appId, appVersion) {
+        return await this.store.activateAppVersion(appId, appVersion);
+    }
+    async saveAsJSON(originalPath, schema) {
+        const json = JSON.stringify(schema, null, 2);
+        const newPath = path.join(path.dirname(originalPath), `.hotmesh.${schema.app.id}.${schema.app.version}.json`);
+        await fs.writeFile(newPath, json, 'utf8');
+    }
+}
+export { CompilerService };

package/build/esm/services/compiler/validator.d.ts

@@ -0,0 +1,32 @@
+import { StoreService } from '../store';
+import { MappingStatements } from "../../types/map";
+import { HotMeshManifest } from "../../types/hotmesh";
+import { RedisClient, RedisMulti } from "../../types/redis";
+declare class Validator {
+    manifest: HotMeshManifest | null;
+    activityIds: string[];
+    mappingStatements: MappingStatements;
+    store: StoreService<RedisClient, RedisMulti> | null;
+    static SYS_VARS: string[];
+    constructor(manifest: HotMeshManifest);
+    /**
+     * validate the manifest file
+     */
+    validate(store: StoreService<RedisClient, RedisMulti>): Promise<void>;
+    validateActivityIds(): void;
+    isMappingStatement(value: string): boolean;
+    extractMappingStatements(obj: any, result: MappingStatements, currentActivityId: string): void;
+    getMappingStatements(): void;
+    validateReferencedActivityIds(): void;
+    isFunction(value: string): boolean;
+    validateMappingStatements(): void;
+    validateTransitions(): void;
+    validateTransitionConditions(): void;
+    validateStats(): void;
+    validateSchemas(): void;
+    validateUniqueHandledTopics(): void;
+    validateGraphPublishSubscribe(): void;
+    validateHooks(): void;
+    validateConditionalStatements(): void;
+}
+export { Validator };