@kapeta/local-cluster-service 0.0.60

package/src/operatorManager.js

@@ -0,0 +1,180 @@
+const ClusterConfiguration = require('@kapeta/local-cluster-config');
+const serviceManager = require('./serviceManager');
+const storageService = require('./storageService');
+const containerManager = require('./containerManager');
+const _ = require('lodash');
+const mkdirp = require('mkdirp');
+const Path = require('path');
+const md5 = require('md5');
+
+const KIND_OPERATOR = 'core/resource-type-operator';
+
+class Operator {
+    constructor(data) {
+        this._data = data;
+    }
+
+    getData() {
+        return this._data;
+    }
+
+    getCredentials() {
+        return this._data.credentials;
+    }
+}
+
+class OperatorManager {
+
+    constructor() {
+        this._mountDir = Path.join(storageService.getKapetaBasedir(), 'mounts');
+
+        mkdirp.sync(this._mountDir);
+    }
+
+    _getMountPoint(operatorType, mountName) {
+        return Path.join(this._mountDir, operatorType, mountName);
+    }
+
+    /**
+     * Get operator definition for resource type
+     *
+     * @param resourceType
+     * @return {Operator}
+     */
+    getOperator(resourceType) {
+        const operators = ClusterConfiguration.getDefinitions(KIND_OPERATOR);
+
+        const operator = _.find(operators, (operator) => operator.definition &&
+            operator.definition.metadata &&
+            operator.definition.metadata.name &&
+            operator.definition.metadata.name.toLowerCase() === resourceType.toLowerCase());
+
+        if (!operator) {
+            throw new Error('Unknown resource type: ' + resourceType);
+        }
+
+        if (!operator.definition.spec ||
+            !operator.definition.spec.local) {
+            throw new Error('Operator missing local definition: ' + resourceType);
+        }
+
+        return new Operator(operator.definition.spec.local);
+    }
+
+    /**
+     * Get information about a specific resource
+     *
+     * @param {string} systemId
+     * @param {string} fromServiceId
+     * @param {string} resourceType
+     * @param {string} portType
+     * @param {string} name
+     * @returns {Promise<{host: string, port: (*|string), type: *, protocol: *, credentials: *}>}
+     */
+    async getResourceInfo(systemId, fromServiceId, resourceType, portType, name) {
+
+        const operator = this.getOperator(resourceType);
+
+        const credentials = operator.getCredentials();
+
+        const container = await this.ensureResource(systemId, resourceType);
+
+        const portInfo = await container.getPort(portType);
+
+        if (!portInfo) {
+            throw new Error('Unknown resource port type : ' + resourceType + '#' + portType);
+        }
+
+        const dbName = name + '_' + fromServiceId.replace(/[^a-z0-9]/gi, '');
+
+        return {
+            host: 'localhost',
+            port: portInfo.hostPort,
+            type: portType,
+            protocol: portInfo.protocol,
+            options: {
+                dbName
+            },
+            credentials
+        };
+    }
+
+    /**
+     * Ensure we have a running operator of given type
+     *
+     * @param systemId
+     * @param resourceType
+     * @return {Promise<ContainerInfo>}
+     */
+    async ensureResource(systemId, resourceType) {
+        const operator = this.getOperator(resourceType);
+
+        const operatorData = operator.getData();
+
+        const portTypes = Object.keys(operatorData.ports);
+
+        portTypes.sort();
+
+        const containerBaseName = 'kapeta-resource';
+
+        const nameParts = [resourceType.toLowerCase()];
+
+        const ports = {};
+
+        for(let i = 0 ; i < portTypes.length; i++) {
+            const portType = portTypes[i];
+            let containerPortInfo = operatorData.ports[portType];
+            const hostPort = await serviceManager.ensureServicePort(resourceType, portType);
+
+            if (typeof containerPortInfo === 'number' ||
+                typeof containerPortInfo === 'string') {
+                containerPortInfo = {port: containerPortInfo, type: 'tcp'};
+            }
+
+            if (!containerPortInfo.type) {
+                containerPortInfo.type = 'tcp';
+            }
+
+            const portId = containerPortInfo.port + '/' + containerPortInfo.type;
+            nameParts.push(portType + '-' + portId + '-' + hostPort);
+
+            ports[portId] = {
+                type: portType,
+                hostPort
+            };
+        }
+
+        const mounts = {};
+
+        _.forEach(operatorData.mounts, (containerPath, mountName) => {
+            const hostPath = this._getMountPoint(resourceType, mountName);
+            mkdirp.sync(hostPath);
+            mounts[containerPath] = hostPath;
+        });
+
+        const containerName = containerBaseName + '-' + md5(nameParts.join('_'));
+        let container = await containerManager.get(containerName);
+
+        const isRunning = container ? await container.isRunning() : false;
+        if (container && !isRunning) {
+            await container.start();
+        }
+
+        if (!container) {
+
+            container = await containerManager.run(
+                operatorData.image,
+                containerName,
+                {
+                    mounts,
+                    ports,
+                    health: operatorData.health,
+                    env: operatorData.env
+                });
+        }
+
+        return container;
+    }
+}
+
+module.exports = new OperatorManager();

package/src/providerManager.js

@@ -0,0 +1,84 @@
+const _ = require('lodash');
+const FS = require('fs');
+const Path = require('path');
+const Glob = require("glob");
+
+const ClusterConfiguration = require('@kapeta/local-cluster-config');
+
+class ProviderManager {
+
+    constructor() {
+        this._assetCache = {};
+    }
+
+    getWebProviders() {
+        return ClusterConfiguration
+            .getProviderDefinitions()
+            .filter((providerDefinition) => providerDefinition.hasWeb)
+    }
+
+    getWebAssets() {
+        const webProviders = this.getWebProviders();
+
+        let providerFiles = [];
+        webProviders.map((webProvider) => {
+            return Glob.sync('web/**/*.js', {cwd: webProvider.path}).map((file) => {
+                return {webProvider, file};
+            });
+        }).forEach((webFiles) => {
+            providerFiles.push(...webFiles);
+        });
+
+        return providerFiles;
+    }
+
+    loadAssets() {
+        this.getWebAssets().forEach((asset) => {
+            const providerId = asset.webProvider.definition.metadata.name;
+            const file = asset.file;
+            const assetId = `${providerId}/${asset.webProvider.version}/${file}`;
+            this._assetCache[assetId] = Path.join(asset.webProvider.path, file);
+        })
+    }
+
+
+    /**
+     * Returns all public (web) javascript for available providers.
+     *
+     * Provides frontend / applications with the implementation of the frontends for the
+     * providers.
+     *
+     */
+    getPublicJS() {
+        this.loadAssets();
+        const includes = Object.keys(this._assetCache).map((assetId) => {
+            return `${ClusterConfiguration.getClusterServiceAddress()}/providers/asset/${assetId}`
+        });
+
+        return `Kapeta.setPluginPaths(${JSON.stringify(includes)});`
+    }
+
+    getAsset(id) {
+        if (_.isEmpty(this._assetCache)) {
+            this.loadAssets();
+        }
+        if (this._assetCache[id]) {
+            return FS.readFileSync(this._assetCache[id]).toString();
+        }
+        return null;
+    }
+}
+
+const providerDefinitions = ClusterConfiguration.getProviderDefinitions();
+
+if (providerDefinitions.length > 0) {
+    console.log('## Loaded the following providers ##');
+    providerDefinitions.forEach(providerDefinition => {
+        console.log(' - %s[%s:%s]', providerDefinition.definition.kind, providerDefinition.definition.metadata.name, providerDefinition.version);
+        console.log('   from %s', providerDefinition.path);
+    })
+} else {
+    console.log('## No providers found ##');
+}
+
+module.exports = new ProviderManager();

package/src/providers/routes.js

@@ -0,0 +1,26 @@
+const Router = require('express-promise-router').default;
+const providerManager = require('../providerManager');
+
+const router = new Router();
+
+router.use('/', require('../middleware/cors'));
+
+/**
+ * Get all local assets available
+ */
+router.get('/all.js', (req, res) => {
+    res.send(providerManager.getPublicJS());
+});
+
+router.get('/asset/*', (req, res) => {
+    const assetId = req.params[0];
+    const result = providerManager.getAsset(assetId)
+    if (!result) {
+        res.status(404).send('');
+    } else {
+        res.send(result);
+    }
+});
+
+
+module.exports = router;

package/src/proxy/routes.js

@@ -0,0 +1,125 @@
+const Router = require('express-promise-router').default;
+const _ = require('lodash');
+
+const router = new Router();
+const serviceManager = require('../serviceManager');
+const clusterService = require('../clusterService');
+const assetManager = require('../assetManager');
+
+/**
+ * @var {{[key:string]:ProxyRequestHandler}}
+ */
+const TYPE_HANDLERS = {
+    rest: require('./types/rest'),
+    web: require('./types/web')
+};
+
+function getResource(resources, resourceName) {
+    return _.find(resources, (resource) => {
+        return (resource.metadata.name.toLowerCase() === resourceName.toLowerCase());
+    });
+}
+
+router.use('/:systemId/:consumerInstanceId/:consumerResourceName', require('../middleware/stringBody'));
+
+router.all('/:systemId/:consumerInstanceId/:consumerResourceName/:type/*', async (req, res) => {
+
+    try {
+
+        const typeHandler = TYPE_HANDLERS[req.params.type.toLowerCase()];
+        if (!typeHandler) {
+            res.status(401).send({error: 'Unknown connection type: ' + req.params.type});
+            return;
+        }
+
+        const plan = await assetManager.getPlan(req.params.systemId);
+
+        // We can find the connection by the consumer information alone since
+        // only 1 provider can be connected to a consumer resource at a time
+        const connection = _.find(plan.spec.connections, (connection) => {
+            return connection.to.blockId.toLowerCase() === req.params.consumerInstanceId.toLowerCase() &&
+                connection.to.resourceName.toLowerCase() === req.params.consumerResourceName.toLowerCase();
+        });
+
+        if (!connection) {
+            res.status(401).send({error:`No connection found for consumer "${req.params.consumerInstanceId}::${req.params.consumerResourceName}"`});
+            return;
+        }
+
+        const toBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
+            return blockInstance.id.toLowerCase() === connection.to.blockId.toLowerCase();
+        });
+
+        if (!toBlockInstance) {
+            res.status(401).send({error:`Block instance not found "${req.params.consumerInstanceId}`});
+            return;
+        }
+
+        const toBlockAsset = await assetManager.getAsset(toBlockInstance.block.ref);
+
+        const toResource = getResource(toBlockAsset.data.spec.consumers, req.params.consumerResourceName);
+
+        if (!toResource) {
+            res.status(401).send({error:`Block resource not found "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`});
+            return;
+        }
+
+        const basePath = clusterService.getProxyPath(
+            req.params.systemId,
+            req.params.consumerInstanceId,
+            req.params.consumerResourceName,
+            req.params.type
+        );
+
+        /*
+         Get the path the consumer requested.
+         Note that this might not match the path the destination is expecting so we need to identify the method
+         that is being called and identify the destination path from the connection.
+         */
+        const consumerPath = req.originalUrl.substr(basePath.length - 1);
+
+        const fromBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
+            return blockInstance.id.toLowerCase() === connection.from.blockId.toLowerCase();
+        });
+
+        if (!fromBlockInstance) {
+            res.status(401).send({error:`Block instance not found "${connection.from.blockId}`});
+            return;
+        }
+
+        const fromBlockAsset = await assetManager.getAsset(fromBlockInstance.block.ref);
+
+        const fromResource = getResource(fromBlockAsset.data.spec.providers, connection.from.resourceName);
+
+        if (!fromResource) {
+            res.status(401).send({error:`Block resource not found "${connection.from.blockId}::${connection.from.resourceName}`});
+            return;
+        }
+
+
+        //Get target address
+        let address = await serviceManager.getProviderAddress(
+            req.params.systemId,
+            connection.from.blockId,
+            req.params.type
+        );
+
+        while(address.endsWith('/')) {
+            address = address.substr(0, address.length - 1);
+        }
+
+        typeHandler(req, res, {
+            consumerPath,
+            address,
+            toResource,
+            fromResource,
+            connection
+        });
+
+    } catch(err) {
+        res.status(400).send({error: err.message});
+    }
+
+});
+
+module.exports = router;

package/src/proxy/types/rest.js

@@ -0,0 +1,163 @@
+const request = require('request');
+const Path = require('path');
+const _ = require('lodash');
+
+const pathTemplateParser = require('../../utils/pathTemplateParser');
+const networkManager = require('../../networkManager');
+const socketManager = require('../../socketManager');
+
+
+function getRestMethodId(restResource, httpMethod, httpPath) {
+    return _.findKey(restResource.spec.methods, (method) => {
+        let methodType = method.method ? method.method.toUpperCase() : 'GET';
+
+        if (methodType !== httpMethod.toUpperCase()) {
+            return false;
+        }
+
+        let path = method.path;
+
+        if (restResource.spec.basePath) {
+            path = Path.join(restResource.spec.basePath, path);
+        }
+
+        const pathTemplate = pathTemplateParser(path);
+
+        return pathTemplate.matches(httpPath);
+
+    });
+}
+
+/**
+ *
+ * @param req {Request}
+ * @param opts {ProxyRequestInfo}
+ * @return {{consumerMethod: *, providerMethod: *}}
+ */
+function resolveMethods(req, opts) {
+    const consumerMethodId = getRestMethodId(opts.toResource, req.method, opts.consumerPath);
+
+    if (!consumerMethodId) {
+        throw new Error(
+            `Consumer method not found for path "${req.method} ${opts.consumerPath}" in resource "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`
+        );
+    }
+
+    const consumerMethod = _.cloneDeep(opts.toResource.spec.methods[consumerMethodId]);
+
+    if (!consumerMethod) {
+        throw new Error(
+            `Consumer method not found for path "${req.method} ${opts.consumerPath}" in resource "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`
+        );
+    }
+
+    consumerMethod.id = consumerMethodId;
+
+    const providerMethodId = _.findKey(opts.connection.mapping, (mapping) => {
+        return mapping.targetId === consumerMethodId;
+    });
+
+    if (!providerMethodId) {
+        throw new Error(`Connection contained no mapping for consumer method "${consumerMethodId}`);
+    }
+
+    const providerMethod = _.cloneDeep(opts.fromResource.spec.methods[providerMethodId]);
+
+    if (!providerMethod) {
+        throw new Error(
+            `Provider method not found "${providerMethodId}" in resource "${opts.connection.from.blockId}::${opts.connection.from.resourceName}`
+        );
+    }
+
+    providerMethod.id = providerMethodId;
+
+    return {
+        consumerMethod,
+        providerMethod
+    };
+}
+
+/**
+ *
+ * @param req {Request}
+ * @param res {Response}
+ * @param opts {ProxyRequestInfo}
+ */
+module.exports = function proxyRestRequest(req, res, opts) {
+
+    let {consumerMethod, providerMethod} = resolveMethods(req, opts);
+
+    const consumerPathTemplate = pathTemplateParser(consumerMethod.path);
+    const providerPathTemplate = pathTemplateParser(providerMethod.path);
+
+    const pathVariables = consumerPathTemplate.parse(opts.consumerPath);
+
+    let providerPath = providerPathTemplate.create(pathVariables);
+
+    if (!providerPath.startsWith('/')) {
+        providerPath = '/' + providerPath;
+    }
+
+    const requestHeaders = _.clone(req.headers);
+
+    delete requestHeaders['content-length'];
+    delete requestHeaders['content-encoding'];
+    delete requestHeaders['connection'];
+    delete requestHeaders['host'];
+    delete requestHeaders['origin'];
+
+
+    console.log('Route to provider: %s => %s', opts.consumerPath, opts.address + providerPath);
+
+    const reqOpts = {
+        method: providerMethod.method || 'GET',
+        url: opts.address + providerPath,
+        body: req.stringBody,
+        headers: requestHeaders
+    };
+
+    const traffic = networkManager.addRequest(
+        req.params.systemId,
+        opts.connection,
+        reqOpts,
+        consumerMethod.id,
+        providerMethod.id
+    );
+
+    socketManager.emit(traffic.connectionId, 'traffic_start', traffic);
+
+    request(reqOpts, function(err, response, responseBody) {
+        if (err) {
+            traffic.asError(err);
+            socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+
+            res.status(500).send({error: '' + err});
+            return;
+        }
+
+        const responseHeaders = _.clone(response.headers);
+
+        delete responseHeaders['content-length'];
+        delete responseHeaders['content-encoding'];
+        delete responseHeaders['connection'];
+
+        res.set(responseHeaders);
+
+        res.status(response.statusCode);
+
+        traffic.withResponse({
+            code: response.statusCode,
+            headers: response.headers,
+            body: responseBody
+        });
+
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+
+        if (responseBody) {
+            res.send(responseBody);
+        } else {
+            res.end();
+        }
+    });
+
+};

package/src/proxy/types/web.js

@@ -0,0 +1,83 @@
+const request = require('request');
+const _ = require('lodash');
+
+const networkManager = require('../../networkManager');
+const socketManager = require('../../socketManager');
+
+/**
+ *
+ * @param req {Request}
+ * @param res {Response}
+ * @param opts {ProxyRequestInfo}
+ */
+module.exports = function proxyRestRequest(req, res, opts) {
+
+    console.log('Route to provider: %s => %s', opts.consumerPath, opts.address);
+
+    const requestHeaders = _.clone(req.headers);
+
+    delete requestHeaders['content-length'];
+    delete requestHeaders['content-encoding'];
+    delete requestHeaders['connection'];
+    delete requestHeaders['host'];
+    delete requestHeaders['origin'];
+
+    const sourceBasePath = opts.fromResource.spec.path;
+    const targetBasePath = opts.toResource.spec.path;
+    let path = opts.consumerPath;
+    if (opts.consumerPath.startsWith(sourceBasePath)) {
+        path = path.replace(sourceBasePath, targetBasePath);
+    }
+
+
+    const reqOpts = {
+        method: req.method,
+        url: opts.address + path,
+        headers: requestHeaders,
+        body: req.stringBody
+    };
+
+    console.log('reqOpts', reqOpts);
+
+    const traffic = networkManager.addRequest(
+        req.params.systemId,
+        opts.connection,
+        reqOpts
+    );
+
+    socketManager.emit(traffic.connectionId, 'traffic_start', traffic);
+
+    request(reqOpts, function(err, response, responseBody) {
+        if (err) {
+            traffic.asError(err);
+            socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+
+            res.status(500).send({error: '' + err});
+            return;
+        }
+
+        const responseHeaders = _.clone(response.headers);
+
+        delete responseHeaders['content-length'];
+        delete responseHeaders['content-encoding'];
+        delete responseHeaders['connection'];
+
+        res.set(responseHeaders);
+        res.status(response.statusCode);
+
+        traffic.withResponse({
+            code: response.statusCode,
+            headers: response.headers,
+            body: responseBody
+        });
+
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+
+        if (responseBody) {
+            res.send(responseBody);
+        } else {
+            res.end();
+        }
+    });
+
+};