@kapeta/local-cluster-service 0.0.60

package/src/instanceManager.js

@@ -0,0 +1,416 @@
+const _ = require('lodash');
+const request = require('request');
+
+const {BlockInstanceRunner} = require('@kapeta/local-cluster-executor');
+
+const storageService = require('./storageService');
+const socketManager = require('./socketManager');
+const serviceManager = require('./serviceManager');
+const assetManager = require('./assetManager');
+const containerManager = require('./containerManager');
+
+const CHECK_INTERVAL = 10000;
+const DEFAULT_HEALTH_PORT_TYPE = 'rest';
+
+const EVENT_STATUS_CHANGED = 'status-changed';
+const EVENT_INSTANCE_CREATED = 'instance-created';
+const EVENT_INSTANCE_EXITED = 'instance-exited';
+const EVENT_INSTANCE_LOG = 'instance-log';
+
+const STATUS_STARTING = 'starting';
+const STATUS_READY = 'ready';
+const STATUS_UNHEALTHY = 'unhealthy';
+const STATUS_STOPPED = 'stopped';
+
+const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
+
+class InstanceManager {
+    constructor() {
+        this._interval = setInterval(() => this._checkInstances(), CHECK_INTERVAL);
+        /**
+         * Contains an array of running instances that have self-registered with this
+         * cluster service. This is done by the Kapeta SDKs
+         *
+         * @type {any[]}
+         * @private
+         */
+        this._instances = storageService.section('instances', []);
+        /**
+         * Contains the process info for the instances started by this manager. In memory only
+         * so can't be relied on for knowing everything that's running.
+         *
+         * @type {{[systemId:string]:{[instanceId:string]:ProcessInfo}}}
+         * @private
+         */
+        this._processes = {};
+
+        this._checkInstances();
+    }
+
+    _save() {
+        storageService.put('instances', this._instances);
+    }
+
+    async _checkInstances() {
+        let changed = false;
+        for (let i = 0; i < this._instances.length; i++) {
+            const instance = this._instances[i];
+
+            const newStatus = await this._getInstanceStatus(instance);
+
+            if (newStatus === STATUS_UNHEALTHY &&
+                instance.status === STATUS_STARTING) {
+                // If instance is starting we consider unhealthy an indication
+                // that it is still starting
+                continue;
+            }
+
+            if (instance.status !== newStatus) {
+                instance.status = newStatus;
+                this._emit(instance.systemId, EVENT_STATUS_CHANGED, instance);
+                changed = true;
+            }
+        }
+
+        if (changed) {
+            this._save();
+        }
+    }
+
+    async _isRunning(instance) {
+        if (!instance.pid) {
+            return;
+        }
+
+        if (instance.type === 'docker') {
+            const container = await containerManager.get(instance.pid);
+            return await container.isRunning()
+        }
+
+        //Otherwise its just a normal process.
+        //TODO: Handle for Windows
+        try {
+            return process.kill(instance.pid, 0)
+        } catch (err) {
+            return err.code === 'EPERM';
+        }
+    }
+
+    async _getInstanceStatus(instance) {
+        if (instance.status === STATUS_STOPPED) {
+            //Will only change when it reregisters
+            return STATUS_STOPPED;
+        }
+
+        if (!await this._isRunning(instance)) {
+            return STATUS_STOPPED;
+        }
+
+        if (!instance.health) {
+            //No health url means we assume it's healthy as soon as it's running
+            return STATUS_READY;
+        }
+
+        return new Promise((resolve) => {
+            request(instance.health, (err, response) => {
+                if (err) {
+                    resolve(STATUS_UNHEALTHY);
+                    return;
+                }
+
+                if (response.responseCode > 399) {
+                    resolve(STATUS_UNHEALTHY);
+                    return;
+                }
+
+                resolve(STATUS_READY);
+            });
+        });
+    }
+
+    getInstances() {
+        if (!this._instances) {
+            return [];
+        }
+
+        return [...this._instances];
+    }
+
+    getInstancesForPlan(systemId) {
+        if (!this._instances) {
+            return [];
+        }
+
+        return this._instances.filter(instance => instance.systemId === systemId);
+    }
+
+    /**
+     * Get instance information
+     *
+     * @param {string} systemId
+     * @param {string} instanceId
+     * @return {*}
+     */
+    getInstance(systemId, instanceId) {
+        return _.find(this._instances, {systemId, instanceId});
+    }
+
+    /**
+     *
+     * @param {string} systemId
+     * @param {string} instanceId
+     * @param {{health:string,pid:string,type:'docker'|'local',portType?:string}} info
+     * @return {Promise<void>}
+     */
+    async registerInstance(systemId, instanceId, info) {
+        let instance = this.getInstance(systemId, instanceId);
+
+        //Get target address
+        let address = await serviceManager.getProviderAddress(
+            systemId,
+            instanceId,
+            info.portType ?? DEFAULT_HEALTH_PORT_TYPE
+        );
+
+        let healthUrl = null;
+        let health = info.health;
+        if (health) {
+            if (health.startsWith('/')) {
+                health = health.substring(1);
+            }
+            healthUrl = address + health;
+        }
+
+        if (instance) {
+            instance.status = STATUS_STARTING;
+            instance.pid = info.pid;
+            instance.type = info.type;
+            instance.health = healthUrl;
+            this._emit(systemId, EVENT_STATUS_CHANGED, instance);
+        } else {
+            instance = {
+                systemId,
+                instanceId,
+                status: STATUS_STARTING,
+                pid: info.pid,
+                type: info.type,
+                health: healthUrl
+            };
+
+            this._instances.push(instance);
+
+            this._emit(systemId, EVENT_INSTANCE_CREATED, instance);
+        }
+
+        this._save();
+    }
+
+    setInstanceAsStopped(systemId, instanceId) {
+        const instance = _.find(this._instances, {systemId, instanceId});
+        if (instance) {
+            instance.status = STATUS_STOPPED;
+            instance.pid = null;
+            instance.health = null;
+            this._emit(systemId, EVENT_STATUS_CHANGED, instance);
+            this._save();
+        }
+    }
+
+    _emit(systemId, type, payload) {
+        socketManager.emit(`${systemId}/instances`, type, payload);
+    }
+
+    /**
+     *
+     * @param planRef
+     * @return {Promise<ProcessInfo[]>}
+     */
+    async createProcessesForPlan(planRef) {
+        await this.stopAllForPlan(planRef);
+
+        const plan = assetManager.getPlan(planRef);
+        if (!plan) {
+            throw new Error('Plan not found: ' + planRef);
+        }
+
+        if (!plan.spec.blocks) {
+            console.warn('No blocks found in plan', planRef);
+            return [];
+        }
+
+        let processes = [];
+        let errors = [];
+        for(let blockInstance of Object.values(plan.spec.blocks)) {
+            try {
+                processes.push(await this.createProcess(planRef, blockInstance.id));
+            } catch (e) {
+                errors.push(e);
+            }
+        }
+
+        if (errors.length > 0) {
+            throw errors[0];
+        }
+
+        return processes;
+    }
+
+    async _stopInstance(instance) {
+        if (!instance.pid) {
+            return;
+        }
+
+        if (instance.status === 'stopped') {
+            return;
+        }
+
+        try {
+            if (instance.type === 'docker') {
+                const container = await containerManager.get(instance.pid);
+                await container.stop();
+                return;
+            }
+            process.kill(instance.pid, 'SIGTERM');
+        } catch (e) {
+            console.error('Failed to stop process', e);
+        }
+    }
+
+    async stopAllForPlan(planRef) {
+        if (this._processes[planRef]) {
+            for(let instance of Object.values(this._processes[planRef])) {
+                await instance.stop();
+            }
+
+            this._processes[planRef] = {};
+        }
+
+        //Also stop instances not being maintained by the cluster service
+        const instancesForPlan = this._instances
+            .filter(instance => instance.systemId === planRef);
+
+        for(let instance of instancesForPlan) {
+            await this._stopInstance(instance);
+        }
+    }
+
+    /**
+     *
+     * @param planRef
+     * @param instanceId
+     * @return {Promise<PromiseInfo>}
+     */
+    async createProcess(planRef, instanceId) {
+        const plan = assetManager.getPlan(planRef);
+        if (!plan) {
+            throw new Error('Plan not found: ' + planRef);
+        }
+
+        const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, {id: instanceId}) : null;
+        if (!blockInstance) {
+            throw new Error('Block instance not found: ' + instanceId);
+        }
+
+        const blockRef = blockInstance.block.ref;
+
+        const blockAsset = assetManager.getAsset(blockRef);
+
+        if (!blockAsset) {
+            throw new Error('Block not found: ' + blockRef);
+        }
+
+        if (!this._processes[planRef]) {
+            this._processes[planRef] = {};
+        }
+
+        await this.stopProcess(planRef, instanceId);
+
+        const runner = new BlockInstanceRunner(planRef);
+        const startTime = Date.now();
+        const process = await runner.start(blockRef, instanceId);
+        //emit stdout/stderr via sockets
+        process.output.on("data", (data) => {
+            const payload = {source: "stdout", level: "INFO", message: data.toString(), time: Date.now()};
+            this._emit(instanceId, EVENT_INSTANCE_LOG, payload);
+        });
+
+        process.output.on('exit', (exitCode) => {
+            const timeRunning = Date.now() - startTime;
+            const instance = this.getInstance(planRef, instanceId);
+            if (instance.status === STATUS_READY) {
+                //It's already been running
+                return;
+            }
+
+            if (exitCode === 143 ||
+                exitCode === 137) {
+                //Process got SIGTERM (143) or SIGKILL (137)
+                //TODO: Windows?
+                return;
+            }
+
+            if (exitCode !== 0 || timeRunning < MIN_TIME_RUNNING) {
+                this._emit(blockInstance.id, EVENT_INSTANCE_EXITED, {
+                    error: "Failed to start instance",
+                    status: EVENT_INSTANCE_EXITED,
+                    instanceId: blockInstance.id
+                })
+            }
+        });
+
+        await this.registerInstance(planRef, instanceId, {
+            type: process.type,
+            pid: process.pid,
+            health: null
+        });
+
+        return this._processes[planRef][instanceId] = process;
+    }
+
+    /**
+     *
+     * @param {string} planRef
+     * @param {string} instanceId
+     * @return {ProcessInfo|null}
+     */
+    getProcessForInstance(planRef, instanceId) {
+        if (!this._processes[planRef]) {
+            return null;
+        }
+
+        return this._processes[planRef][instanceId];
+    }
+
+    async stopProcess(planRef, instanceId) {
+        if (!this._processes[planRef]) {
+            return;
+        }
+
+        if (this._processes[planRef][instanceId]) {
+            await this._processes[planRef][instanceId].stop();
+            delete this._processes[planRef][instanceId];
+        }
+    }
+
+    async stopAllProcesses() {
+        for(let processesForPlan of Object.values(this._processes)) {
+            for(let processInfo of Object.values(processesForPlan)) {
+                await processInfo.stop();
+            }
+        }
+        this._processes = {};
+
+        for(let instance of this._instances) {
+            await this._stopInstance(instance);
+        }
+    }
+}
+
+
+const instanceManager = new InstanceManager();
+
+process.on('exit', async () => {
+    await instanceManager.stopAllProcesses();
+});
+
+module.exports = instanceManager;

package/src/instances/routes.js

@@ -0,0 +1,117 @@
+const Router = require('express-promise-router').default;
+
+const instanceManager = require('../instanceManager');
+
+
+const router = new Router();
+router.use('/', require('../middleware/cors'));
+
+/**
+ * Get all instances
+ */
+router.get('/', (req, res) => {
+    res.send(instanceManager.getInstances());
+});
+
+/**
+ * Get all instances
+ */
+router.get('/:systemId/instances', (req, res) => {
+    res.send(instanceManager.getInstancesForPlan(req.params.systemId));
+});
+
+/**
+ * Start all instances in a plan
+ */
+router.post('/:systemId/start', async (req, res) => {
+    const processes = await instanceManager.createProcessesForPlan(req.params.systemId);
+
+    res.status(202).send({
+        ok:true,
+        processes: processes.map(p => {
+            return {pid:p.pid, type:p.type};
+        })
+    });
+});
+
+/**
+ * Stop all instances in plan
+ */
+router.post('/:systemId/stop', async (req, res) => {
+    await instanceManager.stopAllForPlan(req.params.systemId);
+
+    res.status(202).send({
+        ok:true
+    });
+});
+
+/**
+ * Start single instance in a plan
+ */
+router.post('/:systemId/:instanceId/start', async (req, res) => {
+    const process = await instanceManager.createProcess(req.params.systemId, req.params.instanceId);
+
+    res.status(202).send({
+        ok:true,
+        pid: process.pid,
+        type: process.type
+    });
+});
+
+/**
+ * Stop single instance in a plan
+ */
+router.post('/:systemId/:instanceId/stop', async (req, res) => {
+    await instanceManager.stopProcess(req.params.systemId, req.params.instanceId);
+
+    res.status(202).send({ok:true});
+});
+
+
+/**
+ * Get logs for instance in a plan
+ */
+router.get('/:systemId/:instanceId/logs', (req, res) => {
+    const processInfo = instanceManager.getProcessForInstance(req.params.systemId, req.params.instanceId);
+    if (!processInfo) {
+        res.status(404).send({ok:false});
+        return;
+    }
+
+    res.status(202).send({
+        logs: processInfo.logs()
+    });
+});
+
+router.use('/', require('../middleware/stringBody'));
+
+
+router.use('/', require('../middleware/kapeta'));
+
+/**
+ * Updates the full configuration for a given service.
+ */
+router.put('/', async (req, res) => {
+
+    let instance = JSON.parse(req.stringBody);
+
+    await instanceManager.registerInstance(
+        req.kapeta.systemId,
+        req.kapeta.instanceId,
+        instance
+    );
+
+    res.status(202).send({ok:true});
+});
+
+/**
+ * Delete instance
+ */
+router.delete('/', async (req, res) => {
+    await instanceManager.setInstanceAsStopped(req.kapeta.systemId, req.kapeta.instanceId);
+
+    res.status(202).send({ok:true});
+});
+
+
+module.exports = router;

package/src/middleware/cors.js

@@ -0,0 +1,7 @@
+module.exports = function cors(req, res, next) {
+    res.set('Access-Control-Allow-Origin', req.headers.origin);
+    res.set('Access-Control-Allow-Methods', 'GET, PUT, POST, DELETE, HEAD, PATCH');
+    res.set('Access-Control-Allow-Headers', '*');
+
+    next();
+};

package/src/middleware/kapeta.js

@@ -0,0 +1,20 @@
+
+module.exports = function kapeta(req, res, next) {
+
+    let blockRef = req.headers['x-kapeta-block'];
+    let systemId = req.headers['x-kapeta-system'];
+    let instanceId = req.headers['x-kapeta-instance'];
+
+    if (!blockRef) {
+        res.status(400).send({error: 'Missing X-Kapeta-Block header.'});
+        return;
+    }
+
+    req.kapeta = {
+        blockRef,
+        instanceId,
+        systemId
+    };
+
+    next();
+};

package/src/middleware/stringBody.js

@@ -0,0 +1,11 @@
+module.exports = function stringBody (req, res, next) {
+
+    // push the data to body
+    const body = [];
+    req.on('data', (chunk) => {
+        body.push(chunk);
+    }).on('end', () => {
+        req.stringBody = Buffer.concat(body).toString();
+        next();
+    });
+};

package/src/networkManager.js

@@ -0,0 +1,120 @@
+const uuid = require('node-uuid');
+class NetworkManager {
+
+    static toConnectionId(connection) {
+        return [
+            connection.from.blockId,
+            connection.from.resourceName,
+            connection.to.blockId,
+            connection.to.resourceName
+        ].join('_');
+    }
+
+    constructor() {
+        this._connections = {};
+        this._sources = {};
+        this._targets = {};
+    }
+
+    _ensureSystem(systemId) {
+        if (!this._connections[systemId]) {
+            this._connections[systemId] = {};
+        }
+
+        if (!this._sources[systemId]) {
+            this._sources[systemId] = {};
+        }
+
+        if (!this._targets[systemId]) {
+            this._targets[systemId] = {};
+        }
+    }
+
+    _ensureConnection(systemId, connectionId) {
+        this._ensureSystem(systemId);
+
+        if (!this._connections[systemId][connectionId]) {
+            this._connections[systemId][connectionId] = [];
+        }
+
+        return this._connections[systemId][connectionId];
+    }
+
+    _ensureSource(systemId, sourceBlockInstanceId) {
+        this._ensureSystem(systemId);
+
+        if (!this._sources[systemId][sourceBlockInstanceId]) {
+            this._sources[systemId][sourceBlockInstanceId] = [];
+        }
+
+        return this._sources[systemId][sourceBlockInstanceId];
+    }
+
+    _ensureTarget(systemId, targetBlockInstanceId) {
+        this._ensureSystem(systemId);
+
+        if (!this._targets[systemId][targetBlockInstanceId]) {
+            this._targets[systemId][targetBlockInstanceId] = [];
+        }
+
+        return this._targets[systemId][targetBlockInstanceId];
+    }
+
+    addRequest(systemId, connection, request, consumerMethodId, providerMethodId) {
+
+        const traffic = new Traffic(connection, request, consumerMethodId, providerMethodId);
+
+        this._ensureConnection(systemId, traffic.connectionId).push(traffic);
+        this._ensureSource(systemId, connection.from.blockId).push(traffic);
+        this._ensureTarget(systemId, connection.to.blockId).push(traffic);
+
+        return traffic;
+    }
+
+    getTrafficForConnection(systemId, connectionId) {
+        return this._ensureConnection(systemId, connectionId);
+    }
+
+    getTrafficForSource(systemId, blockInstanceId) {
+
+        return this._ensureSource(systemId, blockInstanceId);
+    }
+
+    getTrafficForTarget(systemId, blockInstanceId) {
+        return this._ensureTarget(systemId, blockInstanceId);
+    }
+}
+
+
+class Traffic {
+
+    constructor(connection, request, consumerMethodId, providerMethodId) {
+        this.id = uuid.v4();
+        this.connectionId = NetworkManager.toConnectionId(connection);
+        this.consumerMethodId = consumerMethodId;
+        this.providerMethodId = providerMethodId;
+        this.request = request;
+        this.response = null;
+        this.error = null;
+        this.ended = null;
+        this.created = new Date().getTime();
+    }
+
+    asError(err) {
+        this.ended = new Date().getTime();
+        this.response = {
+            code: 0,
+            headers: {},
+            body: null
+        };
+        this.error = err + '';
+    }
+
+    withResponse(response) {
+        this.ended = new Date().getTime();
+        this.response = response;
+    }
+
+}
+
+module.exports = new NetworkManager();