@kapeta/local-cluster-service 0.6.0 → 0.7.0

package/src/instances/routes.ts

@@ -0,0 +1,164 @@
+import Router from 'express-promise-router';
+import { instanceManager } from '../instanceManager';
+import { serviceManager } from '../serviceManager';
+import { corsHandler } from '../middleware/cors';
+import { NextFunction, Request, Response } from 'express';
+import { kapetaHeaders, KapetaRequest } from '../middleware/kapeta';
+import { stringBody } from '../middleware/stringBody';
+import { EnvironmentType, KapetaBodyRequest } from '../types';
+
+const router = Router();
+router.use('/', corsHandler);
+router.use('/', kapetaHeaders);
+/**
+ * Get all instances
+ */
+router.get('/', (req: Request, res: Response) => {
+    res.send(instanceManager.getInstances());
+});
+
+/**
+ * Get all instances
+ */
+router.get('/:systemId/instances', (req: Request, res: Response) => {
+    res.send(instanceManager.getInstancesForPlan(req.params.systemId));
+});
+
+/**
+ * Start all instances in a plan
+ */
+router.post('/:systemId/start', async (req: Request, res: Response) => {
+    const processes = await instanceManager.createProcessesForPlan(req.params.systemId);
+
+    res.status(202).send({
+        ok: true,
+        processes: processes.map((p) => {
+            return { pid: p.pid, type: p.type };
+        }),
+    });
+});
+
+/**
+ * Stop all instances in plan
+ */
+router.post('/:systemId/stop', async (req: Request, res: Response) => {
+    await instanceManager.stopAllForPlan(req.params.systemId);
+
+    res.status(202).send({
+        ok: true,
+    });
+});
+
+/**
+ * Start single instance in a plan
+ */
+router.post('/:systemId/:instanceId/start', async (req: Request, res: Response) => {
+    const process = await instanceManager.createProcess(req.params.systemId, req.params.instanceId);
+
+    res.status(202).send({
+        ok: true,
+        pid: process.pid,
+        type: process.type,
+    });
+});
+
+/**
+ * Stop single instance in a plan
+ */
+router.post('/:systemId/:instanceId/stop', async (req: Request, res: Response) => {
+    await instanceManager.stopProcess(req.params.systemId, req.params.instanceId);
+
+    res.status(202).send({ ok: true });
+});
+
+/**
+ * Get logs for instance in a plan
+ */
+router.get('/:systemId/:instanceId/logs', (req: Request, res: Response) => {
+    const processInfo = instanceManager.getProcessForInstance(req.params.systemId, req.params.instanceId);
+    if (!processInfo) {
+        res.status(404).send({ ok: false });
+        return;
+    }
+
+    res.status(202).send({
+        logs: processInfo.logs(),
+    });
+});
+
+/**
+ * Get public address for instance in a plan if available
+ */
+router.get('/:systemId/:instanceId/address/public', (req: Request, res: Response) => {
+    const instance = instanceManager.getInstance(req.params.systemId, req.params.instanceId);
+    if (!instance) {
+        res.status(404).send({ ok: false });
+        return;
+    }
+
+    if (!instance.address) {
+        res.status(400).send({ error: `Instance does not have an address. Make sure it's running.` });
+        return;
+    }
+
+    res.status(200).send(instance.address);
+});
+
+/**
+ * Get public address for particular resource on instance in a plan if available
+ */
+router.get(
+    '/:systemId/:instanceId/provider/:portType/:resourceName/address/public',
+    (req: KapetaRequest, res: Response) => {
+        res.send(
+            serviceManager.getConsumerAddress(
+                req.params.systemId,
+                req.params.instanceId,
+                req.params.resourceName,
+                req.params.portType,
+                req.kapeta?.environment
+            )
+        );
+    }
+);
+
+router.use('/', stringBody);
+router.use('/', (req: KapetaBodyRequest, res: Response, next: NextFunction) => {
+    if (!req.kapeta!.blockRef) {
+        res.status(400).send({ error: 'Missing X-Kapeta-Block header.' });
+        return;
+    }
+    next();
+});
+
+/**
+ * Updates the full configuration for a given service.
+ */
+router.put('/', async (req: KapetaBodyRequest, res: Response) => {
+    let instance = req.stringBody ? JSON.parse(req.stringBody) : null;
+    if (req.kapeta!.environment === 'docker') {
+        //A bit hacky but we want to avoid overwriting the docker PID with a process PID
+        const oldInstance = instanceManager.getInstance(req.kapeta!.systemId, req.kapeta!.instanceId);
+        if (oldInstance) {
+            instance.pid = oldInstance.pid;
+        }
+        instance.type = 'docker';
+    } else if (req.kapeta!.environment === 'process') {
+        instance.type = 'process';
+    }
+
+    await instanceManager.registerInstance(req.kapeta!.systemId, req.kapeta!.instanceId, instance);
+
+    res.status(202).send({ ok: true });
+});
+
+/**
+ * Delete instance
+ */
+router.delete('/', async (req: KapetaRequest, res: Response) => {
+    await instanceManager.setInstanceAsStopped(req.kapeta!.systemId, req.kapeta!.instanceId);
+
+    res.status(202).send({ ok: true });
+});
+
+export default router;

package/src/middleware/cors.ts

@@ -0,0 +1,9 @@
+import { NextFunction, Request, Response } from 'express';
+
+export function corsHandler(req: Request, res: Response, next: NextFunction) {
+    res.set('Access-Control-Allow-Origin', req.headers.origin);
+    res.set('Access-Control-Allow-Methods', 'GET, PUT, POST, DELETE, HEAD, PATCH');
+    res.set('Access-Control-Allow-Headers', '*');
+
+    next();
+}

package/src/middleware/kapeta.ts

@@ -0,0 +1,27 @@
+import { NextFunction, Request, Response } from 'express';
+import { EnvironmentType } from '../types';
+
+export interface KapetaRequest extends Request {
+    kapeta?: {
+        blockRef: string;
+        instanceId: string;
+        systemId: string;
+        environment?: EnvironmentType;
+    };
+}
+
+export function kapetaHeaders(req: KapetaRequest, res: Response, next: NextFunction) {
+    let blockRef: string = req.headers['x-kapeta-block'] as string;
+    let systemId: string = req.headers['x-kapeta-system'] as string;
+    let instanceId: string = req.headers['x-kapeta-instance'] as string;
+    let environment: string = req.headers['x-kapeta-environment'] as string;
+
+    req.kapeta = {
+        blockRef,
+        instanceId,
+        systemId,
+        environment: environment ? (environment as EnvironmentType) : undefined,
+    };
+
+    next();
+}

package/src/middleware/stringBody.ts

@@ -0,0 +1,16 @@
+import { NextFunction, Request, Response } from 'express';
+
+export type StringBodyRequest = Request<any> & {
+    stringBody?: string;
+};
+
+export function stringBody(req: StringBodyRequest, res: Response, next: NextFunction) {
+    // push the data to body
+    const body: Buffer[] = [];
+    req.on('data', (chunk) => {
+        body.push(chunk);
+    }).on('end', () => {
+        req.stringBody = Buffer.concat(body).toString();
+        next();
+    });
+}

package/src/networkManager.ts

@@ -0,0 +1,137 @@
+import uuid from 'node-uuid';
+import { Connection, SimpleRequest, SimpleResponse } from './types';
+import express from 'express';
+
+class NetworkManager {
+    private _connections: { [systemId: string]: { [connectionId: string]: Traffic[] } };
+    private _sources: { [systemId: string]: { [instanceId: string]: Traffic[] } };
+    private _targets: { [systemId: string]: { [instanceId: string]: Traffic[] } };
+
+    static toConnectionId(connection: Connection) {
+        return [
+            connection.provider.blockId,
+            connection.provider.resourceName,
+            connection.consumer.blockId,
+            connection.consumer.resourceName,
+        ].join('_');
+    }
+
+    constructor() {
+        this._connections = {};
+        this._sources = {};
+        this._targets = {};
+    }
+
+    _ensureSystem(systemId: string) {
+        if (!this._connections[systemId]) {
+            this._connections[systemId] = {};
+        }
+
+        if (!this._sources[systemId]) {
+            this._sources[systemId] = {};
+        }
+
+        if (!this._targets[systemId]) {
+            this._targets[systemId] = {};
+        }
+    }
+
+    _ensureConnection(systemId: string, connectionId: string) {
+        this._ensureSystem(systemId);
+
+        if (!this._connections[systemId][connectionId]) {
+            this._connections[systemId][connectionId] = [];
+        }
+
+        return this._connections[systemId][connectionId];
+    }
+
+    _ensureSource(systemId: string, sourceBlockInstanceId: string) {
+        this._ensureSystem(systemId);
+
+        if (!this._sources[systemId][sourceBlockInstanceId]) {
+            this._sources[systemId][sourceBlockInstanceId] = [];
+        }
+
+        return this._sources[systemId][sourceBlockInstanceId];
+    }
+
+    _ensureTarget(systemId: string, targetBlockInstanceId: string) {
+        this._ensureSystem(systemId);
+
+        if (!this._targets[systemId][targetBlockInstanceId]) {
+            this._targets[systemId][targetBlockInstanceId] = [];
+        }
+
+        return this._targets[systemId][targetBlockInstanceId];
+    }
+
+    addRequest(
+        systemId: string,
+        connection: Connection,
+        request: SimpleRequest,
+        consumerMethodId?: string,
+        providerMethodId?: string
+    ) {
+        const traffic = new Traffic(connection, request, consumerMethodId, providerMethodId);
+
+        this._ensureConnection(systemId, traffic.connectionId).push(traffic);
+        this._ensureSource(systemId, connection.provider.blockId).push(traffic);
+        this._ensureTarget(systemId, connection.consumer.blockId).push(traffic);
+
+        return traffic;
+    }
+
+    getTrafficForConnection(systemId: string, connectionId: string) {
+        return this._ensureConnection(systemId, connectionId);
+    }
+
+    getTrafficForSource(systemId: string, blockInstanceId: string) {
+        return this._ensureSource(systemId, blockInstanceId);
+    }
+
+    getTrafficForTarget(systemId: string, blockInstanceId: string) {
+        return this._ensureTarget(systemId, blockInstanceId);
+    }
+}
+
+class Traffic {
+    public readonly id: string;
+    public readonly connectionId: string;
+    public readonly consumerMethodId: string | undefined;
+    public readonly providerMethodId: string | undefined;
+    public readonly created: number;
+    public readonly request: SimpleRequest;
+    public ended: null | number;
+    public error: null | string;
+    public response: SimpleResponse | null;
+
+    constructor(connection: Connection, request: SimpleRequest, consumerMethodId?: string, providerMethodId?: string) {
+        this.id = uuid.v4();
+        this.connectionId = NetworkManager.toConnectionId(connection);
+        this.consumerMethodId = consumerMethodId;
+        this.providerMethodId = providerMethodId;
+        this.request = request;
+        this.response = null;
+        this.error = null;
+        this.ended = null;
+        this.created = new Date().getTime();
+    }
+
+    asError(err: any) {
+        this.ended = new Date().getTime();
+        this.response = {
+            code: 0,
+            headers: {},
+            body: null,
+        };
+        this.error = err + '';
+    }
+
+    withResponse(response: SimpleResponse) {
+        this.ended = new Date().getTime();
+        this.response = response;
+    }
+}
+
+export const networkManager = new NetworkManager();

package/src/operatorManager.ts

@@ -0,0 +1,221 @@
+import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
+import _ from 'lodash';
+import Path from 'path';
+import md5 from 'md5';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import { serviceManager } from './serviceManager';
+import { storageService } from './storageService';
+import { ContainerInfo, containerManager } from './containerManager';
+import FSExtra from 'fs-extra';
+import { AnyMap, EnvironmentType, OperatorInfo } from './types';
+import { BlockInstance, BlockResource, Resource } from '@kapeta/schemas';
+
+const KIND_OPERATOR = 'core/resource-type-operator';
+
+class Operator {
+    private _data: any;
+    constructor(data: any) {
+        this._data = data;
+    }
+
+    getData() {
+        return this._data;
+    }
+
+    getCredentials() {
+        return this._data.credentials;
+    }
+}
+
+class OperatorManager {
+    private _mountDir: string;
+
+    constructor() {
+        this._mountDir = Path.join(storageService.getKapetaBasedir(), 'mounts');
+
+        FSExtra.mkdirpSync(this._mountDir);
+    }
+
+    _getMountPoint(operatorType: string, mountName: string) {
+        return Path.join(this._mountDir, operatorType, mountName);
+    }
+
+    /**
+     * Get operator definition for resource type
+     *
+     * @param {string} resourceType
+     * @param {string} version
+     * @return {Operator}
+     */
+    getOperator(resourceType: string, version: string) {
+        const operators = ClusterConfiguration.getDefinitions(KIND_OPERATOR);
+
+        const operator: DefinitionInfo | undefined = operators.find(
+            (operator) =>
+                operator.definition &&
+                operator.definition.metadata &&
+                operator.definition.metadata.name &&
+                operator.definition.metadata.name.toLowerCase() === resourceType.toLowerCase() &&
+                operator.version === version
+        );
+
+        if (!operator) {
+            throw new Error(`Unknown resource type: ${resourceType}:${version}`);
+        }
+
+        if (!operator.definition.spec || !operator.definition.spec.local) {
+            throw new Error(`Operator missing local definition: ${resourceType}:${version}`);
+        }
+
+        return new Operator(operator.definition.spec.local);
+    }
+
+    /**
+     * Get information about a specific consumed resource
+     */
+    async getConsumerResourceInfo(
+        systemId: string,
+        fromServiceId: string,
+        resourceType: string,
+        portType: string,
+        name: string,
+        environment?: EnvironmentType
+    ): Promise<OperatorInfo> {
+        const plans = ClusterConfiguration.getDefinitions('core/plan');
+
+        const planUri = parseKapetaUri(systemId);
+        const currentPlan = plans.find(
+            (plan) => plan.definition.metadata.name === planUri.fullName && plan.version === planUri.version
+        );
+        if (!currentPlan) {
+            throw new Error(`Unknown plan: ${systemId}`);
+        }
+
+        const currentInstance = currentPlan.definition.spec.blocks?.find(
+            (instance: BlockInstance) => instance.id === fromServiceId
+        );
+        if (!currentInstance) {
+            throw new Error(`Unknown instance: ${fromServiceId} in plan ${systemId}`);
+        }
+
+        const blockUri = parseKapetaUri(currentInstance.block.ref);
+        const blockDefinition = ClusterConfiguration.getDefinitions().find(
+            (definition) =>
+                definition.version === blockUri.version && definition.definition.metadata.name === blockUri.fullName
+        );
+
+        if (!blockDefinition) {
+            throw new Error(`Unknown block: ${currentInstance.block.ref} in plan ${systemId}`);
+        }
+
+        const blockResource = blockDefinition.definition.spec?.consumers?.find(
+            (resource: Resource) => resource.metadata.name === name
+        );
+        if (!blockResource) {
+            throw new Error(`Unknown resource: ${name} in block ${currentInstance.block.ref} in plan ${systemId}`);
+        }
+
+        const kindUri = parseKapetaUri(blockResource.kind);
+        const operator = this.getOperator(resourceType, kindUri.version);
+        const credentials = operator.getCredentials();
+        const container = await this.ensureResource(systemId, resourceType, kindUri.version);
+        const portInfo = await container.getPort(portType);
+
+        if (!portInfo) {
+            throw new Error('Unknown resource port type : ' + resourceType + '#' + portType);
+        }
+
+        const dbName = name + '_' + fromServiceId.replace(/[^a-z0-9]/gi, '');
+
+        return {
+            host: environment === 'docker' ? 'host.docker.internal' : '127.0.0.1',
+            port: portInfo.hostPort,
+            type: portType,
+            protocol: portInfo.protocol,
+            options: {
+                dbName,
+            },
+            credentials,
+        };
+    }
+
+    /**
+     * Ensure we have a running operator of given type
+     *
+     * @param {string} systemId
+     * @param {string} resourceType
+     * @param {string} version
+     * @return {Promise<ContainerInfo>}
+     */
+    async ensureResource(systemId: string, resourceType: string, version: string): Promise<ContainerInfo> {
+        const operator = this.getOperator(resourceType, version);
+
+        const operatorData = operator.getData();
+
+        const portTypes = Object.keys(operatorData.ports);
+
+        portTypes.sort();
+
+        const containerBaseName = 'kapeta-resource';
+
+        const nameParts = [resourceType.toLowerCase()];
+
+        const ports: AnyMap = {};
+
+        for (let i = 0; i < portTypes.length; i++) {
+            const portType = portTypes[i];
+            let containerPortInfo = operatorData.ports[portType];
+            const hostPort = await serviceManager.ensureServicePort(resourceType, portType);
+
+            if (typeof containerPortInfo === 'number' || typeof containerPortInfo === 'string') {
+                containerPortInfo = { port: containerPortInfo, type: 'tcp' };
+            }
+
+            if (!containerPortInfo.type) {
+                containerPortInfo.type = 'tcp';
+            }
+
+            const portId = containerPortInfo.port + '/' + containerPortInfo.type;
+            nameParts.push(portType + '-' + portId + '-' + hostPort);
+
+            ports[portId] = {
+                type: portType,
+                hostPort,
+            };
+        }
+
+        const mounts = containerManager.createMounts(resourceType, operatorData.mounts);
+
+        const containerName = containerBaseName + '-' + md5(nameParts.join('_'));
+        let container = await containerManager.get(containerName);
+
+        const isRunning = container ? await container.isRunning() : false;
+        if (container && !isRunning) {
+            await container.start();
+        }
+
+        if (!container) {
+            container = await containerManager.run(operatorData.image, containerName, {
+                mounts,
+                ports,
+                health: operatorData.health,
+                env: operatorData.env,
+                cmd: operatorData.cmd,
+            });
+        }
+
+        try {
+            if (operatorData.health) {
+                await containerManager.waitForHealthy(container.native);
+            } else {
+                await containerManager.waitForReady(container.native);
+            }
+        } catch (e: any) {
+            console.error(e.message);
+        }
+
+        return container;
+    }
+}
+
+export const operatorManager = new OperatorManager();

package/src/progressListener.ts

@@ -0,0 +1,102 @@
+import { spawn } from 'child_process';
+import { SocketManager, socketManager } from './socketManager';
+class ProgressListener {
+    private socketManager: SocketManager;
+
+    constructor(socketManager: SocketManager) {
+        this.socketManager = socketManager;
+    }
+
+    run(command: string, directory?: string): Promise<{ exit: number; signal: NodeJS.Signals | null }> {
+        this.socketManager.emit(`install`, 'install:log', {
+            type: 'info',
+            message: `Running command "${command}"`,
+        });
+
+        return new Promise((resolve, reject) => {
+            const child = spawn(command, {
+                cwd: directory ? directory : process.cwd(),
+                detached: true,
+                shell: true,
+            });
+
+            child.stdout.on('data', (data) => {
+                this.socketManager.emit(`install`, 'install:log', { type: 'info', message: data.toString() });
+            });
+
+            child.stderr.on('data', (data) => {
+                this.socketManager.emit(`install`, 'install:log', { type: 'info', message: data.toString() });
+            });
+
+            child.on('exit', (exit, signal) => {
+                if (exit !== 0) {
+                    this.socketManager.emit(`install`, 'install:log', {
+                        type: 'info',
+                        message: `"${command}" failed: "${exit}"`,
+                    });
+                    reject(new Error(`Command "${command}" exited with code ${exit}`));
+                } else {
+                    this.socketManager.emit(`install`, 'install:log', {
+                        type: 'info',
+                        message: `Command OK: "${command}"`,
+                    });
+                    resolve({ exit, signal });
+                }
+            });
+
+            child.on('error', (err) => {
+                this.socketManager.emit(`install`, 'install:log', {
+                    type: 'info',
+                    message: `"${command}" failed: "${err.message}"`,
+                });
+                reject(err);
+            });
+        });
+    }
+
+    async progress(label: string, callback: () => void | Promise<void>) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'info', message: `${label}: started` });
+        try {
+            const result = await callback();
+            this.socketManager.emit(`install`, 'install:log', { type: 'info', message: `${label}: done` });
+            return result;
+        } catch (e: any) {
+            this.socketManager.emit(`install`, 'install:log', {
+                type: 'info',
+                message: `${label}: failed. ${e.message}`,
+            });
+            throw e;
+        }
+    }
+
+    async check(message: string, ok: boolean | Promise<boolean> | (() => Promise<boolean>)) {
+        const wasOk = await ok;
+        this.socketManager.emit(`install`, 'install:log', { type: 'info', message: `${message}: ${wasOk}` });
+    }
+
+    start(label: string) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'info', message: label });
+    }
+
+    showValue(label: string, value: any) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'info', message: `${label}: ${value}` });
+    }
+
+    error(msg: string, ...args: any[]) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'error', message: msg });
+    }
+
+    warn(msg: string, ...args: any[]) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'warn', message: msg });
+    }
+
+    info(msg: string, ...args: any[]) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'info', message: msg });
+    }
+
+    debug(msg: string, ...args: any[]) {
+        this.socketManager.emit(`install`, 'install:log', { type: 'debug', message: msg });
+    }
+}
+
+export const progressListener = new ProgressListener(socketManager);