@kapeta/local-cluster-service 0.6.0 → 0.7.0

package/src/configManager.ts

@@ -0,0 +1,148 @@
+import { EnrichedAsset } from './assetManager';
+import { BlockInstance } from '@kapeta/schemas';
+import { storageService } from './storageService';
+import { assetManager } from './assetManager';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+
+type AnyMap = { [key: string]: any };
+
+interface MatchedIdentity {
+    systemId: string;
+    instanceId: string;
+}
+
+class ConfigManager {
+    private _config: AnyMap;
+
+    constructor() {
+        this._config = storageService.section('config');
+    }
+
+    _forSystem(systemId: string) {
+        if (!this._config[systemId]) {
+            this._config[systemId] = {};
+        }
+
+        return this._config[systemId];
+    }
+
+    setConfigForSystem(systemId: string, config: AnyMap) {
+        const systemConfig = config || {};
+
+        storageService.put('config', systemId, systemConfig);
+    }
+
+    getConfigForSystem(systemId: string): AnyMap {
+        return this._forSystem(systemId);
+    }
+
+    setConfigForSection(systemId: string, sectionId: string, config: AnyMap) {
+        let systemConfig = this._forSystem(systemId);
+        systemConfig[sectionId] = config || {};
+
+        storageService.put('config', systemId, systemConfig);
+    }
+
+    getConfigForSection(systemId: string, sectionId: string) {
+        const systemConfig = this._forSystem(systemId);
+
+        if (!systemConfig[sectionId]) {
+            systemConfig[sectionId] = {};
+        }
+
+        return systemConfig[sectionId];
+    }
+
+    /**
+     * Try to identify the plan and instance in a plan automatically based on the block reference
+     *
+     * It will:
+     * 1. Go through all plans available in the assets
+     * 2. Look through each plan and see if the plan is referencing the block
+     * 3. If only 1 plan references the block - assume that as the system id
+     * 4. If only 1 instance in 1 plan references the block - assume that as instance id
+     *
+     * In case multiple uses of the same block reference we will prompt to user to choose which instance they want to
+     * use.
+     *
+     * @param blockRef block reference
+     * @param [systemId] plan reference
+     * @returns {Promise<{systemId:string,instanceId:string}>}
+     */
+    async resolveIdentity(blockRef: string, systemId?: string) {
+        const planAssets = assetManager.getPlans();
+
+        const blockUri = parseKapetaUri(blockRef);
+
+        let matchingIdentities: MatchedIdentity[] = [];
+        planAssets.forEach((planAsset: EnrichedAsset) => {
+            if (systemId && planAsset.ref !== systemId) {
+                //Skip plans that do not match systemid if provided
+                return;
+            }
+
+            if (!planAsset.data.spec.blocks) {
+                return;
+            }
+
+            planAsset.data.spec.blocks.forEach((blockInstance: BlockInstance) => {
+                const refUri = parseKapetaUri(blockInstance.block.ref);
+                if (refUri.equals(blockUri)) {
+                    matchingIdentities.push({
+                        systemId: planAsset.ref,
+                        instanceId: blockInstance.id,
+                    });
+                }
+            });
+        });
+
+        if (matchingIdentities.length === 0) {
+            if (systemId) {
+                throw new Error(`No uses of block "${blockRef}" was found in plan: "${systemId}"`);
+            }
+
+            throw new Error(`No uses of block "${blockRef}" was found in any known plan`);
+        }
+
+        if (matchingIdentities.length > 1) {
+            if (systemId) {
+                throw new Error(
+                    `Multiple uses of block "${blockRef}" was found in plan: "${systemId}". Please specify which instance in the plan you wish to run.`
+                );
+            }
+
+            throw new Error(
+                `Multiple uses of block "${blockRef}" was found in 1 or more plan. Please specify which instance in which plan you wish to run.`
+            );
+        }
+
+        return matchingIdentities[0];
+    }
+
+    async verifyIdentity(blockRef: string, systemId: string, instanceId: string) {
+        const planAssets = assetManager.getPlans();
+        const systemUri = systemId ? parseKapetaUri(systemId) : null;
+        const blockUri = parseKapetaUri(blockRef);
+        let found = false;
+        planAssets.forEach((planAsset: EnrichedAsset) => {
+            if (systemUri && !parseKapetaUri(planAsset.ref).equals(systemUri)) {
+                //Skip plans that do not match systemid if provided
+                return;
+            }
+
+            planAsset.data.spec.blocks.forEach((blockInstance: BlockInstance) => {
+                if (blockInstance.id === instanceId && parseKapetaUri(blockInstance.block.ref).equals(blockUri)) {
+                    found = true;
+                }
+            });
+        });
+
+        if (!found) {
+            throw new Error(
+                `Block "${blockRef}" was not found in plan: "${systemId}" using instance id ${instanceId}. Please verify that the provided information is accurate.`
+            );
+        }
+    }
+}
+
+export const configManager = new ConfigManager();

package/src/containerManager.ts

@@ -0,0 +1,466 @@
+import Path from 'path';
+import { storageService } from './storageService';
+import os from 'os';
+import _ from 'lodash';
+import FSExtra, { ReadStream } from 'fs-extra';
+import { Docker } from 'node-docker-api';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { Container } from 'node-docker-api/lib/container';
+
+type StringMap = { [key: string]: string };
+
+export type PortMap = {
+    [key: string]: {
+        containerPort: string;
+        protocol: string;
+        hostPort: string;
+    };
+};
+
+export interface DockerMounts {
+    Target: string;
+    Source: string;
+    Type: string;
+    ReadOnly: boolean;
+    Consistency: string;
+}
+
+interface Health {
+    cmd: string;
+    interval?: number;
+    timeout?: number;
+    retries?: number;
+}
+
+const LABEL_PORT_PREFIX = 'kapeta_port-';
+const NANO_SECOND = 1000000;
+const HEALTH_CHECK_INTERVAL = 2000;
+const HEALTH_CHECK_MAX = 30;
+const IMAGE_PULL_CACHE_TTL = 30 * 60 * 1000;
+const IMAGE_PULL_CACHE: { [key: string]: number } = {};
+
+const promisifyStream = (stream: ReadStream) =>
+    new Promise((resolve, reject) => {
+        stream.on('data', (d) => console.log(d.toString()));
+        stream.on('end', resolve);
+        stream.on('error', reject);
+    });
+
+class ContainerManager {
+    private _docker: Docker | null;
+    private _alive: boolean;
+    private _mountDir: string;
+
+    constructor() {
+        this._docker = null;
+        this._alive = false;
+        this._mountDir = Path.join(storageService.getKapetaBasedir(), 'mounts');
+        FSExtra.mkdirpSync(this._mountDir);
+    }
+
+    async initialize() {
+        // Use the value from cluster-service.yml if configured
+        const dockerConfig = ClusterConfiguration.getDockerConfig();
+        const connectOptions =
+            Object.keys(dockerConfig).length > 0
+                ? [dockerConfig]
+                : [
+                      // use defaults: DOCKER_HOST etc from env, if available
+                      undefined,
+                      // default linux
+                      { socketPath: '/var/run/docker.sock' },
+                      // default macOS
+                      {
+                          socketPath: Path.join(os.homedir(), '.docker/run/docker.sock'),
+                      },
+                      // Default http
+                      { protocol: 'http', host: 'localhost', port: 2375 },
+                      { protocol: 'https', host: 'localhost', port: 2376 },
+                      { protocol: 'http', host: '127.0.0.1', port: 2375 },
+                      { protocol: 'https', host: '127.0.0.1', port: 2376 },
+                  ];
+        for (const opts of connectOptions) {
+            try {
+                const client = new Docker(opts);
+                await client.ping();
+                this._docker = client;
+                this._alive = true;
+                return;
+            } catch (err) {
+                // silently ignore bad configs
+            }
+        }
+
+        throw new Error('Could not connect to docker daemon. Please make sure docker is running and working.');
+    }
+
+    isAlive() {
+        return this._alive;
+    }
+
+    getMountPoint(kind: string, mountName: string) {
+        const kindUri = parseKapetaUri(kind);
+        return Path.join(this._mountDir, kindUri.handle, kindUri.name, mountName);
+    }
+
+    createMounts(kind: string, mountOpts: StringMap): StringMap {
+        const mounts: StringMap = {};
+
+        _.forEach(mountOpts, (containerPath, mountName) => {
+            const hostPath = this.getMountPoint(kind, mountName);
+            FSExtra.mkdirpSync(hostPath);
+            mounts[containerPath] = hostPath;
+        });
+        return mounts;
+    }
+
+    async ping() {
+        try {
+            const pingResult = await this.docker().ping();
+            if (pingResult !== 'OK') {
+                throw new Error(`Ping failed: ${pingResult}`);
+            }
+        } catch (e: any) {
+            throw new Error(
+                `Docker not running. Please start the docker daemon before running this command. Error: ${e.message}`
+            );
+        }
+    }
+
+    docker() {
+        if (!this._docker) {
+            throw new Error(`Docker not running`);
+        }
+        return this._docker;
+    }
+
+    async getContainerByName(containerName: string): Promise<Container | undefined> {
+        const containers = await this.docker().container.list({ all: true });
+        return containers.find((container) => {
+            return (container.data as any).Names.indexOf(`/${containerName}`) > -1;
+        });
+    }
+
+    async pull(image: string, cacheForMS: number = IMAGE_PULL_CACHE_TTL) {
+        let [imageName, tag] = image.split(/:/);
+        if (!tag) {
+            tag = 'latest';
+        }
+
+        if (tag !== 'latest') {
+            if (IMAGE_PULL_CACHE[image]) {
+                const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
+                if (timeSince < cacheForMS) {
+                    return;
+                }
+            }
+
+            const imageTagList = (await this.docker().image.list())
+                .map((image) => image.data as any)
+                .filter((imageData) => !!imageData.RepoTags)
+                .map((imageData) => imageData.RepoTags as string[]);
+
+            if (imageTagList.some((imageTags) => imageTags.indexOf(image) > -1)) {
+                console.log('Image found: %s', image);
+                return;
+            }
+            console.log('Image not found: %s', image);
+        }
+
+        console.log('Pulling image: %s', image);
+        await this.docker()
+            .image.create(
+                {},
+                {
+                    fromImage: imageName,
+                    tag: tag,
+                }
+            )
+            .then((stream) => promisifyStream(stream as ReadStream));
+
+        IMAGE_PULL_CACHE[image] = Date.now();
+
+        console.log('Image pulled: %s', image);
+    }
+
+    toDockerMounts(mounts: StringMap) {
+        const Mounts: DockerMounts[] = [];
+        _.forEach(mounts, (Source, Target) => {
+            Mounts.push({
+                Target,
+                Source,
+                Type: 'bind',
+                ReadOnly: false,
+                Consistency: 'consistent',
+            });
+        });
+
+        return Mounts;
+    }
+
+    toDockerHealth(health: Health) {
+        return {
+            Test: ['CMD-SHELL', health.cmd],
+            Interval: health.interval ? health.interval * NANO_SECOND : 5000 * NANO_SECOND,
+            Timeout: health.timeout ? health.timeout * NANO_SECOND : 15000 * NANO_SECOND,
+            Retries: health.retries || 10,
+        };
+    }
+
+    async run(
+        image: string,
+        name: string,
+        opts: { ports: {}; mounts: {}; env: {}; cmd: string; health: Health }
+    ): Promise<ContainerInfo> {
+        const PortBindings: { [key: string]: any } = {};
+        const Env: string[] = [];
+        const Labels: StringMap = {
+            kapeta: 'true',
+        };
+
+        await this.pull(image);
+
+        const ExposedPorts: { [key: string]: any } = {};
+
+        _.forEach(opts.ports, (portInfo: any, containerPort) => {
+            ExposedPorts['' + containerPort] = {};
+            PortBindings['' + containerPort] = [
+                {
+                    HostPort: '' + portInfo.hostPort,
+                    HostIp: '127.0.0.1',
+                },
+            ];
+
+            Labels[LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
+        });
+
+        const Mounts = this.toDockerMounts(opts.mounts);
+
+        _.forEach(opts.env, (value, name) => {
+            Env.push(name + '=' + value);
+        });
+
+        let HealthCheck = undefined;
+
+        if (opts.health) {
+            HealthCheck = this.toDockerHealth(opts.health);
+        }
+        const dockerContainer = await this.startContainer({
+            name: name,
+            Image: image,
+            Hostname: name + '.kapeta',
+            Labels,
+            Cmd: opts.cmd,
+
+            ExposedPorts,
+            Env,
+            HealthCheck,
+            HostConfig: {
+                PortBindings,
+                Mounts,
+            },
+        });
+
+        if (opts.health) {
+            await this.waitForHealthy(dockerContainer);
+        }
+
+        return new ContainerInfo(dockerContainer);
+    }
+
+    async startContainer(opts: any) {
+        const dockerContainer = await this.docker().container.create(opts);
+        await dockerContainer.start();
+        return dockerContainer;
+    }
+
+    async waitForReady(container: Container, attempt: number = 0): Promise<void> {
+        if (!attempt) {
+            attempt = 0;
+        }
+
+        if (attempt >= HEALTH_CHECK_MAX) {
+            throw new Error('Container did not become ready within the timeout');
+        }
+
+        if (await this._isReady(container)) {
+            return;
+        }
+
+        return new Promise((resolve, reject) => {
+            setTimeout(async () => {
+                try {
+                    await this.waitForReady(container, attempt + 1);
+                    resolve();
+                } catch (err) {
+                    reject(err);
+                }
+            }, HEALTH_CHECK_INTERVAL);
+        });
+    }
+
+    async waitForHealthy(container: Container, attempt?: number): Promise<void> {
+        if (!attempt) {
+            attempt = 0;
+        }
+
+        if (attempt >= HEALTH_CHECK_MAX) {
+            throw new Error('Container did not become healthy within the timeout');
+        }
+
+        if (await this._isHealthy(container)) {
+            return;
+        }
+
+        return new Promise((resolve, reject) => {
+            setTimeout(async () => {
+                try {
+                    await this.waitForHealthy(container, (attempt ?? 0) + 1);
+                    resolve();
+                } catch (err) {
+                    reject(err);
+                }
+            }, HEALTH_CHECK_INTERVAL);
+        });
+    }
+
+    async _isReady(container: Container) {
+        const info: Container = await container.status();
+        const infoData: any = info?.data;
+        if (infoData?.State?.Status === 'exited') {
+            throw new Error('Container exited unexpectedly');
+        }
+        return infoData?.State?.Running ?? false;
+    }
+
+    async _isHealthy(container: Container) {
+        const info = await container.status();
+        const infoData: any = info?.data;
+        return infoData?.State?.Health?.Status === 'healthy';
+    }
+
+    /**
+     *
+     * @param name
+     * @return {Promise<ContainerInfo>}
+     */
+    async get(name: string): Promise<ContainerInfo | null> {
+        let dockerContainer = null;
+
+        try {
+            dockerContainer = await this.docker().container.get(name);
+            await dockerContainer.status();
+        } catch (err) {
+            //Ignore
+            dockerContainer = null;
+        }
+
+        if (!dockerContainer) {
+            return null;
+        }
+
+        return new ContainerInfo(dockerContainer);
+    }
+}
+
+export class ContainerInfo {
+    private readonly _container: Container;
+    /**
+     *
+     * @param {Container} dockerContainer
+     */
+    constructor(dockerContainer: Container) {
+        /**
+         *
+         * @type {Container}
+         * @private
+         */
+        this._container = dockerContainer;
+    }
+
+    get native() {
+        return this._container;
+    }
+
+    async isRunning() {
+        const inspectResult = await this.getStatus();
+
+        if (!inspectResult || !inspectResult.State) {
+            return false;
+        }
+
+        return inspectResult.State.Running || inspectResult.State.Restarting;
+    }
+
+    async start() {
+        await this._container.start();
+    }
+
+    async restart() {
+        await this._container.restart();
+    }
+
+    async stop() {
+        await this._container.stop();
+    }
+
+    async remove(opts?: { force?: boolean }) {
+        await this._container.delete({ force: !!opts?.force });
+    }
+
+    async getPort(type: string) {
+        const ports = await this.getPorts();
+
+        if (ports && ports[type]) {
+            return ports[type];
+        }
+
+        return null;
+    }
+
+    async getStatus() {
+        const result = await this._container.status();
+
+        return result ? (result.data as any) : null;
+    }
+
+    async getPorts(): Promise<PortMap | false> {
+        const inspectResult = await this.getStatus();
+
+        if (!inspectResult || !inspectResult.Config || !inspectResult.Config.Labels) {
+            return false;
+        }
+
+        const portTypes: StringMap = {};
+        const ports: PortMap = {};
+
+        _.forEach(inspectResult.Config.Labels, (portType, name) => {
+            if (!name.startsWith(LABEL_PORT_PREFIX)) {
+                return;
+            }
+
+            const hostPort = name.substr(LABEL_PORT_PREFIX.length);
+
+            portTypes[hostPort] = portType;
+        });
+
+        _.forEach(inspectResult.HostConfig.PortBindings, (portBindings, containerPortSpec) => {
+            let [containerPort, protocol] = containerPortSpec.split(/\//);
+
+            const hostPort = portBindings[0].HostPort;
+
+            const portType = portTypes[hostPort];
+
+            ports[portType] = {
+                containerPort,
+                protocol,
+                hostPort,
+            };
+        });
+
+        return ports;
+    }
+}
+
+export const containerManager = new ContainerManager();

package/src/filesystem/routes.ts

@@ -0,0 +1,74 @@
+import Router from 'express-promise-router';
+import { stringBody, StringBodyRequest } from '../middleware/stringBody';
+import { filesystemManager } from '../filesystemManager';
+import { corsHandler } from '../middleware/cors';
+import { NextFunction, Request, Response } from 'express';
+
+let router = Router();
+
+router.use('/', corsHandler);
+
+router.get('/root', (req: Request, res: Response) => {
+    res.send(filesystemManager.getRootFolder());
+});
+
+router.get('/project/root', (req: Request, res: Response) => {
+    res.send(filesystemManager.getProjectRootFolder());
+});
+
+router.use('/project/root', stringBody);
+
+router.post('/project/root', (req: StringBodyRequest, res: Response) => {
+    filesystemManager.setProjectRootFolder(req.stringBody ?? '');
+    res.sendStatus(204);
+});
+
+router.use('/', (req: Request, res: Response, next: NextFunction) => {
+    if (!req.query.path) {
+        res.status(400).send({ error: 'Missing required query parameter "path"' });
+        return;
+    }
+    next();
+});
+
+router.get('/list', async (req: Request, res: Response) => {
+    let pathArg = req.query.path as string;
+
+    try {
+        res.send(await filesystemManager.readDirectory(pathArg));
+    } catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
+
+router.get('/readfile', async (req: Request, res: Response) => {
+    let pathArg = req.query.path as string;
+    try {
+        res.send(await filesystemManager.readFile(pathArg));
+    } catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
+
+router.put('/mkdir', async (req: Request, res: Response) => {
+    let pathArg = req.query.path as string;
+    try {
+        await filesystemManager.createFolder(pathArg);
+        res.sendStatus(204);
+    } catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
+
+router.use('/writefile', stringBody);
+router.post('/writefile', async (req: StringBodyRequest, res: Response) => {
+    let pathArg = req.query.path as string;
+    try {
+        await filesystemManager.writeFile(pathArg, req.stringBody ?? '');
+        res.sendStatus(204);
+    } catch (err) {
+        res.status(400).send({ error: '' + err });
+    }
+});
+
+export default router;