@kapeta/local-cluster-service 0.0.0-96f91ef

package/src/operatorManager.ts

@@ -0,0 +1,294 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { DefinitionInfo } from '@kapeta/local-cluster-config';
+import Path from 'path';
+import md5 from 'md5';
+import { serviceManager } from './serviceManager';
+import { storageService } from './storageService';
+import {
+    COMPOSE_LABEL_PROJECT,
+    COMPOSE_LABEL_SERVICE,
+    CONTAINER_LABEL_PORT_PREFIX,
+    ContainerInfo,
+    containerManager,
+} from './containerManager';
+import FSExtra from 'fs-extra';
+import { AnyMap, EnvironmentType, KIND_BLOCK_TYPE_OPERATOR, KIND_RESOURCE_OPERATOR, StringMap } from './types';
+import { BlockInstance, LocalInstance, Resource } from '@kapeta/schemas';
+import { definitionsManager } from './definitionsManager';
+import { getDockerHostIp, getRemoteHostForEnvironment, toPortInfo } from './utils/utils';
+import { parseKapetaUri, normalizeKapetaUri } from '@kapeta/nodejs-utils';
+import _ from 'lodash';
+import AsyncLock from 'async-lock';
+import { taskManager } from './taskManager';
+import { ResourceInfo } from '@kapeta/sdk-config';
+
+const KIND_PLAN = 'core/plan';
+
+class Operator {
+    private readonly _data: DefinitionInfo;
+
+    constructor(data: DefinitionInfo) {
+        this._data = data;
+    }
+
+    getLocalData(): LocalInstance {
+        return this._data.definition.spec.local;
+    }
+
+    getDefinitionInfo() {
+        return this._data;
+    }
+
+    getCredentials() {
+        return this._data.definition.spec.local.credentials;
+    }
+}
+
+class OperatorManager {
+    private _mountDir: string;
+
+    private operatorLock: AsyncLock = new AsyncLock();
+
+    constructor() {
+        this._mountDir = Path.join(storageService.getKapetaBasedir(), 'mounts');
+
+        FSExtra.mkdirpSync(this._mountDir);
+    }
+
+    _getMountPoint(operatorType: string, mountName: string) {
+        return Path.join(this._mountDir, operatorType, mountName);
+    }
+
+    /**
+     * Get operator definition for resource type
+     */
+    async getOperator(fullName: string, version: string) {
+        const operators = await definitionsManager.getDefinitions([KIND_RESOURCE_OPERATOR, KIND_BLOCK_TYPE_OPERATOR]);
+
+        const operator: DefinitionInfo | undefined = operators.find(
+            (operator) =>
+                operator.definition &&
+                operator.definition.metadata &&
+                operator.definition.metadata.name &&
+                operator.definition.metadata.name.toLowerCase() === fullName.toLowerCase() &&
+                operator.version === version
+        );
+
+        if (!operator) {
+            throw new Error(`Unknown operator type: ${fullName}:${version}`);
+        }
+
+        if (!operator.definition.spec || !operator.definition.spec.local) {
+            throw new Error(`Operator missing local definition: ${fullName}:${version}`);
+        }
+
+        return new Operator(operator);
+    }
+
+    /**
+     * Get information about a specific consumed resource
+     */
+    async getConsumerResourceInfo(
+        systemId: string,
+        fromServiceId: string,
+        resourceType: string,
+        portType: string,
+        name: string,
+        environment?: EnvironmentType,
+        ensureContainer: boolean = true
+    ): Promise<ResourceInfo<any, any>> {
+        systemId = normalizeKapetaUri(systemId);
+        const plans = await definitionsManager.getDefinitions(KIND_PLAN);
+
+        const planUri = parseKapetaUri(systemId);
+        const currentPlan = plans.find(
+            (plan) => plan.definition.metadata.name === planUri.fullName && plan.version === planUri.version
+        );
+        if (!currentPlan) {
+            throw new Error(`Unknown plan: ${systemId}`);
+        }
+
+        const currentInstance = currentPlan.definition.spec.blocks?.find(
+            (instance: BlockInstance) => instance.id === fromServiceId
+        );
+        if (!currentInstance) {
+            throw new Error(`Unknown instance: ${fromServiceId} in plan ${systemId}`);
+        }
+
+        const blockDefinition = await definitionsManager.getDefinition(currentInstance.block.ref);
+
+        if (!blockDefinition) {
+            throw new Error(`Unknown block: ${currentInstance.block.ref} in plan ${systemId}`);
+        }
+
+        const blockResource = blockDefinition.definition.spec?.consumers?.find((resource: Resource) => {
+            if (resource.metadata.name !== name) {
+                return false;
+            }
+            return parseKapetaUri(resource.kind).fullName === resourceType;
+        });
+
+        if (!blockResource) {
+            throw new Error(`Unknown resource: ${name} in block ${currentInstance.block.ref} in plan ${systemId}`);
+        }
+
+        const kindUri = parseKapetaUri(blockResource.kind);
+        const operator = await this.getOperator(resourceType, kindUri.version);
+        const credentials = operator.getCredentials();
+        if (ensureContainer) {
+            await this.ensureOperator(systemId, resourceType, kindUri.version);
+        }
+
+        const hostPort = await serviceManager.ensureServicePort(systemId, resourceType, portType);
+
+        if (!hostPort) {
+            throw new Error('Unknown resource port type : ' + resourceType + '#' + portType);
+        }
+
+        const dbName = name + '_' + fromServiceId.replace(/[^a-z0-9]/gi, '');
+        const safeName = dbName.replace('_', '-');
+
+        return {
+            host: getRemoteHostForEnvironment(environment),
+            port: hostPort,
+            type: portType,
+            protocol: 'tcp',
+            options: {
+                // expose as fullName since that is not operator specific, but unique
+                fullName: safeName,
+                dbName,
+            },
+            credentials,
+        };
+    }
+
+    async getOperatorPorts(systemId: string, kind: string, version: string) {
+        const operator = await this.getOperator(kind, version);
+
+        const operatorData = operator.getLocalData();
+
+        const portTypes = Object.keys(operatorData.ports);
+
+        portTypes.sort();
+
+        const ports: AnyMap = {};
+
+        for (let i = 0; i < portTypes.length; i++) {
+            const portType = portTypes[i];
+            let containerPortInfo = operatorData.ports[portType];
+            const hostPort = await serviceManager.ensureServicePort(systemId, kind, portType);
+            const portInfo = toPortInfo(containerPortInfo);
+            const portId = portInfo.port + '/' + portInfo.type;
+
+            ports[portId] = {
+                type: portType,
+                hostPort,
+                protocol: portInfo.type,
+            };
+        }
+
+        return ports;
+    }
+
+    /**
+     * Ensure we have a running operator of given type
+     *
+     * @param systemId the plan ref
+     * @param kind the full name - e.g. myhandle/rabbitmq
+     * @param version the version of the operator
+     */
+    async ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo> {
+        systemId = normalizeKapetaUri(systemId);
+
+        const key = `${systemId}#${kind}:${version}`;
+
+        return await this.operatorLock.acquire(key, async () => {
+            const operator = await this.getOperator(kind, version);
+
+            const operatorData = operator.getLocalData();
+
+            const ports = await this.getOperatorPorts(systemId, kind, version);
+
+            const nameParts = [systemId, kind.toLowerCase(), version];
+
+            const containerName = `kapeta-resource-${md5(nameParts.join('_'))}`;
+
+            const PortBindings: { [key: string]: any } = {};
+            const Env: string[] = [];
+
+            const systemUri = parseKapetaUri(systemId);
+
+            const Labels: StringMap = {
+                kapeta: 'true',
+                [COMPOSE_LABEL_PROJECT]: systemUri.id.replace(/[^a-z0-9]/gi, '_'),
+                [COMPOSE_LABEL_SERVICE]: [kind, version].join('_').replace(/[^a-z0-9]/gi, '_'),
+            };
+
+            const operatorMetadata = operator.getDefinitionInfo().definition.metadata;
+
+            const hostIp = getDockerHostIp();
+
+            const ExposedPorts: { [key: string]: any } = {};
+
+            _.forEach(ports, (portInfo: any, containerPort) => {
+                ExposedPorts['' + containerPort] = {};
+                PortBindings['' + containerPort] = [
+                    {
+                        HostPort: '' + portInfo.hostPort,
+                        HostIp: hostIp,
+                    },
+                ];
+
+                Labels[CONTAINER_LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
+            });
+
+            const Mounts = await containerManager.createVolumes(systemId, kind, operatorData.mounts);
+
+            _.forEach(operatorData.env, (value, name) => {
+                Env.push(name + '=' + value);
+            });
+
+            const task = taskManager.add(
+                `operator:ensure:${key}`,
+                async () => {
+                    let HealthCheck = undefined;
+
+                    if (operatorData.health) {
+                        HealthCheck = containerManager.toDockerHealth(operatorData.health);
+                    }
+
+                    const container = await containerManager.ensureContainer({
+                        name: containerName,
+                        Image: operatorData.image,
+                        Hostname: containerName + '.kapeta',
+                        Labels,
+                        Cmd: operatorData.cmd,
+                        ExposedPorts,
+                        Env,
+                        HealthCheck,
+                        HostConfig: {
+                            PortBindings,
+                            Mounts,
+                        },
+                    });
+
+                    await containerManager.waitForReady(container);
+
+                    return new ContainerInfo(container);
+                },
+                {
+                    name: `Ensuring ${operatorMetadata.title ?? operatorMetadata.name}`,
+                    systemId,
+                }
+            );
+
+            return task.wait();
+        });
+    }
+}
+
+export const operatorManager = new OperatorManager();

package/src/progressListener.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { spawn } from '@kapeta/nodejs-process';
+import { socketManager } from './socketManager';
+import { LogEntry } from './types';
+import { format } from 'node:util';
+import { Task } from './taskManager';
+
+export class ProgressListener {
+    private readonly systemId: string | undefined;
+    private readonly instanceId: string | undefined;
+
+    constructor(systemId?: string, instanceId?: string) {
+        this.systemId = systemId;
+        this.instanceId = instanceId;
+    }
+
+    protected emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
+        const logEntry: LogEntry = {
+            ...payload,
+            source: 'stdout',
+            time: Date.now(),
+        };
+        if (this.systemId && this.instanceId) {
+            socketManager.emitInstanceLog(this.systemId, this.instanceId, logEntry);
+            return;
+        }
+
+        if (this.systemId) {
+            socketManager.emitSystemLog(this.systemId, logEntry);
+            return;
+        }
+
+        socketManager.emitGlobalLog(logEntry);
+    }
+
+    run(command: string, directory?: string): Promise<{ exit: number; signal: NodeJS.Signals | null; output: string }> {
+        this.info(`Running command "${command}"`);
+
+        return new Promise(async (resolve, reject) => {
+            try {
+                const chunks: Buffer[] = [];
+                const child = spawn(command, [], {
+                    cwd: directory ? directory : process.cwd(),
+                    shell: true,
+                });
+
+                child.onData((data) => {
+                    this.emitLog({
+                        level: data.type === 'stdout' ? 'INFO' : 'WARN',
+                        message: data.line,
+                    });
+                });
+
+                if (child.process.stdout) {
+                    child.process.stdout.on('data', (data) => {
+                        chunks.push(data);
+                    });
+                }
+
+                child.process.on('exit', (exit, signal) => {
+                    if (exit !== 0) {
+                        this.warn(`Command "${command}" failed: ${exit}`);
+                        reject(new Error(`Command "${command}" exited with code ${exit}`));
+                    } else {
+                        this.info(`Command OK: "${command}"`);
+                        resolve({ exit, signal, output: Buffer.concat(chunks).toString() });
+                    }
+                });
+
+                child.process.on('error', (err) => {
+                    this.warn(`"${command}" failed: "${err.message}"`);
+                    reject(err);
+                });
+
+                await child.wait();
+            } catch (e) {
+                reject(e);
+            }
+        });
+    }
+
+    async progress(label: string, callback: () => void | Promise<void>) {
+        this.info(`${label}: started`);
+        try {
+            const result = await callback();
+            this.info(`${label}: done`);
+            return result;
+        } catch (e: any) {
+            this.warn(`${label}: failed. ${e.message}`);
+            throw e;
+        }
+    }
+
+    async check(message: string, ok: boolean | Promise<boolean> | (() => Promise<boolean>)) {
+        const wasOk = await ok;
+        this.info(`${message}: ${wasOk}`);
+    }
+
+    start(label: string) {
+        this.info(label);
+    }
+
+    showValue(label: string, value: any) {
+        this.info(`${label}: ${value}`);
+    }
+
+    error(msg: string, ...args: any[]) {
+        this.emitLog({
+            message: format(msg, args),
+            level: 'ERROR',
+        });
+    }
+
+    warn(msg: string, ...args: any[]) {
+        this.emitLog({
+            message: format(msg, args),
+            level: 'WARN',
+        });
+    }
+
+    info(msg: string, ...args: any[]) {
+        this.emitLog({
+            message: format(msg, args),
+            level: 'INFO',
+        });
+    }
+
+    debug(msg: string, ...args: any[]) {
+        this.emitLog({
+            message: format(msg, args),
+            level: 'DEBUG',
+        });
+    }
+}
+
+export class TaskProgressListener extends ProgressListener {
+    private readonly task: Task;
+
+    constructor(task: Task) {
+        super();
+        this.task = task;
+    }
+
+    protected emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
+        this.task.addLog(payload.message, payload.level);
+    }
+}

package/src/providerManager.ts

@@ -0,0 +1,97 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Path from 'path';
+import FSExtra from 'fs-extra';
+import { definitionsManager } from './definitionsManager';
+import { cacheManager } from './cacheManager';
+import request from 'request';
+import { DefinitionInfo } from '@kapeta/local-cluster-config';
+
+const PROVIDER_FILE_BASE = 'https://providers.kapeta.com/files';
+
+class ProviderManager {
+    async getWebProviders(): Promise<DefinitionInfo[]> {
+        const providers = await definitionsManager.getProviderDefinitions();
+        return providers.filter((providerDefinition) => providerDefinition.hasWeb);
+    }
+
+    async getProviderWebJS(handle: string, name: string, version: string, sourceMap: boolean = false) {
+        const fullName = `${handle}/${name}`;
+        const id = `${handle}/${name}/${version}/web.js${sourceMap ? '.map' : ''}`;
+
+        const cacheKey = `provider:web:${id}`;
+
+        const file = cacheManager.get<string>(cacheKey);
+        if (file && (await FSExtra.pathExists(file))) {
+            return FSExtra.readFile(file, 'utf8');
+        }
+
+        const providers = await this.getWebProviders();
+        const installedProvider = providers.find((providerDefinition) => {
+            return providerDefinition.definition.metadata.name === fullName && providerDefinition.version === version;
+        });
+
+        if (installedProvider) {
+            //Check locally installed providers
+            const path = Path.join(installedProvider.path, 'web', handle, `${name}.js${sourceMap ? '.map' : ''}`);
+            if (await FSExtra.pathExists(path)) {
+                cacheManager.set(cacheKey, path, 24 * 60 * 60 * 1000);
+                return FSExtra.readFile(path);
+            }
+        }
+
+        if (version === 'local') {
+            return null;
+        }
+
+        const url = `${PROVIDER_FILE_BASE}/${id}`;
+        return new Promise((resolve, reject) => {
+            console.log('Loading provider from %s', url);
+            request.get(url, (error, response, body) => {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                if (response.statusCode === 404) {
+                    resolve(null);
+                    return;
+                }
+
+                if (response.statusCode !== 200) {
+                    reject(new Error(`Failed to load provider from ${url}: ${body}`));
+                    return;
+                }
+
+                resolve(body);
+            });
+        });
+    }
+}
+
+definitionsManager
+    .getProviderDefinitions()
+    .then((providerDefinitions) => {
+        if (providerDefinitions.length > 0) {
+            console.log('## Loaded the following providers ##');
+            providerDefinitions.forEach((providerDefinition) => {
+                console.log(
+                    ' - %s[%s:%s]',
+                    providerDefinition.definition.kind,
+                    providerDefinition.definition.metadata.name,
+                    providerDefinition.version
+                );
+                console.log('   from %s', providerDefinition.path);
+            });
+            console.log('##');
+        } else {
+            console.log('## No providers found ##');
+        }
+    })
+    .catch((e) => {
+        console.error('Failed to load providers', e);
+    });
+
+export const providerManager = new ProviderManager();

package/src/providers/routes.ts

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { providerManager } from '../providerManager';
+
+import { corsHandler } from '../middleware/cors';
+import { Request, Response } from 'express';
+
+const router = Router();
+
+router.use('/', corsHandler);
+
+router.get('/', async (req: Request, res: Response) => {
+    const result = await providerManager.getWebProviders();
+
+    res.send(result);
+});
+
+router.get('/asset/:handle/:name/:version/web.js', async (req: Request, res: Response) => {
+    const { handle, name, version } = req.params;
+    let result = await providerManager.getProviderWebJS(handle, name, version);
+
+    if (!result) {
+        res.status(404).send('');
+    } else {
+        if (version !== 'local') {
+            res.setHeader('Cache-Control', 'max-age=31536000, immutable');
+        }
+        res.send(result.toString().replace(`${name}.js.map`, 'web.js.map'));
+    }
+});
+
+router.get('/asset/:handle/:name/:version/web.js.map', async (req: Request, res: Response) => {
+    const { handle, name, version } = req.params;
+    const result = await providerManager.getProviderWebJS(handle, name, version, true);
+
+    if (!result) {
+        res.status(404).send('');
+    } else {
+        // Only cache successful requests
+        if (version !== 'local') {
+            res.setHeader('Cache-Control', 'max-age=31536000, immutable');
+        }
+        res.send(result);
+    }
+});
+
+export default router;