@kapeta/local-cluster-service 0.0.0-96f91ef

package/src/proxy/routes.ts

@@ -0,0 +1,153 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { Request, Response } from 'express';
+import { Resource } from '@kapeta/schemas';
+import { proxyRestRequest } from './types/rest';
+import { proxyHttpRequest } from './types/web';
+import { ProxyRequestHandler } from '../types';
+import { stringBody, StringBodyRequest } from '../middleware/stringBody';
+import { serviceManager } from '../serviceManager';
+import { clusterService } from '../clusterService';
+import { assetManager } from '../assetManager';
+
+import _ from 'lodash';
+
+const router = Router();
+/**
+ * @var {{[key:string]:ProxyRequestHandler}}
+ */
+const TYPE_HANDLERS: { [p: string]: ProxyRequestHandler } = {
+    rest: proxyRestRequest,
+    http: proxyHttpRequest,
+};
+
+function getResource(resources: Resource[], resourceName: string) {
+    return resources.find((resource) => {
+        return resource.metadata.name.toLowerCase() === resourceName.toLowerCase();
+    });
+}
+
+router.use('/:systemId/:consumerInstanceId/:consumerResourceName', stringBody);
+
+router.all(
+    '/:systemId/:consumerInstanceId/:consumerResourceName/:type/*',
+    async (req: StringBodyRequest, res: Response) => {
+        try {
+            let typeHandler = TYPE_HANDLERS[req.params.type.toLowerCase()];
+            if (!typeHandler) {
+                // Default to http
+                typeHandler = TYPE_HANDLERS['http'];
+            }
+
+            const plan = await assetManager.getPlan(req.params.systemId);
+
+            // We can find the connection by the consumer information alone since
+            // only 1 provider can be connected to a consumer resource at a time
+            const connection = plan.spec.connections.find((connection) => {
+                return (
+                    connection.consumer.blockId.toLowerCase() === req.params.consumerInstanceId.toLowerCase() &&
+                    connection.consumer.resourceName.toLowerCase() === req.params.consumerResourceName.toLowerCase()
+                );
+            });
+
+            if (!connection) {
+                res.status(401).send({
+                    error: `No connection found for consumer "${req.params.consumerInstanceId}::${req.params.consumerResourceName}"`,
+                });
+                return;
+            }
+
+            const toBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
+                return blockInstance.id.toLowerCase() === connection.consumer.blockId.toLowerCase();
+            });
+
+            if (!toBlockInstance) {
+                res.status(401).send({ error: `Block instance not found "${req.params.consumerInstanceId}` });
+                return;
+            }
+
+            const toBlockAsset = await assetManager.getAsset(toBlockInstance.block.ref);
+
+            if (!toBlockAsset) {
+                res.status(401).send({ error: `Block asset not found "${toBlockInstance.block.ref}` });
+                return;
+            }
+
+            const consumerResource = getResource(toBlockAsset.data.spec.consumers, req.params.consumerResourceName);
+
+            if (!consumerResource) {
+                res.status(401).send({
+                    error: `Block resource not found "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`,
+                });
+                return;
+            }
+
+            const basePath = clusterService.getProxyPath(
+                req.params.systemId,
+                req.params.consumerInstanceId,
+                req.params.consumerResourceName,
+                req.params.type
+            );
+
+            const fromBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
+                return blockInstance.id.toLowerCase() === connection.provider.blockId.toLowerCase();
+            });
+
+            if (!fromBlockInstance) {
+                res.status(401).send({ error: `Block instance not found "${connection.provider.blockId}` });
+                return;
+            }
+
+            const fromBlockAsset = await assetManager.getAsset(fromBlockInstance.block.ref);
+
+            if (!fromBlockAsset) {
+                res.status(401).send({ error: `Block asset not found "${fromBlockInstance.block.ref}` });
+                return;
+            }
+
+            const providerResource = getResource(fromBlockAsset.data.spec.providers, connection.provider.resourceName);
+
+            if (!providerResource) {
+                res.status(401).send({
+                    error: `Block resource not found "${connection.provider.blockId}::${connection.provider.resourceName}`,
+                });
+                return;
+            }
+
+            //Get target address
+            let address = await serviceManager.getProviderAddress(
+                req.params.systemId,
+                connection.provider.blockId,
+                req.params.type
+            );
+
+            while (address.endsWith('/')) {
+                address = address.substring(0, address.length - 1);
+            }
+
+            /*
+         Get the path the consumer requested.
+         Note that this might not match the path the destination is expecting so we need to identify the method
+         that is being called and identify the destination path from the connection.
+         */
+            const consumerPath = req.originalUrl.substring(basePath.length - 1);
+
+            typeHandler(req, res, {
+                consumerPath,
+                address,
+                consumerResource,
+                providerResource,
+                connection,
+            });
+        } catch (err: any) {
+            console.warn('Failed to process proxy request', err);
+            res.status(400).send({ error: err.message });
+        }
+    }
+);
+
+export default router;

package/src/proxy/types/rest.ts

@@ -0,0 +1,172 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import _ from 'lodash';
+import request from 'request';
+import Path from 'path';
+import { pathTemplateParser } from '../../utils/pathTemplateParser';
+import { networkManager } from '../../networkManager';
+
+import { socketManager } from '../../socketManager';
+import { Request, Response } from 'express';
+import { ProxyRequestInfo, SimpleRequest, StringMap } from '../../types';
+import { StringBodyRequest } from '../../middleware/stringBody';
+import { Resource } from '@kapeta/schemas';
+import { stringify } from 'qs';
+
+export function getRestMethodId(restResource: Resource, httpMethod: string, httpPath: string) {
+    return _.findKey(restResource.spec.methods, (method) => {
+        let methodType = method.method ? method.method.toUpperCase() : 'GET';
+
+        if (methodType.toUpperCase() !== httpMethod.toUpperCase()) {
+            return false;
+        }
+
+        let path = method.path;
+
+        if (restResource.spec.basePath) {
+            path = Path.join(restResource.spec.basePath, path);
+        }
+
+        const pathTemplate = pathTemplateParser(path);
+
+        return pathTemplate.matches(httpPath);
+    });
+}
+
+/**
+ *
+ * @param req {Request}
+ * @param opts {ProxyRequestInfo}
+ * @return {{consumerMethod: *, providerMethod: *}}
+ */
+function resolveMethods(req: Request, opts: ProxyRequestInfo) {
+    const consumerMethodId = getRestMethodId(opts.consumerResource, req.method, opts.consumerPath);
+
+    if (!consumerMethodId) {
+        throw new Error(
+            `Consumer method not found for path "${req.method} ${opts.consumerPath}" in resource "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`
+        );
+    }
+
+    const consumerMethod = _.cloneDeep(opts.consumerResource.spec.methods[consumerMethodId]);
+
+    if (!consumerMethod) {
+        throw new Error(
+            `Consumer method not found for path "${req.method} ${opts.consumerPath}" in resource "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`
+        );
+    }
+
+    consumerMethod.id = consumerMethodId;
+
+    const providerMethodId = _.findKey(opts.connection.mapping, (mapping) => {
+        return mapping.targetId === consumerMethodId;
+    });
+
+    if (!providerMethodId) {
+        throw new Error(`Connection contained no mapping for consumer method "${consumerMethodId}`);
+    }
+
+    const providerMethod = _.cloneDeep(opts.providerResource.spec.methods[providerMethodId]);
+
+    if (!providerMethod) {
+        throw new Error(
+            `Provider method not found "${providerMethodId}" in resource "${opts.connection.provider.blockId}::${opts.connection.provider.resourceName}`
+        );
+    }
+
+    providerMethod.id = providerMethodId;
+
+    return {
+        consumerMethod,
+        providerMethod,
+    };
+}
+
+export function proxyRestRequest(req: StringBodyRequest, res: Response, opts: ProxyRequestInfo) {
+    let { consumerMethod, providerMethod } = resolveMethods(req, opts);
+
+    const consumerPathTemplate = pathTemplateParser(consumerMethod.path);
+    const providerPathTemplate = pathTemplateParser(providerMethod.path);
+
+    const pathVariables = consumerPathTemplate.parse(opts.consumerPath);
+    if (!pathVariables) {
+        res.status(400).send({
+            error: `Path did not match any patterns: "${opts.consumerPath}"`,
+        });
+        return;
+    }
+
+    let providerPath = providerPathTemplate.create(pathVariables);
+
+    if (!providerPath.startsWith('/')) {
+        providerPath = '/' + providerPath;
+    }
+
+    if (!_.isEmpty(req.query)) {
+        providerPath += '?' + stringify(req.query, { arrayFormat: 'repeat' });
+    }
+
+    const requestHeaders = _.clone(req.headers);
+
+    delete requestHeaders['content-length'];
+    delete requestHeaders['content-encoding'];
+    delete requestHeaders['connection'];
+    delete requestHeaders['host'];
+    delete requestHeaders['origin'];
+
+    console.log('Proxy request to provider: %s => %s [rest]', opts.consumerPath, opts.address + providerPath);
+
+    const reqOpts: SimpleRequest = {
+        method: providerMethod.method || 'GET',
+        url: opts.address + providerPath,
+        body: req.stringBody,
+        headers: requestHeaders as StringMap,
+    };
+
+    const traffic = networkManager.addRequest(
+        req.params.systemId,
+        opts.connection,
+        reqOpts,
+        consumerMethod.id,
+        providerMethod.id
+    );
+
+    socketManager.emit(traffic.connectionId, 'traffic_start', traffic);
+
+    request(reqOpts, function (err, response, responseBody) {
+        if (err) {
+            traffic.asError(err);
+            socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+
+            res.status(500).send({ error: '' + err });
+            return;
+        }
+
+        const responseHeaders = _.clone(response.headers);
+
+        delete responseHeaders['content-length'];
+        delete responseHeaders['content-encoding'];
+        delete responseHeaders['connection'];
+
+        res.set(responseHeaders);
+
+        res.status(response.statusCode);
+
+        traffic.withResponse({
+            code: response.statusCode,
+            headers: response.headers as StringMap,
+            body: responseBody,
+        });
+
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+
+        if (responseBody) {
+            res.write(responseBody);
+        }
+
+        res.end();
+    });
+}

package/src/proxy/types/web.ts

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import request from 'request';
+import _ from 'lodash';
+import { networkManager } from '../../networkManager';
+import { socketManager } from '../../socketManager';
+import { Response } from 'express';
+import { ProxyRequestInfo, SimpleRequest, StringMap } from '../../types';
+import { StringBodyRequest } from '../../middleware/stringBody';
+import { stringify } from 'qs';
+
+export function proxyHttpRequest(req: StringBodyRequest, res: Response, opts: ProxyRequestInfo) {
+    const requestHeaders = _.clone(req.headers);
+
+    delete requestHeaders['content-length'];
+    delete requestHeaders['content-encoding'];
+    delete requestHeaders['connection'];
+    delete requestHeaders['host'];
+    delete requestHeaders['origin'];
+
+    const sourceBasePath = opts.consumerResource.spec.path;
+    const targetBasePath = opts.providerResource.spec.path;
+    let path = opts.consumerPath;
+    if (opts.consumerPath.startsWith(sourceBasePath)) {
+        path = path.replace(sourceBasePath, targetBasePath);
+    }
+
+    if (!_.isEmpty(req.query)) {
+        path += '?' + stringify(req.query, { arrayFormat: 'repeat' });
+    }
+
+    console.log('Proxy request to provider: %s => %s%s [http]', opts.consumerPath, opts.address, path);
+
+    const reqOpts: SimpleRequest = {
+        method: req.method,
+        url: opts.address + path,
+        headers: requestHeaders as StringMap,
+        body: req.stringBody,
+    };
+
+    const traffic = networkManager.addRequest(req.params.systemId, opts.connection, reqOpts);
+
+    socketManager.emit(traffic.connectionId, 'traffic_start', traffic);
+    const proxyReq = request(reqOpts);
+
+    proxyReq.on('error', function (err) {
+        traffic.asError(err);
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+        if (!res.headersSent) {
+            res.status(500).send({ error: '' + err });
+        }
+    });
+
+    proxyReq.on('response', function (response) {
+        //TODO: Include the response body in the traffic object when it is not a stream
+        traffic.withResponse({
+            code: response.statusCode,
+            headers: response.headers as StringMap,
+            body: null,
+        });
+
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+    });
+
+    //We need to pipe the proxy response to the client response to handle sockets and event streams
+    proxyReq.pipe(res);
+}

package/src/repositoryManager.ts

@@ -0,0 +1,291 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import os from 'node:os';
+import { socketManager } from './socketManager';
+import { DependencyReference, Dependency, resolveDependencies } from '@kapeta/schemas';
+import { Actions, AssetVersion, Config, RegistryService } from '@kapeta/nodejs-registry-utils';
+import { definitionsManager } from './definitionsManager';
+import { Task, taskManager } from './taskManager';
+import { normalizeKapetaUri, parseKapetaUri, parseVersion } from '@kapeta/nodejs-utils';
+import { TaskProgressListener } from './progressListener';
+import { RepositoryWatcher } from './RepositoryWatcher';
+import { SourceOfChange } from './types';
+import { cacheManager } from './cacheManager';
+import { EventEmitter } from 'node:events';
+import { DefinitionInfo } from '@kapeta/local-cluster-config';
+
+const EVENT_DEFAULT_PROVIDERS_START = 'default-providers-start';
+const EVENT_DEFAULT_PROVIDERS_END = 'default-providers-end';
+
+const DEFAULT_PROVIDERS = [
+    'kapeta/block-type-service',
+    'kapeta/block-type-frontend',
+    'kapeta/block-type-gateway-http',
+    'kapeta/resource-type-rest-api',
+    'kapeta/resource-type-rest-client',
+    'kapeta/resource-type-web-page',
+    'kapeta/resource-type-web-fragment',
+    'kapeta/resource-type-mongodb',
+    'kapeta/resource-type-postgresql',
+    'kapeta/resource-type-smtp-client',
+    'kapeta/language-target-react-ts',
+    'kapeta/language-target-nodejs',
+];
+
+class RepositoryManager extends EventEmitter {
+    private _registryService: RegistryService;
+    private watcher: RepositoryWatcher;
+
+    constructor() {
+        super();
+        this._registryService = new RegistryService(Config.data.registry.url);
+        this.watcher = new RepositoryWatcher();
+        this.listenForChanges();
+
+        this.watcher.on('change', (file: string, source: SourceOfChange) => {
+            this.emit('change', file, source);
+        });
+    }
+
+    listenForChanges() {
+        this.watcher.watch();
+    }
+
+    async stopListening() {
+        return this.watcher.unwatch();
+    }
+
+    /**
+     * Setting the source of change helps us know
+     * how to react to changes in the UI.
+     */
+    setSourceOfChangeFor(file: string, source: SourceOfChange) {
+        return this.watcher.setSourceOfChangeFor(file, source);
+    }
+
+    clearSourceOfChangeFor(file: string) {
+        return this.watcher.clearSourceOfChangeFor(file);
+    }
+
+    public async ensureDefaultProviders(): Promise<void> {
+        socketManager.emitGlobal(EVENT_DEFAULT_PROVIDERS_START, { providers: DEFAULT_PROVIDERS });
+        const tasks = await this.scheduleInstallation(DEFAULT_PROVIDERS);
+        Promise.allSettled(tasks.map((t) => t.wait())).then(() => {
+            socketManager.emitGlobal(EVENT_DEFAULT_PROVIDERS_END, {});
+        });
+    }
+
+    /**
+     * Will go through all available assets and get a list of
+     * providers that are not referenced anywhere.
+     *
+     * It will also make sure to not include the latest version of an asset.
+     *
+     */
+    public async getUnusedProviders(): Promise<string[]> {
+        const allDefinitions: DefinitionInfo[] = await definitionsManager.getDefinitions();
+        const blocks: DefinitionInfo[] = [];
+        const plans: DefinitionInfo[] = [];
+        const providerMap = new Map<string, DefinitionInfo>();
+        const providerVersions: { [name: string]: Set<string> } = {};
+        const unusedProviders = new Set<string>();
+        allDefinitions.forEach((d) => {
+            if (d.definition.kind === 'core/plan') {
+                plans.push(d);
+                return;
+            }
+
+            if (d.definition.kind.startsWith('core/')) {
+                const ref = normalizeKapetaUri(`${d.definition.metadata.name}:${d.version}`);
+                providerMap.set(ref, d);
+                if (!providerVersions[d.definition.metadata.name]) {
+                    providerVersions[d.definition.metadata.name] = new Set<string>();
+                }
+                providerVersions[d.definition.metadata.name].add(d.version);
+                unusedProviders.add(ref);
+                return;
+            }
+            blocks.push(d);
+        });
+
+        const latestVersions: { [name: string]: string } = {};
+        Object.entries(providerVersions).forEach(([name, versions]) => {
+            const versionArray = Array.from(versions);
+            versionArray.sort((a, b) => {
+                return parseVersion(a).compareTo(parseVersion(b)) * -1;
+            });
+            latestVersions[name] = versionArray[0];
+        });
+
+        function markDependencyAsUsed(dep: DependencyReference) {
+            const uri = parseKapetaUri(dep.name);
+            const ref = uri.toNormalizedString();
+            if (unusedProviders.has(ref)) {
+                unusedProviders.delete(ref);
+            }
+        }
+
+        plans.forEach((plan) => {
+            const dependencies = resolveDependencies(plan.definition);
+            dependencies.forEach(markDependencyAsUsed);
+        });
+
+        blocks.forEach((block) => {
+            const blockTypeKind = normalizeKapetaUri(block.definition.kind);
+            unusedProviders.delete(blockTypeKind);
+            const blockTypeProvider = providerMap.get(blockTypeKind);
+            if (!blockTypeProvider) {
+                console.warn('No provider found for block type', block.definition.kind);
+                return;
+            }
+            const dependencies = resolveDependencies(block.definition, blockTypeProvider.definition);
+            dependencies.forEach(markDependencyAsUsed);
+        });
+
+        return Array.from(unusedProviders).filter((ref) => {
+            const uri = parseKapetaUri(ref);
+            if (uri.version == 'local') {
+                // Don't delete local assets
+                return false;
+            }
+
+            // Don't delete the latest version of an asset
+            return latestVersions[uri.fullName] !== uri.version;
+        });
+    }
+
+    public async getUpdatableAssets(allNames: string[]): Promise<string[]> {
+        const names = Array.from(new Set<string>(allNames));
+
+        const currentVersions = await Promise.all(
+            names.map((name) => definitionsManager.getLatestDefinition(name).catch(() => undefined))
+        );
+
+        const latestVersions = await Promise.all(
+            names.map((name) => this._registryService.getLatestVersion(name).catch(() => undefined))
+        );
+
+        return names
+            .map((name, index) => {
+                const currentVersion: DefinitionInfo | undefined = currentVersions[index];
+                const latestVersion: AssetVersion | undefined = latestVersions[index];
+                if (!currentVersion || !latestVersion) {
+                    // Shouldn't happen unless the registry is down or an asset was deleted
+                    return undefined;
+                }
+
+                const ref = normalizeKapetaUri(`${name}:${latestVersion.version}`);
+
+                if (currentVersion.version === latestVersion.version) {
+                    return undefined;
+                }
+
+                return ref;
+            })
+            .filter((ref) => !!ref) as string[];
+    }
+
+    private async scheduleInstallation(refs: string[]): Promise<Task[]> {
+        //We make sure to only install one asset at a time - otherwise unexpected things might happen
+        const createInstaller = (ref: string) => {
+            return async (task: Task) => {
+                if (await definitionsManager.exists(ref)) {
+                    return;
+                }
+
+                const progressListener = new TaskProgressListener(task);
+                //console.log(`Installing asset: ${ref}`);
+                //Auto-install missing asset
+                try {
+                    //We change to a temp dir to avoid issues with the current working directory
+                    process.chdir(os.tmpdir());
+                    await Actions.install(progressListener, [ref], {});
+                } catch (e) {
+                    console.error(`Failed to install asset: ${ref}`, e);
+                    throw e;
+                }
+                cacheManager.flush();
+                if (await definitionsManager.exists(ref)) {
+                    return;
+                }
+                throw new Error(`Failed to find asset after installation: ${ref}. Please try again`);
+            };
+        };
+
+        const tasks: Task[] = [];
+
+        while (refs.length > 0) {
+            let ref = refs.shift();
+            if (!ref) {
+                continue;
+            }
+            ref = normalizeKapetaUri(ref);
+
+            if (await definitionsManager.exists(ref)) {
+                continue;
+            }
+
+            const task = taskManager.add(`asset:install:${ref}`, createInstaller(ref), {
+                name: `Installing ${ref}`,
+                group: 'asset:install:', //Group prevents multiple tasks from running at the same time
+            });
+
+            tasks.push(task);
+        }
+
+        return tasks;
+    }
+
+    async ensureAsset(
+        handle: string,
+        name: string,
+        version: string,
+        wait: boolean = true
+    ): Promise<undefined | Task[]> {
+        const fullName = `${handle}/${name}`;
+        const ref = `${fullName}:${version}`;
+
+        if (version === 'local') {
+            //TODO: Get dependencies for local asset
+            return;
+        }
+
+        const installedAsset = await definitionsManager.getDefinition(`${fullName}:${version}`);
+
+        let assetVersion;
+        try {
+            assetVersion = await this._registryService.getVersion(fullName, version);
+            if (!assetVersion) {
+                return;
+            }
+        } catch (e) {
+            console.warn(`Unable to resolve asset: ${ref}`, e);
+            if (installedAsset) {
+                return;
+            }
+            throw e;
+        }
+
+        let tasks: Task[] | undefined = undefined;
+        if (!installedAsset) {
+            tasks = await this.scheduleInstallation([ref]);
+        } else {
+            //Ensure dependencies are installed
+            const refs = assetVersion.dependencies.map((dep: Dependency) => dep.name);
+            if (refs.length > 0) {
+                tasks = await this.scheduleInstallation(refs);
+            }
+        }
+
+        if (tasks && wait) {
+            await Promise.all(tasks.map((t) => t.wait()));
+        }
+
+        return tasks;
+    }
+}
+
+export const repositoryManager = new RepositoryManager();