@kapeta/local-cluster-service 0.6.0 → 0.7.0

package/src/{providerManager.js → providerManager.ts}

@@ -1,52 +1,36 @@
-const FS = require('fs');
-const Path = require('path');
-const FSExtra = require('fs-extra');
-const repositoryManager = require('./repositoryManager');
-const ClusterConfiguration = require('@kapeta/local-cluster-config').default;
+import Path from 'path';
+import FSExtra from 'fs-extra';
+import { repositoryManager } from './repositoryManager';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { StringMap } from './types';
 
 class ProviderManager {
+    private _webAssetCache: StringMap;
     constructor() {
         this._webAssetCache = {};
     }
 
     getWebProviders() {
-        return ClusterConfiguration.getProviderDefinitions().filter(
-            (providerDefinition) => providerDefinition.hasWeb
-        );
+        return ClusterConfiguration.getProviderDefinitions().filter((providerDefinition) => providerDefinition.hasWeb);
     }
 
-    async getAsset(handle, name, version, sourceMap = false) {
+    async getAsset(handle: string, name: string, version: string, sourceMap: boolean = false) {
         const fullName = `${handle}/${name}`;
-        const id = `${handle}/${name}/${version}/web.js${
-            sourceMap ? '.map' : ''
-        }`;
+        const id = `${handle}/${name}/${version}/web.js${sourceMap ? '.map' : ''}`;
 
-        if (
-            this._webAssetCache[id] &&
-            (await FSExtra.pathExists(this._webAssetCache[id]))
-        ) {
+        if (this._webAssetCache[id] && (await FSExtra.pathExists(this._webAssetCache[id]))) {
             return FSExtra.readFile(this._webAssetCache[id], 'utf8');
         }
 
         await repositoryManager.ensureAsset(handle, name, version);
 
-        const installedProvider = this.getWebProviders().find(
-            (providerDefinition) => {
-                return (
-                    providerDefinition.definition.metadata.name === fullName &&
-                    providerDefinition.version === version
-                );
-            }
-        );
+        const installedProvider = this.getWebProviders().find((providerDefinition) => {
+            return providerDefinition.definition.metadata.name === fullName && providerDefinition.version === version;
+        });
 
         if (installedProvider) {
             //Check locally installed providers
-            const path = Path.join(
-                installedProvider.path,
-                'web',
-                handle,
-                `${name}.js${sourceMap ? '.map' : ''}`
-            );
+            const path = Path.join(installedProvider.path, 'web', handle, `${name}.js${sourceMap ? '.map' : ''}`);
             if (await FSExtra.pathExists(path)) {
                 this._webAssetCache[id] = path;
 
@@ -75,4 +59,4 @@ if (providerDefinitions.length > 0) {
     console.log('## No providers found ##');
 }
 
-module.exports = new ProviderManager();
+export const providerManager = new ProviderManager();

package/src/providers/routes.ts

@@ -0,0 +1,46 @@
+import Router from 'express-promise-router';
+import { providerManager } from '../providerManager';
+
+import { corsHandler } from '../middleware/cors';
+import { Request, Response } from 'express';
+
+const router = Router();
+
+router.use('/', corsHandler);
+
+router.get('/', async (req: Request, res: Response) => {
+    const result = await providerManager.getWebProviders();
+
+    res.send(result);
+});
+
+router.get('/asset/:handle/:name/:version/web.js', async (req: Request, res: Response) => {
+    const { handle, name, version } = req.params;
+    let result = await providerManager.getAsset(handle, name, version);
+
+    if (!result) {
+        res.status(404).send('');
+    } else {
+        if (version !== 'local') {
+            res.setHeader('Cache-Control', 'max-age=31536000, immutable');
+        }
+        res.send(result.toString().replace(`${name}.js.map`, 'web.js.map'));
+    }
+});
+
+router.get('/asset/:handle/:name/:version/web.js.map', async (req: Request, res: Response) => {
+    const { handle, name, version } = req.params;
+    const result = await providerManager.getAsset(handle, name, version, true);
+
+    if (!result) {
+        res.status(404).send('');
+    } else {
+        // Only cache successful requests
+        if (version !== 'local') {
+            res.setHeader('Cache-Control', 'max-age=31536000, immutable');
+        }
+        res.send(result);
+    }
+});
+
+export default router;

package/src/proxy/routes.ts

@@ -0,0 +1,148 @@
+import Router from 'express-promise-router';
+import { Request, Response } from 'express';
+import { Resource } from '@kapeta/schemas';
+import { proxyRestRequest } from './types/rest';
+import { proxyWebRequest } from './types/web';
+import { ProxyRequestHandler } from '../types';
+import { stringBody, StringBodyRequest } from '../middleware/stringBody';
+import { serviceManager } from '../serviceManager';
+import { clusterService } from '../clusterService';
+import { assetManager } from '../assetManager';
+
+import _ from 'lodash';
+
+const router = Router();
+/**
+ * @var {{[key:string]:ProxyRequestHandler}}
+ */
+const TYPE_HANDLERS: { [p: string]: ProxyRequestHandler } = {
+    rest: proxyRestRequest,
+    web: proxyWebRequest,
+};
+
+function getResource(resources: Resource[], resourceName: string) {
+    return resources.find((resource) => {
+        return resource.metadata.name.toLowerCase() === resourceName.toLowerCase();
+    });
+}
+
+router.use('/:systemId/:consumerInstanceId/:consumerResourceName', stringBody);
+
+router.all(
+    '/:systemId/:consumerInstanceId/:consumerResourceName/:type/*',
+    async (req: StringBodyRequest, res: Response) => {
+        try {
+            const typeHandler = TYPE_HANDLERS[req.params.type.toLowerCase()];
+            if (!typeHandler) {
+                res.status(401).send({ error: 'Unknown connection type: ' + req.params.type });
+                return;
+            }
+
+            const plan = await assetManager.getPlan(req.params.systemId);
+
+            // We can find the connection by the consumer information alone since
+            // only 1 provider can be connected to a consumer resource at a time
+            const connection = _.find(plan.spec.connections, (connection) => {
+                return (
+                    connection.consumer.blockId.toLowerCase() === req.params.consumerInstanceId.toLowerCase() &&
+                    connection.consumer.resourceName.toLowerCase() === req.params.consumerResourceName.toLowerCase()
+                );
+            });
+
+            if (!connection) {
+                res.status(401).send({
+                    error: `No connection found for consumer "${req.params.consumerInstanceId}::${req.params.consumerResourceName}"`,
+                });
+                return;
+            }
+
+            const toBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
+                return blockInstance.id.toLowerCase() === connection.consumer.blockId.toLowerCase();
+            });
+
+            if (!toBlockInstance) {
+                res.status(401).send({ error: `Block instance not found "${req.params.consumerInstanceId}` });
+                return;
+            }
+
+            const toBlockAsset = await assetManager.getAsset(toBlockInstance.block.ref);
+
+            if (!toBlockAsset) {
+                res.status(401).send({ error: `Block asset not found "${toBlockInstance.block.ref}` });
+                return;
+            }
+
+            const consumerResource = getResource(toBlockAsset.data.spec.consumers, req.params.consumerResourceName);
+
+            if (!consumerResource) {
+                res.status(401).send({
+                    error: `Block resource not found "${req.params.consumerInstanceId}::${req.params.consumerResourceName}`,
+                });
+                return;
+            }
+
+            const basePath = clusterService.getProxyPath(
+                req.params.systemId,
+                req.params.consumerInstanceId,
+                req.params.consumerResourceName,
+                req.params.type
+            );
+
+            const fromBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
+                return blockInstance.id.toLowerCase() === connection.provider.blockId.toLowerCase();
+            });
+
+            if (!fromBlockInstance) {
+                res.status(401).send({ error: `Block instance not found "${connection.provider.blockId}` });
+                return;
+            }
+
+            const fromBlockAsset = await assetManager.getAsset(fromBlockInstance.block.ref);
+
+            if (!fromBlockAsset) {
+                res.status(401).send({ error: `Block asset not found "${fromBlockInstance.block.ref}` });
+                return;
+            }
+
+            const providerResource = getResource(fromBlockAsset.data.spec.providers, connection.provider.resourceName);
+
+            if (!providerResource) {
+                res.status(401).send({
+                    error: `Block resource not found "${connection.provider.blockId}::${connection.provider.resourceName}`,
+                });
+                return;
+            }
+
+            //Get target address
+            let address = await serviceManager.getProviderAddress(
+                req.params.systemId,
+                connection.provider.blockId,
+                req.params.type
+            );
+
+            while (address.endsWith('/')) {
+                address = address.substring(0, address.length - 1);
+            }
+
+            /*
+         Get the path the consumer requested.
+         Note that this might not match the path the destination is expecting so we need to identify the method
+         that is being called and identify the destination path from the connection.
+         */
+            const consumerPath = req.originalUrl.substring(basePath.length - 1);
+
+            typeHandler(req, res, {
+                consumerPath,
+                address,
+                consumerResource,
+                providerResource,
+                connection,
+            });
+        } catch (err: any) {
+            console.warn('Failed to process proxy request', err);
+            res.status(400).send({ error: err.message });
+        }
+    }
+);
+
+export default router;

package/src/proxy/types/{rest.js → rest.ts}

@@ -1,13 +1,16 @@
-const request = require('request');
-const Path = require('path');
-const _ = require('lodash');
-
-const pathTemplateParser = require('../../utils/pathTemplateParser');
-const networkManager = require('../../networkManager');
-const socketManager = require('../../socketManager');
-
-
-function getRestMethodId(restResource, httpMethod, httpPath) {
+import _ from 'lodash';
+import request from 'request';
+import Path from 'path';
+import { pathTemplateParser } from '../../utils/pathTemplateParser';
+import { networkManager } from '../../networkManager';
+
+import { socketManager } from '../../socketManager';
+import { Request, Response } from 'express';
+import { ProxyRequestInfo, SimpleRequest, StringMap } from '../../types';
+import { StringBodyRequest } from '../../middleware/stringBody';
+import { Resource } from '@kapeta/schemas';
+
+function getRestMethodId(restResource: Resource, httpMethod: string, httpPath: string) {
     return _.findKey(restResource.spec.methods, (method) => {
         let methodType = method.method ? method.method.toUpperCase() : 'GET';
 
@@ -24,7 +27,6 @@ function getRestMethodId(restResource, httpMethod, httpPath) {
         const pathTemplate = pathTemplateParser(path);
 
         return pathTemplate.matches(httpPath);
-
     });
 }
 
@@ -34,7 +36,7 @@ function getRestMethodId(restResource, httpMethod, httpPath) {
  * @param opts {ProxyRequestInfo}
  * @return {{consumerMethod: *, providerMethod: *}}
  */
-function resolveMethods(req, opts) {
+function resolveMethods(req: Request, opts: ProxyRequestInfo) {
     const consumerMethodId = getRestMethodId(opts.consumerResource, req.method, opts.consumerPath);
 
     if (!consumerMethodId) {
@@ -73,24 +75,23 @@ function resolveMethods(req, opts) {
 
     return {
         consumerMethod,
-        providerMethod
+        providerMethod,
     };
 }
 
-/**
- *
- * @param req {Request}
- * @param res {Response}
- * @param opts {ProxyRequestInfo}
- */
-module.exports = function proxyRestRequest(req, res, opts) {
-
-    let {consumerMethod, providerMethod} = resolveMethods(req, opts);
+export function proxyRestRequest(req: StringBodyRequest, res: Response, opts: ProxyRequestInfo) {
+    let { consumerMethod, providerMethod } = resolveMethods(req, opts);
 
     const consumerPathTemplate = pathTemplateParser(consumerMethod.path);
     const providerPathTemplate = pathTemplateParser(providerMethod.path);
 
     const pathVariables = consumerPathTemplate.parse(opts.consumerPath);
+    if (!pathVariables) {
+        res.status(400).send({
+            error: `Path did not match any patterns: "${opts.consumerPath}"`,
+        });
+        return;
+    }
 
     let providerPath = providerPathTemplate.create(pathVariables);
 
@@ -108,11 +109,11 @@ module.exports = function proxyRestRequest(req, res, opts) {
 
     console.log('Proxy request to provider: %s => %s [rest]', opts.consumerPath, opts.address + providerPath);
 
-    const reqOpts = {
+    const reqOpts: SimpleRequest = {
         method: providerMethod.method || 'GET',
         url: opts.address + providerPath,
         body: req.stringBody,
-        headers: requestHeaders
+        headers: requestHeaders as StringMap,
     };
 
     const traffic = networkManager.addRequest(
@@ -125,12 +126,12 @@ module.exports = function proxyRestRequest(req, res, opts) {
 
     socketManager.emit(traffic.connectionId, 'traffic_start', traffic);
 
-    request(reqOpts, function(err, response, responseBody) {
+    request(reqOpts, function (err, response, responseBody) {
         if (err) {
             traffic.asError(err);
             socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
 
-            res.status(500).send({error: '' + err});
+            res.status(500).send({ error: '' + err });
             return;
         }
 
@@ -146,8 +147,8 @@ module.exports = function proxyRestRequest(req, res, opts) {
 
         traffic.withResponse({
             code: response.statusCode,
-            headers: response.headers,
-            body: responseBody
+            headers: response.headers as StringMap,
+            body: responseBody,
         });
 
         socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
@@ -158,5 +159,4 @@ module.exports = function proxyRestRequest(req, res, opts) {
 
         res.end();
     });
-
-};
+}

package/src/proxy/types/web.ts

@@ -0,0 +1,60 @@
+import request from 'request';
+import _ from 'lodash';
+import { networkManager } from '../../networkManager';
+import { socketManager } from '../../socketManager';
+import { Request, Response } from 'express';
+import { ProxyRequestInfo, SimpleRequest, StringMap } from '../../types';
+import { StringBodyRequest } from '../../middleware/stringBody';
+
+export function proxyWebRequest(req: StringBodyRequest, res: Response, opts: ProxyRequestInfo) {
+    const requestHeaders = _.clone(req.headers);
+
+    delete requestHeaders['content-length'];
+    delete requestHeaders['content-encoding'];
+    delete requestHeaders['connection'];
+    delete requestHeaders['host'];
+    delete requestHeaders['origin'];
+
+    const sourceBasePath = opts.consumerResource.spec.path;
+    const targetBasePath = opts.providerResource.spec.path;
+    let path = opts.consumerPath;
+    if (opts.consumerPath.startsWith(sourceBasePath)) {
+        path = path.replace(sourceBasePath, targetBasePath);
+    }
+
+    console.log('Proxy request to provider: %s => %s%s [web]', opts.consumerPath, opts.address, path);
+
+    const reqOpts: SimpleRequest = {
+        method: req.method,
+        url: opts.address + path,
+        headers: requestHeaders as StringMap,
+        body: req.stringBody,
+    };
+
+    const traffic = networkManager.addRequest(req.params.systemId, opts.connection, reqOpts);
+
+    socketManager.emit(traffic.connectionId, 'traffic_start', traffic);
+    const proxyReq = request(reqOpts);
+
+    proxyReq.on('error', function (err) {
+        traffic.asError(err);
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+        if (!res.headersSent) {
+            res.status(500).send({ error: '' + err });
+        }
+    });
+
+    proxyReq.on('response', function (response) {
+        //TODO: Include the response body in the traffic object when it is not a stream
+        traffic.withResponse({
+            code: response.statusCode,
+            headers: response.headers as StringMap,
+            body: null,
+        });
+
+        socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
+    });
+
+    //We need to pipe the proxy response to the client response to handle sockets and event streams
+    proxyReq.pipe(res);
+}

package/src/{repositoryManager.js → repositoryManager.ts}

@@ -1,21 +1,24 @@
-const ClusterConfiguration = require('@kapeta/local-cluster-config').default;
-const FS = require('node:fs');
-const FSExtra = require('fs-extra');
-const Path = require('node:path');
-const socketManager = require('./socketManager');
-const {
-    Actions,
-    RegistryService,
-    Config,
-} = require('@kapeta/nodejs-registry-utils');
-const progressListener = require('./progressListener');
-const os = require('os');
-const { parseKapetaUri } = require('@kapeta/nodejs-utils');
-const INSTALL_ATTEMPTED = {};
+import FS from 'node:fs';
+import os from 'node:os';
+import Path from 'node:path';
+import FSExtra, { FSWatcher } from 'fs-extra';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import { socketManager } from './socketManager';
+import { progressListener } from './progressListener';
+import { Dependency } from '@kapeta/schemas';
+import { Actions, Config, RegistryService } from '@kapeta/nodejs-registry-utils';
+
+const INSTALL_ATTEMPTED: { [p: string]: boolean } = {};
 
 class RepositoryManager {
+    private changeEventsEnabled: boolean;
+    private _registryService: RegistryService;
+    private _cache: { [key: string]: boolean };
+    private watcher?: FSWatcher;
+    private _installQueue: (() => Promise<void>)[];
+    private _processing: boolean = false;
     constructor() {
-        this.watcher = null;
         this.changeEventsEnabled = true;
         this.listenForChanges();
         this._registryService = new RegistryService(Config.data.registry.url);
@@ -23,7 +26,7 @@ class RepositoryManager {
         this._installQueue = [];
     }
 
-    setChangeEventsEnabled(enabled) {
+    setChangeEventsEnabled(enabled: boolean) {
         this.changeEventsEnabled = enabled;
     }
 
@@ -35,22 +38,17 @@ class RepositoryManager {
 
         let allDefinitions = ClusterConfiguration.getDefinitions();
 
-        console.log(
-            'Watching local repository for provider changes: %s',
-            baseDir
-        );
+        console.log('Watching local repository for provider changes: %s', baseDir);
         try {
             this.watcher = FS.watch(baseDir, { recursive: true });
         } catch (e) {
             // Fallback to run without watch mode due to potential platform issues.
             // https://nodejs.org/docs/latest/api/fs.html#caveats
-            console.log(
-                'Unable to watch for changes. Changes to assets will not update automatically.'
-            );
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.');
             return;
         }
         this.watcher.on('change', (eventType, filename) => {
-            const [handle, name, version] = filename.split(/\//g);
+            const [handle, name, version] = filename.toString().split(/\//g);
             if (!name || !version) {
                 return;
             }
@@ -59,21 +57,11 @@ class RepositoryManager {
                 return;
             }
 
-            const ymlPath = Path.join(
-                baseDir,
-                handle,
-                name,
-                version,
-                'kapeta.yml'
-            );
+            const ymlPath = Path.join(baseDir, handle, name, version, 'kapeta.yml');
             const newDefinitions = ClusterConfiguration.getDefinitions();
 
-            const newDefinition = newDefinitions.find(
-                (d) => d.ymlPath === ymlPath
-            );
-            let currentDefinition = allDefinitions.find(
-                (d) => d.ymlPath === ymlPath
-            );
+            const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
+            let currentDefinition = allDefinitions.find((d) => d.ymlPath === ymlPath);
             const ymlExists = FS.existsSync(ymlPath);
             let type;
             if (ymlExists) {
@@ -112,32 +100,23 @@ class RepositoryManager {
     }
 
     stopListening() {
+        if (!this.watcher) {
+            return;
+        }
         this.watcher.close();
-        this.watcher = null;
+        this.watcher = undefined;
     }
 
-    /**
-     *
-     * @param {string[]} refs
-     * @return {Promise<void>}
-     * @private
-     */
-    async _install(refs) {
+    private async _install(refs: string[]): Promise<void> {
         //We make sure to only install one asset at a time - otherwise unexpected things might happen
-        const out = new Promise((resolve, reject) => {
+        const out = new Promise<void>((resolve, reject) => {
             this._installQueue.push(async () => {
                 try {
-                    const normalizedRefs = refs.map(
-                        (ref) => parseKapetaUri(ref).id
-                    );
-                    const filteredRefs = normalizedRefs.filter(
-                        (ref) => !INSTALL_ATTEMPTED[ref]
-                    );
+                    const normalizedRefs = refs.map((ref) => parseKapetaUri(ref).id);
+                    const filteredRefs = normalizedRefs.filter((ref) => !INSTALL_ATTEMPTED[ref]);
                     console.log(filteredRefs);
                     if (filteredRefs.length > 0) {
-                        filteredRefs.forEach(
-                            (ref) => (INSTALL_ATTEMPTED[ref] = true)
-                        );
+                        filteredRefs.forEach((ref) => (INSTALL_ATTEMPTED[ref] = true));
                         //Auto-install missing asset
                         try {
                             //We change to a temp dir to avoid issues with the current working directory
@@ -148,16 +127,12 @@ class RepositoryManager {
                                 type: 'start',
                                 refs,
                             });
-                            await Actions.install(
-                                progressListener,
-                                normalizedRefs,
-                                {}
-                            );
+                            await Actions.install(progressListener, normalizedRefs, {});
                             socketManager.emit(`install`, 'install:action', {
                                 type: 'done',
                                 refs,
                             });
-                        } catch (e) {
+                        } catch (e: any) {
                             socketManager.emit(`install`, 'install:action', {
                                 type: 'failed',
                                 refs,
@@ -189,27 +164,27 @@ class RepositoryManager {
         try {
             while (this._installQueue.length > 0) {
                 const item = this._installQueue.shift();
-
-                await item();
+                if (item) {
+                    await item();
+                }
             }
         } finally {
             this._processing = false;
         }
     }
 
-    async ensureAsset(handle, name, version) {
+    async ensureAsset(handle: string, name: string, version: string) {
         const fullName = `${handle}/${name}`;
         const ref = `${fullName}:${version}`;
 
         if (version === 'local') {
             //TODO: Get dependencies for local asset
-            return null;
+            return;
         }
 
         const definitions = ClusterConfiguration.getDefinitions();
         const installedAsset = definitions.find(
-            (d) =>
-                d.definition.metadata.name === fullName && d.version === version
+            (d) => d.definition.metadata.name === fullName && d.version === version
         );
 
         if (installedAsset && this._cache[ref] === true) {
@@ -222,10 +197,7 @@ class RepositoryManager {
 
         let assetVersion;
         try {
-            assetVersion = await this._registryService.getVersion(
-                fullName,
-                version
-            );
+            assetVersion = await this._registryService.getVersion(fullName, version);
             if (!assetVersion) {
                 this._cache[ref] = false;
                 return;
@@ -244,11 +216,11 @@ class RepositoryManager {
             await this._install([ref]);
         } else {
             //Ensure dependencies are installed
-            const refs = assetVersion.dependencies.map((dep) => dep.name);
+            const refs = assetVersion.dependencies.map((dep: Dependency) => dep.name);
             console.log(`Auto-installing dependencies: ${refs.join(', ')}`);
             await this._install(refs);
         }
     }
 }
 
-module.exports = new RepositoryManager();
+export const repositoryManager = new RepositoryManager();