@kapeta/local-cluster-service 0.0.60

package/.github/workflows/pr-check.yml

@@ -0,0 +1,18 @@
+name: Check
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+      - run: npm install
+        env:
+          CI: true

package/.github/workflows/publish.yml

@@ -0,0 +1,20 @@
+name: Publish to Kapeta Public NPM
+on:
+  push:
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          scope: '@kapeta'
+
+      - run: npm install
+      - run: npm publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch Program",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/start.js"
+        }
+    ]
+}

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 blockwarecom
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,32 @@
+## Local cluster service for Kapeta
+
+This service is a multi-functional service for simulating a "real" cluster - specifically during
+local development.
+
+### Features
+
+#### Configuration Service
+Provides configuration management for local services to simplify configuring local instances and
+also auto-generates configuration as part of its service discovery and routing capabilities.
+
+#### Service Discovery
+Also provides simple service-discovery through its control over configuration - by simply controlling
+where services find other services. This is also how it injects itself as a MITM proxy for all local 
+traffic and how we intend to achieve "local -> remote" and "remote -> local" tunneling in the future.  
+
+#### Local Proxy
+The service also provides a local proxy server that enables fine-grained routing and traffic-inspection.
+The only protocol currently supported is HTTP and REST-JSON but the intention is to add support for 
+several others such as MySQL, PostgreSQL, MongoDB, Redis and more.  
+
+#### Local Metrics (Not implemented)
+The local cluster service should also support metrics reporting from the 
+local instances to make testing and checking metrics for your local environment straight-forward.
+
+#### Remote Tunnel (Not implemented)
+It's able to connect to a remote cluster and override certain endpoints conditionally in that cluster to
+make them point to itself.
+
+This is to allow these scenarios:
+- Testing in production for a subset of users (Or just your own)
+- Development against a team or private sandbox to avoid running every service locally 

package/definitions.d.ts

@@ -0,0 +1,42 @@
+interface Request {
+    method: string
+    stringBody: string
+    headers: {[key:string]:string}
+    params: {[key:string]:string}
+}
+
+interface Response {
+    headers: {[key:string]:string}
+    status: (status:number) => void
+    send: (body:any) => void
+    end: () => void
+    set: (headers:{[key:string]:string}) => void
+}
+
+interface ResourceRef {
+    blockId:string
+    resourceName:string
+}
+
+declare function ProxyRequestHandler(req:Request, res:Response, info:ProxyRequestInfo);
+
+
+interface Connection {
+    mapping: any
+    from: ResourceRef
+    to: ResourceRef
+}
+
+interface ResourceInfo {
+    spec:any
+    metadata:any
+    kind:string
+}
+
+interface ProxyRequestInfo {
+    address: string
+    connection:Connection
+    fromResource:ResourceInfo
+    toResource:ResourceInfo
+    consumerPath:string
+}

package/index.js

@@ -0,0 +1,127 @@
+const clusterService = require('./src/clusterService');
+const storageService = require('./src/storageService');
+const serviceManager = require('./src/serviceManager');
+const socketManager = require('./src/socketManager');
+const containerManager = require('./src/containerManager');
+const express = require('express');
+const HTTP = require('http');
+const {Server} = require("socket.io");
+
+let currentServer = null;
+
+function createServer() {
+    const app = express();
+    app.use('/traffic', require('./src/traffic/routes'));
+    app.use('/proxy', require('./src/proxy/routes'));
+    app.use('/config', require('./src/config/routes'));
+    app.use('/instances', require('./src/instances/routes'));
+    app.use('/identities', require('./src/identities/routes'));
+    app.use('/files', require('./src/filesystem/routes'));
+    app.use('/assets', require('./src/assets/routes'));
+    app.use('/providers', require('./src/providers/routes'));
+    app.use('/', (err, req, res, next) => {
+        console.error('Request failed: %s %s', req.method, req.originalUrl, err.stack);
+        res.status(500).send({
+            ok: false,
+            error: err.error ?? err.message
+        })
+    });
+    const server = HTTP.createServer(app);
+
+    //socket 
+    const io = new Server(server, {
+        cors: {
+            //TODO: This should'nt be hardcoded but also shouldn't be "*"
+            origin: "http://localhost:8080"
+        }
+    });
+    socketManager.setIo(io);
+    return server;
+}
+
+module.exports = {
+
+    isRunning: function() {
+        return !!currentServer;
+    },
+
+    getCurrentPort: function() {
+        if (!currentServer) {
+            return -1;
+        }
+
+        return currentServer.port;
+    },
+
+    /**
+     * Starts the local cluster service.
+     * @return {Promise<Integer>} resolves when listening is done with port number. Rejects if listening failed.
+     */
+    start: async function() {
+        if (currentServer) {
+            throw new Error('Server already started');
+        }
+
+        try {
+            await containerManager.ping()
+        } catch (e) {
+            throw new Error('Could not ping docker runtime: ' + e.toString() + '. Make sure docker is running and working.');
+        }
+
+        const clusterPort = storageService.get('cluster','port');
+        if (clusterPort) {
+            clusterService.setClusterServicePort(clusterPort);
+        }
+
+        const clusterHost = storageService.get('cluster','host');
+        if (clusterHost) {
+            clusterService.setClusterServiceHost(clusterHost);
+        }
+
+        await clusterService.init();
+
+        currentServer = createServer();
+
+        const port = clusterService.getClusterServicePort();
+
+        const host = clusterService.getClusterServiceHost();
+
+        if (clusterPort !== port) {
+            storageService.put('cluster','port', port);
+        }
+
+        if (clusterHost !== host) {
+            storageService.put('cluster','host', host);
+        }
+
+        return new Promise((resolve, reject) => {
+
+            currentServer.once('error', (err) => {
+                currentServer.close();
+                currentServer = null;
+                reject(err);
+            });
+
+            currentServer.listen(port, host, () => resolve({host,port}));
+            currentServer.host = host;
+            currentServer.port = port;
+        });
+    },
+
+    /**
+     * Stops any currently running cluster services.
+     * @return {Promise<boolean>} Returns true if the service was stopped - false if no service was running.
+     */
+    stop: function() {
+        if (currentServer) {
+            return new Promise(function(resolve) {
+                currentServer.close(() => resolve(true));
+                currentServer = null;
+            });
+        }
+
+        return Promise.resolve(false);
+    },
+    getServices: () => serviceManager.getServices()
+};
+

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@kapeta/local-cluster-service",
+  "version": "0.0.60",
+  "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
+  "main": "index.js",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/blockwarecom/local-cluster-service.git"
+  },
+  "keywords": [
+    "kapeta"
+  ],
+  "author": "Henrik Hofmeister <hh@kapeta.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/blockwarecom/local-cluster-service/issues"
+  },
+  "scripts": {
+    "start": "nodemon start.js"
+  },
+  "homepage": "https://github.com/blockwarecom/local-cluster-service#readme",
+  "dependencies": {
+    "@kapeta/codegen": "<2",
+    "@kapeta/local-cluster-config": "<2",
+    "@kapeta/local-cluster-executor": "<2",
+    "@kapeta/nodejs-api-client": "<2",
+    "@kapeta/nodejs-utils": "<2",
+    "@kapeta/sdk-config": "<2",
+    "express": "4.17.1",
+    "express-promise-router": "^4.1.1",
+    "fs-extra": "^11.1.0",
+    "glob": "7.1.6",
+    "lodash": "^4.17.15",
+    "md5": "2.2.1",
+    "mkdirp": "0.5.1",
+    "node-docker-api": "1.1.22",
+    "node-uuid": "^1.4.8",
+    "request": "2.88.2",
+    "socket.io": "^4.5.2",
+    "yaml": "^1.6.0"
+  },
+  "devDependencies": {
+    "nodemon": "^2.0.2"
+  }
+}

package/src/assetManager.js

@@ -0,0 +1,249 @@
+const Path = require("node:path");
+const FS = require('node:fs');
+const FSExtra = require('fs-extra');
+const YAML = require('yaml');
+const ClusterConfiguration = require('@kapeta/local-cluster-config');
+const codeGeneratorManager = require('./codeGeneratorManager');
+const socketManager = require('./socketManager');
+
+function makeSymLink(directory, versionTarget) {
+    FSExtra.mkdirpSync(Path.dirname(versionTarget));
+    FSExtra.createSymlinkSync(directory, versionTarget);
+}
+
+function enrichAsset(asset) {
+    return {
+        ref: `kapeta://${asset.definition.metadata.name}:${asset.version}`,
+        editable: asset.version === 'local', //Only local versions are editable
+        exists: true,
+        version: asset.version,
+        kind: asset.definition.kind,
+        data: asset.definition,
+        path: asset.path,
+        ymlPath: asset.ymlPath
+    }
+}
+
+function compareRefs(a, b) {
+    const [aProtocol, aId] = parseRef(a);
+    const [bProtocol, bId] = parseRef(b);
+
+    return aProtocol === bProtocol && aId === bId;
+}
+function parseRef(ref) {
+    let out = ref.split(/:\/\//,2);
+
+    if (out.length === 1) {
+        return [
+            'kapeta',
+            ref.toLowerCase()
+        ]
+    }
+    return [
+        out[0].toLowerCase(),
+        out[1].toLowerCase()
+    ];
+}
+
+class AssetManager {
+
+    constructor() {
+        this.watcher = null;
+        this.listenForChanges();
+    }
+
+    listenForChanges() {
+        const baseDir = ClusterConfiguration.getRepositoryBasedir();
+        if (!FS.existsSync(baseDir)) {
+            FSExtra.mkdirpSync(baseDir);
+        }
+
+        let currentWebDefinitions = ClusterConfiguration
+            .getProviderDefinitions()
+            .filter(d => d.hasWeb);
+
+        console.log('Watching local repository for provider changes: %s', baseDir);
+        try {
+            this.watcher = FS.watch(baseDir, { recursive: true });
+        } catch (e) {
+            // Fallback to run without watch mode due to potential platform issues.
+            // https://nodejs.org/docs/latest/api/fs.html#caveats
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.');
+            return;
+        }
+        this.watcher.on('change', (eventType, filename) => {
+            const [handle, name, version] = filename.split(/\//g);
+            if (!name || !version) {
+                return;
+            }
+
+            const ymlPath = Path.join(baseDir, handle, name, version, 'kapeta.yml');
+            const newWebDefinitions = ClusterConfiguration
+                .getProviderDefinitions()
+                .filter(d => d.hasWeb);
+
+            const newWebDefinition = newWebDefinitions.find(d => d.ymlPath === ymlPath);
+            let currentWebDefinition = currentWebDefinitions.find(d => d.ymlPath === ymlPath);
+            const ymlExists = FS.existsSync(ymlPath);
+            let type;
+            if (ymlExists) {
+                if (currentWebDefinition) {
+                    type = 'updated';
+                } else if (newWebDefinition) {
+                    type = 'added';
+                    currentWebDefinition = newWebDefinition;
+                } else {
+                    //Other definition was added / updated - ignore
+                    return;
+                }
+            } else {
+                if (currentWebDefinition) {
+                    //Something was removed
+                    type = 'removed';
+                } else {
+                    //Other definition was removed - ignore
+                    return;
+                }
+            }
+
+            const payload = {type, definition: currentWebDefinition?.definition, asset: {handle, name, version} };
+
+            currentWebDefinitions = newWebDefinitions
+
+            socketManager.emit(`assets`, 'changed', payload);
+        });
+    }
+
+    stopListening() {
+        this.watcher.close();
+        this.watcher = null;
+    }
+
+
+    /**
+     *
+     * @param {string[]} [assetKinds]
+     * @returns {{path: *, ref: string, data: *, editable: boolean, kind: *, exists: boolean}[]}
+     */
+    getAssets(assetKinds) {
+        if (!assetKinds) {
+            const blockTypeProviders = ClusterConfiguration.getDefinitions('core/block-type');
+            assetKinds = blockTypeProviders.map(p => {
+                return `${p.definition.metadata.name}:${p.version}`
+            });
+            assetKinds.push('core/plan');
+        }
+
+        const assets = ClusterConfiguration.getDefinitions(assetKinds);
+
+        return assets.map(enrichAsset);
+    }
+
+    getPlans() {
+        return this.getAssets(['core/plan']);
+    }
+
+    getPlan(ref) {
+        const asset = this.getAsset(ref);
+
+        if ('core/plan' !== asset.kind) {
+            throw new Error('Asset was not a plan: ' + ref);
+        }
+
+        return asset.data;
+    }
+
+    getAsset(ref) {
+        const asset = this.getAssets().find(a => compareRefs(a.ref,ref));
+        if (!asset) {
+            throw new Error('Asset not found: ' + ref);
+        }
+
+        return asset;
+    }
+
+    async createAsset(path, yaml) {
+        if (FS.existsSync(path)) {
+            throw new Error('File already exists: ' + path);
+        }
+
+        const dirName = Path.dirname(path);
+        if (!FS.existsSync(dirName)) {
+            FSExtra.mkdirpSync(dirName);
+        }
+
+        FS.writeFileSync(path, YAML.stringify(yaml));
+
+        const asset = await this.importFile(path);
+
+        if (codeGeneratorManager.canGenerateCode(yaml)) {
+            await codeGeneratorManager.generate(path, yaml);
+        }
+
+        return asset;
+    }
+
+    async updateAsset(ref, yaml) {
+        const asset = this.getAsset(ref);
+        if (!asset) {
+            throw new Error('Attempted to update unknown asset: ' + ref);
+        }
+
+        if (!asset.editable) {
+            throw new Error('Attempted to update read-only asset: ' + ref);
+        }
+
+        if (!asset.ymlPath) {
+            throw new Error('Attempted to update corrupted asset: ' + ref);
+        }
+
+        FS.writeFileSync(asset.ymlPath, YAML.stringify(yaml));
+
+        if (codeGeneratorManager.canGenerateCode(yaml)) {
+            await codeGeneratorManager.generate(asset.ymlPath, yaml);
+        } else {
+            console.log('Could not generate code for %s', yaml.kind ? yaml.kind : 'unknown yaml');
+        }
+    }
+
+    hasAsset(ref) {
+        return !!this.getAsset(ref);
+    }
+
+    async importFile(filePath) {
+        if (filePath.startsWith('file://')) {
+            filePath = filePath.substring('file://'.length);
+        }
+
+        if (!FS.existsSync(filePath)) {
+            throw new Error('File not found: ' + filePath);
+        }
+
+        const assetInfos = YAML.parseAllDocuments(FS.readFileSync(filePath).toString())
+            .map(doc => doc.toJSON());
+
+        const assetInfo = assetInfos[0];
+        const version = 'local';
+        const [handle, name] = assetInfo.metadata.name.split('/');
+
+        const target = ClusterConfiguration.getRepositoryAssetPath(handle, name, version);
+        if (!FS.existsSync(target)) {
+            makeSymLink(Path.dirname(filePath), target);
+        }
+
+        const refs = assetInfos.map(assetInfo => `kapeta://${assetInfo.metadata.name}:${version}`);
+
+        return this.getAssets().filter(a => refs.some(ref => compareRefs(ref, a.ref)));
+    }
+
+    unregisterAsset(ref) {
+        const asset = this.getAsset(ref);
+        if (!asset) {
+            throw new Error('Asset does not exists: ' + ref);
+        }
+        //Remove from repository. If its local it is just a symlink - so no unchecked code is removed.
+        FSExtra.removeSync(asset.path);
+    }
+}
+
+module.exports = new AssetManager();