@kapeta/local-cluster-service 0.0.0-96f91ef

package/dist/esm/test/proxy/types/rest.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+export {};

package/dist/esm/test/proxy/types/rest.test.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const rest_1 = require("../../../src/proxy/types/rest");
+describe('getRestMethodId', () => {
+    it('should match @Query in url', () => {
+        const restResource = new TestResource();
+        const restMethodId = (0, rest_1.getRestMethodId)(restResource, 'POST', '/names?name=Ib');
+        expect(restMethodId).toBeDefined();
+    });
+});
+class TestResource {
+    kind = '';
+    metadata = new TestResourceMetaData();
+    get spec() {
+        return {
+            methods: [
+                {
+                    responseType: {
+                        ref: 'Name[]',
+                    },
+                    method: 'GET',
+                    path: '/names',
+                    arguments: {},
+                },
+                {
+                    responseType: {
+                        ref: 'Name',
+                    },
+                    method: 'POST',
+                    path: '/names',
+                    arguments: {
+                        name: {
+                            type: 'string',
+                            transport: 'QUERY',
+                        },
+                    },
+                },
+            ],
+        };
+    }
+}
+class TestResourceMetaData {
+    name = '';
+}

package/dist/esm/test/utils/pathTemplateParser.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+export {};

package/dist/esm/test/utils/pathTemplateParser.test.js

@@ -0,0 +1,27 @@
+"use strict";
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const pathTemplateParser_1 = require("../../src/utils/pathTemplateParser");
+describe('pathTemplateParser', () => {
+    it('must return parameters for query parameters from parse', () => {
+        const template = new pathTemplateParser_1.PathTemplate('/names');
+        const parse = template.parse('/names?name=Ib#test');
+        console.log(parse);
+        expect(parse).toBeTruthy();
+    });
+    it('must return parameters defined in url', () => {
+        const template = new pathTemplateParser_1.PathTemplate('/names/{identityId}');
+        const parse = template.parse('/names/idn_xxyyzz');
+        expect(parse).toBeTruthy();
+        expect(parse).toMatchObject({ identityId: 'idn_xxyyzz' });
+    });
+    it('must return parameters defined in url regardless of query parameters', () => {
+        const template = new pathTemplateParser_1.PathTemplate('/names/{identityId}');
+        const parse = template.parse('/names/idn_xxyyzz?name=Ib#test');
+        expect(parse).toBeTruthy();
+        expect(parse).toMatchObject({ identityId: 'idn_xxyyzz' });
+    });
+});

package/index.ts

@@ -0,0 +1,280 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { clusterService } from './src/clusterService';
+import { storageService } from './src/storageService';
+import { serviceManager } from './src/serviceManager';
+import { socketManager } from './src/socketManager';
+import { containerManager } from './src/containerManager';
+import express from 'express';
+import HTTP from 'http';
+import { Server } from 'socket.io';
+
+import TrafficRoutes from './src/traffic/routes';
+import ProxyRoutes from './src/proxy/routes';
+import ConfigRoutes from './src/config/routes';
+import InstancesRoutes from './src/instances/routes';
+import IdentitiesRoutes from './src/identities/routes';
+import FilesystemRoutes from './src/filesystem/routes';
+import AssetsRoutes from './src/assets/routes';
+import ProviderRoutes from './src/providers/routes';
+import AttachmentRoutes from './src/attachments/routes';
+import TaskRoutes from './src/tasks/routes';
+import APIRoutes from './src/api';
+import AIRoutes from './src/ai/routes';
+import { isLinux } from './src/utils/utils';
+import request from 'request';
+import { repositoryManager } from './src/repositoryManager';
+import { taskManager } from './src/taskManager';
+import { ensureCLI, ensureCLICommands } from './src/utils/commandLineUtils';
+import { defaultProviderInstaller } from './src/utils/DefaultProviderInstaller';
+import { authManager } from './src/authManager';
+import { codeGeneratorManager } from './src/codeGeneratorManager';
+import * as Sentry from '@sentry/node';
+import { assetManager } from './src/assetManager';
+
+Sentry.init({
+    dsn: 'https://0b7cc946d82c591473d6f95fff5e210b@o4505820837249024.ingest.sentry.io/4506212692000768',
+    enabled: process.env.NODE_ENV !== 'development',
+    // Performance Monitoring on every ~20th request
+    tracesSampleRate: 0.05,
+    // Set sampling rate for profiling - this is relative to tracesSampleRate
+    profilesSampleRate: 1.0,
+});
+
+export type LocalClusterService = HTTP.Server & { host?: string; port?: number };
+
+export type StartResult = { host: string; port: number; dockerStatus: boolean };
+
+let currentServer: LocalClusterService | null = null;
+
+function createServer() {
+    const app = express();
+
+    Sentry.addIntegration(new Sentry.Integrations.Http({ tracing: true }));
+    Sentry.addIntegration(new Sentry.Integrations.Express({ app }));
+
+    // This causes node < 20 to crash on request.
+    //app.use(Sentry.Handlers.requestHandler());
+
+    // TracingHandler creates a trace for every incoming request
+    app.use(Sentry.Handlers.tracingHandler());
+
+    app.use('/traffic', TrafficRoutes);
+    app.use('/proxy', ProxyRoutes);
+    app.use('/config', ConfigRoutes);
+    app.use('/instances', InstancesRoutes);
+    app.use('/identities', IdentitiesRoutes);
+    app.use('/files', FilesystemRoutes);
+    app.use('/assets', AssetsRoutes);
+    app.use('/providers', ProviderRoutes);
+    app.use('/attachments', AttachmentRoutes);
+    app.use('/tasks', TaskRoutes);
+    app.use('/api', APIRoutes);
+    app.use('/ai', AIRoutes);
+
+    app.get('/status', async (req, res) => {
+        res.send({
+            ok: true,
+            dockerStatus: await containerManager.checkAlive(),
+            socketStatus: socketManager.isAlive(),
+        });
+    });
+
+    app.get('/ping', async (req, res) => {
+        res.send({
+            ok: true,
+        });
+    });
+
+    app.use('/', (req: express.Request, res: express.Response) => {
+        console.error('Invalid request: %s %s', req.method, req.originalUrl);
+        res.status(400).json({
+            ok: false,
+            error: 'Unknown',
+        });
+    });
+
+    app.use(Sentry.Handlers.errorHandler());
+
+    /**
+     * Central error handler, allows us to return a consistent JSON response wrapper with the error.
+     */
+    app.use((err: any, req: express.Request, res: express.Response, next: express.NextFunction) => {
+        console.error('Error handling request: %s %s', req.method, req.originalUrl, err);
+        res.status(err.status || 500).json({
+            ok: false,
+            error: err.message || 'Unknown error',
+            stack: process.env.NODE_ENV !== 'production' ? err.stack : undefined,
+        });
+    });
+
+    const server = HTTP.createServer(app);
+
+    //socket
+    const io = new Server(server, {
+        cors: {
+            //TODO: This should'nt be hardcoded but also shouldn't be "*"
+            origin: 'http://localhost:8080',
+        },
+    });
+    socketManager.setIo(io);
+    return server;
+}
+
+export default {
+    isRunning: function () {
+        return !!currentServer;
+    },
+
+    getCurrentPort: function () {
+        if (!currentServer) {
+            return -1;
+        }
+
+        return currentServer.port;
+    },
+
+    ping: async function (host: string, port: number): Promise<{ ok: boolean }> {
+        return new Promise((resolve, reject) => {
+            request.get(`http://${host}:${port}/ping`, (err, res, body) => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+
+                resolve(JSON.parse(body));
+            });
+        });
+    },
+
+    /**
+     * Starts the local cluster service.
+     * resolves when listening is done with port number. Rejects if listening failed.
+     */
+    start: async function (): Promise<StartResult> {
+        if (currentServer) {
+            throw new Error('Server already started');
+        }
+
+        try {
+            console.log('Testing docker runtime...');
+            await containerManager.initialize();
+        } catch (e: any) {
+            console.error(
+                'Could not ping docker runtime: ' + e.toString() + '. Make sure docker is running and working.'
+            );
+        }
+
+        await defaultProviderInstaller.checkForDefault();
+        await codeGeneratorManager.initialize();
+
+        const clusterPort = storageService.get('cluster', 'port');
+        if (clusterPort) {
+            clusterService.setClusterServicePort(clusterPort);
+        }
+
+        const clusterHost = storageService.get('cluster', 'host');
+        if (clusterHost) {
+            clusterService.setClusterServiceHost(clusterHost);
+        }
+
+        let pingResult = undefined;
+        try {
+            pingResult = await this.ping(clusterHost, clusterPort);
+        } catch (e: any) {
+            //Ignore - expected to not be running since we're starting it
+        }
+
+        if (pingResult?.ok) {
+            throw new Error(`Cluster service already running on: ${clusterHost}:${clusterPort}.`);
+        }
+
+        await clusterService.init();
+
+        authManager.listenForChanges();
+
+        currentServer = createServer();
+
+        const port = clusterService.getClusterServicePort();
+
+        const host = clusterService.getClusterServiceHost();
+
+        if (clusterPort !== port) {
+            storageService.put('cluster', 'port', port);
+        }
+
+        if (clusterHost !== host) {
+            storageService.put('cluster', 'host', host);
+        }
+
+        return new Promise((resolve, reject) => {
+            if (!currentServer) {
+                reject(new Error(`Current server wasn't set`));
+                return;
+            }
+            currentServer.once('error', (err) => {
+                if (currentServer) {
+                    currentServer.close();
+                    currentServer = null;
+                }
+                reject(err);
+            });
+
+            // On Linux we need to bind to 0.0.0.0 to be able to connect to it from docker containers.
+            // TODO: This might pose a security risk - so we should authenticate all requests using a
+            //       shared secret/nonce that we pass around.
+            const bindHost = isLinux() ? '0.0.0.0' : host;
+
+            currentServer.listen(port, bindHost, async () => {
+                try {
+                    const ensureCLITask = await ensureCLI();
+                    if (ensureCLITask) {
+                        await taskManager.waitFor((t) => t === ensureCLITask);
+                    }
+                } catch (e: any) {
+                    console.error('Failed to install CLI.', e);
+                }
+
+                try {
+                    await ensureCLICommands();
+                } catch (error) {
+                    console.error('Failed to ensure default CLI commands', error);
+                }
+
+                try {
+                    // Start installation process for all default providers
+                    await repositoryManager.ensureDefaultProviders();
+                } catch (e: any) {
+                    console.error('Failed to install default providers.', e);
+                }
+
+                assetManager.startUpgradeInterval();
+
+                resolve({ host, port, dockerStatus: containerManager.isAlive() });
+            });
+            currentServer.host = host;
+            currentServer.port = port;
+        });
+    },
+
+    /**
+     * Stops any currently running cluster services.
+     * @return {Promise<boolean>} Returns true if the service was stopped - false if no service was running.
+     */
+    stop: function () {
+        if (currentServer) {
+            return new Promise(function (resolve) {
+                if (currentServer) {
+                    currentServer.close(() => resolve(true));
+                    currentServer = null;
+                }
+            });
+        }
+
+        return Promise.resolve(false);
+    },
+    getServices: () => serviceManager.getServices(),
+};

package/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    transform: {
+        '^.+\\.ts?$': 'ts-jest',
+    },
+    transformIgnorePatterns: ['<rootDir>/node_modules/'],
+};

package/package.json

@@ -0,0 +1,134 @@
+{
+    "name": "@kapeta/local-cluster-service",
+    "version": "0.0.0-96f91ef",
+    "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
+    "type": "commonjs",
+    "exports": {
+        ".": {
+            "import": {
+                "types": "./dist/esm/index.d.ts",
+                "default": "./dist/esm/index.js"
+            },
+            "require": {
+                "types": "./dist/cjs/index.d.cts",
+                "default": "./dist/cjs/index.js"
+            }
+        }
+    },
+    "types": "./dist/esm/index.d.ts",
+    "main": "./dist/cjs/index.js",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/kapetacom/local-cluster-service.git"
+    },
+    "keywords": [
+        "kapeta"
+    ],
+    "license": "BUSL-1.1",
+    "author": {
+        "email": "engineering@kapeta.com",
+        "name": "Kapeta",
+        "url": "https://kapeta.com"
+    },
+    "bugs": {
+        "url": "https://github.com/kapetacom/local-cluster-service/issues"
+    },
+    "scripts": {
+        "start": "node ./dist/cjs/start.js",
+        "dev": "npm run start:dev",
+        "start:dev": "nodemon -e js,ts,json ./start.ts",
+        "test": "jest",
+        "clean": "rm -rf ./dist",
+        "build:esm": "tsc --module nodenext --moduleResolution nodenext --outDir ./dist/esm && echo '{\"type\":\"module\"}' > ./dist/esm/package.json",
+        "build:cjs": "tsc  --outDir ./dist/cjs && echo '{\"type\":\"commonjs\"}' > ./dist/cjs/package.json",
+        "build": "npm run clean && npm run build:esm && npm run build:cjs",
+        "format": "prettier --write .",
+        "lint": "tsc --noEmit && eslint src/**/*.ts",
+        "prepublishOnly": "npm run build"
+    },
+    "homepage": "https://github.com/kapetacom/local-cluster-service#readme",
+    "dependencies": {
+        "@kapeta/codegen": "^1.3.0",
+        "@kapeta/local-cluster-config": "^0.4.0",
+        "@kapeta/nodejs-api-client": ">=0.2.0 <2",
+        "@kapeta/nodejs-process": "^1.2.0",
+        "@kapeta/nodejs-registry-utils": ">=0.11.1 <2",
+        "@kapeta/nodejs-utils": "<2",
+        "@kapeta/schemas": "^3.5.0",
+        "@kapeta/sdk-config": "^2.1.1",
+        "@kapeta/web-microfrontend": "^1.2.5",
+        "@sentry/node": "^7.94.1",
+        "@types/dockerode": "^3.3.19",
+        "@types/stream-json": "^1.7.3",
+        "async-lock": "^1.4.0",
+        "chokidar": "^3.5.3",
+        "dockerode": "^3.3.5",
+        "express": "4.17.1",
+        "express-promise-router": "^4.1.1",
+        "fs-extra": "^11.1.0",
+        "glob": "^7.1.6",
+        "gunzip-maybe": "^1.4.2",
+        "lodash": "^4.17.15",
+        "md5": "2.2.1",
+        "node-cache": "^5.1.2",
+        "node-uuid": "^1.4.8",
+        "qs": "^6.11.2",
+        "request": "2.88.2",
+        "socket.io": "^4.5.2",
+        "stream-json": "^1.8.0",
+        "tar-stream": "^3.1.6",
+        "typescript": "^5.1.6",
+        "yaml": "^1.6.0",
+        "@kapeta/config-mapper": "^1.1.1"
+    },
+    "devDependencies": {
+        "@kapeta/eslint-config": "^0.7.0",
+        "@kapeta/prettier-config": "^0.6.2",
+        "@tsconfig/node18": "^18.2.0",
+        "@types/async-lock": "^1.4.0",
+        "@types/express": "^4.17.17",
+        "@types/fs-extra": "^11.0.1",
+        "@types/glob": "^8.1.0",
+        "@types/gunzip-maybe": "^1.4.0",
+        "@types/jest": "^29.5.4",
+        "@types/lodash": "^4.14.195",
+        "@types/md5": "^2.3.2",
+        "@types/node": "^20.5.8",
+        "@types/node-uuid": "^0.0.29",
+        "@types/request": "^2.48.8",
+        "@types/tar-stream": "^2.2.2",
+        "eslint": "^8.42.0",
+        "eslint-config-prettier": "^8.8.0",
+        "jest": "^29.6.4",
+        "nodemon": "^2.0.2",
+        "prettier": "^2.8.8",
+        "ts-jest": "^29.1.1",
+        "ts-node": "^10.9.1"
+    },
+    "prettier": "@kapeta/prettier-config",
+    "release": {
+        "plugins": [
+            "@semantic-release/commit-analyzer",
+            "@semantic-release/release-notes-generator",
+            [
+                "@semantic-release/changelog",
+                {
+                    "changelogFile": "CHANGELOG.md"
+                }
+            ],
+            "@semantic-release/npm",
+            "@semantic-release/github",
+            [
+                "@semantic-release/git",
+                {
+                    "assets": [
+                        "CHANGELOG.md",
+                        "package.json",
+                        "package-lock.json"
+                    ],
+                    "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+                }
+            ]
+        ]
+    }
+}