@kapeta/local-cluster-service 0.0.0-96f91ef

package/src/assets/routes.ts

@@ -0,0 +1,183 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { Request, Response } from 'express';
+import YAML from 'yaml';
+import { assetManager } from '../assetManager';
+
+import { corsHandler } from '../middleware/cors';
+
+import { stringBody, StringBodyRequest } from '../middleware/stringBody';
+import { definitionsManager } from '../definitionsManager';
+
+function parseBody(req: StringBodyRequest) {
+    switch (req.headers['content-type']) {
+        case 'application/json':
+        case 'application/x-json':
+        case 'text/json':
+            return JSON.parse(req.stringBody ?? '{}');
+
+        case 'application/yaml':
+        case 'application/x-yaml':
+        case 'text/yaml':
+        case 'text/x-yaml':
+        default:
+            return YAML.parse(req.stringBody ?? '{}');
+    }
+}
+
+const router = Router();
+
+router.use('/', corsHandler);
+router.use('/', stringBody);
+
+/**
+ * Get all local assets available
+ */
+router.get('/', async (req: Request, res: Response) => {
+    res.send(await assetManager.getAssets([]));
+});
+
+/**
+ * Get single asset
+ */
+router.get('/read', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    const ensure = req.query.ensure !== 'false';
+
+    try {
+        const asset = await assetManager.getAsset(req.query.ref as string, true, ensure);
+        if (asset) {
+            res.send(asset);
+        } else {
+            res.status(404).send({ error: 'Asset not found' });
+        }
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Get all versions for asset name
+ */
+router.get('/versions', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    const ensure = req.query.ensure !== 'false';
+
+    try {
+        const versions = await definitionsManager.getVersions(req.query.ref as string);
+        if (versions) {
+            res.send(versions);
+        } else {
+            res.status(404).send({ error: 'Asset not found' });
+        }
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Creates a new local file and registers it as an asset
+ */
+router.post('/create', async (req: Request, res: Response) => {
+    if (!req.query.path) {
+        res.status(400).send({ error: 'Query parameter "path" is missing' });
+        return;
+    }
+
+    const content = parseBody(req);
+
+    try {
+        const assets = await assetManager.createAsset(req.query.path as string, content, 'user');
+
+        res.status(200).send(assets);
+    } catch (err: any) {
+        console.log('Failed while creating asset', req.query.path, err.message);
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Updates reference with new content
+ */
+router.put('/update', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    const content = parseBody(req);
+
+    try {
+        await assetManager.updateAsset(req.query.ref as string, content, 'user');
+
+        res.sendStatus(204);
+    } catch (err: any) {
+        console.log('Failed while updating asset', req.query.ref, err.message);
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Unregisters an asset (doesn't delete the asset)
+ */
+router.delete('/', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        await assetManager.unregisterAsset(req.query.ref as string);
+
+        res.status(204).send();
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Registers an existing file as an asset
+ */
+router.put('/import', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        const assets = await assetManager.importFile(req.query.ref as string);
+
+        res.status(200).send(assets);
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+router.put('/install', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        const tasks = await assetManager.installAsset(req.query.ref as string, !!req.query.wait);
+        const taskIds = tasks?.map((t) => t.id) ?? [];
+        res.status(200).send(taskIds);
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+export default router;

package/src/attachments/routes.ts

@@ -0,0 +1,79 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+
+import { corsHandler } from '../middleware/cors';
+import { Request, Response } from 'express';
+import { storageService } from '../storageService';
+
+const router = Router();
+const api = new KapetaAPI();
+
+const DEFAULT_REGISTRY_BASE = 'https://registry.kapeta.com';
+
+function getBaseUrl() {
+    const endpoint = storageService.get('endpoints', 'registry', DEFAULT_REGISTRY_BASE);
+    return `${endpoint}/v1/registry`;
+}
+
+router.use('/', corsHandler);
+
+router.put('/:handle/:name', async (req: Request, res: Response) => {
+    if (!api.hasToken()) {
+        res.status(401).send({
+            status: 401,
+            error: 'Unauthorized',
+        });
+        return;
+    }
+
+    const endpoint = getBaseUrl();
+    if (!req.headers['content-type']) {
+        res.status(400).send({
+            status: 400,
+            error: 'Missing content-type header',
+        });
+        return;
+    }
+
+    if (!req.headers['content-length']) {
+        res.status(400).send({
+            status: 400,
+            error: 'Missing content-length header',
+        });
+        return;
+    }
+
+    if (!req.headers['content-disposition']) {
+        res.status(400).send({
+            status: 400,
+            error: 'Missing content-disposition header',
+        });
+        return;
+    }
+
+    try {
+        const { handle, name } = req.params;
+        const url = `${endpoint}/${handle}/${name}/attachments`;
+        const result = await api.send<{ url: string }>({
+            method: 'PUT',
+            url,
+            auth: true,
+            headers: {
+                'content-type': req.headers['content-type'],
+                'content-length': req.headers['content-length'],
+                'content-disposition': req.headers['content-disposition'],
+            },
+            body: req,
+        });
+        res.send(result);
+    } catch (e: any) {
+        res.status(e.status ?? 500).send(e);
+    }
+});
+
+export default router;

package/src/authManager.ts

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { EventEmitter } from 'node:events';
+import Path from 'node:path';
+import chokidar, { FSWatcher } from 'chokidar';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { WatchEventName } from './types';
+import { definitionsManager } from './definitionsManager';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { socketManager } from './socketManager';
+
+class AuthManager extends EventEmitter {
+    private watcher?: FSWatcher;
+
+    private hadToken: boolean;
+
+    constructor() {
+        super();
+        this.hadToken = this.hasToken();
+    }
+
+    public listenForChanges() {
+        const parentDir = Path.dirname(ClusterConfiguration.getKapetaBasedir());
+        //We watch the parent dir to catch changes to the base dir itself
+        this.watcher = chokidar.watch(parentDir, {
+            followSymlinks: false,
+            ignorePermissionErrors: true,
+            disableGlobbing: true,
+            persistent: true,
+            ignoreInitial: true,
+            depth: 1,
+            ignored: (path) => {
+                return !path.startsWith(ClusterConfiguration.getKapetaBasedir());
+            },
+        });
+        this.watcher.add(ClusterConfiguration.getKapetaBasedir());
+        this.watcher.on('all', this.handleFileChange.bind(this));
+        this.watcher.on('error', (error) => {
+            console.log('Error watching repository', error);
+        });
+        this.watcher.on('ready', () => {
+            console.log('Watching for auth changes: %s', ClusterConfiguration.getKapetaBasedir());
+        });
+    }
+
+    private hasToken() {
+        const api = new KapetaAPI();
+        return api.hasToken();
+    }
+
+    private async handleFileChange(eventName: WatchEventName, path: string) {
+        const hasTokenNow = this.hasToken();
+        if (this.hadToken !== hasTokenNow) {
+            socketManager.emitGlobal('auth-change', {});
+            if (hasTokenNow) {
+                // Clear the cache in case we need to rewrite the sample plan
+                definitionsManager.clearCache();
+            }
+            this.hadToken = hasTokenNow;
+        }
+    }
+}
+
+export const authManager = new AuthManager();

package/src/cacheManager.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import NodeCache from 'node-cache';
+
+const DEFAULT_CACHE_TTL = 60 * 1000; // 1 min
+export interface CacheEntry<T = any> {
+    expires: number;
+    data: T;
+}
+export class CacheManager {
+    private cache: NodeCache = new NodeCache();
+
+    public flush() {
+        this.cache.flushAll();
+    }
+
+    public doCached<T>(key: string, getter: () => T, ttl = DEFAULT_CACHE_TTL) {
+        const data = this.cache.get<T>(key);
+        if (data !== undefined) {
+            return data;
+        }
+        const result = getter();
+        this.cache.set(key, result, ttl);
+        return result;
+    }
+
+    public get<T>(key: string): T | undefined {
+        return this.cache.get<T>(key);
+    }
+
+    public set<T>(key: string, data: T, ttl = DEFAULT_CACHE_TTL) {
+        this.cache.set(key, data, ttl);
+    }
+
+    public has(key: string): boolean {
+        return this.cache.has(key);
+    }
+
+    public remove(key: string) {
+        return this.cache.del(key);
+    }
+
+    public removePrefix(key: string) {
+        const keys = this.cache.keys();
+        for (const k of keys) {
+            if (k.startsWith(key)) {
+                this.remove(k);
+            }
+        }
+    }
+}
+
+export const cacheManager = new CacheManager();
+
+export const doCached = <T>(key: string, getter: () => T, ttl = DEFAULT_CACHE_TTL) =>
+    cacheManager.doCached(key, getter, ttl);

package/src/clusterService.ts

@@ -0,0 +1,142 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { normalizeKapetaUri } from '@kapeta/nodejs-utils';
+
+const net = require('net');
+const DEFAULT_SERVER_PORT = 35100;
+const DEFAULT_START_PORT = 40000;
+const DEFAULT_HOST = '127.0.0.1';
+
+class ClusterService {
+    private _port: number;
+    private _currentPort: number;
+    private _initialized: boolean;
+    private _reservedPorts: number[];
+    private _host: string;
+
+    constructor() {
+        this._port = DEFAULT_SERVER_PORT;
+        this._currentPort = DEFAULT_START_PORT;
+        this._initialized = false;
+        this._reservedPorts = [];
+        this._host = DEFAULT_HOST;
+    }
+
+    reservePort(port: number | string) {
+        const intPort = parseInt(port as string);
+        if (this._reservedPorts.indexOf(intPort) > -1) {
+            throw new Error('Port already reserved: ' + intPort);
+        }
+
+        this._reservedPorts.push(intPort);
+    }
+
+    async init() {
+        if (this._initialized) {
+            return;
+        }
+
+        this._initialized = true;
+        await this._findClusterServicePort();
+    }
+
+    async _findClusterServicePort() {
+        while (true) {
+            const isUsed = await this._checkIfPortIsUsed(this._port);
+            if (!isUsed) {
+                break;
+            }
+
+            this._port++;
+        }
+    }
+
+    /**
+     * Gets next available port
+     * @return {Promise<number>}
+     */
+    async getNextAvailablePort() {
+        while (true) {
+            while (this._reservedPorts.indexOf(this._currentPort) > -1) {
+                this._currentPort++;
+            }
+
+            const nextPort = this._currentPort++;
+            const isUsed = await this._checkIfPortIsUsed(nextPort);
+            if (!isUsed) {
+                return nextPort;
+            }
+        }
+    }
+
+    _checkIfPortIsUsed(port: number, host: string = this._host) {
+        return new Promise((resolve, reject) => {
+            const server = net.createServer();
+
+            server.once('error', function (err: any) {
+                if (err.code === 'EADDRINUSE') {
+                    server.close();
+                    resolve(true);
+                    return;
+                }
+
+                server.close();
+                reject(err);
+            });
+
+            server.once('listening', function () {
+                server.close();
+                resolve(false);
+            });
+
+            server.listen(port, host);
+        });
+    }
+
+    /**
+     * The port of this local cluster service itself
+     */
+    getClusterServicePort() {
+        return this._port;
+    }
+
+    /*
+     *Gets the host name ( 127.0.0.1 ) on which Express JS is listening
+     */
+    getClusterServiceHost() {
+        return this._host;
+    }
+
+    /**
+     * Set the port to be used for this local service
+     * @param port
+     */
+    setClusterServicePort(port: number) {
+        this._port = port;
+    }
+
+    setClusterServiceHost(host: string) {
+        this._host = host;
+    }
+
+    /**
+     * Gets that proxy path of a given request
+     *
+     * @param systemId
+     * @param consumerInstanceId
+     * @param consumerResourceName
+     * @param portType
+     * @return {string}
+     */
+    getProxyPath(systemId: string, consumerInstanceId: string, consumerResourceName: string, portType: string) {
+        systemId = normalizeKapetaUri(systemId);
+        return `/proxy/${encodeURIComponent(systemId)}/${encodeURIComponent(consumerInstanceId)}/${encodeURIComponent(
+            consumerResourceName
+        )}/${encodeURIComponent(portType)}/`;
+    }
+}
+
+export const clusterService = new ClusterService();

package/src/codeGeneratorManager.ts

@@ -0,0 +1,109 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Path from 'path';
+import { registry as Targets, BlockCodeGenerator, CodeWriter } from '@kapeta/codegen';
+import { BlockDefinition } from '@kapeta/schemas';
+import { definitionsManager } from './definitionsManager';
+import { Definition } from '@kapeta/local-cluster-config';
+import { assetManager } from './assetManager';
+import { normalizeKapetaUri, parseKapetaUri } from '@kapeta/nodejs-utils';
+import { repositoryManager } from './repositoryManager';
+
+const TARGET_KIND = 'core/language-target';
+const BLOCK_TYPE_KIND = 'core/block-type';
+
+class CodeGeneratorManager {
+    private async ensureLanguageTargetInRegistry(path: string, version: string, definition: Definition) {
+        const key = `${definition.metadata.name}:${version}`;
+
+        try {
+            if (await Targets.get(key)) {
+                return;
+            }
+        } catch (e) {}
+
+        try {
+            const target = require(path);
+            if (target.default) {
+                Targets.register(key, target.default);
+            } else {
+                Targets.register(key, target);
+            }
+        } catch (e) {
+            console.error('Failed to load target: %s', key, e);
+        }
+    }
+    async reload() {
+        Targets.reset();
+        const languageTargets = await definitionsManager.getDefinitions(TARGET_KIND);
+        for (const languageTarget of languageTargets) {
+            await this.ensureLanguageTargetInRegistry(
+                languageTarget.path,
+                languageTarget.version,
+                languageTarget.definition
+            );
+        }
+    }
+
+    async initialize() {
+        await this.reload();
+        repositoryManager.on('change', async () => {
+            // Reload code generators when the repository changes
+            try {
+                await this.reload();
+            } catch (e) {
+                console.error('Failed to reload code generators', e);
+            }
+        });
+    }
+
+    async canGenerateCode(yamlContent: Definition): Promise<boolean> {
+        if (!yamlContent.spec?.target?.kind || !yamlContent.kind) {
+            //Not all block types have targets
+            return false;
+        }
+
+        const kindUri = parseKapetaUri(yamlContent.kind);
+
+        const blockTypes = await definitionsManager.getDefinitions(BLOCK_TYPE_KIND);
+        const blockTypeKinds = blockTypes.map(
+            (blockType) => parseKapetaUri(blockType.definition.metadata.name).fullName
+        );
+        return blockTypeKinds.includes(kindUri.fullName);
+    }
+
+    async generate(yamlFile: string, yamlContent: Definition) {
+        if (!yamlContent.spec.target?.kind) {
+            //Not all block types have targets
+            return;
+        }
+
+        const targetRef = normalizeKapetaUri(yamlContent.spec.target?.kind);
+
+        // Automatically downloads target if not available
+        const targetAsset = await assetManager.getAsset(targetRef);
+
+        if (!targetAsset) {
+            console.error('Language target not found: %s', yamlContent.spec.target?.kind);
+            return;
+        }
+
+        await this.ensureLanguageTargetInRegistry(targetAsset?.path, targetAsset?.version, targetAsset?.data);
+        const baseDir = Path.dirname(yamlFile);
+        console.log('Generating code for path: %s', baseDir);
+        const codeGenerator = new BlockCodeGenerator(yamlContent as BlockDefinition);
+
+        const output = await codeGenerator.generate();
+        const writer = new CodeWriter(baseDir, {});
+        const assets = writer.write(output);
+
+        await codeGenerator.postprocess(baseDir, assets);
+
+        console.log('Code generated for path: %s', baseDir);
+    }
+}
+
+export const codeGeneratorManager = new CodeGeneratorManager();