@kapeta/local-cluster-service 0.6.1 → 0.7.0

package/src/assetManager.ts

@@ -0,0 +1,191 @@
+import Path from 'node:path';
+import FS from 'node:fs';
+import FSExtra from 'fs-extra';
+import YAML from 'yaml';
+import NodeCache from 'node-cache';
+import ClusterConfiguration, { Definition, DefinitionInfo } from '@kapeta/local-cluster-config';
+import { codeGeneratorManager } from './codeGeneratorManager';
+import { progressListener } from './progressListener';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import { repositoryManager } from './repositoryManager';
+import { BlockDefinition } from '@kapeta/schemas';
+import { Actions } from '@kapeta/nodejs-registry-utils';
+
+export interface EnrichedAsset {
+    ref: string;
+    editable: boolean;
+    exists: boolean;
+    version: string;
+    kind: string;
+    data: Definition;
+    path: string;
+    ymlPath: string;
+}
+
+function enrichAsset(asset: DefinitionInfo): EnrichedAsset {
+    return {
+        ref: `kapeta://${asset.definition.metadata.name}:${asset.version}`,
+        editable: asset.version === 'local', //Only local versions are editable
+        exists: true,
+        version: asset.version,
+        kind: asset.definition.kind,
+        data: asset.definition,
+        path: asset.path,
+        ymlPath: asset.ymlPath,
+    };
+}
+
+function compareRefs(a: string, b: string) {
+    const [aProtocol, aId] = parseRef(a);
+    const [bProtocol, bId] = parseRef(b);
+
+    return aProtocol === bProtocol && aId === bId;
+}
+
+function parseRef(ref: string) {
+    let out = ref.split(/:\/\//, 2);
+
+    if (out.length === 1) {
+        return ['kapeta', ref.toLowerCase()];
+    }
+    return [out[0].toLowerCase(), out[1].toLowerCase()];
+}
+
+class AssetManager {
+    private cache: NodeCache;
+
+    constructor() {
+        this.cache = new NodeCache({
+            stdTTL: 60 * 60, // 1 hour
+        });
+    }
+
+    /**
+     *
+     * @param {string[]} [assetKinds]
+     * @returns {{path: *, ref: string, data: *, editable: boolean, kind: *, exists: boolean}[]}
+     */
+    getAssets(assetKinds?: string[]): EnrichedAsset[] {
+        if (!assetKinds) {
+            const blockTypeProviders = ClusterConfiguration.getDefinitions([
+                'core/block-type',
+                'core/block-type-operator',
+            ]);
+            assetKinds = blockTypeProviders.map((p) => {
+                return `${p.definition.metadata.name}:${p.version}`;
+            });
+            assetKinds.push('core/plan');
+        }
+
+        const assets = ClusterConfiguration.getDefinitions(assetKinds);
+
+        return assets.map(enrichAsset);
+    }
+
+    getPlans(): EnrichedAsset[] {
+        return this.getAssets(['core/plan']);
+    }
+
+    async getPlan(ref: string, noCache: boolean = false) {
+        const asset = await this.getAsset(ref, noCache);
+
+        if ('core/plan' !== asset?.kind) {
+            throw new Error('Asset was not a plan: ' + ref);
+        }
+
+        return asset.data;
+    }
+
+    async getAsset(ref: string, noCache: boolean = false): Promise<EnrichedAsset | undefined> {
+        const cacheKey = `getAsset:${ref}`;
+        if (!noCache && this.cache.has(cacheKey)) {
+            return this.cache.get(cacheKey);
+        }
+        const uri = parseKapetaUri(ref);
+        await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
+
+        let asset = ClusterConfiguration.getDefinitions()
+            .map(enrichAsset)
+            .find((a) => parseKapetaUri(a.ref).equals(uri));
+
+        if (!asset) {
+            throw new Error('Asset not found: ' + ref);
+        }
+        this.cache.set(cacheKey, asset);
+        return asset;
+    }
+
+    async createAsset(path: string, yaml: BlockDefinition): Promise<EnrichedAsset[]> {
+        if (FS.existsSync(path)) {
+            throw new Error('File already exists: ' + path);
+        }
+
+        const dirName = Path.dirname(path);
+        if (!FS.existsSync(dirName)) {
+            FSExtra.mkdirpSync(dirName);
+        }
+
+        FS.writeFileSync(path, YAML.stringify(yaml));
+
+        const asset = await this.importFile(path);
+
+        if (codeGeneratorManager.canGenerateCode(yaml)) {
+            await codeGeneratorManager.generate(path, yaml);
+        }
+        this.cache.flushAll();
+        return asset;
+    }
+
+    async updateAsset(ref: string, yaml: BlockDefinition) {
+        const asset = await this.getAsset(ref, true);
+        if (!asset) {
+            throw new Error('Attempted to update unknown asset: ' + ref);
+        }
+
+        if (!asset.editable) {
+            throw new Error('Attempted to update read-only asset: ' + ref);
+        }
+
+        if (!asset.ymlPath) {
+            throw new Error('Attempted to update corrupted asset: ' + ref);
+        }
+
+        FS.writeFileSync(asset.ymlPath, YAML.stringify(yaml));
+        this.cache.flushAll();
+        if (codeGeneratorManager.canGenerateCode(yaml)) {
+            await codeGeneratorManager.generate(asset.ymlPath, yaml);
+        } else {
+            console.log('Could not generate code for %s', yaml.kind ? yaml.kind : 'unknown yaml');
+        }
+    }
+
+    async importFile(filePath: string) {
+        if (filePath.startsWith('file://')) {
+            filePath = filePath.substring('file://'.length);
+        }
+
+        if (!FS.existsSync(filePath)) {
+            throw new Error('File not found: ' + filePath);
+        }
+
+        const assetInfos = YAML.parseAllDocuments(FS.readFileSync(filePath).toString()).map((doc) => doc.toJSON());
+
+        await Actions.link(progressListener, Path.dirname(filePath));
+
+        const version = 'local';
+        const refs = assetInfos.map((assetInfo) => `kapeta://${assetInfo.metadata.name}:${version}`);
+        this.cache.flushAll();
+        return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
+    }
+
+    async unregisterAsset(ref: string) {
+        const asset = await this.getAsset(ref, true);
+        if (!asset) {
+            throw new Error('Asset does not exists: ' + ref);
+        }
+        this.cache.flushAll();
+        await Actions.uninstall(progressListener, [asset.ref]);
+    }
+}
+
+export const assetManager = new AssetManager();

package/src/assets/routes.ts

@@ -0,0 +1,132 @@
+import Router from 'express-promise-router';
+import { Request, Response } from 'express';
+import YAML from 'yaml';
+import { assetManager } from '../assetManager';
+
+import { corsHandler } from '../middleware/cors';
+
+import { stringBody, StringBodyRequest } from '../middleware/stringBody';
+
+function parseBody(req: StringBodyRequest) {
+    switch (req.headers['content-type']) {
+        case 'application/json':
+        case 'application/x-json':
+        case 'text/json':
+            return JSON.parse(req.stringBody ?? '{}');
+
+        case 'application/yaml':
+        case 'application/x-yaml':
+        case 'text/yaml':
+        case 'text/x-yaml':
+        default:
+            return YAML.parse(req.stringBody ?? '{}');
+    }
+}
+
+const router = Router();
+
+router.use('/', corsHandler);
+router.use('/', stringBody);
+
+/**
+ * Get all local assets available
+ */
+router.get('/', (req: Request, res: Response) => {
+    res.send(assetManager.getAssets());
+});
+
+/**
+ * Get single asset
+ */
+router.get('/read', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        res.send(await assetManager.getAsset(req.query.ref as string, true));
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Creates a new local file and registers it as an asset
+ */
+router.post('/create', async (req: Request, res: Response) => {
+    if (!req.query.path) {
+        res.status(400).send({ error: 'Query parameter "path" is missing' });
+        return;
+    }
+
+    const content = parseBody(req);
+
+    try {
+        const assets = await assetManager.createAsset(req.query.path as string, content);
+
+        res.status(200).send(assets);
+    } catch (err: any) {
+        console.log('Failed while creating asset', req.query.path, err.message);
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Updates reference with new content
+ */
+router.put('/update', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    const content = parseBody(req);
+
+    try {
+        await assetManager.updateAsset(req.query.ref as string, content);
+
+        res.sendStatus(204);
+    } catch (err: any) {
+        console.log('Failed while updating asset', req.query.ref, err.message);
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Unregisters an asset (doesn't delete the asset)
+ */
+router.delete('/', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        await assetManager.unregisterAsset(req.query.ref as string);
+
+        res.status(204).send();
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Registers an existing file as an asset
+ */
+router.put('/import', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        const assets = await assetManager.importFile(req.query.ref as string);
+
+        res.status(200).send(assets);
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
+export default router;

package/src/clusterService.ts

@@ -0,0 +1,134 @@
+const net = require('net');
+const DEFAULT_SERVER_PORT = 35100;
+const DEFAULT_START_PORT = 40000;
+const DEFAULT_HOST = '127.0.0.1';
+
+class ClusterService {
+    private _port: number;
+    private _currentPort: number;
+    private _initialized: boolean;
+    private _reservedPorts: number[];
+    private _host: string;
+
+    constructor() {
+        this._port = DEFAULT_SERVER_PORT;
+        this._currentPort = DEFAULT_START_PORT;
+        this._initialized = false;
+        this._reservedPorts = [];
+        this._host = DEFAULT_HOST;
+    }
+
+    reservePort(port: number | string) {
+        const intPort = parseInt(port as string);
+        if (this._reservedPorts.indexOf(intPort) > -1) {
+            throw new Error('Port already reserved: ' + intPort);
+        }
+
+        this._reservedPorts.push(intPort);
+    }
+
+    async init() {
+        if (this._initialized) {
+            return;
+        }
+
+        this._initialized = true;
+        await this._findClusterServicePort();
+    }
+
+    async _findClusterServicePort() {
+        while (true) {
+            const isUsed = await this._checkIfPortIsUsed(this._port);
+            if (!isUsed) {
+                break;
+            }
+
+            this._port++;
+        }
+    }
+
+    /**
+     * Gets next available port
+     * @return {Promise<number>}
+     */
+    async getNextAvailablePort() {
+        while (true) {
+            while (this._reservedPorts.indexOf(this._currentPort) > -1) {
+                this._currentPort++;
+            }
+
+            const nextPort = this._currentPort++;
+            const isUsed = await this._checkIfPortIsUsed(nextPort);
+            if (!isUsed) {
+                return nextPort;
+            }
+        }
+    }
+
+    _checkIfPortIsUsed(port: number, host: string = this._host) {
+        return new Promise((resolve, reject) => {
+            const server = net.createServer();
+
+            server.once('error', function (err: any) {
+                if (err.code === 'EADDRINUSE') {
+                    server.close();
+                    resolve(true);
+                    return;
+                }
+
+                server.close();
+                reject(err);
+            });
+
+            server.once('listening', function () {
+                server.close();
+                resolve(false);
+            });
+
+            server.listen(port, host);
+        });
+    }
+
+    /**
+     * The port of this local cluster service itself
+     */
+    getClusterServicePort() {
+        return this._port;
+    }
+
+    /*
+     *Gets the host name ( 127.0.0.1 ) on which Express JS is listening
+     */
+    getClusterServiceHost() {
+        return this._host;
+    }
+
+    /**
+     * Set the port to be used for this local service
+     * @param port
+     */
+    setClusterServicePort(port: number) {
+        this._port = port;
+    }
+
+    setClusterServiceHost(host: string) {
+        this._host = host;
+    }
+
+    /**
+     * Gets that proxy path of a given request
+     *
+     * @param systemId
+     * @param consumerInstanceId
+     * @param consumerResourceName
+     * @param portType
+     * @return {string}
+     */
+    getProxyPath(systemId: string, consumerInstanceId: string, consumerResourceName: string, portType: string) {
+        return `/proxy/${encodeURIComponent(systemId)}/${encodeURIComponent(consumerInstanceId)}/${encodeURIComponent(
+            consumerResourceName
+        )}/${encodeURIComponent(portType)}/`;
+    }
+}
+
+export const clusterService = new ClusterService();

package/src/codeGeneratorManager.ts

@@ -0,0 +1,57 @@
+import Path from 'path';
+import { registry as Targets, BlockCodeGenerator, CodeWriter } from '@kapeta/codegen';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { BlockDefinition } from '@kapeta/schemas';
+
+const TARGET_KIND = 'core/language-target';
+const BLOCK_TYPE_KIND = 'core/block-type';
+
+class CodeGeneratorManager {
+    async reload() {
+        Targets.reset();
+        const languageTargets = ClusterConfiguration.getDefinitions(TARGET_KIND);
+        for (const languageTarget of languageTargets) {
+            const key = `${languageTarget.definition.metadata.name}:${languageTarget.version}`;
+            try {
+                const target = require(languageTarget.path);
+                if (target.default) {
+                    Targets.register(key, target.default);
+                } else {
+                    Targets.register(key, target);
+                }
+            } catch (e) {
+                console.error('Failed to load target: %s', key, e);
+            }
+        }
+    }
+
+    canGenerateCode(yamlContent: BlockDefinition): boolean {
+        if (!yamlContent.spec.target?.kind) {
+            //Not all block types have targets
+            return false;
+        }
+
+        const blockTypes = ClusterConfiguration.getDefinitions(BLOCK_TYPE_KIND);
+        const blockTypeKinds = blockTypes.map(
+            (blockType) => blockType.definition.metadata.name.toLowerCase() + ':' + blockType.version
+        );
+        return !!(yamlContent && yamlContent.kind && blockTypeKinds.indexOf(yamlContent.kind.toLowerCase()) > -1);
+    }
+
+    async generate(yamlFile: string, yamlContent: BlockDefinition) {
+        const baseDir = Path.dirname(yamlFile);
+        console.log('Generating code for path: %s', baseDir);
+        const codeGenerator = new BlockCodeGenerator(yamlContent);
+
+        const output = await codeGenerator.generate();
+        const writer = new CodeWriter(baseDir, {});
+        const assets = writer.write(output);
+
+        await codeGenerator.postprocess(baseDir, assets);
+
+        console.log('Code generated for path: %s', baseDir);
+    }
+}
+
+export const codeGeneratorManager = new CodeGeneratorManager();
+codeGeneratorManager.reload();

package/src/config/routes.ts

@@ -0,0 +1,159 @@
+import Router from 'express-promise-router';
+import { configManager } from '../configManager';
+import { serviceManager } from '../serviceManager';
+import { operatorManager } from '../operatorManager';
+import { instanceManager } from '../instanceManager';
+import { StringBodyRequest } from '../middleware/stringBody';
+import { corsHandler } from '../middleware/cors';
+import { kapetaHeaders, KapetaRequest } from '../middleware/kapeta';
+import { stringBody } from '../middleware/stringBody';
+import { EnvironmentType, KapetaBodyRequest } from '../types';
+import { Response } from 'express';
+
+const router = Router();
+const SYSTEM_ID = '$plan';
+
+router.use('/', corsHandler);
+router.use('/', kapetaHeaders);
+router.use('/', stringBody);
+
+/**
+ * Returns the full configuration for a given service.
+ */
+router.get('/instance', (req: KapetaBodyRequest, res: Response) => {
+    const config = req.kapeta!.instanceId
+        ? configManager.getConfigForSection(req.kapeta!.systemId, req.kapeta!.instanceId)
+        : configManager.getConfigForSystem(req.kapeta!.systemId);
+
+    res.send(config);
+});
+
+/**
+ * Updates the full configuration for a given service.
+ */
+router.put('/instance', async (req: KapetaBodyRequest, res) => {
+    try {
+        let config = JSON.parse(req.stringBody ?? '{}');
+        if (!config) {
+            config = {};
+        }
+
+        if (req.kapeta!.instanceId) {
+            configManager.setConfigForSection(req.kapeta!.systemId, req.kapeta!.instanceId, config);
+            //Restart the instance if it is running after config change
+            await instanceManager.restartIfRunning(req.kapeta!.systemId, req.kapeta!.instanceId);
+        } else {
+            configManager.setConfigForSystem(req.kapeta!.systemId, config);
+        }
+    } catch (err: any) {
+        console.error('Failed to update instance config', err);
+        res.status(400).send({ error: err.message });
+        return;
+    }
+
+    res.status(202).send({ ok: true });
+});
+
+/**
+ * Returns the full configuration for a plan
+ */
+router.get('/system', (req: KapetaRequest, res) => {
+    const config = configManager.getConfigForSection(req.kapeta!.systemId, SYSTEM_ID);
+
+    res.send(config);
+});
+
+/**
+ * Updates the full configuration for a plan
+ */
+router.put('/system', (req: KapetaBodyRequest, res) => {
+    let config = JSON.parse(req.stringBody ?? '{}');
+    if (!config) {
+        config = {};
+    }
+    configManager.setConfigForSection(req.kapeta!.systemId, SYSTEM_ID, config);
+    res.status(202).send({ ok: true });
+});
+
+/**
+ * Resolves and checks the identity of a block instance
+ */
+router.get('/identity', async (req: KapetaRequest, res) => {
+    const identity = {
+        systemId: req.kapeta!.systemId,
+        instanceId: req.kapeta!.instanceId,
+    };
+
+    if (!req.kapeta!.blockRef) {
+        res.status(400).send({ error: 'Missing required header "X-Kapeta-Block"' });
+        return;
+    }
+
+    try {
+        if (!identity.systemId || !identity.instanceId) {
+            const { systemId, instanceId } = await configManager.resolveIdentity(
+                req.kapeta!.blockRef,
+                identity.systemId
+            );
+            identity.systemId = systemId;
+            identity.instanceId = instanceId;
+        } else {
+            await configManager.verifyIdentity(req.kapeta!.blockRef, identity.systemId, identity.instanceId);
+        }
+
+        res.send(identity);
+    } catch (err: any) {
+        console.warn('Failed to resolve identity', err);
+        res.status(400).send({ error: err.message });
+    }
+});
+
+/**
+ * Services call this to request a free port. If a service has
+ * already called the endpoint the same port is returned.
+ */
+router.get('/provides/:type', async (req: KapetaRequest, res) => {
+    //Get service port
+    res.send(
+        '' + (await serviceManager.ensureServicePort(req.kapeta!.systemId, req.kapeta!.instanceId, req.params.type))
+    );
+});
+
+/**
+ * Used by services to get info for consumed operator resource.
+ *
+ * If the operator resource is not already available this will cause it to start an instance and
+ * assign port numbers to it etc.
+ */
+router.get('/consumes/resource/:resourceType/:portType/:name', async (req: KapetaRequest, res) => {
+    const operatorInfo = await operatorManager.getConsumerResourceInfo(
+        req.kapeta!.systemId,
+        req.kapeta!.instanceId,
+        req.params.resourceType,
+        req.params.portType,
+        req.params.name,
+        req.kapeta!.environment
+    );
+
+    res.send(operatorInfo);
+});
+
+/**
+ * Used by services to get address for their clients.
+ *
+ * If the remote service is not already registered with a port - we do that here
+ * to handle clients for services that hasn't started yet.
+ */
+router.get('/consumes/:resourceName/:type', (req: KapetaRequest, res) => {
+    res.send(
+        serviceManager.getConsumerAddress(
+            req.kapeta!.systemId,
+            req.kapeta!.instanceId,
+            req.params.resourceName,
+            req.params.type,
+            req.kapeta!.environment
+        )
+    );
+});
+
+export default router;