@kapeta/local-cluster-service 0.8.2 → 0.9.0

package/index.ts

@@ -17,7 +17,7 @@ import AssetsRoutes from './src/assets/routes';
 import ProviderRoutes from './src/providers/routes';
 import AttachmentRoutes from './src/attachments/routes';
 import { getBindHost } from './src/utils/utils';
-import request from "request";
+import request from 'request';
 
 export type LocalClusterService = HTTP.Server & { host?: string; port?: number };
 
@@ -40,13 +40,13 @@ function createServer() {
         res.send({
             ok: true,
             dockerStatus: await containerManager.checkAlive(),
-            socketStatus: socketManager.isAlive()
+            socketStatus: socketManager.isAlive(),
         });
     });
 
     app.get('/ping', async (req, res) => {
         res.send({
-            ok: true
+            ok: true,
         });
     });
 
@@ -54,7 +54,7 @@ function createServer() {
         console.error('Invalid request: %s %s', req.method, req.originalUrl);
         res.status(400).send({
             ok: false,
-            error: 'Unknown'
+            error: 'Unknown',
         });
     });
 
@@ -84,7 +84,7 @@ export default {
         return currentServer.port;
     },
 
-    ping: async function(host:string, port:number): Promise<{ ok:boolean }> {
+    ping: async function (host: string, port: number): Promise<{ ok: boolean }> {
         return new Promise((resolve, reject) => {
             request.get(`http://${host}:${port}/ping`, (err, res, body) => {
                 if (err) {
@@ -93,8 +93,8 @@ export default {
                 }
 
                 resolve(JSON.parse(body));
-            })
-        })
+            });
+        });
     },
 
     /**
@@ -166,7 +166,9 @@ export default {
 
             const bindHost = getBindHost(host);
 
-            currentServer.listen(port, bindHost, () => resolve({ host, port, dockerStatus: containerManager.isAlive() }));
+            currentServer.listen(port, bindHost, () =>
+                resolve({ host, port, dockerStatus: containerManager.isAlive() })
+            );
             currentServer.host = host;
             currentServer.port = port;
         });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.8.2",
+    "version": "0.9.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -67,6 +67,7 @@
         "yaml": "^1.6.0"
     },
     "devDependencies": {
+        "@kapeta/eslint-config": "^0.6.1",
         "@kapeta/prettier-config": "^0.6.0",
         "@tsconfig/node18": "^18.2.0",
         "@types/express": "^4.17.17",

package/src/assetManager.ts

@@ -10,6 +10,8 @@ import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { repositoryManager } from './repositoryManager';
 import { BlockDefinition } from '@kapeta/schemas';
 import { Actions } from '@kapeta/nodejs-registry-utils';
+import { definitionsManager } from './definitionsManager';
+import { normalizeKapetaUri } from './utils/utils';
 
 export interface EnrichedAsset {
     ref: string;
@@ -67,7 +69,7 @@ class AssetManager {
      */
     getAssets(assetKinds?: string[]): EnrichedAsset[] {
         if (!assetKinds) {
-            const blockTypeProviders = ClusterConfiguration.getDefinitions([
+            const blockTypeProviders = definitionsManager.getDefinitions([
                 'core/block-type',
                 'core/block-type-operator',
             ]);
@@ -77,7 +79,7 @@ class AssetManager {
             assetKinds.push('core/plan');
         }
 
-        const assets = ClusterConfiguration.getDefinitions(assetKinds);
+        const assets = definitionsManager.getDefinitions(assetKinds);
 
         return assets.map(enrichAsset);
     }
@@ -97,6 +99,7 @@ class AssetManager {
     }
 
     async getAsset(ref: string, noCache: boolean = false): Promise<EnrichedAsset | undefined> {
+        ref = normalizeKapetaUri(ref);
         const cacheKey = `getAsset:${ref}`;
         if (!noCache && this.cache.has(cacheKey)) {
             return this.cache.get(cacheKey);
@@ -104,10 +107,10 @@ class AssetManager {
         const uri = parseKapetaUri(ref);
         await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
 
-        let asset = ClusterConfiguration.getDefinitions()
+        let asset = definitionsManager
+            .getDefinitions()
             .map(enrichAsset)
             .find((a) => parseKapetaUri(a.ref).equals(uri));
-
         if (!asset) {
             throw new Error('Asset not found: ' + ref);
         }

package/src/attachments/routes.ts

@@ -3,7 +3,7 @@ import { KapetaAPI } from '@kapeta/nodejs-api-client';
 
 import { corsHandler } from '../middleware/cors';
 import { Request, Response } from 'express';
-import {storageService} from "../storageService";
+import { storageService } from '../storageService';
 
 const router = Router();
 const api = new KapetaAPI();
@@ -22,7 +22,7 @@ router.put('/:handle/:name', async (req: Request, res: Response) => {
     if (!req.headers['content-type']) {
         res.status(400).send({
             status: 400,
-            error: 'Missing content-type header'
+            error: 'Missing content-type header',
         });
         return;
     }
@@ -30,7 +30,7 @@ router.put('/:handle/:name', async (req: Request, res: Response) => {
     if (!req.headers['content-length']) {
         res.status(400).send({
             status: 400,
-            error: 'Missing content-length header'
+            error: 'Missing content-length header',
         });
         return;
     }
@@ -38,15 +38,15 @@ router.put('/:handle/:name', async (req: Request, res: Response) => {
     if (!req.headers['content-disposition']) {
         res.status(400).send({
             status: 400,
-            error: 'Missing content-disposition header'
+            error: 'Missing content-disposition header',
         });
         return;
     }
 
     try {
-        const {handle, name} = req.params;
+        const { handle, name } = req.params;
         const url = `${endpoint}/${handle}/${name}/attachments`;
-        const result = await api.send<{url:string}>({
+        const result = await api.send<{ url: string }>({
             method: 'PUT',
             url,
             auth: true,
@@ -55,10 +55,10 @@ router.put('/:handle/:name', async (req: Request, res: Response) => {
                 'content-length': req.headers['content-length'],
                 'content-disposition': req.headers['content-disposition'],
             },
-            body: req
+            body: req,
         });
         res.send(result);
-    } catch (e:any) {
+    } catch (e: any) {
         res.status(e.status ?? 500).send(e);
     }
 });

package/src/clusterService.ts

@@ -1,3 +1,5 @@
+import { normalizeKapetaUri } from './utils/utils';
+
 const net = require('net');
 const DEFAULT_SERVER_PORT = 35100;
 const DEFAULT_START_PORT = 40000;
@@ -125,6 +127,7 @@ class ClusterService {
      * @return {string}
      */
     getProxyPath(systemId: string, consumerInstanceId: string, consumerResourceName: string, portType: string) {
+        systemId = normalizeKapetaUri(systemId);
         return `/proxy/${encodeURIComponent(systemId)}/${encodeURIComponent(consumerInstanceId)}/${encodeURIComponent(
             consumerResourceName
         )}/${encodeURIComponent(portType)}/`;

package/src/codeGeneratorManager.ts

@@ -2,6 +2,7 @@ import Path from 'path';
 import { registry as Targets, BlockCodeGenerator, CodeWriter } from '@kapeta/codegen';
 import ClusterConfiguration from '@kapeta/local-cluster-config';
 import { BlockDefinition } from '@kapeta/schemas';
+import { definitionsManager } from './definitionsManager';
 
 const TARGET_KIND = 'core/language-target';
 const BLOCK_TYPE_KIND = 'core/block-type';
@@ -9,7 +10,7 @@ const BLOCK_TYPE_KIND = 'core/block-type';
 class CodeGeneratorManager {
     async reload() {
         Targets.reset();
-        const languageTargets = ClusterConfiguration.getDefinitions(TARGET_KIND);
+        const languageTargets = definitionsManager.getDefinitions(TARGET_KIND);
         for (const languageTarget of languageTargets) {
             const key = `${languageTarget.definition.metadata.name}:${languageTarget.version}`;
             try {
@@ -31,7 +32,7 @@ class CodeGeneratorManager {
             return false;
         }
 
-        const blockTypes = ClusterConfiguration.getDefinitions(BLOCK_TYPE_KIND);
+        const blockTypes = definitionsManager.getDefinitions(BLOCK_TYPE_KIND);
         const blockTypeKinds = blockTypes.map(
             (blockType) => blockType.definition.metadata.name.toLowerCase() + ':' + blockType.version
         );

package/src/config/routes.ts

@@ -41,7 +41,7 @@ router.put('/instance', async (req: KapetaBodyRequest, res) => {
         if (req.kapeta!.instanceId) {
             configManager.setConfigForSection(req.kapeta!.systemId, req.kapeta!.instanceId, config);
             //Restart the instance if it is running after config change
-            await instanceManager.restartIfRunning(req.kapeta!.systemId, req.kapeta!.instanceId);
+            await instanceManager.restart(req.kapeta!.systemId, req.kapeta!.instanceId);
         } else {
             configManager.setConfigForSystem(req.kapeta!.systemId, config);
         }

package/src/configManager.ts

@@ -3,6 +3,7 @@ import { BlockInstance } from '@kapeta/schemas';
 import { storageService } from './storageService';
 import { assetManager } from './assetManager';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import { normalizeKapetaUri } from './utils/utils';
 
 type AnyMap = { [key: string]: any };
 
@@ -19,6 +20,7 @@ class ConfigManager {
     }
 
     _forSystem(systemId: string) {
+        systemId = normalizeKapetaUri(systemId);
         if (!this._config[systemId]) {
             this._config[systemId] = {};
         }
@@ -27,16 +29,19 @@ class ConfigManager {
     }
 
     setConfigForSystem(systemId: string, config: AnyMap) {
+        systemId = normalizeKapetaUri(systemId);
         const systemConfig = config || {};
 
         storageService.put('config', systemId, systemConfig);
     }
 
     getConfigForSystem(systemId: string): AnyMap {
+        systemId = normalizeKapetaUri(systemId);
         return this._forSystem(systemId);
     }
 
     setConfigForSection(systemId: string, sectionId: string, config: AnyMap) {
+        systemId = normalizeKapetaUri(systemId);
         let systemConfig = this._forSystem(systemId);
         systemConfig[sectionId] = config || {};
 
@@ -44,6 +49,7 @@ class ConfigManager {
     }
 
     getConfigForSection(systemId: string, sectionId: string) {
+        systemId = normalizeKapetaUri(systemId);
         const systemConfig = this._forSystem(systemId);
 
         if (!systemConfig[sectionId]) {
@@ -70,6 +76,10 @@ class ConfigManager {
      * @returns {Promise<{systemId:string,instanceId:string}>}
      */
     async resolveIdentity(blockRef: string, systemId?: string) {
+        blockRef = normalizeKapetaUri(blockRef);
+        if (systemId) {
+            systemId = normalizeKapetaUri(systemId);
+        }
         const planAssets = assetManager.getPlans();
 
         const blockUri = parseKapetaUri(blockRef);
@@ -89,7 +99,7 @@ class ConfigManager {
                 const refUri = parseKapetaUri(blockInstance.block.ref);
                 if (refUri.equals(blockUri)) {
                     matchingIdentities.push({
-                        systemId: planAsset.ref,
+                        systemId: normalizeKapetaUri(planAsset.ref),
                         instanceId: blockInstance.id,
                     });
                 }
@@ -120,6 +130,8 @@ class ConfigManager {
     }
 
     async verifyIdentity(blockRef: string, systemId: string, instanceId: string) {
+        blockRef = normalizeKapetaUri(blockRef);
+        systemId = normalizeKapetaUri(systemId);
         const planAssets = assetManager.getPlans();
         const systemUri = systemId ? parseKapetaUri(systemId) : null;
         const blockUri = parseKapetaUri(blockRef);

package/src/containerManager.ts

@@ -27,6 +27,25 @@ export interface DockerMounts {
     Consistency: string;
 }
 
+interface DockerState {
+    Status: 'created' | 'running' | 'paused' | 'restarting' | 'removing' | 'exited' | 'dead';
+    Running: boolean;
+    Paused: boolean;
+    Restarting: boolean;
+    OOMKilled: boolean;
+    Dead: boolean;
+    Pid: number;
+    ExitCode: number;
+    Error: string;
+    StartedAt: string;
+    FinishedAt: string;
+    Health?: {
+        Status: 'starting' | 'healthy' | 'unhealthy' | 'none';
+        FailingStreak: number;
+        Log: any[] | null;
+    };
+}
+
 interface Health {
     cmd: string;
     interval?: number;
@@ -36,11 +55,13 @@ interface Health {
 
 const LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
-const HEALTH_CHECK_INTERVAL = 2000;
-const HEALTH_CHECK_MAX = 30;
+const HEALTH_CHECK_INTERVAL = 3000;
+const HEALTH_CHECK_MAX = 20;
 const IMAGE_PULL_CACHE_TTL = 30 * 60 * 1000;
 const IMAGE_PULL_CACHE: { [key: string]: number } = {};
 
+export const HEALTH_CHECK_TIMEOUT = HEALTH_CHECK_INTERVAL * HEALTH_CHECK_MAX * 2;
+
 const promisifyStream = (stream: ReadStream) =>
     new Promise((resolve, reject) => {
         stream.on('data', (d) => console.log(d.toString()));
@@ -116,7 +137,7 @@ class ContainerManager {
         }
 
         try {
-            await this._docker.ping()
+            await this._docker.ping();
             this._alive = true;
         } catch (e) {
             this._alive = false;
@@ -165,11 +186,17 @@ class ContainerManager {
         return this._docker;
     }
 
-    async getContainerByName(containerName: string): Promise<Container | undefined> {
+    async getContainerByName(containerName: string): Promise<ContainerInfo | undefined> {
         const containers = await this.docker().container.list({ all: true });
-        return containers.find((container) => {
-            return (container.data as any).Names.indexOf(`/${containerName}`) > -1;
+        const out = containers.find((container) => {
+            const containerData = container.data as any;
+            return containerData.Names.indexOf(`/${containerName}`) > -1;
         });
+
+        if (out) {
+            return new ContainerInfo(out);
+        }
+        return undefined;
     }
 
     async pull(image: string, cacheForMS: number = IMAGE_PULL_CACHE_TTL) {
@@ -371,18 +398,28 @@ class ContainerManager {
     }
 
     async _isReady(container: Container) {
-        const info: Container = await container.status();
+        let info: Container;
+        try {
+            info = await container.status();
+        } catch (err) {
+            return false;
+        }
         const infoData: any = info?.data;
-        if (infoData?.State?.Status === 'exited') {
+        const state = infoData?.State as DockerState;
+        if (state?.Status === 'exited' || state?.Status === 'removing' || state?.Status === 'dead') {
             throw new Error('Container exited unexpectedly');
         }
         return infoData?.State?.Running ?? false;
     }
 
     async _isHealthy(container: Container) {
-        const info = await container.status();
-        const infoData: any = info?.data;
-        return infoData?.State?.Health?.Status === 'healthy';
+        try {
+            const info = await container.status();
+            const infoData: any = info?.data;
+            return infoData?.State?.Health?.Status === 'healthy';
+        } catch (err) {
+            return false;
+        }
     }
 
     /**
@@ -429,7 +466,7 @@ export class ContainerInfo {
     }
 
     async isRunning() {
-        const inspectResult = await this.getStatus();
+        const inspectResult = await this.inspect();
 
         if (!inspectResult || !inspectResult.State) {
             return false;
@@ -464,14 +501,24 @@ export class ContainerInfo {
         return null;
     }
 
-    async getStatus() {
-        const result = await this._container.status();
+    async inspect() {
+        try {
+            const result = await this._container.status();
+
+            return result ? (result.data as any) : null;
+        } catch (err) {
+            return null;
+        }
+    }
+
+    async status() {
+        const result = await this.inspect();
 
-        return result ? (result.data as any) : null;
+        return result.State as DockerState;
     }
 
     async getPorts(): Promise<PortMap | false> {
-        const inspectResult = await this.getStatus();
+        const inspectResult = await this.inspect();
 
         if (!inspectResult || !inspectResult.Config || !inspectResult.Config.Labels) {
             return false;

package/src/definitionsManager.ts

@@ -0,0 +1,54 @@
+import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
+
+const CACHE_TTL = 60 * 1000; // 1 min
+
+interface DefinitionCacheEntry {
+    expires: number;
+    definitions: DefinitionInfo[];
+}
+
+class DefinitionsManager {
+    private cache: { [key: string]: DefinitionCacheEntry } = {};
+
+    private getKey(kindFilter?: string | string[]) {
+        if (kindFilter) {
+            if (Array.isArray(kindFilter)) {
+                return kindFilter.join(',');
+            }
+            return kindFilter;
+        }
+        return 'none';
+    }
+
+    public clearCache() {
+        this.cache = {};
+    }
+
+    private doCached(key: string, getter: () => DefinitionInfo[]) {
+        if (this.cache[key]) {
+            if (this.cache[key].expires > Date.now()) {
+                return this.cache[key].definitions;
+            }
+            delete this.cache[key];
+        }
+
+        this.cache[key] = {
+            expires: Date.now() + CACHE_TTL,
+            definitions: getter(),
+        };
+
+        return this.cache[key].definitions;
+    }
+
+    public getDefinitions(kindFilter?: string | string[]) {
+        const key = this.getKey(kindFilter);
+
+        return this.doCached(key, () => ClusterConfiguration.getDefinitions(kindFilter));
+    }
+
+    public getProviderDefinitions() {
+        return this.doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
+    }
+}
+
+export const definitionsManager = new DefinitionsManager();

package/src/filesystemManager.ts

@@ -17,9 +17,7 @@ function isFile(path: string) {
 class FilesystemManager {
     async writeFile(path: string, data: string | Buffer) {
         const dirName = Path.dirname(path);
-        console.log('Dir name', dirName, path);
         if (!FS.existsSync(dirName)) {
-            console.log('Making folder', dirName);
             FSExtra.mkdirpSync(dirName, {});
         }
         FS.writeFileSync(path, data);

package/src/identities/routes.ts

@@ -12,16 +12,15 @@ router.use('/', corsHandler);
 router.get('/current', async (req: Request, res: Response) => {
     try {
         res.send(await api.getCurrentIdentity());
-    } catch (e:any) {
+    } catch (e: any) {
         res.status(e.status ?? 500).send(e);
     }
-
 });
 
 router.get('/:identityId/memberships', async (req: Request, res: Response) => {
     try {
         res.send(await api.getMemberships(req.params.identityId));
-    } catch (e:any) {
+    } catch (e: any) {
         res.status(e.status ?? 500).send(e);
     }
 });