@kapeta/local-cluster-service 0.16.8 → 0.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.16.8",
+    "version": "0.18.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -34,8 +34,8 @@
         "dev": "nodemon -e js,ts,json ./start.ts",
         "test": "echo its fine",
         "clean": "rm -rf ./dist",
-        "build:esm": "tsc --module esnext --outDir ./dist/esm && echo '{\"type\":\"module\"}' > ./dist/esm/package.json",
-        "build:cjs": "tsc --module commonjs --outDir ./dist/cjs && echo '{\"type\":\"commonjs\"}' > ./dist/cjs/package.json",
+        "build:esm": "tsc --module nodenext --moduleResolution nodenext --outDir ./dist/esm && echo '{\"type\":\"module\"}' > ./dist/esm/package.json",
+        "build:cjs": "tsc  --outDir ./dist/cjs && echo '{\"type\":\"commonjs\"}' > ./dist/cjs/package.json",
         "build": "npm run clean && npm run build:esm && npm run build:cjs",
         "format": "prettier --write .",
         "lint": "tsc --noEmit && eslint src/**/*.ts",
@@ -57,7 +57,8 @@
         "express": "4.17.1",
         "express-promise-router": "^4.1.1",
         "fs-extra": "^11.1.0",
-        "glob": "7.1.6",
+        "glob": "^7.1.6",
+        "gunzip-maybe": "^1.4.2",
         "lodash": "^4.17.15",
         "md5": "2.2.1",
         "node-cache": "^5.1.2",
@@ -66,6 +67,7 @@
         "request": "2.88.2",
         "request-promise": "4.2.6",
         "socket.io": "^4.5.2",
+        "tar-stream": "^3.1.6",
         "typescript": "^5.1.6",
         "yaml": "^1.6.0"
     },
@@ -76,10 +78,14 @@
         "@types/async-lock": "^1.4.0",
         "@types/express": "^4.17.17",
         "@types/fs-extra": "^11.0.1",
+        "@types/glob": "^8.1.0",
+        "@types/gunzip-maybe": "^1.4.0",
         "@types/lodash": "^4.14.195",
         "@types/md5": "^2.3.2",
+        "@types/node": "^20.5.8",
         "@types/node-uuid": "^0.0.29",
         "@types/request": "^2.48.8",
+        "@types/tar-stream": "^2.2.2",
         "eslint": "^8.42.0",
         "eslint-config-prettier": "^8.8.0",
         "nodemon": "^2.0.2",

package/src/assetManager.ts

@@ -64,9 +64,9 @@ class AssetManager {
      * @param {string[]} [assetKinds]
      * @returns {{path: *, ref: string, data: *, editable: boolean, kind: *, exists: boolean}[]}
      */
-    getAssets(assetKinds?: string[]): EnrichedAsset[] {
+    async getAssets(assetKinds?: string[]): Promise<EnrichedAsset[]> {
         if (!assetKinds) {
-            const blockTypeProviders = definitionsManager.getDefinitions([
+            const blockTypeProviders = await definitionsManager.getDefinitions([
                 'core/block-type',
                 'core/block-type-operator',
             ]);
@@ -76,12 +76,12 @@ class AssetManager {
             assetKinds.push('core/plan');
         }
 
-        const assets = definitionsManager.getDefinitions(assetKinds);
+        const assets = await definitionsManager.getDefinitions(assetKinds);
 
         return assets.map(enrichAsset);
     }
 
-    getPlans(): EnrichedAsset[] {
+    async getPlans(): Promise<EnrichedAsset[]> {
         return this.getAssets(['core/plan']);
     }
 
@@ -110,18 +110,18 @@ class AssetManager {
             await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, true);
         }
 
-        let asset = definitionsManager
-            .getDefinitions()
-            .map(enrichAsset)
-            .find((a) => parseKapetaUri(a.ref).equals(uri));
-        if (autoFetch && !asset) {
+        const definitionInfo = await definitionsManager.getDefinition(ref);
+        if (autoFetch && !definitionInfo) {
             throw new Error('Asset not found: ' + ref);
         }
-        if (asset) {
+
+        if (definitionInfo) {
+            const asset = enrichAsset(definitionInfo);
             cacheManager.set(cacheKey, asset, CACHE_TTL);
+            return asset;
         }
 
-        return asset;
+        return undefined;
     }
 
     async createAsset(
@@ -151,7 +151,7 @@ class AssetManager {
 
         const ref = `kapeta://${yaml.metadata.name}:local`;
 
-        this.maybeGenerateCode(ref, path, yaml);
+        await this.maybeGenerateCode(ref, path, yaml);
 
         return asset;
     }
@@ -178,7 +178,7 @@ class AssetManager {
         cacheManager.remove(toKey(ref));
         definitionsManager.clearCache();
 
-        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
+        await this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
     }
 
     async importFile(filePath: string) {
@@ -206,7 +206,9 @@ class AssetManager {
 
         definitionsManager.clearCache();
 
-        return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
+        const assets = await this.getAssets();
+
+        return assets.filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
     }
 
     async unregisterAsset(ref: string) {
@@ -236,9 +238,9 @@ class AssetManager {
         return await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
     }
 
-    private maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
+    private async maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
         ref = normalizeKapetaUri(ref);
-        if (codeGeneratorManager.canGenerateCode(block)) {
+        if (await codeGeneratorManager.canGenerateCode(block)) {
             const assetTitle = block.metadata.title ? block.metadata.title : parseKapetaUri(block.metadata.name).name;
             taskManager.add(
                 `codegen:${ref}`,

package/src/assets/routes.ts

@@ -31,8 +31,8 @@ router.use('/', stringBody);
 /**
  * Get all local assets available
  */
-router.get('/', (req: Request, res: Response) => {
-    res.send(assetManager.getAssets([]));
+router.get('/', async (req: Request, res: Response) => {
+    res.send(await assetManager.getAssets([]));
 });
 
 /**

package/src/authManager.ts

@@ -0,0 +1,62 @@
+import { EventEmitter } from 'node:events';
+import Path from 'node:path';
+import chokidar, { FSWatcher } from 'chokidar';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { WatchEventName } from './types';
+import { definitionsManager } from './definitionsManager';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { socketManager } from './socketManager';
+
+class AuthManager extends EventEmitter {
+    private watcher?: FSWatcher;
+
+    private hadToken: boolean;
+
+    constructor() {
+        super();
+        this.hadToken = this.hasToken();
+    }
+
+    public listenForChanges() {
+        const parentDir = Path.dirname(ClusterConfiguration.getKapetaBasedir());
+        //We watch the parent dir to catch changes to the base dir itself
+        this.watcher = chokidar.watch(parentDir, {
+            followSymlinks: false,
+            ignorePermissionErrors: true,
+            disableGlobbing: true,
+            persistent: true,
+            ignoreInitial: true,
+            depth: 1,
+            ignored: (path) => {
+                return !path.startsWith(ClusterConfiguration.getKapetaBasedir());
+            },
+        });
+        this.watcher.add(ClusterConfiguration.getKapetaBasedir());
+        this.watcher.on('all', this.handleFileChange.bind(this));
+        this.watcher.on('error', (error) => {
+            console.log('Error watching repository', error);
+        });
+        this.watcher.on('ready', () => {
+            console.log('Watching for auth changes: %s', ClusterConfiguration.getKapetaBasedir());
+        });
+    }
+
+    private hasToken() {
+        const api = new KapetaAPI();
+        return api.hasToken();
+    }
+
+    private async handleFileChange(eventName: WatchEventName, path: string) {
+        const hasTokenNow = this.hasToken();
+        if (this.hadToken !== hasTokenNow) {
+            socketManager.emitGlobal('auth-change', {});
+            if (hasTokenNow) {
+                // Clear the cache in case we need to rewrite the sample plan
+                definitionsManager.clearCache();
+            }
+            this.hadToken = hasTokenNow;
+        }
+    }
+}
+
+export const authManager = new AuthManager();

package/src/codeGeneratorManager.ts

@@ -10,7 +10,7 @@ const BLOCK_TYPE_KIND = 'core/block-type';
 class CodeGeneratorManager {
     async reload() {
         Targets.reset();
-        const languageTargets = definitionsManager.getDefinitions(TARGET_KIND);
+        const languageTargets = await definitionsManager.getDefinitions(TARGET_KIND);
         for (const languageTarget of languageTargets) {
             const key = `${languageTarget.definition.metadata.name}:${languageTarget.version}`;
             try {
@@ -26,13 +26,13 @@ class CodeGeneratorManager {
         }
     }
 
-    canGenerateCode(yamlContent: BlockDefinition): boolean {
+    async canGenerateCode(yamlContent: BlockDefinition): Promise<boolean> {
         if (!yamlContent.spec.target?.kind) {
             //Not all block types have targets
             return false;
         }
 
-        const blockTypes = definitionsManager.getDefinitions(BLOCK_TYPE_KIND);
+        const blockTypes = await definitionsManager.getDefinitions(BLOCK_TYPE_KIND);
         const blockTypeKinds = blockTypes.map(
             (blockType) => blockType.definition.metadata.name.toLowerCase() + ':' + blockType.version
         );

package/src/configManager.ts

@@ -80,7 +80,7 @@ class ConfigManager {
         if (systemId) {
             systemId = normalizeKapetaUri(systemId);
         }
-        const planAssets = assetManager.getPlans();
+        const planAssets = await assetManager.getPlans();
 
         const blockUri = parseKapetaUri(blockRef);
 
@@ -132,7 +132,7 @@ class ConfigManager {
     async verifyIdentity(blockRef: string, systemId: string, instanceId: string) {
         blockRef = normalizeKapetaUri(blockRef);
         systemId = normalizeKapetaUri(systemId);
-        const planAssets = assetManager.getPlans();
+        const planAssets = await assetManager.getPlans();
         const systemUri = systemId ? parseKapetaUri(systemId) : null;
         const blockUri = parseKapetaUri(blockRef);
         let found = false;

package/src/definitionsManager.ts

@@ -1,39 +1,154 @@
 import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { cacheManager, doCached } from './cacheManager';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { Plan } from '@kapeta/schemas';
+import FS from 'fs-extra';
+import { normalizeKapetaUri } from './utils/utils';
+import YAML from 'yaml';
+import { Actions } from '@kapeta/nodejs-registry-utils';
+import { ProgressListener } from './progressListener';
+import Path from 'path';
+
+export const SAMPLE_PLAN_NAME = 'kapeta/sample-nodejs-plan';
+
+function applyHandleChange(definition: DefinitionInfo, targetHandle: string) {
+    const originalUri = parseKapetaUri(definition.definition.metadata.name);
+    definition.definition.metadata.name = `${targetHandle}/${originalUri.name}`;
+    return definition;
+}
+
+function normalizeFilters(kindFilter?: string | string[]) {
+    let resolvedFilters: string[] = [];
+
+    if (kindFilter) {
+        if (Array.isArray(kindFilter)) {
+            resolvedFilters = [...kindFilter];
+        } else {
+            resolvedFilters = [kindFilter];
+        }
+    }
+
+    return resolvedFilters.map((k) => k.toLowerCase());
+}
 
 class DefinitionsManager {
-    private getHash(kindFilter?: string | string[]) {
-        if (kindFilter) {
-            if (Array.isArray(kindFilter)) {
-                return kindFilter.join(',');
+    private async resolveDefinitionsAndSamples() {
+        const definitions = ClusterConfiguration.getDefinitions();
+        const samplePlan = definitions.find(
+            (d) => d.version === 'local' && d.definition.metadata.name === SAMPLE_PLAN_NAME
+        );
+
+        if (!samplePlan) {
+            return definitions;
+        }
+
+        // We will only rewrite the sample plan once since we change the handle to be the users handle
+        const api = new KapetaAPI();
+        if (!api.hasToken()) {
+            // Not logged in yet, so we can't rewrite the sample plan
+            return definitions;
+        }
+        const profile = await api.getCurrentIdentity();
+        if (!profile) {
+            // Not logged in yet, so we can't rewrite the sample plan
+            return definitions;
+        }
+
+        console.log('Rewriting sample plan to use handle %s', profile.handle);
+
+        applyHandleChange(samplePlan, profile.handle);
+
+        const planDef = samplePlan.definition as Plan;
+
+        const blockRefs = new Set<string>();
+
+        planDef.spec.blocks.forEach((b) => {
+            const blockUri = parseKapetaUri(b.block.ref);
+            if (blockUri.version === 'local') {
+                blockRefs.add(blockUri.id);
+                b.block.ref = normalizeKapetaUri(`${profile.handle}/${blockUri.name}:local`);
             }
-            return kindFilter;
+        });
+
+        // Rewrite all blocks that are referenced by the sample plan
+        const rewrittenBlocks = Array.from(blockRefs)
+            .map((ref) =>
+                definitions.find(
+                    (d) => normalizeKapetaUri(d.definition.metadata.name + ':' + d.version) === normalizeKapetaUri(ref)
+                )
+            )
+            .filter((d) => d !== undefined)
+            .map((d) => applyHandleChange(d!, profile.handle));
+
+        // Persist the rewritten assets
+        const progressListener = new ProgressListener();
+        const rewrittenAssets = [samplePlan, ...rewrittenBlocks];
+        const originalRefs = [`${SAMPLE_PLAN_NAME}:local`, ...Array.from(blockRefs)];
+
+        // Store the original paths on the assets - we'll need them later
+        for (const asset of rewrittenAssets) {
+            asset.path = await FS.readlink(asset.path);
+            asset.ymlPath = Path.join(asset.path, Path.basename(asset.ymlPath));
+        }
+
+        // Uninstall the original assets
+        // This removes the symlinks
+        console.log('Uninstalling original assets', originalRefs);
+        try {
+            await Actions.uninstall(progressListener, originalRefs);
+        } catch (err) {
+            console.warn('Failed to uninstall original assets', err);
         }
-        return 'none';
+
+        for (const asset of rewrittenAssets) {
+            console.log('Updating %s ', asset.ymlPath);
+            await FS.writeFile(asset.ymlPath, YAML.stringify(asset.definition));
+
+            console.log('Linking %s ', asset.path);
+            await Actions.link(progressListener, asset.path);
+        }
+
+        console.log('Rewrite done for sample plan');
+
+        // Return the rewritten definitions
+        return ClusterConfiguration.getDefinitions();
     }
 
-    private getFullKey(kindFilter?: string | string[]) {
-        return `definitionsManager:${this.getHash(kindFilter)}`;
+    private applyFilters(definitions: DefinitionInfo[], kindFilter: string[]): DefinitionInfo[] {
+        if (kindFilter.length === 0) {
+            return definitions;
+        }
+
+        return definitions.filter((d) => {
+            return kindFilter.includes(d.definition.kind.toLowerCase());
+        });
     }
 
-    public getDefinitions(kindFilter?: string | string[]): DefinitionInfo[] {
-        const key = this.getFullKey(kindFilter);
+    public async getDefinitions(kindFilter?: string | string[]): Promise<DefinitionInfo[]> {
+        kindFilter = normalizeFilters(kindFilter);
+
+        const definitions = await doCached<Promise<DefinitionInfo[]>>('definitionsManager:all', () =>
+            this.resolveDefinitionsAndSamples()
+        );
 
-        return doCached<DefinitionInfo[]>(key, () => ClusterConfiguration.getDefinitions(kindFilter));
+        return this.applyFilters(definitions, kindFilter);
     }
 
-    public exists(ref: string) {
-        return !!this.getDefinition(ref);
+    public async exists(ref: string) {
+        return !!(await this.getDefinition(ref));
     }
 
-    public getProviderDefinitions(): DefinitionInfo[] {
-        return doCached<DefinitionInfo[]>('providers', () => ClusterConfiguration.getProviderDefinitions());
+    public async getProviderDefinitions(): Promise<DefinitionInfo[]> {
+        return doCached<DefinitionInfo[]>('definitionsManager:providers', () =>
+            ClusterConfiguration.getProviderDefinitions()
+        );
     }
 
-    public getDefinition(ref: string) {
+    public async getDefinition(ref: string) {
         const uri = parseKapetaUri(ref);
-        return this.getDefinitions().find((d) => {
+        const definitions = await this.getDefinitions();
+        return definitions.find((d) => {
             if (!uri.version) {
                 return d.definition.metadata.name === uri.fullName;
             }

package/src/instanceManager.ts

@@ -22,7 +22,7 @@ const DEFAULT_HEALTH_PORT_TYPE = 'rest';
 const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
 
 export class InstanceManager {
-    private _interval: NodeJS.Timer | undefined = undefined;
+    private _interval: any = undefined;
 
     private readonly _instances: InstanceInfo[] = [];
 
@@ -54,14 +54,14 @@ export class InstanceManager {
         return [...this._instances];
     }
 
-    public getInstancesForPlan(systemId: string) {
+    public async getInstancesForPlan(systemId: string) {
         if (!this._instances) {
             return [];
         }
 
         systemId = normalizeKapetaUri(systemId);
 
-        const planInfo = definitionsManager.getDefinition(systemId);
+        const planInfo = await definitionsManager.getDefinition(systemId);
 
         if (!planInfo) {
             return [];
@@ -458,9 +458,9 @@ export class InstanceManager {
 
             const blockSpec = blockAsset.data.spec as BlockDefinitionSpec;
             if (blockSpec.consumers) {
-                const promises = blockSpec.consumers.map((consumer) => {
+                const promises = blockSpec.consumers.map(async (consumer) => {
                     const consumerUri = parseKapetaUri(consumer.kind);
-                    const asset = definitionsManager.getDefinition(consumer.kind);
+                    const asset = await definitionsManager.getDefinition(consumer.kind);
                     if (!asset) {
                         // Definition not found
                         return Promise.resolve();

package/src/instances/routes.ts

@@ -21,8 +21,8 @@ router.get('/', (req: Request, res: Response) => {
 /**
  * Get all instances
  */
-router.get('/:systemId/instances', (req: Request, res: Response) => {
-    res.send(instanceManager.getInstancesForPlan(req.params.systemId));
+router.get('/:systemId/instances', async (req: Request, res: Response) => {
+    res.send(await instanceManager.getInstancesForPlan(req.params.systemId));
 });
 
 /**

package/src/operatorManager.ts

@@ -15,6 +15,7 @@ import AsyncLock from 'async-lock';
 import { taskManager } from './taskManager';
 
 export const KIND_OPERATOR = 'core/resource-type-operator';
+const KIND_PLAN = 'core/plan';
 
 class Operator {
     private readonly _data: DefinitionInfo;
@@ -57,8 +58,8 @@ class OperatorManager {
      * @param {string} version
      * @return {Operator}
      */
-    getOperator(resourceType: string, version: string) {
-        const operators = definitionsManager.getDefinitions(KIND_OPERATOR);
+    async getOperator(resourceType: string, version: string) {
+        const operators = await definitionsManager.getDefinitions(KIND_OPERATOR);
 
         const operator: DefinitionInfo | undefined = operators.find(
             (operator) =>
@@ -92,7 +93,7 @@ class OperatorManager {
         environment?: EnvironmentType
     ): Promise<OperatorInfo> {
         systemId = normalizeKapetaUri(systemId);
-        const plans = definitionsManager.getDefinitions('core/plan');
+        const plans = await definitionsManager.getDefinitions(KIND_PLAN);
 
         const planUri = parseKapetaUri(systemId);
         const currentPlan = plans.find(
@@ -109,13 +110,7 @@ class OperatorManager {
             throw new Error(`Unknown instance: ${fromServiceId} in plan ${systemId}`);
         }
 
-        const blockUri = parseKapetaUri(currentInstance.block.ref);
-        const blockDefinition = definitionsManager
-            .getDefinitions()
-            .find(
-                (definition) =>
-                    definition.version === blockUri.version && definition.definition.metadata.name === blockUri.fullName
-            );
+        const blockDefinition = await definitionsManager.getDefinition(currentInstance.block.ref);
 
         if (!blockDefinition) {
             throw new Error(`Unknown block: ${currentInstance.block.ref} in plan ${systemId}`);
@@ -129,7 +124,7 @@ class OperatorManager {
         }
 
         const kindUri = parseKapetaUri(blockResource.kind);
-        const operator = this.getOperator(resourceType, kindUri.version);
+        const operator = await this.getOperator(resourceType, kindUri.version);
         const credentials = operator.getCredentials();
         const container = await this.ensureResource(systemId, resourceType, kindUri.version);
         const portInfo = await container.getPort(portType);
@@ -164,7 +159,7 @@ class OperatorManager {
         systemId = normalizeKapetaUri(systemId);
         const key = `${systemId}#${resourceType}:${version}`;
         return await this.operatorLock.acquire(key, async () => {
-            const operator = this.getOperator(resourceType, version);
+            const operator = await this.getOperator(resourceType, version);
 
             const operatorData = operator.getLocalData();
 

package/src/providerManager.ts

@@ -3,12 +3,14 @@ import FSExtra from 'fs-extra';
 import { definitionsManager } from './definitionsManager';
 import { cacheManager } from './cacheManager';
 import request from 'request';
+import { DefinitionInfo } from '@kapeta/local-cluster-config';
 
 const PROVIDER_FILE_BASE = 'https://providers.kapeta.com/files';
 
 class ProviderManager {
-    getWebProviders() {
-        return definitionsManager.getProviderDefinitions().filter((providerDefinition) => providerDefinition.hasWeb);
+    async getWebProviders(): Promise<DefinitionInfo[]> {
+        const providers = await definitionsManager.getProviderDefinitions();
+        return providers.filter((providerDefinition) => providerDefinition.hasWeb);
     }
 
     async getProviderWebJS(handle: string, name: string, version: string, sourceMap: boolean = false) {
@@ -22,7 +24,8 @@ class ProviderManager {
             return FSExtra.readFile(file, 'utf8');
         }
 
-        const installedProvider = this.getWebProviders().find((providerDefinition) => {
+        const providers = await this.getWebProviders();
+        const installedProvider = providers.find((providerDefinition) => {
             return providerDefinition.definition.metadata.name === fullName && providerDefinition.version === version;
         });
 
@@ -63,22 +66,27 @@ class ProviderManager {
     }
 }
 
-const providerDefinitions = definitionsManager.getProviderDefinitions();
-
-if (providerDefinitions.length > 0) {
-    console.log('## Loaded the following providers ##');
-    providerDefinitions.forEach((providerDefinition) => {
-        console.log(
-            ' - %s[%s:%s]',
-            providerDefinition.definition.kind,
-            providerDefinition.definition.metadata.name,
-            providerDefinition.version
-        );
-        console.log('   from %s', providerDefinition.path);
+definitionsManager
+    .getProviderDefinitions()
+    .then((providerDefinitions) => {
+        if (providerDefinitions.length > 0) {
+            console.log('## Loaded the following providers ##');
+            providerDefinitions.forEach((providerDefinition) => {
+                console.log(
+                    ' - %s[%s:%s]',
+                    providerDefinition.definition.kind,
+                    providerDefinition.definition.metadata.name,
+                    providerDefinition.version
+                );
+                console.log('   from %s', providerDefinition.path);
+            });
+            console.log('##');
+        } else {
+            console.log('## No providers found ##');
+        }
+    })
+    .catch((e) => {
+        console.error('Failed to load providers', e);
     });
-    console.log('##');
-} else {
-    console.log('## No providers found ##');
-}
 
 export const providerManager = new ProviderManager();