@kapeta/local-cluster-service 0.17.0 → 0.19.0

package/src/assetManager.ts

@@ -64,9 +64,9 @@ class AssetManager {
      * @param {string[]} [assetKinds]
      * @returns {{path: *, ref: string, data: *, editable: boolean, kind: *, exists: boolean}[]}
      */
-    getAssets(assetKinds?: string[]): EnrichedAsset[] {
+    async getAssets(assetKinds?: string[]): Promise<EnrichedAsset[]> {
         if (!assetKinds) {
-            const blockTypeProviders = definitionsManager.getDefinitions([
+            const blockTypeProviders = await definitionsManager.getDefinitions([
                 'core/block-type',
                 'core/block-type-operator',
             ]);
@@ -76,12 +76,12 @@ class AssetManager {
             assetKinds.push('core/plan');
         }
 
-        const assets = definitionsManager.getDefinitions(assetKinds);
+        const assets = await definitionsManager.getDefinitions(assetKinds);
 
         return assets.map(enrichAsset);
     }
 
-    getPlans(): EnrichedAsset[] {
+    async getPlans(): Promise<EnrichedAsset[]> {
         return this.getAssets(['core/plan']);
     }
 
@@ -110,18 +110,18 @@ class AssetManager {
             await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, true);
         }
 
-        let asset = definitionsManager
-            .getDefinitions()
-            .map(enrichAsset)
-            .find((a) => parseKapetaUri(a.ref).equals(uri));
-        if (autoFetch && !asset) {
+        const definitionInfo = await definitionsManager.getDefinition(ref);
+        if (autoFetch && !definitionInfo) {
             throw new Error('Asset not found: ' + ref);
         }
-        if (asset) {
+
+        if (definitionInfo) {
+            const asset = enrichAsset(definitionInfo);
             cacheManager.set(cacheKey, asset, CACHE_TTL);
+            return asset;
         }
 
-        return asset;
+        return undefined;
     }
 
     async createAsset(
@@ -151,7 +151,7 @@ class AssetManager {
 
         const ref = `kapeta://${yaml.metadata.name}:local`;
 
-        this.maybeGenerateCode(ref, path, yaml);
+        await this.maybeGenerateCode(ref, path, yaml);
 
         return asset;
     }
@@ -178,7 +178,7 @@ class AssetManager {
         cacheManager.remove(toKey(ref));
         definitionsManager.clearCache();
 
-        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
+        await this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
     }
 
     async importFile(filePath: string) {
@@ -206,7 +206,9 @@ class AssetManager {
 
         definitionsManager.clearCache();
 
-        return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
+        const assets = await this.getAssets();
+
+        return assets.filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
     }
 
     async unregisterAsset(ref: string) {
@@ -236,9 +238,9 @@ class AssetManager {
         return await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
     }
 
-    private maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
+    private async maybeGenerateCode(ref: string, ymlPath: string, block: BlockDefinition) {
         ref = normalizeKapetaUri(ref);
-        if (codeGeneratorManager.canGenerateCode(block)) {
+        if (await codeGeneratorManager.canGenerateCode(block)) {
             const assetTitle = block.metadata.title ? block.metadata.title : parseKapetaUri(block.metadata.name).name;
             taskManager.add(
                 `codegen:${ref}`,

package/src/assets/routes.ts

@@ -31,8 +31,8 @@ router.use('/', stringBody);
 /**
  * Get all local assets available
  */
-router.get('/', (req: Request, res: Response) => {
-    res.send(assetManager.getAssets([]));
+router.get('/', async (req: Request, res: Response) => {
+    res.send(await assetManager.getAssets([]));
 });
 
 /**

package/src/authManager.ts

@@ -0,0 +1,62 @@
+import { EventEmitter } from 'node:events';
+import Path from 'node:path';
+import chokidar, { FSWatcher } from 'chokidar';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { WatchEventName } from './types';
+import { definitionsManager } from './definitionsManager';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { socketManager } from './socketManager';
+
+class AuthManager extends EventEmitter {
+    private watcher?: FSWatcher;
+
+    private hadToken: boolean;
+
+    constructor() {
+        super();
+        this.hadToken = this.hasToken();
+    }
+
+    public listenForChanges() {
+        const parentDir = Path.dirname(ClusterConfiguration.getKapetaBasedir());
+        //We watch the parent dir to catch changes to the base dir itself
+        this.watcher = chokidar.watch(parentDir, {
+            followSymlinks: false,
+            ignorePermissionErrors: true,
+            disableGlobbing: true,
+            persistent: true,
+            ignoreInitial: true,
+            depth: 1,
+            ignored: (path) => {
+                return !path.startsWith(ClusterConfiguration.getKapetaBasedir());
+            },
+        });
+        this.watcher.add(ClusterConfiguration.getKapetaBasedir());
+        this.watcher.on('all', this.handleFileChange.bind(this));
+        this.watcher.on('error', (error) => {
+            console.log('Error watching repository', error);
+        });
+        this.watcher.on('ready', () => {
+            console.log('Watching for auth changes: %s', ClusterConfiguration.getKapetaBasedir());
+        });
+    }
+
+    private hasToken() {
+        const api = new KapetaAPI();
+        return api.hasToken();
+    }
+
+    private async handleFileChange(eventName: WatchEventName, path: string) {
+        const hasTokenNow = this.hasToken();
+        if (this.hadToken !== hasTokenNow) {
+            socketManager.emitGlobal('auth-change', {});
+            if (hasTokenNow) {
+                // Clear the cache in case we need to rewrite the sample plan
+                definitionsManager.clearCache();
+            }
+            this.hadToken = hasTokenNow;
+        }
+    }
+}
+
+export const authManager = new AuthManager();

package/src/codeGeneratorManager.ts

@@ -10,7 +10,7 @@ const BLOCK_TYPE_KIND = 'core/block-type';
 class CodeGeneratorManager {
     async reload() {
         Targets.reset();
-        const languageTargets = definitionsManager.getDefinitions(TARGET_KIND);
+        const languageTargets = await definitionsManager.getDefinitions(TARGET_KIND);
         for (const languageTarget of languageTargets) {
             const key = `${languageTarget.definition.metadata.name}:${languageTarget.version}`;
             try {
@@ -26,13 +26,13 @@ class CodeGeneratorManager {
         }
     }
 
-    canGenerateCode(yamlContent: BlockDefinition): boolean {
+    async canGenerateCode(yamlContent: BlockDefinition): Promise<boolean> {
         if (!yamlContent.spec.target?.kind) {
             //Not all block types have targets
             return false;
         }
 
-        const blockTypes = definitionsManager.getDefinitions(BLOCK_TYPE_KIND);
+        const blockTypes = await definitionsManager.getDefinitions(BLOCK_TYPE_KIND);
         const blockTypeKinds = blockTypes.map(
             (blockType) => blockType.definition.metadata.name.toLowerCase() + ':' + blockType.version
         );

package/src/configManager.ts

@@ -80,7 +80,7 @@ class ConfigManager {
         if (systemId) {
             systemId = normalizeKapetaUri(systemId);
         }
-        const planAssets = assetManager.getPlans();
+        const planAssets = await assetManager.getPlans();
 
         const blockUri = parseKapetaUri(blockRef);
 
@@ -132,7 +132,7 @@ class ConfigManager {
     async verifyIdentity(blockRef: string, systemId: string, instanceId: string) {
         blockRef = normalizeKapetaUri(blockRef);
         systemId = normalizeKapetaUri(systemId);
-        const planAssets = assetManager.getPlans();
+        const planAssets = await assetManager.getPlans();
         const systemUri = systemId ? parseKapetaUri(systemId) : null;
         const blockUri = parseKapetaUri(blockRef);
         let found = false;

package/src/containerManager.ts

@@ -13,6 +13,7 @@ import { getBlockInstanceContainerName } from './utils/utils';
 import { InstanceInfo, LogEntry, LogSource } from './types';
 import { KapetaAPI } from '@kapeta/nodejs-api-client';
 import { taskManager, Task } from './taskManager';
+import { EventEmitter } from 'node:events';
 
 type StringMap = { [key: string]: string };
 
@@ -78,6 +79,7 @@ class ContainerManager {
     private _mountDir: string;
     private _version: string;
     private _lastDockerAccessCheck: number = 0;
+    private logStreams: { [p: string]: { stream?: ClosableLogStream; timer?: NodeJS.Timeout } } = {};
 
     constructor() {
         this._docker = null;
@@ -607,6 +609,193 @@ class ContainerManager {
 
         return containerInfo.getLogs();
     }
+
+    async stopLogListening(systemId: string, instanceId: string) {
+        const containerName = getBlockInstanceContainerName(systemId, instanceId);
+        if (this.logStreams[containerName]) {
+            if (this.logStreams[containerName]?.timer) {
+                clearTimeout(this.logStreams[containerName].timer);
+            }
+
+            console.log('Stopped listening for logs on container: %s', containerName);
+            try {
+                const stream = this.logStreams[containerName].stream;
+                if (stream) {
+                    await stream.close();
+                }
+            } catch (err) {
+                // Ignore
+            }
+            delete this.logStreams[containerName];
+        }
+    }
+
+    async ensureLogListening(systemId: string, instanceId: string, handler: (log: LogEntry) => void) {
+        const containerName = getBlockInstanceContainerName(systemId, instanceId);
+        try {
+            if (this.logStreams[containerName]?.stream) {
+                // Already listening - will shut itself down
+                return;
+            }
+
+            if (this.logStreams[containerName]?.timer) {
+                clearTimeout(this.logStreams[containerName].timer);
+            }
+
+            const tryLater = () => {
+                this.logStreams[containerName] = {
+                    timer: setTimeout(() => {
+                        // Keep trying until user decides to not listen anymore
+                        this.ensureLogListening(systemId, instanceId, handler);
+                    }, 5000),
+                };
+            };
+
+            const containerInfo = await this.getContainerByName(containerName);
+            if (!containerInfo || !(await containerInfo.isRunning())) {
+                // Container not currently running - try again in 5 seconds
+                tryLater();
+                return;
+            }
+
+            const stream = await containerInfo.getLogStream();
+            stream.onLog((log) => {
+                try {
+                    handler(log);
+                } catch (err) {
+                    console.warn('Error handling log', err);
+                }
+            });
+            stream.onEnd(() => {
+                // We get here if the container is stopped
+                delete this.logStreams[containerName];
+                tryLater();
+            });
+            stream.onError((err) => {
+                // We get here if the container crashes
+                delete this.logStreams[containerName];
+                tryLater();
+            });
+
+            this.logStreams[containerName] = {
+                stream,
+            };
+        } catch (err) {
+            // Ignore
+        }
+    }
+}
+
+function readLogBuffer(logBuffer: Buffer) {
+    const out: LogEntry[] = [];
+    let offset = 0;
+    while (offset < logBuffer.length) {
+        try {
+            // Read the docker log format - explained here:
+            // https://docs.docker.com/engine/api/v1.41/#operation/ContainerAttach
+            // or here : https://ahmet.im/blog/docker-logs-api-binary-format-explained/
+
+            // First byte is stream type
+            const streamTypeInt = logBuffer.readInt8(offset);
+            const streamType: LogSource = streamTypeInt === 1 ? 'stdout' : 'stderr';
+            if (streamTypeInt !== 1 && streamTypeInt !== 2) {
+                console.error('Unknown stream type: %s', streamTypeInt, out[out.length - 1]);
+                break;
+            }
+
+            // Bytes 4-8 is frame size
+            const messageLength = logBuffer.readInt32BE(offset + 4);
+
+            // After that is the message - with the message length
+            const dataWithoutStreamType = logBuffer.subarray(offset + 8, offset + 8 + messageLength);
+            const raw = dataWithoutStreamType.toString();
+
+            // Split the message into date and message
+            const firstSpaceIx = raw.indexOf(' ');
+            const dateString = raw.substring(0, firstSpaceIx);
+            const line = raw.substring(firstSpaceIx + 1);
+            offset = offset + messageLength + 8;
+            if (!dateString) {
+                break;
+            }
+            out.push({
+                time: new Date(dateString).getTime(),
+                message: line,
+                level: 'INFO',
+                source: streamType,
+            });
+        } catch (err) {
+            console.error('Error parsing log entry', err);
+            offset = logBuffer.length;
+            break;
+        }
+    }
+    return out;
+}
+
+class ClosableLogStream {
+    private readonly stream: FSExtra.ReadStream;
+
+    private readonly eventEmitter: EventEmitter;
+
+    constructor(stream: FSExtra.ReadStream) {
+        this.stream = stream;
+        this.eventEmitter = new EventEmitter();
+        stream.on('data', (data) => {
+            const logs = readLogBuffer(data as Buffer);
+            logs.forEach((log) => {
+                this.eventEmitter.emit('log', log);
+            });
+        });
+
+        stream.on('end', () => {
+            this.eventEmitter.emit('end');
+        });
+
+        stream.on('error', (error) => {
+            this.eventEmitter.emit('error', error);
+        });
+
+        stream.on('close', () => {
+            this.eventEmitter.emit('end');
+        });
+    }
+
+    onLog(listener: (log: LogEntry) => void) {
+        this.eventEmitter.on('log', listener);
+        return () => {
+            this.eventEmitter.removeListener('log', listener);
+        };
+    }
+
+    onEnd(listener: () => void) {
+        this.eventEmitter.on('end', listener);
+        return () => {
+            this.eventEmitter.removeListener('end', listener);
+        };
+    }
+
+    onError(listener: (error: Error) => void) {
+        this.eventEmitter.on('error', listener);
+        return () => {
+            this.eventEmitter.removeListener('error', listener);
+        };
+    }
+
+    close() {
+        return new Promise<void>((resolve, reject) => {
+            try {
+                this.stream.close((err) => {
+                    if (err) {
+                        console.warn('Error closing log stream', err);
+                    }
+                    resolve();
+                });
+            } catch (err) {
+                // Ignore
+            }
+        });
+    }
 }
 
 export class ContainerInfo {
@@ -718,57 +907,38 @@ export class ContainerInfo {
         return ports;
     }
 
+    async getLogStream() {
+        try {
+            const logStream = (await this.native.logs({
+                stdout: true,
+                stderr: true,
+                follow: true,
+                tail: 0,
+                timestamps: true,
+            })) as ReadStream;
+
+            return new ClosableLogStream(logStream);
+        } catch (err) {
+            console.log('Error getting log stream', err);
+            throw err;
+        }
+    }
+
     async getLogs(): Promise<LogEntry[]> {
         const logStream = (await this.native.logs({
             stdout: true,
             stderr: true,
             follow: false,
-            tail: 100,
             timestamps: true,
         })) as ReadStream;
 
-        const out = [] as LogEntry[];
+        const chunks: Buffer[] = [];
         await promisifyStream(logStream, (data) => {
-            const buf = data as Buffer;
-            let offset = 0;
-            while (offset < buf.length) {
-                try {
-                    // Read the docker log format - explained here:
-                    // https://docs.docker.com/engine/api/v1.41/#operation/ContainerAttach
-                    // or here : https://ahmet.im/blog/docker-logs-api-binary-format-explained/
-
-                    // First byte is stream type
-                    const streamTypeInt = buf.readInt8(offset);
-                    const streamType: LogSource = streamTypeInt === 1 ? 'stdout' : 'stderr';
-
-                    // Bytes 4-8 is frame size
-                    const messageLength = buf.readInt32BE(offset + 4);
-
-                    // After that is the message - with the message length
-                    const dataWithoutStreamType = buf.subarray(offset + 8, offset + 8 + messageLength);
-                    const raw = dataWithoutStreamType.toString();
-
-                    // Split the message into date and message
-                    const firstSpaceIx = raw.indexOf(' ');
-                    const dateString = raw.substring(0, firstSpaceIx);
-                    const line = raw.substring(firstSpaceIx + 1);
-                    offset = offset + messageLength + 8;
-                    if (!dateString) {
-                        continue;
-                    }
-                    out.push({
-                        time: new Date(dateString).getTime(),
-                        message: line,
-                        level: 'INFO',
-                        source: streamType,
-                    });
-                } catch (err) {
-                    console.error('Error parsing log entry', err);
-                    offset = buf.length;
-                }
-            }
+            chunks.push(data as Buffer);
         });
 
+        const out = readLogBuffer(Buffer.concat(chunks));
+
         if (out.length === 0) {
             out.push({
                 time: Date.now(),

package/src/definitionsManager.ts

@@ -1,39 +1,154 @@
 import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { cacheManager, doCached } from './cacheManager';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { Plan } from '@kapeta/schemas';
+import FS from 'fs-extra';
+import { normalizeKapetaUri } from './utils/utils';
+import YAML from 'yaml';
+import { Actions } from '@kapeta/nodejs-registry-utils';
+import { ProgressListener } from './progressListener';
+import Path from 'path';
+
+export const SAMPLE_PLAN_NAME = 'kapeta/sample-nodejs-plan';
+
+function applyHandleChange(definition: DefinitionInfo, targetHandle: string) {
+    const originalUri = parseKapetaUri(definition.definition.metadata.name);
+    definition.definition.metadata.name = `${targetHandle}/${originalUri.name}`;
+    return definition;
+}
+
+function normalizeFilters(kindFilter?: string | string[]) {
+    let resolvedFilters: string[] = [];
+
+    if (kindFilter) {
+        if (Array.isArray(kindFilter)) {
+            resolvedFilters = [...kindFilter];
+        } else {
+            resolvedFilters = [kindFilter];
+        }
+    }
+
+    return resolvedFilters.map((k) => k.toLowerCase());
+}
 
 class DefinitionsManager {
-    private getHash(kindFilter?: string | string[]) {
-        if (kindFilter) {
-            if (Array.isArray(kindFilter)) {
-                return kindFilter.join(',');
+    private async resolveDefinitionsAndSamples() {
+        const definitions = ClusterConfiguration.getDefinitions();
+        const samplePlan = definitions.find(
+            (d) => d.version === 'local' && d.definition.metadata.name === SAMPLE_PLAN_NAME
+        );
+
+        if (!samplePlan) {
+            return definitions;
+        }
+
+        // We will only rewrite the sample plan once since we change the handle to be the users handle
+        const api = new KapetaAPI();
+        if (!api.hasToken()) {
+            // Not logged in yet, so we can't rewrite the sample plan
+            return definitions;
+        }
+        const profile = await api.getCurrentIdentity();
+        if (!profile) {
+            // Not logged in yet, so we can't rewrite the sample plan
+            return definitions;
+        }
+
+        console.log('Rewriting sample plan to use handle %s', profile.handle);
+
+        applyHandleChange(samplePlan, profile.handle);
+
+        const planDef = samplePlan.definition as Plan;
+
+        const blockRefs = new Set<string>();
+
+        planDef.spec.blocks.forEach((b) => {
+            const blockUri = parseKapetaUri(b.block.ref);
+            if (blockUri.version === 'local') {
+                blockRefs.add(blockUri.id);
+                b.block.ref = normalizeKapetaUri(`${profile.handle}/${blockUri.name}:local`);
             }
-            return kindFilter;
+        });
+
+        // Rewrite all blocks that are referenced by the sample plan
+        const rewrittenBlocks = Array.from(blockRefs)
+            .map((ref) =>
+                definitions.find(
+                    (d) => normalizeKapetaUri(d.definition.metadata.name + ':' + d.version) === normalizeKapetaUri(ref)
+                )
+            )
+            .filter((d) => d !== undefined)
+            .map((d) => applyHandleChange(d!, profile.handle));
+
+        // Persist the rewritten assets
+        const progressListener = new ProgressListener();
+        const rewrittenAssets = [samplePlan, ...rewrittenBlocks];
+        const originalRefs = [`${SAMPLE_PLAN_NAME}:local`, ...Array.from(blockRefs)];
+
+        // Store the original paths on the assets - we'll need them later
+        for (const asset of rewrittenAssets) {
+            asset.path = await FS.readlink(asset.path);
+            asset.ymlPath = Path.join(asset.path, Path.basename(asset.ymlPath));
+        }
+
+        // Uninstall the original assets
+        // This removes the symlinks
+        console.log('Uninstalling original assets', originalRefs);
+        try {
+            await Actions.uninstall(progressListener, originalRefs);
+        } catch (err) {
+            console.warn('Failed to uninstall original assets', err);
         }
-        return 'none';
+
+        for (const asset of rewrittenAssets) {
+            console.log('Updating %s ', asset.ymlPath);
+            await FS.writeFile(asset.ymlPath, YAML.stringify(asset.definition));
+
+            console.log('Linking %s ', asset.path);
+            await Actions.link(progressListener, asset.path);
+        }
+
+        console.log('Rewrite done for sample plan');
+
+        // Return the rewritten definitions
+        return ClusterConfiguration.getDefinitions();
     }
 
-    private getFullKey(kindFilter?: string | string[]) {
-        return `definitionsManager:${this.getHash(kindFilter)}`;
+    private applyFilters(definitions: DefinitionInfo[], kindFilter: string[]): DefinitionInfo[] {
+        if (kindFilter.length === 0) {
+            return definitions;
+        }
+
+        return definitions.filter((d) => {
+            return kindFilter.includes(d.definition.kind.toLowerCase());
+        });
     }
 
-    public getDefinitions(kindFilter?: string | string[]): DefinitionInfo[] {
-        const key = this.getFullKey(kindFilter);
+    public async getDefinitions(kindFilter?: string | string[]): Promise<DefinitionInfo[]> {
+        kindFilter = normalizeFilters(kindFilter);
+
+        const definitions = await doCached<Promise<DefinitionInfo[]>>('definitionsManager:all', () =>
+            this.resolveDefinitionsAndSamples()
+        );
 
-        return doCached<DefinitionInfo[]>(key, () => ClusterConfiguration.getDefinitions(kindFilter));
+        return this.applyFilters(definitions, kindFilter);
     }
 
-    public exists(ref: string) {
-        return !!this.getDefinition(ref);
+    public async exists(ref: string) {
+        return !!(await this.getDefinition(ref));
     }
 
-    public getProviderDefinitions(): DefinitionInfo[] {
-        return doCached<DefinitionInfo[]>('providers', () => ClusterConfiguration.getProviderDefinitions());
+    public async getProviderDefinitions(): Promise<DefinitionInfo[]> {
+        return doCached<DefinitionInfo[]>('definitionsManager:providers', () =>
+            ClusterConfiguration.getProviderDefinitions()
+        );
     }
 
-    public getDefinition(ref: string) {
+    public async getDefinition(ref: string) {
         const uri = parseKapetaUri(ref);
-        return this.getDefinitions().find((d) => {
+        const definitions = await this.getDefinitions();
+        return definitions.find((d) => {
             if (!uri.version) {
                 return d.definition.metadata.name === uri.fullName;
             }