@kapeta/local-cluster-service 0.15.3 → 0.16.1

package/dist/esm/src/cacheManager.js

@@ -0,0 +1,31 @@
+import NodeCache from 'node-cache';
+const DEFAULT_CACHE_TTL = 60 * 1000; // 1 min
+class CacheManager {
+    cache = new NodeCache();
+    flush() {
+        this.cache.flushAll();
+    }
+    doCached(key, getter, ttl = DEFAULT_CACHE_TTL) {
+        const data = this.cache.get(key);
+        if (data !== undefined) {
+            return data;
+        }
+        const result = getter();
+        this.cache.set(key, result, ttl);
+        return result;
+    }
+    get(key) {
+        return this.cache.get(key);
+    }
+    set(key, data, ttl = DEFAULT_CACHE_TTL) {
+        this.cache.set(key, data, ttl);
+    }
+    has(key) {
+        return this.cache.has(key);
+    }
+    remove(key) {
+        return this.cache.del(key);
+    }
+}
+export const cacheManager = new CacheManager();
+export const doCached = (key, getter, ttl = DEFAULT_CACHE_TTL) => cacheManager.doCached(key, getter, ttl);

package/dist/esm/src/definitionsManager.d.ts

@@ -1,9 +1,7 @@
 import { DefinitionInfo } from '@kapeta/local-cluster-config';
 declare class DefinitionsManager {
-    private cache;
-    private getKey;
-    clearCache(): void;
-    private doCached;
+    private getHash;
+    private getFullKey;
     getDefinitions(kindFilter?: string | string[]): DefinitionInfo[];
     exists(ref: string): boolean;
     getProviderDefinitions(): DefinitionInfo[];

package/dist/esm/src/definitionsManager.js

@@ -1,9 +1,8 @@
 import ClusterConfiguration from '@kapeta/local-cluster-config';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
-const CACHE_TTL = 60 * 1000; // 1 min
+import { doCached } from './cacheManager';
 class DefinitionsManager {
-    cache = {};
-    getKey(kindFilter) {
+    getHash(kindFilter) {
         if (kindFilter) {
             if (Array.isArray(kindFilter)) {
                 return kindFilter.join(',');
@@ -12,31 +11,18 @@ class DefinitionsManager {
         }
         return 'none';
     }
-    clearCache() {
-        this.cache = {};
-    }
-    doCached(key, getter) {
-        if (this.cache[key]) {
-            if (this.cache[key].expires > Date.now()) {
-                return this.cache[key].definitions;
-            }
-            delete this.cache[key];
-        }
-        this.cache[key] = {
-            expires: Date.now() + CACHE_TTL,
-            definitions: getter(),
-        };
-        return this.cache[key].definitions;
+    getFullKey(kindFilter) {
+        return `DefinitionsManager:${this.getHash(kindFilter)}`;
     }
     getDefinitions(kindFilter) {
-        const key = this.getKey(kindFilter);
-        return this.doCached(key, () => ClusterConfiguration.getDefinitions(kindFilter));
+        const key = this.getFullKey(kindFilter);
+        return doCached(key, () => ClusterConfiguration.getDefinitions(kindFilter));
     }
     exists(ref) {
         return !!this.getDefinition(ref);
     }
     getProviderDefinitions() {
-        return this.doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
+        return doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
     }
     getDefinition(ref) {
         const uri = parseKapetaUri(ref);

package/dist/esm/src/instanceManager.js

@@ -51,7 +51,10 @@ export class InstanceManager {
             return [];
         }
         const plan = planInfo.definition;
-        const instanceIds = plan.spec.blocks.map((block) => block.id);
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const instanceIds = plan.spec?.blocks?.map((block) => block.id) || [];
         return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
     }
     getInstance(systemId, instanceId) {

package/dist/esm/src/repositoryManager.d.ts

@@ -1,13 +1,18 @@
 import { Task } from './taskManager';
+import { SourceOfChange } from './types';
 declare class RepositoryManager {
-    private changeEventsEnabled;
     private _registryService;
     private _cache;
-    private watcher?;
+    private watcher;
     constructor();
-    setChangeEventsEnabled(enabled: boolean): void;
     listenForChanges(): void;
-    stopListening(): void;
+    stopListening(): Promise<void>;
+    /**
+     * Setting the source of change helps us know
+     * how to react to changes in the UI.
+     */
+    setSourceOfChangeFor(file: string, source: SourceOfChange): Promise<void>;
+    clearSourceOfChangeFor(file: string): Promise<void>;
     ensureDefaultProviders(): void;
     private _install;
     ensureAsset(handle: string, name: string, version: string, wait?: boolean): Promise<undefined | Task[]>;

package/dist/esm/src/repositoryManager.js

@@ -1,17 +1,12 @@
-import FS from 'node:fs';
 import os from 'node:os';
-import Path from 'node:path';
-import watch from 'recursive-watch';
-import FSExtra from 'fs-extra';
-import ClusterConfiguration from '@kapeta/local-cluster-config';
-import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { socketManager } from './socketManager';
 import { Actions, Config, RegistryService } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
 import { taskManager } from './taskManager';
 import { normalizeKapetaUri } from './utils/utils';
-import { assetManager } from './assetManager';
 import { ProgressListener } from './progressListener';
+import { RepositoryWatcher } from './RepositoryWatcher';
+import { cacheManager } from './cacheManager';
 const EVENT_DEFAULT_PROVIDERS_START = 'default-providers-start';
 const EVENT_DEFAULT_PROVIDERS_END = 'default-providers-end';
 const DEFAULT_PROVIDERS = [
@@ -30,92 +25,30 @@ const DEFAULT_PROVIDERS = [
 ];
 const INSTALL_ATTEMPTED = {};
 class RepositoryManager {
-    changeEventsEnabled;
     _registryService;
     _cache;
     watcher;
     constructor() {
-        this.changeEventsEnabled = true;
-        this.listenForChanges();
         this._registryService = new RegistryService(Config.data.registry.url);
         this._cache = {};
-    }
-    setChangeEventsEnabled(enabled) {
-        this.changeEventsEnabled = enabled;
+        this.watcher = new RepositoryWatcher();
+        this.listenForChanges();
     }
     listenForChanges() {
-        const baseDir = ClusterConfiguration.getRepositoryBasedir();
-        if (!FS.existsSync(baseDir)) {
-            FSExtra.mkdirpSync(baseDir);
-        }
-        let allDefinitions = ClusterConfiguration.getDefinitions();
-        console.log('Watching local repository for provider changes: %s', baseDir);
-        try {
-            this.watcher = watch(baseDir, (filename) => {
-                if (!filename) {
-                    return;
-                }
-                const [handle, name, version] = filename.toString().split(/\//g);
-                if (!name || !version) {
-                    return;
-                }
-                if (!this.changeEventsEnabled) {
-                    return;
-                }
-                const ymlPath = Path.join(baseDir, handle, name, version, 'kapeta.yml');
-                const newDefinitions = ClusterConfiguration.getDefinitions();
-                const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
-                let currentDefinition = allDefinitions.find((d) => d.ymlPath === ymlPath);
-                const ymlExists = FS.existsSync(ymlPath);
-                let type;
-                if (ymlExists) {
-                    if (currentDefinition) {
-                        type = 'updated';
-                    }
-                    else if (newDefinition) {
-                        type = 'added';
-                        currentDefinition = newDefinition;
-                    }
-                    else {
-                        //Other definition was added / updated - ignore
-                        return;
-                    }
-                }
-                else {
-                    if (currentDefinition) {
-                        const ref = parseKapetaUri(`${currentDefinition.definition.metadata.name}:${currentDefinition.version}`).id;
-                        delete INSTALL_ATTEMPTED[ref];
-                        //Something was removed
-                        type = 'removed';
-                    }
-                    else {
-                        //Other definition was removed - ignore
-                        return;
-                    }
-                }
-                const payload = {
-                    type,
-                    definition: currentDefinition?.definition,
-                    asset: { handle, name, version },
-                };
-                allDefinitions = newDefinitions;
-                socketManager.emit(`assets`, 'changed', payload);
-                definitionsManager.clearCache();
-            });
-        }
-        catch (e) {
-            // Fallback to run without watch mode due to potential platform issues.
-            // https://nodejs.org/docs/latest/api/fs.html#caveats
-            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
-            return;
-        }
+        this.watcher.watch();
     }
-    stopListening() {
-        if (!this.watcher) {
-            return;
-        }
-        this.watcher();
-        this.watcher = undefined;
+    async stopListening() {
+        return this.watcher.unwatch();
+    }
+    /**
+     * Setting the source of change helps us know
+     * how to react to changes in the UI.
+     */
+    setSourceOfChangeFor(file, source) {
+        return this.watcher.setSourceOfChangeFor(file, source);
+    }
+    clearSourceOfChangeFor(file) {
+        return this.watcher.clearSourceOfChangeFor(file);
     }
     ensureDefaultProviders() {
         socketManager.emitGlobal(EVENT_DEFAULT_PROVIDERS_START, { providers: DEFAULT_PROVIDERS });
@@ -140,19 +73,13 @@ class RepositoryManager {
                 try {
                     //We change to a temp dir to avoid issues with the current working directory
                     process.chdir(os.tmpdir());
-                    //Disable change events while installing
-                    this.setChangeEventsEnabled(false);
                     await Actions.install(new ProgressListener(), [ref], {});
                 }
                 catch (e) {
                     console.error(`Failed to install asset: ${ref}`, e);
                     throw e;
                 }
-                finally {
-                    this.setChangeEventsEnabled(true);
-                }
-                definitionsManager.clearCache();
-                assetManager.clearCache();
+                cacheManager.flush();
                 //console.log(`Asset installed: ${ref}`);
             };
         };

package/dist/esm/src/types.d.ts

@@ -8,6 +8,8 @@ export type StringMap = {
 export type AnyMap = {
     [key: string]: any;
 };
+export type SourceOfChange = 'user' | 'filesystem';
+export type WatchEventName = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
 export type LogLevel = 'ERROR' | 'WARN' | 'INFO' | 'DEBUG' | 'TRACE' | 'FATAL';
 export type LogSource = 'stdout' | 'stderr';
 export type EnvironmentType = 'docker' | 'process';

package/dist/esm/src/utils/utils.js

@@ -21,7 +21,7 @@ export function readYML(path) {
         return YAML.parse(rawYaml.toString());
     }
     catch (err) {
-        throw new Error('Failed to parse plan YAML: ' + err);
+        throw new Error(`Failed to parse plan YAML: ${err}`);
     }
 }
 export function isWindows() {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.15.3",
+    "version": "0.16.1",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -53,6 +53,7 @@
         "@kapeta/sdk-config": "<2",
         "@kapeta/web-microfrontend": "^0.2.1",
         "async-lock": "^1.4.0",
+        "chokidar": "^3.5.3",
         "express": "4.17.1",
         "express-promise-router": "^4.1.1",
         "fs-extra": "^11.1.0",
@@ -62,7 +63,6 @@
         "node-cache": "^5.1.2",
         "node-docker-api": "1.1.22",
         "node-uuid": "^1.4.8",
-        "recursive-watch": "^1.1.4",
         "request": "2.88.2",
         "request-promise": "4.2.6",
         "socket.io": "^4.5.2",

package/src/RepositoryWatcher.ts

@@ -0,0 +1,302 @@
+import chokidar, { FSWatcher } from 'chokidar';
+import ClusterConfiguration, { Definition, DefinitionInfo } from '@kapeta/local-cluster-config';
+import FS from 'fs-extra';
+import Path from 'node:path';
+import YAML from 'yaml';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import _ from 'lodash';
+import { socketManager } from './socketManager';
+import { definitionsManager } from './definitionsManager';
+import { assetManager } from './assetManager';
+import { SourceOfChange, WatchEventName } from './types';
+import { cacheManager } from './cacheManager';
+
+interface AssetIdentity {
+    handle: string;
+    name: string;
+    version: string;
+}
+const KAPETA_YML_RX = /^kapeta.ya?ml$/;
+export class RepositoryWatcher {
+    private watcher?: FSWatcher;
+    private disabled: boolean = false;
+    private readonly baseDir: string;
+    private allDefinitions: DefinitionInfo[] = [];
+    private symbolicLinks: { [link: string]: string } = {};
+    private sourceOfChange: Map<string, SourceOfChange> = new Map();
+    constructor() {
+        this.baseDir = ClusterConfiguration.getRepositoryBasedir();
+    }
+
+    setDisabled(disabled: boolean) {
+        this.disabled = disabled;
+    }
+    public watch() {
+        if (!FS.existsSync(this.baseDir)) {
+            FS.mkdirpSync(this.baseDir);
+        }
+
+        this.allDefinitions = ClusterConfiguration.getDefinitions();
+
+        try {
+            this.watcher = chokidar.watch(this.baseDir, {
+                followSymlinks: false,
+                ignorePermissionErrors: true,
+                disableGlobbing: true,
+                persistent: true,
+                depth: 2,
+                ignored: (path) => this.ignoreFile(path),
+            });
+            this.watcher.on('all', this.handleFileChange.bind(this));
+            this.watcher.on('error', (error) => {
+                console.log('Error watching repository', error);
+            });
+            this.watcher.on('ready', () => {
+                console.log('Watching local repository for provider changes: %s', this.baseDir);
+            });
+        } catch (e) {
+            // Fallback to run without watch mode due to potential platform issues.
+            // https://nodejs.org/docs/latest/api/fs.html#caveats
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
+            return;
+        }
+    }
+
+    async setSourceOfChangeFor(file: string, source: SourceOfChange) {
+        this.sourceOfChange.set(file, source);
+        const realPath = await FS.realpath(file);
+        if (realPath !== file) {
+            this.sourceOfChange.set(realPath, source);
+        }
+    }
+
+    async clearSourceOfChangeFor(file: string) {
+        this.sourceOfChange.delete(file);
+        const realPath = await FS.realpath(file);
+        if (realPath !== file) {
+            this.sourceOfChange.delete(realPath);
+        }
+    }
+
+    public async unwatch() {
+        if (!this.watcher) {
+            return;
+        }
+        this.symbolicLinks = {};
+        await this.watcher.close();
+        this.watcher = undefined;
+    }
+
+    private async getAssetIdentity(path: string): Promise<AssetIdentity | undefined> {
+        const baseName = Path.basename(path);
+        let handle, name, version;
+        if (path.startsWith(this.baseDir)) {
+            const relativePath = Path.relative(this.baseDir, path);
+            // Inside the repo we can use the path to determine the handle, name and version
+            [handle, name, version] = relativePath.split(/\//g);
+            if (!handle || !name || !version) {
+                // Do nothing with this
+                return;
+            }
+
+            return {
+                handle,
+                name,
+                version,
+            };
+        }
+
+        if (!KAPETA_YML_RX.test(baseName)) {
+            // Do nothing with this
+            return;
+        }
+        // Outside the repo we need to use the file content to determine the handle, name
+        // Version is always 'local'
+        version = 'local';
+
+        try {
+            const definition: Definition = YAML.parse((await FS.readFile(path)).toString());
+            const uri = parseKapetaUri(definition.metadata.name);
+            handle = uri.handle;
+            name = uri.name;
+            return {
+                handle,
+                name,
+                version,
+            };
+        } catch (e) {
+            // Ignore issues in the YML file
+            return;
+        }
+    }
+
+    private async handleFileChange(eventName: WatchEventName, path: string) {
+        if (!path) {
+            return;
+        }
+
+        //console.log('File changed', eventName, path);
+
+        const assetIdentity = await this.getAssetIdentity(path);
+        if (!assetIdentity) {
+            return;
+        }
+
+        if (this.disabled) {
+            return;
+        }
+
+        // If this is false it's because we're watching a symlink target
+        const withinRepo = path.startsWith(this.baseDir);
+        if (withinRepo && assetIdentity.version === 'local' && path.endsWith('/local')) {
+            // This is likely a symlink target
+            if (eventName === 'add') {
+                //console.log('Checking if we should add symlink target', handle, name, version, path);
+                await this.addSymlinkTarget(path);
+            }
+
+            if (eventName === 'unlink') {
+                await this.removeSymlinkTarget(path);
+            }
+
+            if (eventName === 'change') {
+                await this.updateSymlinkTarget(path);
+            }
+        }
+
+        const sourceOfChange = this.sourceOfChange.get(path) ?? 'filesystem';
+        await this.checkForChange(assetIdentity, sourceOfChange);
+
+        // We consume the sourceOfChange when the file is changed
+        this.sourceOfChange.delete(path);
+    }
+
+    private async checkForChange(assetIdentity: AssetIdentity, sourceOfChange: SourceOfChange) {
+        const ymlPath = Path.join(
+            this.baseDir,
+            assetIdentity.handle,
+            assetIdentity.name,
+            assetIdentity.version,
+            'kapeta.yml'
+        );
+        const newDefinitions = ClusterConfiguration.getDefinitions();
+        const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
+        let currentDefinition = this.allDefinitions.find((d) => d.ymlPath === ymlPath);
+        const ymlExists = await this.exists(ymlPath);
+        let type;
+        if (ymlExists) {
+            if (currentDefinition) {
+                if (newDefinition && _.isEqual(currentDefinition, newDefinition)) {
+                    //Definition was not changed
+                    return;
+                }
+                type = 'updated';
+            } else if (newDefinition) {
+                type = 'added';
+                currentDefinition = newDefinition;
+            } else {
+                //Other definition was added / updated - ignore
+                return;
+            }
+        } else {
+            if (currentDefinition) {
+                const ref = parseKapetaUri(
+                    `${currentDefinition.definition.metadata.name}:${currentDefinition.version}`
+                ).id;
+                //Something was removed
+                type = 'removed';
+            } else {
+                //Other definition was removed - ignore
+                return;
+            }
+        }
+
+        const payload = {
+            type,
+            definition: newDefinition?.definition ?? currentDefinition?.definition,
+            asset: assetIdentity,
+            sourceOfChange,
+        };
+
+        this.allDefinitions = newDefinitions;
+
+        //console.log('Asset changed', payload);
+        socketManager.emitGlobal('asset-change', payload);
+
+        cacheManager.flush();
+    }
+
+    private async exists(path: string): Promise<boolean> {
+        try {
+            await FS.access(path);
+            return true;
+        } catch (e) {
+            return false;
+        }
+    }
+    private async removeSymlinkTarget(path: string) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Unwatching symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+        }
+    }
+
+    private async updateSymlinkTarget(path: string) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Updating symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+            await this.addSymlinkTarget(path);
+        }
+    }
+
+    private async addSymlinkTarget(path: string) {
+        try {
+            // Make sure we're not watching the symlink target
+            await this.removeSymlinkTarget(path);
+            const stat = await FS.lstat(path);
+            if (stat.isSymbolicLink()) {
+                const realPath = `${await FS.realpath(path)}/kapeta.yml`;
+                if (await this.exists(realPath)) {
+                    //console.log('Watching symlink target %s => %s', path, realPath);
+                    this.watcher?.add(realPath);
+                    this.symbolicLinks[path] = realPath;
+                }
+            }
+        } catch (e) {
+            // Ignore
+            console.warn('Failed to check local symlink target', e);
+        }
+    }
+
+    private ignoreFile(path: string) {
+        if (!path.startsWith(this.baseDir)) {
+            return false;
+        }
+        if (path.includes('/node_modules/')) {
+            return true;
+        }
+
+        const filename = Path.basename(path);
+        if (filename.startsWith('.')) {
+            return true;
+        }
+
+        const relativePath = Path.relative(this.baseDir, path).split(Path.sep);
+
+        try {
+            if (FS.statSync(path).isDirectory()) {
+                if (relativePath.length > 3) {
+                    return true;
+                }
+                return false;
+            }
+        } catch (e) {
+            // Didn't exist - dont ignore
+            return false;
+        }
+
+        return !/^kapeta\.ya?ml$/.test(filename);
+    }
+}