@kapeta/local-cluster-service 0.15.3 → 0.16.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [0.16.0](https://github.com/kapetacom/local-cluster-service/compare/v0.15.3...v0.16.0) (2023-08-11)
+
+
+### Features
+
+* Use chokidar lib for watching for changes on disk ([#60](https://github.com/kapetacom/local-cluster-service/issues/60)) ([f2af855](https://github.com/kapetacom/local-cluster-service/commit/f2af85554fc2a23133ce27a4f8989cabdea097d7))
+
 ## [0.15.3](https://github.com/kapetacom/local-cluster-service/compare/v0.15.2...v0.15.3) (2023-08-10)
 
 

package/dist/cjs/src/RepositoryWatcher.d.ts

@@ -0,0 +1,22 @@
+export declare class RepositoryWatcher {
+    private watcher?;
+    private disabled;
+    private readonly baseDir;
+    private allDefinitions;
+    private symbolicLinks;
+    private ignoredFiles;
+    constructor();
+    setDisabled(disabled: boolean): void;
+    watch(): void;
+    ignoreChangesFor(file: string): Promise<void>;
+    resumeChangedFor(file: string): Promise<void>;
+    unwatch(): Promise<void>;
+    private getAssetIdentity;
+    private handleFileChange;
+    private checkForChange;
+    private exists;
+    private removeSymlinkTarget;
+    private updateSymlinkTarget;
+    private addSymlinkTarget;
+    private ignoreFile;
+}

package/dist/cjs/src/RepositoryWatcher.js

@@ -0,0 +1,273 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepositoryWatcher = void 0;
+const chokidar_1 = __importDefault(require("chokidar"));
+const local_cluster_config_1 = __importDefault(require("@kapeta/local-cluster-config"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const node_path_1 = __importDefault(require("node:path"));
+const yaml_1 = __importDefault(require("yaml"));
+const nodejs_utils_1 = require("@kapeta/nodejs-utils");
+const lodash_1 = __importDefault(require("lodash"));
+const socketManager_1 = require("./socketManager");
+const definitionsManager_1 = require("./definitionsManager");
+const assetManager_1 = require("./assetManager");
+function clearAllCaches() {
+    definitionsManager_1.definitionsManager.clearCache();
+    assetManager_1.assetManager.clearCache();
+}
+const KAPETA_YML_RX = /^kapeta.ya?ml$/;
+class RepositoryWatcher {
+    watcher;
+    disabled = false;
+    baseDir;
+    allDefinitions = [];
+    symbolicLinks = {};
+    ignoredFiles = new Set();
+    constructor() {
+        this.baseDir = local_cluster_config_1.default.getRepositoryBasedir();
+    }
+    setDisabled(disabled) {
+        this.disabled = disabled;
+    }
+    watch() {
+        if (!fs_extra_1.default.existsSync(this.baseDir)) {
+            fs_extra_1.default.mkdirpSync(this.baseDir);
+        }
+        this.allDefinitions = local_cluster_config_1.default.getDefinitions();
+        try {
+            this.watcher = chokidar_1.default.watch(this.baseDir, {
+                followSymlinks: false,
+                ignorePermissionErrors: true,
+                disableGlobbing: true,
+                persistent: true,
+                depth: 2,
+                ignored: (path) => this.ignoreFile(path),
+            });
+            this.watcher.on('all', this.handleFileChange.bind(this));
+            this.watcher.on('error', (error) => {
+                console.log('Error watching repository', error);
+            });
+            this.watcher.on('ready', () => {
+                console.log('Watching local repository for provider changes: %s', this.baseDir);
+            });
+        }
+        catch (e) {
+            // Fallback to run without watch mode due to potential platform issues.
+            // https://nodejs.org/docs/latest/api/fs.html#caveats
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
+            return;
+        }
+    }
+    async ignoreChangesFor(file) {
+        this.ignoredFiles.add(file);
+        const realPath = await fs_extra_1.default.realpath(file);
+        if (realPath !== file) {
+            this.ignoredFiles.add(realPath);
+        }
+    }
+    async resumeChangedFor(file) {
+        this.ignoredFiles.delete(file);
+        const realPath = await fs_extra_1.default.realpath(file);
+        if (realPath !== file) {
+            this.ignoredFiles.delete(realPath);
+        }
+    }
+    async unwatch() {
+        if (!this.watcher) {
+            return;
+        }
+        this.symbolicLinks = {};
+        await this.watcher.close();
+        this.watcher = undefined;
+    }
+    async getAssetIdentity(path) {
+        const baseName = node_path_1.default.basename(path);
+        let handle, name, version;
+        if (path.startsWith(this.baseDir)) {
+            const relativePath = node_path_1.default.relative(this.baseDir, path);
+            // Inside the repo we can use the path to determine the handle, name and version
+            [handle, name, version] = relativePath.split(/\//g);
+            if (!handle || !name || !version) {
+                // Do nothing with this
+                return;
+            }
+            return {
+                handle,
+                name,
+                version,
+            };
+        }
+        if (!KAPETA_YML_RX.test(baseName)) {
+            // Do nothing with this
+            return;
+        }
+        // Outside the repo we need to use the file content to determine the handle, name
+        // Version is always 'local'
+        version = 'local';
+        try {
+            const definition = yaml_1.default.parse((await fs_extra_1.default.readFile(path)).toString());
+            const uri = (0, nodejs_utils_1.parseKapetaUri)(definition.metadata.name);
+            handle = uri.handle;
+            name = uri.name;
+            return {
+                handle,
+                name,
+                version,
+            };
+        }
+        catch (e) {
+            // Ignore issues in the YML file
+            return;
+        }
+    }
+    async handleFileChange(eventName, path) {
+        if (!path) {
+            return;
+        }
+        if (this.ignoredFiles.has(path)) {
+            return;
+        }
+        //console.log('File changed', eventName, path);
+        const assetIdentity = await this.getAssetIdentity(path);
+        if (!assetIdentity) {
+            return;
+        }
+        if (this.disabled) {
+            return;
+        }
+        // If this is false it's because we're watching a symlink target
+        const withinRepo = path.startsWith(this.baseDir);
+        if (withinRepo && assetIdentity.version === 'local' && path.endsWith('/local')) {
+            // This is likely a symlink target
+            if (eventName === 'add') {
+                //console.log('Checking if we should add symlink target', handle, name, version, path);
+                await this.addSymlinkTarget(path);
+            }
+            if (eventName === 'unlink') {
+                await this.removeSymlinkTarget(path);
+            }
+            if (eventName === 'change') {
+                await this.updateSymlinkTarget(path);
+            }
+        }
+        await this.checkForChange(assetIdentity);
+    }
+    async checkForChange(assetIdentity) {
+        const ymlPath = node_path_1.default.join(this.baseDir, assetIdentity.handle, assetIdentity.name, assetIdentity.version, 'kapeta.yml');
+        const newDefinitions = local_cluster_config_1.default.getDefinitions();
+        const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
+        let currentDefinition = this.allDefinitions.find((d) => d.ymlPath === ymlPath);
+        const ymlExists = await this.exists(ymlPath);
+        let type;
+        if (ymlExists) {
+            if (currentDefinition) {
+                if (newDefinition && lodash_1.default.isEqual(currentDefinition, newDefinition)) {
+                    //Definition was not changed
+                    return;
+                }
+                type = 'updated';
+            }
+            else if (newDefinition) {
+                type = 'added';
+                currentDefinition = newDefinition;
+            }
+            else {
+                //Other definition was added / updated - ignore
+                return;
+            }
+        }
+        else {
+            if (currentDefinition) {
+                const ref = (0, nodejs_utils_1.parseKapetaUri)(`${currentDefinition.definition.metadata.name}:${currentDefinition.version}`).id;
+                //Something was removed
+                type = 'removed';
+            }
+            else {
+                //Other definition was removed - ignore
+                return;
+            }
+        }
+        const payload = {
+            type,
+            definition: newDefinition?.definition ?? currentDefinition?.definition,
+            asset: assetIdentity,
+        };
+        this.allDefinitions = newDefinitions;
+        //console.log('Asset changed', payload);
+        socketManager_1.socketManager.emitGlobal('asset-change', payload);
+        clearAllCaches();
+    }
+    async exists(path) {
+        try {
+            await fs_extra_1.default.access(path);
+            return true;
+        }
+        catch (e) {
+            return false;
+        }
+    }
+    async removeSymlinkTarget(path) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Unwatching symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+        }
+    }
+    async updateSymlinkTarget(path) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Updating symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+            await this.addSymlinkTarget(path);
+        }
+    }
+    async addSymlinkTarget(path) {
+        try {
+            // Make sure we're not watching the symlink target
+            await this.removeSymlinkTarget(path);
+            const stat = await fs_extra_1.default.lstat(path);
+            if (stat.isSymbolicLink()) {
+                const realPath = `${await fs_extra_1.default.realpath(path)}/kapeta.yml`;
+                if (await this.exists(realPath)) {
+                    //console.log('Watching symlink target %s => %s', path, realPath);
+                    this.watcher?.add(realPath);
+                    this.symbolicLinks[path] = realPath;
+                }
+            }
+        }
+        catch (e) {
+            // Ignore
+            console.warn('Failed to check local symlink target', e);
+        }
+    }
+    ignoreFile(path) {
+        if (!path.startsWith(this.baseDir)) {
+            return false;
+        }
+        if (path.includes('/node_modules/')) {
+            return true;
+        }
+        const filename = node_path_1.default.basename(path);
+        if (filename.startsWith('.')) {
+            return true;
+        }
+        const relativePath = node_path_1.default.relative(this.baseDir, path).split(node_path_1.default.sep);
+        try {
+            if (fs_extra_1.default.statSync(path).isDirectory()) {
+                if (relativePath.length > 3) {
+                    return true;
+                }
+                return false;
+            }
+        }
+        catch (e) {
+            // Didn't exist - dont ignore
+            return false;
+        }
+        return !/^kapeta\.ya?ml$/.test(filename);
+    }
+}
+exports.RepositoryWatcher = RepositoryWatcher;

package/dist/cjs/src/assetManager.js

@@ -5,7 +5,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.assetManager = void 0;
 const node_path_1 = __importDefault(require("node:path"));
-const node_fs_1 = __importDefault(require("node:fs"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const yaml_1 = __importDefault(require("yaml"));
 const node_cache_1 = __importDefault(require("node-cache"));
@@ -17,6 +16,10 @@ const nodejs_registry_utils_1 = require("@kapeta/nodejs-registry-utils");
 const definitionsManager_1 = require("./definitionsManager");
 const utils_1 = require("./utils/utils");
 const taskManager_1 = require("./taskManager");
+function clearAllCaches() {
+    definitionsManager_1.definitionsManager.clearCache();
+    exports.assetManager.clearCache();
+}
 function enrichAsset(asset) {
     return {
         ref: `kapeta://${asset.definition.metadata.name}:${asset.version}`,
@@ -103,19 +106,16 @@ class AssetManager {
         return asset;
     }
     async createAsset(path, yaml) {
-        if (node_fs_1.default.existsSync(path)) {
+        if (await fs_extra_1.default.pathExists(path)) {
             throw new Error('File already exists: ' + path);
         }
         const dirName = node_path_1.default.dirname(path);
-        if (!node_fs_1.default.existsSync(dirName)) {
-            fs_extra_1.default.mkdirpSync(dirName);
+        if (!(await fs_extra_1.default.pathExists(dirName))) {
+            await fs_extra_1.default.mkdirp(dirName);
         }
-        console.log('Wrote to ' + path);
-        node_fs_1.default.writeFileSync(path, yaml_1.default.stringify(yaml));
+        await fs_extra_1.default.writeFile(path, yaml_1.default.stringify(yaml));
         const asset = await this.importFile(path);
-        console.log('Imported');
-        this.cache.flushAll();
-        definitionsManager_1.definitionsManager.clearCache();
+        clearAllCaches();
         const ref = `kapeta://${yaml.metadata.name}:local`;
         this.maybeGenerateCode(ref, path, yaml);
         return asset;
@@ -131,11 +131,23 @@ class AssetManager {
         if (!asset.ymlPath) {
             throw new Error('Attempted to update corrupted asset: ' + ref);
         }
-        console.log('Wrote to ' + asset.ymlPath);
-        node_fs_1.default.writeFileSync(asset.ymlPath, yaml_1.default.stringify(yaml));
-        this.cache.flushAll();
-        definitionsManager_1.definitionsManager.clearCache();
-        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
+        const path = asset.ymlPath;
+        try {
+            await repositoryManager_1.repositoryManager.ignoreChangesFor(path);
+            await fs_extra_1.default.writeFile(asset.ymlPath, yaml_1.default.stringify(yaml));
+            console.log('Wrote to ' + asset.ymlPath);
+            clearAllCaches();
+            this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
+        }
+        finally {
+            //We need to wait a bit for the disk to settle before we can resume watching
+            setTimeout(async () => {
+                try {
+                    await repositoryManager_1.repositoryManager.resumeChangedFor(path);
+                }
+                catch (e) { }
+            }, 500);
+        }
     }
     maybeGenerateCode(ref, ymlPath, block) {
         ref = (0, utils_1.normalizeKapetaUri)(ref);
@@ -154,14 +166,15 @@ class AssetManager {
         if (filePath.startsWith('file://')) {
             filePath = filePath.substring('file://'.length);
         }
-        if (!node_fs_1.default.existsSync(filePath)) {
+        if (!(await fs_extra_1.default.pathExists(filePath))) {
             throw new Error('File not found: ' + filePath);
         }
-        const assetInfos = yaml_1.default.parseAllDocuments(node_fs_1.default.readFileSync(filePath).toString()).map((doc) => doc.toJSON());
+        const content = await fs_extra_1.default.readFile(filePath);
+        const assetInfos = yaml_1.default.parseAllDocuments(content.toString()).map((doc) => doc.toJSON());
         await nodejs_registry_utils_1.Actions.link(new progressListener_1.ProgressListener(), node_path_1.default.dirname(filePath));
         const version = 'local';
         const refs = assetInfos.map((assetInfo) => `kapeta://${assetInfo.metadata.name}:${version}`);
-        this.cache.flushAll();
+        clearAllCaches();
         return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
     }
     async unregisterAsset(ref) {
@@ -169,7 +182,7 @@ class AssetManager {
         if (!asset) {
             throw new Error('Asset does not exists: ' + ref);
         }
-        this.cache.flushAll();
+        clearAllCaches();
         await nodejs_registry_utils_1.Actions.uninstall(new progressListener_1.ProgressListener(), [asset.ref]);
     }
     async installAsset(ref) {

package/dist/cjs/src/instanceManager.js

@@ -57,7 +57,10 @@ class InstanceManager {
             return [];
         }
         const plan = planInfo.definition;
-        const instanceIds = plan.spec.blocks.map((block) => block.id);
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const instanceIds = plan.spec?.blocks?.map((block) => block.id) || [];
         return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
     }
     getInstance(systemId, instanceId) {

package/dist/cjs/src/repositoryManager.d.ts

@@ -1,13 +1,13 @@
 import { Task } from './taskManager';
 declare class RepositoryManager {
-    private changeEventsEnabled;
     private _registryService;
     private _cache;
-    private watcher?;
+    private watcher;
     constructor();
-    setChangeEventsEnabled(enabled: boolean): void;
     listenForChanges(): void;
-    stopListening(): void;
+    stopListening(): Promise<void>;
+    ignoreChangesFor(file: string): Promise<void>;
+    resumeChangedFor(file: string): Promise<void>;
     ensureDefaultProviders(): void;
     private _install;
     ensureAsset(handle: string, name: string, version: string, wait?: boolean): Promise<undefined | Task[]>;

package/dist/cjs/src/repositoryManager.js

@@ -4,20 +4,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.repositoryManager = void 0;
-const node_fs_1 = __importDefault(require("node:fs"));
 const node_os_1 = __importDefault(require("node:os"));
-const node_path_1 = __importDefault(require("node:path"));
-const recursive_watch_1 = __importDefault(require("recursive-watch"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const local_cluster_config_1 = __importDefault(require("@kapeta/local-cluster-config"));
-const nodejs_utils_1 = require("@kapeta/nodejs-utils");
 const socketManager_1 = require("./socketManager");
 const nodejs_registry_utils_1 = require("@kapeta/nodejs-registry-utils");
 const definitionsManager_1 = require("./definitionsManager");
 const taskManager_1 = require("./taskManager");
 const utils_1 = require("./utils/utils");
-const assetManager_1 = require("./assetManager");
 const progressListener_1 = require("./progressListener");
+const RepositoryWatcher_1 = require("./RepositoryWatcher");
+const assetManager_1 = require("./assetManager");
+function clearAllCaches() {
+    definitionsManager_1.definitionsManager.clearCache();
+    assetManager_1.assetManager.clearCache();
+}
 const EVENT_DEFAULT_PROVIDERS_START = 'default-providers-start';
 const EVENT_DEFAULT_PROVIDERS_END = 'default-providers-end';
 const DEFAULT_PROVIDERS = [
@@ -36,92 +35,26 @@ const DEFAULT_PROVIDERS = [
 ];
 const INSTALL_ATTEMPTED = {};
 class RepositoryManager {
-    changeEventsEnabled;
     _registryService;
     _cache;
     watcher;
     constructor() {
-        this.changeEventsEnabled = true;
-        this.listenForChanges();
         this._registryService = new nodejs_registry_utils_1.RegistryService(nodejs_registry_utils_1.Config.data.registry.url);
         this._cache = {};
-    }
-    setChangeEventsEnabled(enabled) {
-        this.changeEventsEnabled = enabled;
+        this.watcher = new RepositoryWatcher_1.RepositoryWatcher();
+        this.listenForChanges();
     }
     listenForChanges() {
-        const baseDir = local_cluster_config_1.default.getRepositoryBasedir();
-        if (!node_fs_1.default.existsSync(baseDir)) {
-            fs_extra_1.default.mkdirpSync(baseDir);
-        }
-        let allDefinitions = local_cluster_config_1.default.getDefinitions();
-        console.log('Watching local repository for provider changes: %s', baseDir);
-        try {
-            this.watcher = (0, recursive_watch_1.default)(baseDir, (filename) => {
-                if (!filename) {
-                    return;
-                }
-                const [handle, name, version] = filename.toString().split(/\//g);
-                if (!name || !version) {
-                    return;
-                }
-                if (!this.changeEventsEnabled) {
-                    return;
-                }
-                const ymlPath = node_path_1.default.join(baseDir, handle, name, version, 'kapeta.yml');
-                const newDefinitions = local_cluster_config_1.default.getDefinitions();
-                const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
-                let currentDefinition = allDefinitions.find((d) => d.ymlPath === ymlPath);
-                const ymlExists = node_fs_1.default.existsSync(ymlPath);
-                let type;
-                if (ymlExists) {
-                    if (currentDefinition) {
-                        type = 'updated';
-                    }
-                    else if (newDefinition) {
-                        type = 'added';
-                        currentDefinition = newDefinition;
-                    }
-                    else {
-                        //Other definition was added / updated - ignore
-                        return;
-                    }
-                }
-                else {
-                    if (currentDefinition) {
-                        const ref = (0, nodejs_utils_1.parseKapetaUri)(`${currentDefinition.definition.metadata.name}:${currentDefinition.version}`).id;
-                        delete INSTALL_ATTEMPTED[ref];
-                        //Something was removed
-                        type = 'removed';
-                    }
-                    else {
-                        //Other definition was removed - ignore
-                        return;
-                    }
-                }
-                const payload = {
-                    type,
-                    definition: currentDefinition?.definition,
-                    asset: { handle, name, version },
-                };
-                allDefinitions = newDefinitions;
-                socketManager_1.socketManager.emit(`assets`, 'changed', payload);
-                definitionsManager_1.definitionsManager.clearCache();
-            });
-        }
-        catch (e) {
-            // Fallback to run without watch mode due to potential platform issues.
-            // https://nodejs.org/docs/latest/api/fs.html#caveats
-            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
-            return;
-        }
+        this.watcher.watch();
     }
-    stopListening() {
-        if (!this.watcher) {
-            return;
-        }
-        this.watcher();
-        this.watcher = undefined;
+    async stopListening() {
+        return this.watcher.unwatch();
+    }
+    ignoreChangesFor(file) {
+        return this.watcher.ignoreChangesFor(file);
+    }
+    resumeChangedFor(file) {
+        return this.watcher.resumeChangedFor(file);
     }
     ensureDefaultProviders() {
         socketManager_1.socketManager.emitGlobal(EVENT_DEFAULT_PROVIDERS_START, { providers: DEFAULT_PROVIDERS });
@@ -146,19 +79,13 @@ class RepositoryManager {
                 try {
                     //We change to a temp dir to avoid issues with the current working directory
                     process.chdir(node_os_1.default.tmpdir());
-                    //Disable change events while installing
-                    this.setChangeEventsEnabled(false);
                     await nodejs_registry_utils_1.Actions.install(new progressListener_1.ProgressListener(), [ref], {});
                 }
                 catch (e) {
                     console.error(`Failed to install asset: ${ref}`, e);
                     throw e;
                 }
-                finally {
-                    this.setChangeEventsEnabled(true);
-                }
-                definitionsManager_1.definitionsManager.clearCache();
-                assetManager_1.assetManager.clearCache();
+                clearAllCaches();
                 //console.log(`Asset installed: ${ref}`);
             };
         };

package/dist/esm/src/RepositoryWatcher.d.ts

@@ -0,0 +1,22 @@
+export declare class RepositoryWatcher {
+    private watcher?;
+    private disabled;
+    private readonly baseDir;
+    private allDefinitions;
+    private symbolicLinks;
+    private ignoredFiles;
+    constructor();
+    setDisabled(disabled: boolean): void;
+    watch(): void;
+    ignoreChangesFor(file: string): Promise<void>;
+    resumeChangedFor(file: string): Promise<void>;
+    unwatch(): Promise<void>;
+    private getAssetIdentity;
+    private handleFileChange;
+    private checkForChange;
+    private exists;
+    private removeSymlinkTarget;
+    private updateSymlinkTarget;
+    private addSymlinkTarget;
+    private ignoreFile;
+}