@kapeta/local-cluster-service 0.15.2 → 0.16.0

package/dist/esm/src/RepositoryWatcher.js

@@ -0,0 +1,266 @@
+import chokidar from 'chokidar';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import FS from 'fs-extra';
+import Path from 'node:path';
+import YAML from 'yaml';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import _ from 'lodash';
+import { socketManager } from './socketManager';
+import { definitionsManager } from './definitionsManager';
+import { assetManager } from './assetManager';
+function clearAllCaches() {
+    definitionsManager.clearCache();
+    assetManager.clearCache();
+}
+const KAPETA_YML_RX = /^kapeta.ya?ml$/;
+export class RepositoryWatcher {
+    watcher;
+    disabled = false;
+    baseDir;
+    allDefinitions = [];
+    symbolicLinks = {};
+    ignoredFiles = new Set();
+    constructor() {
+        this.baseDir = ClusterConfiguration.getRepositoryBasedir();
+    }
+    setDisabled(disabled) {
+        this.disabled = disabled;
+    }
+    watch() {
+        if (!FS.existsSync(this.baseDir)) {
+            FS.mkdirpSync(this.baseDir);
+        }
+        this.allDefinitions = ClusterConfiguration.getDefinitions();
+        try {
+            this.watcher = chokidar.watch(this.baseDir, {
+                followSymlinks: false,
+                ignorePermissionErrors: true,
+                disableGlobbing: true,
+                persistent: true,
+                depth: 2,
+                ignored: (path) => this.ignoreFile(path),
+            });
+            this.watcher.on('all', this.handleFileChange.bind(this));
+            this.watcher.on('error', (error) => {
+                console.log('Error watching repository', error);
+            });
+            this.watcher.on('ready', () => {
+                console.log('Watching local repository for provider changes: %s', this.baseDir);
+            });
+        }
+        catch (e) {
+            // Fallback to run without watch mode due to potential platform issues.
+            // https://nodejs.org/docs/latest/api/fs.html#caveats
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
+            return;
+        }
+    }
+    async ignoreChangesFor(file) {
+        this.ignoredFiles.add(file);
+        const realPath = await FS.realpath(file);
+        if (realPath !== file) {
+            this.ignoredFiles.add(realPath);
+        }
+    }
+    async resumeChangedFor(file) {
+        this.ignoredFiles.delete(file);
+        const realPath = await FS.realpath(file);
+        if (realPath !== file) {
+            this.ignoredFiles.delete(realPath);
+        }
+    }
+    async unwatch() {
+        if (!this.watcher) {
+            return;
+        }
+        this.symbolicLinks = {};
+        await this.watcher.close();
+        this.watcher = undefined;
+    }
+    async getAssetIdentity(path) {
+        const baseName = Path.basename(path);
+        let handle, name, version;
+        if (path.startsWith(this.baseDir)) {
+            const relativePath = Path.relative(this.baseDir, path);
+            // Inside the repo we can use the path to determine the handle, name and version
+            [handle, name, version] = relativePath.split(/\//g);
+            if (!handle || !name || !version) {
+                // Do nothing with this
+                return;
+            }
+            return {
+                handle,
+                name,
+                version,
+            };
+        }
+        if (!KAPETA_YML_RX.test(baseName)) {
+            // Do nothing with this
+            return;
+        }
+        // Outside the repo we need to use the file content to determine the handle, name
+        // Version is always 'local'
+        version = 'local';
+        try {
+            const definition = YAML.parse((await FS.readFile(path)).toString());
+            const uri = parseKapetaUri(definition.metadata.name);
+            handle = uri.handle;
+            name = uri.name;
+            return {
+                handle,
+                name,
+                version,
+            };
+        }
+        catch (e) {
+            // Ignore issues in the YML file
+            return;
+        }
+    }
+    async handleFileChange(eventName, path) {
+        if (!path) {
+            return;
+        }
+        if (this.ignoredFiles.has(path)) {
+            return;
+        }
+        //console.log('File changed', eventName, path);
+        const assetIdentity = await this.getAssetIdentity(path);
+        if (!assetIdentity) {
+            return;
+        }
+        if (this.disabled) {
+            return;
+        }
+        // If this is false it's because we're watching a symlink target
+        const withinRepo = path.startsWith(this.baseDir);
+        if (withinRepo && assetIdentity.version === 'local' && path.endsWith('/local')) {
+            // This is likely a symlink target
+            if (eventName === 'add') {
+                //console.log('Checking if we should add symlink target', handle, name, version, path);
+                await this.addSymlinkTarget(path);
+            }
+            if (eventName === 'unlink') {
+                await this.removeSymlinkTarget(path);
+            }
+            if (eventName === 'change') {
+                await this.updateSymlinkTarget(path);
+            }
+        }
+        await this.checkForChange(assetIdentity);
+    }
+    async checkForChange(assetIdentity) {
+        const ymlPath = Path.join(this.baseDir, assetIdentity.handle, assetIdentity.name, assetIdentity.version, 'kapeta.yml');
+        const newDefinitions = ClusterConfiguration.getDefinitions();
+        const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
+        let currentDefinition = this.allDefinitions.find((d) => d.ymlPath === ymlPath);
+        const ymlExists = await this.exists(ymlPath);
+        let type;
+        if (ymlExists) {
+            if (currentDefinition) {
+                if (newDefinition && _.isEqual(currentDefinition, newDefinition)) {
+                    //Definition was not changed
+                    return;
+                }
+                type = 'updated';
+            }
+            else if (newDefinition) {
+                type = 'added';
+                currentDefinition = newDefinition;
+            }
+            else {
+                //Other definition was added / updated - ignore
+                return;
+            }
+        }
+        else {
+            if (currentDefinition) {
+                const ref = parseKapetaUri(`${currentDefinition.definition.metadata.name}:${currentDefinition.version}`).id;
+                //Something was removed
+                type = 'removed';
+            }
+            else {
+                //Other definition was removed - ignore
+                return;
+            }
+        }
+        const payload = {
+            type,
+            definition: newDefinition?.definition ?? currentDefinition?.definition,
+            asset: assetIdentity,
+        };
+        this.allDefinitions = newDefinitions;
+        //console.log('Asset changed', payload);
+        socketManager.emitGlobal('asset-change', payload);
+        clearAllCaches();
+    }
+    async exists(path) {
+        try {
+            await FS.access(path);
+            return true;
+        }
+        catch (e) {
+            return false;
+        }
+    }
+    async removeSymlinkTarget(path) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Unwatching symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+        }
+    }
+    async updateSymlinkTarget(path) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Updating symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+            await this.addSymlinkTarget(path);
+        }
+    }
+    async addSymlinkTarget(path) {
+        try {
+            // Make sure we're not watching the symlink target
+            await this.removeSymlinkTarget(path);
+            const stat = await FS.lstat(path);
+            if (stat.isSymbolicLink()) {
+                const realPath = `${await FS.realpath(path)}/kapeta.yml`;
+                if (await this.exists(realPath)) {
+                    //console.log('Watching symlink target %s => %s', path, realPath);
+                    this.watcher?.add(realPath);
+                    this.symbolicLinks[path] = realPath;
+                }
+            }
+        }
+        catch (e) {
+            // Ignore
+            console.warn('Failed to check local symlink target', e);
+        }
+    }
+    ignoreFile(path) {
+        if (!path.startsWith(this.baseDir)) {
+            return false;
+        }
+        if (path.includes('/node_modules/')) {
+            return true;
+        }
+        const filename = Path.basename(path);
+        if (filename.startsWith('.')) {
+            return true;
+        }
+        const relativePath = Path.relative(this.baseDir, path).split(Path.sep);
+        try {
+            if (FS.statSync(path).isDirectory()) {
+                if (relativePath.length > 3) {
+                    return true;
+                }
+                return false;
+            }
+        }
+        catch (e) {
+            // Didn't exist - dont ignore
+            return false;
+        }
+        return !/^kapeta\.ya?ml$/.test(filename);
+    }
+}

package/dist/esm/src/assetManager.js

@@ -1,16 +1,19 @@
 import Path from 'node:path';
-import FS from 'node:fs';
-import FSExtra from 'fs-extra';
+import FS from 'fs-extra';
 import YAML from 'yaml';
 import NodeCache from 'node-cache';
 import { codeGeneratorManager } from './codeGeneratorManager';
-import { progressListener } from './progressListener';
+import { ProgressListener } from './progressListener';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { repositoryManager } from './repositoryManager';
 import { Actions } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
 import { normalizeKapetaUri } from './utils/utils';
 import { taskManager } from './taskManager';
+function clearAllCaches() {
+    definitionsManager.clearCache();
+    assetManager.clearCache();
+}
 function enrichAsset(asset) {
     return {
         ref: `kapeta://${asset.definition.metadata.name}:${asset.version}`,
@@ -97,19 +100,16 @@ class AssetManager {
         return asset;
     }
     async createAsset(path, yaml) {
-        if (FS.existsSync(path)) {
+        if (await FS.pathExists(path)) {
             throw new Error('File already exists: ' + path);
         }
         const dirName = Path.dirname(path);
-        if (!FS.existsSync(dirName)) {
-            FSExtra.mkdirpSync(dirName);
+        if (!(await FS.pathExists(dirName))) {
+            await FS.mkdirp(dirName);
         }
-        console.log('Wrote to ' + path);
-        FS.writeFileSync(path, YAML.stringify(yaml));
+        await FS.writeFile(path, YAML.stringify(yaml));
         const asset = await this.importFile(path);
-        console.log('Imported');
-        this.cache.flushAll();
-        definitionsManager.clearCache();
+        clearAllCaches();
         const ref = `kapeta://${yaml.metadata.name}:local`;
         this.maybeGenerateCode(ref, path, yaml);
         return asset;
@@ -125,11 +125,23 @@ class AssetManager {
         if (!asset.ymlPath) {
             throw new Error('Attempted to update corrupted asset: ' + ref);
         }
-        console.log('Wrote to ' + asset.ymlPath);
-        FS.writeFileSync(asset.ymlPath, YAML.stringify(yaml));
-        this.cache.flushAll();
-        definitionsManager.clearCache();
-        this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
+        const path = asset.ymlPath;
+        try {
+            await repositoryManager.ignoreChangesFor(path);
+            await FS.writeFile(asset.ymlPath, YAML.stringify(yaml));
+            console.log('Wrote to ' + asset.ymlPath);
+            clearAllCaches();
+            this.maybeGenerateCode(asset.ref, asset.ymlPath, yaml);
+        }
+        finally {
+            //We need to wait a bit for the disk to settle before we can resume watching
+            setTimeout(async () => {
+                try {
+                    await repositoryManager.resumeChangedFor(path);
+                }
+                catch (e) { }
+            }, 500);
+        }
     }
     maybeGenerateCode(ref, ymlPath, block) {
         ref = normalizeKapetaUri(ref);
@@ -148,14 +160,15 @@ class AssetManager {
         if (filePath.startsWith('file://')) {
             filePath = filePath.substring('file://'.length);
         }
-        if (!FS.existsSync(filePath)) {
+        if (!(await FS.pathExists(filePath))) {
             throw new Error('File not found: ' + filePath);
         }
-        const assetInfos = YAML.parseAllDocuments(FS.readFileSync(filePath).toString()).map((doc) => doc.toJSON());
-        await Actions.link(progressListener, Path.dirname(filePath));
+        const content = await FS.readFile(filePath);
+        const assetInfos = YAML.parseAllDocuments(content.toString()).map((doc) => doc.toJSON());
+        await Actions.link(new ProgressListener(), Path.dirname(filePath));
         const version = 'local';
         const refs = assetInfos.map((assetInfo) => `kapeta://${assetInfo.metadata.name}:${version}`);
-        this.cache.flushAll();
+        clearAllCaches();
         return this.getAssets().filter((a) => refs.some((ref) => compareRefs(ref, a.ref)));
     }
     async unregisterAsset(ref) {
@@ -163,8 +176,8 @@ class AssetManager {
         if (!asset) {
             throw new Error('Asset does not exists: ' + ref);
         }
-        this.cache.flushAll();
-        await Actions.uninstall(progressListener, [asset.ref]);
+        clearAllCaches();
+        await Actions.uninstall(new ProgressListener(), [asset.ref]);
     }
     async installAsset(ref) {
         const asset = await this.getAsset(ref, true, false);

package/dist/esm/src/containerManager.js

@@ -11,7 +11,6 @@ import md5 from 'md5';
 import { getBlockInstanceContainerName } from './utils/utils';
 import { KapetaAPI } from '@kapeta/nodejs-api-client';
 import { taskManager } from './taskManager';
-const EVENT_IMAGE_PULL = 'docker-image-pull';
 export const CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;

package/dist/esm/src/identities/routes.js

@@ -2,10 +2,10 @@ import Router from 'express-promise-router';
 import { KapetaAPI } from '@kapeta/nodejs-api-client';
 import { corsHandler } from '../middleware/cors';
 const router = Router();
-const api = new KapetaAPI();
 router.use('/', corsHandler);
 router.get('/current', async (req, res) => {
     try {
+        const api = new KapetaAPI();
         if (api.hasToken()) {
             res.send(await api.getCurrentIdentity());
         }
@@ -19,6 +19,7 @@ router.get('/current', async (req, res) => {
 });
 router.get('/:identityId/memberships', async (req, res) => {
     try {
+        const api = new KapetaAPI();
         if (api.hasToken()) {
             res.send(await api.getMemberships(req.params.identityId));
         }

package/dist/esm/src/instanceManager.d.ts

@@ -38,7 +38,5 @@ export declare class InstanceManager {
     private checkInstances;
     private getExternalStatus;
     private requestInstanceStatus;
-    private emitSystemEvent;
-    private emitInstanceEvent;
 }
 export declare const instanceManager: InstanceManager;

package/dist/esm/src/instanceManager.js

@@ -3,7 +3,7 @@ import request from 'request';
 import AsyncLock from 'async-lock';
 import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
 import { storageService } from './storageService';
-import { socketManager } from './socketManager';
+import { EVENT_INSTANCE_CREATED, EVENT_INSTANCE_EXITED, EVENT_STATUS_CHANGED, socketManager, } from './socketManager';
 import { serviceManager } from './serviceManager';
 import { assetManager } from './assetManager';
 import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
@@ -16,10 +16,6 @@ import { definitionsManager } from './definitionsManager';
 import { Task, taskManager } from './taskManager';
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
-const EVENT_STATUS_CHANGED = 'status-changed';
-const EVENT_INSTANCE_CREATED = 'instance-created';
-const EVENT_INSTANCE_EXITED = 'instance-exited';
-const EVENT_INSTANCE_LOG = 'instance-log';
 const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
 export class InstanceManager {
     _interval = undefined;
@@ -55,7 +51,10 @@ export class InstanceManager {
             return [];
         }
         const plan = planInfo.definition;
-        const instanceIds = plan.spec.blocks.map((block) => block.id);
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const instanceIds = plan.spec?.blocks?.map((block) => block.id) || [];
         return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
     }
     getInstance(systemId, instanceId) {
@@ -115,11 +114,11 @@ export class InstanceManager {
         if (existingInstance) {
             const ix = this._instances.indexOf(existingInstance);
             this._instances.splice(ix, 1, instance);
-            this.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+            socketManager.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
         }
         else {
             this._instances.push(instance);
-            this.emitSystemEvent(instance.systemId, EVENT_INSTANCE_CREATED, instance);
+            socketManager.emitSystemEvent(instance.systemId, EVENT_INSTANCE_CREATED, instance);
         }
         this.save();
         return instance;
@@ -162,7 +161,7 @@ export class InstanceManager {
                 if (healthUrl) {
                     instance.health = healthUrl;
                 }
-                this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
             }
             else {
                 //If instance was not found - then we're receiving an externally started instance
@@ -178,7 +177,7 @@ export class InstanceManager {
                     address,
                 };
                 this._instances.push(instance);
-                this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
+                socketManager.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
             }
             this.save();
             return instance;
@@ -203,7 +202,7 @@ export class InstanceManager {
                 instance.status = InstanceStatus.STOPPED;
                 instance.pid = null;
                 instance.health = null;
-                this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
                 this.save();
             }
         });
@@ -261,7 +260,7 @@ export class InstanceManager {
                 instance.desiredStatus = DesiredInstanceStatus.STOP;
             }
             instance.status = InstanceStatus.STOPPING;
-            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
             console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
             this.save();
             try {
@@ -272,7 +271,7 @@ export class InstanceManager {
                         try {
                             await container.stop();
                             instance.status = InstanceStatus.STOPPED;
-                            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                            socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
                             this.save();
                         }
                         catch (e) {
@@ -291,7 +290,7 @@ export class InstanceManager {
                 }
                 process.kill(instance.pid, 'SIGTERM');
                 instance.status = InstanceStatus.STOPPED;
-                this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
                 this.save();
             }
             catch (e) {
@@ -416,8 +415,8 @@ export class InstanceManager {
                         status: InstanceStatus.FAILED,
                         errorMessage: e.message ?? 'Failed to start - Check logs for details.',
                     });
-                    this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
-                    this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
+                    socketManager.emitInstanceLog(systemId, instanceId, logs[0]);
+                    socketManager.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
                         error: `Failed to start instance: ${e.message}`,
                         status: EVENT_INSTANCE_EXITED,
                         instanceId: blockInstance.id,
@@ -506,7 +505,7 @@ export class InstanceManager {
                         const oldStatus = instance.status;
                         instance.status = newStatus;
                         console.log('Instance status changed: %s %s: %s -> %s', instance.systemId, instance.instanceId, oldStatus, instance.status);
-                        this.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+                        socketManager.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
                         changed = true;
                     }
                 }
@@ -638,24 +637,6 @@ export class InstanceManager {
             });
         });
     }
-    emitSystemEvent(systemId, type, payload) {
-        systemId = normalizeKapetaUri(systemId);
-        try {
-            socketManager.emit(`${systemId}/instances`, type, payload);
-        }
-        catch (e) {
-            console.warn('Failed to emit instance event: %s', e.message);
-        }
-    }
-    emitInstanceEvent(systemId, instanceId, type, payload) {
-        systemId = normalizeKapetaUri(systemId);
-        try {
-            socketManager.emit(`${systemId}/instances/${instanceId}`, type, payload);
-        }
-        catch (e) {
-            console.warn('Failed to emit instance event: %s', e.message);
-        }
-    }
 }
 export const instanceManager = new InstanceManager();
 process.on('exit', async () => {

package/dist/esm/src/progressListener.d.ts

@@ -1,8 +1,9 @@
 /// <reference types="node" />
-import { SocketManager } from './socketManager';
-declare class ProgressListener {
-    private socketManager;
-    constructor(socketManager: SocketManager);
+export declare class ProgressListener {
+    private readonly systemId;
+    private readonly instanceId;
+    constructor(systemId?: string, instanceId?: string);
+    private emitLog;
     run(command: string, directory?: string): Promise<{
         exit: number;
         signal: NodeJS.Signals | null;
@@ -17,5 +18,3 @@ declare class ProgressListener {
     info(msg: string, ...args: any[]): void;
     debug(msg: string, ...args: any[]): void;
 }
-export declare const progressListener: ProgressListener;
-export {};