@kapeta/local-cluster-service 0.0.67 → 0.0.68

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kapeta/local-cluster-service",
-  "version": "0.0.67",
+  "version": "0.0.68",
   "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
   "main": "index.js",
   "repository": {
@@ -24,6 +24,7 @@
     "@kapeta/local-cluster-config": "<2",
     "@kapeta/local-cluster-executor": "<2",
     "@kapeta/nodejs-api-client": "<2",
+    "@kapeta/nodejs-registry-utils": "^0.0.2",
     "@kapeta/nodejs-utils": "<2",
     "@kapeta/sdk-config": "<2",
     "express": "4.17.1",
@@ -36,6 +37,7 @@
     "node-docker-api": "1.1.22",
     "node-uuid": "^1.4.8",
     "request": "2.88.2",
+    "request-promise": "4.2.6",
     "socket.io": "^4.5.2",
     "yaml": "^1.6.0"
   },

package/src/assetManager.js

@@ -3,13 +3,11 @@ const FS = require('node:fs');
 const FSExtra = require('fs-extra');
 const YAML = require('yaml');
 const ClusterConfiguration = require('@kapeta/local-cluster-config');
+const {Actions} = require('@kapeta/nodejs-registry-utils');
 const codeGeneratorManager = require('./codeGeneratorManager');
-const socketManager = require('./socketManager');
-
-function makeSymLink(directory, versionTarget) {
-    FSExtra.mkdirpSync(Path.dirname(versionTarget));
-    FSExtra.createSymlinkSync(directory, versionTarget);
-}
+const progressListener = require('./progressListener');
+const {parseKapetaUri} = require("@kapeta/nodejs-utils");
+const repositoryManager = require("./repositoryManager");
 
 function enrichAsset(asset) {
     return {
@@ -47,77 +45,6 @@ function parseRef(ref) {
 
 class AssetManager {
 
-    constructor() {
-        this.watcher = null;
-        this.listenForChanges();
-    }
-
-    listenForChanges() {
-        const baseDir = ClusterConfiguration.getRepositoryBasedir();
-        if (!FS.existsSync(baseDir)) {
-            FSExtra.mkdirpSync(baseDir);
-        }
-
-        let currentWebDefinitions = ClusterConfiguration
-            .getProviderDefinitions()
-            .filter(d => d.hasWeb);
-
-        console.log('Watching local repository for provider changes: %s', baseDir);
-        try {
-            this.watcher = FS.watch(baseDir, { recursive: true });
-        } catch (e) {
-            // Fallback to run without watch mode due to potential platform issues.
-            // https://nodejs.org/docs/latest/api/fs.html#caveats
-            console.log('Unable to watch for changes. Changes to assets will not update automatically.');
-            return;
-        }
-        this.watcher.on('change', (eventType, filename) => {
-            const [handle, name, version] = filename.split(/\//g);
-            if (!name || !version) {
-                return;
-            }
-
-            const ymlPath = Path.join(baseDir, handle, name, version, 'kapeta.yml');
-            const newWebDefinitions = ClusterConfiguration
-                .getProviderDefinitions()
-                .filter(d => d.hasWeb);
-
-            const newWebDefinition = newWebDefinitions.find(d => d.ymlPath === ymlPath);
-            let currentWebDefinition = currentWebDefinitions.find(d => d.ymlPath === ymlPath);
-            const ymlExists = FS.existsSync(ymlPath);
-            let type;
-            if (ymlExists) {
-                if (currentWebDefinition) {
-                    type = 'updated';
-                } else if (newWebDefinition) {
-                    type = 'added';
-                    currentWebDefinition = newWebDefinition;
-                } else {
-                    //Other definition was added / updated - ignore
-                    return;
-                }
-            } else {
-                if (currentWebDefinition) {
-                    //Something was removed
-                    type = 'removed';
-                } else {
-                    //Other definition was removed - ignore
-                    return;
-                }
-            }
-
-            const payload = {type, definition: currentWebDefinition?.definition, asset: {handle, name, version} };
-
-            currentWebDefinitions = newWebDefinitions
-
-            socketManager.emit(`assets`, 'changed', payload);
-        });
-    }
-
-    stopListening() {
-        this.watcher.close();
-        this.watcher = null;
-    }
 
 
     /**
@@ -143,8 +70,8 @@ class AssetManager {
         return this.getAssets(['core/plan']);
     }
 
-    getPlan(ref) {
-        const asset = this.getAsset(ref);
+    async getPlan(ref) {
+        const asset = await this.getAsset(ref);
 
         if ('core/plan' !== asset.kind) {
             throw new Error('Asset was not a plan: ' + ref);
@@ -153,8 +80,15 @@ class AssetManager {
         return asset.data;
     }
 
-    getAsset(ref) {
-        const asset = this.getAssets().find(a => compareRefs(a.ref,ref));
+    async getAsset(ref) {
+        const uri = parseKapetaUri(ref);
+
+        await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
+
+        let asset = ClusterConfiguration.getDefinitions()
+            .map(enrichAsset)
+            .find(a => parseKapetaUri(a.ref).equals(uri));
+
         if (!asset) {
             throw new Error('Asset not found: ' + ref);
         }
@@ -184,7 +118,7 @@ class AssetManager {
     }
 
     async updateAsset(ref, yaml) {
-        const asset = this.getAsset(ref);
+        const asset = await this.getAsset(ref);
         if (!asset) {
             throw new Error('Attempted to update unknown asset: ' + ref);
         }
@@ -206,10 +140,6 @@ class AssetManager {
         }
     }
 
-    hasAsset(ref) {
-        return !!this.getAsset(ref);
-    }
-
     async importFile(filePath) {
         if (filePath.startsWith('file://')) {
             filePath = filePath.substring('file://'.length);
@@ -222,27 +152,21 @@ class AssetManager {
         const assetInfos = YAML.parseAllDocuments(FS.readFileSync(filePath).toString())
             .map(doc => doc.toJSON());
 
-        const assetInfo = assetInfos[0];
-        const version = 'local';
-        const [handle, name] = assetInfo.metadata.name.split('/');
-
-        const target = ClusterConfiguration.getRepositoryAssetPath(handle, name, version);
-        if (!FS.existsSync(target)) {
-            makeSymLink(Path.dirname(filePath), target);
-        }
+        await Actions.link(progressListener, Path.dirname(filePath));
 
+        const version = 'local';
         const refs = assetInfos.map(assetInfo => `kapeta://${assetInfo.metadata.name}:${version}`);
 
         return this.getAssets().filter(a => refs.some(ref => compareRefs(ref, a.ref)));
     }
 
-    unregisterAsset(ref) {
-        const asset = this.getAsset(ref);
+    async unregisterAsset(ref) {
+        const asset = await this.getAsset(ref);
         if (!asset) {
             throw new Error('Asset does not exists: ' + ref);
         }
-        //Remove from repository. If its local it is just a symlink - so no unchecked code is removed.
-        FSExtra.removeSync(asset.path);
+
+        await Actions.uninstall(progressListener, asset.path);
     }
 }
 

package/src/assets/routes.js

@@ -34,14 +34,14 @@ router.get('/', (req, res) => {
 /**
  * Get single asset
  */
-router.get('/read', (req, res) => {
+router.get('/read', async (req, res) => {
     if (!req.query.ref) {
         res.status(400).send({error:'Query parameter "ref" is missing'});
         return;
     }
 
     try {
-        res.send(assetManager.getAsset(req.query.ref));
+        res.send(await assetManager.getAsset(req.query.ref));
     } catch(err) {
         res.status(400).send({error: err.message});
     }

package/src/instanceManager.js

@@ -228,7 +228,7 @@ class InstanceManager {
     async createProcessesForPlan(planRef) {
         await this.stopAllForPlan(planRef);
 
-        const plan = assetManager.getPlan(planRef);
+        const plan = await assetManager.getPlan(planRef);
         if (!plan) {
             throw new Error('Plan not found: ' + planRef);
         }
@@ -301,7 +301,7 @@ class InstanceManager {
      * @return {Promise<PromiseInfo>}
      */
     async createProcess(planRef, instanceId) {
-        const plan = assetManager.getPlan(planRef);
+        const plan = await assetManager.getPlan(planRef);
         if (!plan) {
             throw new Error('Plan not found: ' + planRef);
         }
@@ -313,7 +313,7 @@ class InstanceManager {
 
         const blockRef = blockInstance.block.ref;
 
-        const blockAsset = assetManager.getAsset(blockRef);
+        const blockAsset = await assetManager.getAsset(blockRef);
 
         if (!blockAsset) {
             throw new Error('Block not found: ' + blockRef);

package/src/progressListener.js

@@ -0,0 +1,82 @@
+const {spawn} = require("child_process");
+const socketManager = require("./socketManager");
+/**
+ *
+ * @type {ProgressListener}
+ */
+module.exports = {
+    run: (command, directory) => {
+        socketManager.emit(
+            `install`,
+            'install:log',
+            {
+                type: 'info',
+                message: `Running command "${command}"`
+            }
+        );
+
+        return new Promise((resolve, reject) => {
+            const child = spawn(command, {
+                cwd: directory ? directory : process.cwd(),
+                detached: true,
+                shell: true
+            });
+
+            child.stdout.on('data', (data) => {
+                socketManager.emit(`install`, 'install:log', {type: 'info', message: data.toString()});
+            });
+
+            child.stderr.on('data', (data) => {
+                socketManager.emit(`install`, 'install:log', {type: 'info', message: data.toString()});
+            });
+
+            child.on('exit', (exit, signal) => {
+                if (exit !== 0) {
+                    socketManager.emit(`install`, 'install:log', {type: 'info', message: `"${command}" failed: "${exit}"`});
+                    reject(new Error(`Command "${command}" exited with code ${exit}`));
+                } else {
+                    socketManager.emit(`install`, 'install:log', {type: 'info', message: `Command OK: "${command}"`});
+                    resolve({exit, signal});
+                }
+            });
+
+            child.on('error', (err) => {
+                socketManager.emit(`install`, 'install:log', {type: 'info', message: `"${command}" failed: "${err.message}"`});
+                reject(err);
+            });
+        });
+    },
+    progress: async (label, callback) => {
+        socketManager.emit(`install`, 'install:log', {type: 'info', message: `${label}: started`});
+        try {
+            const result = await callback();
+            socketManager.emit(`install`, 'install:log', {type: 'info', message: `${label}: done`});
+            return result;
+        } catch (e) {
+            socketManager.emit(`install`, 'install:log', {type: 'info', message: `${label}: failed. ${e.message}`});
+            throw e;
+        }
+    },
+    check: async (message, ok) => {
+        const wasOk = await ok;
+        socketManager.emit(`install`, 'install:log', {type: 'info', message: `${message}: ${wasOk}`});
+    },
+    start: (label) => {
+        socketManager.emit(`install`, 'install:log', {type: 'info', message: label});
+    },
+    showValue: (label, value) => {
+        socketManager.emit(`install`, 'install:log', {type: 'info', message: `${label}: ${value}`});
+    },
+    error: (msg, ...args) => {
+        socketManager.emit(`install`, 'install:log', {type: 'error', message: msg});
+    },
+    warn: (msg, ...args) => {
+        socketManager.emit(`install`, 'install:log', {type: 'warn', message: msg});
+    },
+    info: (msg, ...args) => {
+        socketManager.emit(`install`, 'install:log', {type: 'info', message: msg});
+    },
+    debug: (msg, ...args) => {
+        socketManager.emit(`install`, 'install:log', {type: 'debug', message: msg});
+    },
+}

package/src/providerManager.js

@@ -1,14 +1,13 @@
-const _ = require('lodash');
 const FS = require('fs');
 const Path = require('path');
-const Glob = require("glob");
-
+const FSExtra = require('fs-extra');
+const repositoryManager = require('./repositoryManager')
 const ClusterConfiguration = require('@kapeta/local-cluster-config');
 
 class ProviderManager {
 
     constructor() {
-        this._assetCache = {};
+        this._webAssetCache = {};
     }
 
     getWebProviders() {
@@ -17,54 +16,32 @@ class ProviderManager {
             .filter((providerDefinition) => providerDefinition.hasWeb)
     }
 
-    getWebAssets() {
-        const webProviders = this.getWebProviders();
+    async getAsset(handle, name, version, sourceMap = false) {
+        const fullName = `${handle}/${name}`;
+        const id = `${handle}/${name}/${version}/web.js${sourceMap ? '.map' : ''}`;
+        
+        if (this._webAssetCache[id] &&
+            await FSExtra.exists(this._webAssetCache[id])) {
+            return FSExtra.read(this._webAssetCache[id]);
+        }
 
-        let providerFiles = [];
-        webProviders.map((webProvider) => {
-            return Glob.sync('web/**/*.js', {cwd: webProvider.path}).map((file) => {
-                return {webProvider, file};
-            });
-        }).forEach((webFiles) => {
-            providerFiles.push(...webFiles);
-        });
+        await repositoryManager.ensureAsset(handle, name, version);
 
-        return providerFiles;
-    }
-
-    loadAssets() {
-        this.getWebAssets().forEach((asset) => {
-            const providerId = asset.webProvider.definition.metadata.name;
-            const file = asset.file;
-            const assetId = `${providerId}/${asset.webProvider.version}/${file}`;
-            this._assetCache[assetId] = Path.join(asset.webProvider.path, file);
+        const installedProvider = this.getWebProviders().find((providerDefinition) => {
+            return providerDefinition.definition.metadata.name === fullName &&
+                providerDefinition.version === version;
         })
-    }
 
+        if (installedProvider) {
+            //Check locally installed providers
+            const path = Path.join(installedProvider.path, 'web', handle, `${name}.js${sourceMap ? '.map' : ''}`);
+            if (await FSExtra.exists(path)) {
+                this._webAssetCache[id] = path;
 
-    /**
-     * Returns all public (web) javascript for available providers.
-     *
-     * Provides frontend / applications with the implementation of the frontends for the
-     * providers.
-     *
-     */
-    getPublicJS() {
-        this.loadAssets();
-        const includes = Object.keys(this._assetCache).map((assetId) => {
-            return `${ClusterConfiguration.getClusterServiceAddress()}/providers/asset/${assetId}`
-        });
-
-        return `Kapeta.setPluginPaths(${JSON.stringify(includes)});`
-    }
-
-    getAsset(id) {
-        if (_.isEmpty(this._assetCache)) {
-            this.loadAssets();
-        }
-        if (this._assetCache[id]) {
-            return FS.readFileSync(this._assetCache[id]).toString();
+                return FSExtra.read(path);
+            }
         }
+
         return null;
     }
 }

package/src/providers/routes.js

@@ -5,16 +5,38 @@ const router = new Router();
 
 router.use('/', require('../middleware/cors'));
 
-/**
- * Get all local assets available
- */
-router.get('/all.js', (req, res) => {
-    res.send(providerManager.getPublicJS());
+router.get('/', async (req, res) => {
+    const result = await providerManager.getWebProviders();
+
+    res.send(result);
+});
+
+router.get('/asset/:handle/:name/:version/web.js', async (req, res) => {
+
+    const {handle, name, version} = req.params;
+    let result = await providerManager.getAsset(handle, name, version);
+
+    if (version !== 'local') {
+        res.setHeader('Cache-Control', 'max-age=31536000, immutable');
+    }
+
+    if (!result) {
+        res.status(404).send('');
+    } else {
+        res.send(result
+            .replace(`${name}.js.map`, 'web.js.map')
+        );
+    }
 });
 
-router.get('/asset/*', (req, res) => {
-    const assetId = req.params[0];
-    const result = providerManager.getAsset(assetId)
+router.get('/asset/:handle/:name/:version/web.js.map', async (req, res) => {
+
+    const {handle, name, version} = req.params;
+    const result = await providerManager.getAsset(handle, name, version, true);
+    if (version !== 'local') {
+        res.setHeader('Cache-Control', 'max-age=31536000, immutable');
+    }
+
     if (!result) {
         res.status(404).send('');
     } else {

package/src/repositoryManager.js

@@ -0,0 +1,201 @@
+const ClusterConfiguration = require("@kapeta/local-cluster-config");
+const FS = require("node:fs");
+const FSExtra = require("fs-extra");
+const Path = require("node:path");
+const socketManager = require("./socketManager");
+const {Actions, RegistryService, Config} = require("@kapeta/nodejs-registry-utils");
+const progressListener = require("./progressListener");
+const os = require("os");
+const {parseKapetaUri} = require("@kapeta/nodejs-utils");
+const INSTALL_ATTEMPTED = {};
+
+class RepositoryManager {
+
+    constructor() {
+        this.watcher = null;
+        this.changeEventsEnabled = true;
+        this.listenForChanges();
+        this._registryService = new RegistryService(
+            Config.data.registry.url
+        );
+        this._cache = {};
+        this._installQueue = [];
+    }
+
+    setChangeEventsEnabled(enabled) {
+        this.changeEventsEnabled = enabled;
+    }
+
+    listenForChanges() {
+        const baseDir = ClusterConfiguration.getRepositoryBasedir();
+        if (!FS.existsSync(baseDir)) {
+            FSExtra.mkdirpSync(baseDir);
+        }
+
+        let allDefinitions = ClusterConfiguration
+            .getDefinitions();
+
+        console.log('Watching local repository for provider changes: %s', baseDir);
+        try {
+            this.watcher = FS.watch(baseDir, { recursive: true });
+        } catch (e) {
+            // Fallback to run without watch mode due to potential platform issues.
+            // https://nodejs.org/docs/latest/api/fs.html#caveats
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.');
+            return;
+        }
+        this.watcher.on('change', (eventType, filename) => {
+            const [handle, name, version] = filename.split(/\//g);
+            if (!name || !version) {
+                return;
+            }
+
+            if (!this.changeEventsEnabled) {
+                return;
+            }
+
+            const ymlPath = Path.join(baseDir, handle, name, version, 'kapeta.yml');
+            const newDefinitions = ClusterConfiguration.getDefinitions();
+
+            const newDefinition = newDefinitions.find(d => d.ymlPath === ymlPath);
+            let currentDefinition = allDefinitions.find(d => d.ymlPath === ymlPath);
+            const ymlExists = FS.existsSync(ymlPath);
+            let type;
+            if (ymlExists) {
+                if (currentDefinition) {
+                    type = 'updated';
+                } else if (newDefinition) {
+                    type = 'added';
+                    currentDefinition = newDefinition;
+                } else {
+                    //Other definition was added / updated - ignore
+                    return;
+                }
+            } else {
+                if (currentDefinition) {
+                    const ref = parseKapetaUri(
+                        `${currentDefinition.definition.metadata.name}:${currentDefinition.version}`
+                    ).id;
+                    delete INSTALL_ATTEMPTED[ref];
+                    //Something was removed
+                    type = 'removed';
+                } else {
+                    //Other definition was removed - ignore
+                    return;
+                }
+            }
+
+            const payload = {type, definition: currentDefinition?.definition, asset: {handle, name, version} };
+
+            allDefinitions = newDefinitions
+            socketManager.emit(`assets`, 'changed', payload);
+        });
+    }
+
+    stopListening() {
+        this.watcher.close();
+        this.watcher = null;
+    }
+
+    /**
+     *
+     * @param {string[]} refs
+     * @return {Promise<void>}
+     * @private
+     */
+    async _install(refs) {
+        //We make sure to only install one asset at a time - otherwise unexpected things might happen
+        const out = new Promise((resolve, reject) => {
+            this._installQueue.push(async () => {
+                try {
+                    const normalizedRefs = refs.map(ref => parseKapetaUri(ref).id)
+                    const filteredRefs = normalizedRefs.filter(ref => !INSTALL_ATTEMPTED[ref]);
+                    if (filteredRefs.length > 0) {
+                        filteredRefs.forEach(ref => INSTALL_ATTEMPTED[ref] = true);
+                        //Auto-install missing asset
+                        try {
+                            //We change to a temp dir to avoid issues with the current working directory
+                            process.chdir(os.tmpdir());
+                            //Disable change events while installing
+                            this.setChangeEventsEnabled(false);
+                            socketManager.emit(`install`, 'install:action', {type: 'start', refs});
+                            await Actions.install(progressListener, normalizedRefs, {});
+                            socketManager.emit(`install`, 'install:action', {type: 'done', refs});
+                        } catch (e) {
+                            socketManager.emit(`install`, 'install:action', {type: 'failed', refs, error: e.message});
+                        } finally {
+                            this.setChangeEventsEnabled(true);
+                        }
+                    }
+                    resolve();
+                } catch (e) {
+                    reject(e);
+                } finally {
+                    this._processNext().catch(e => console.error(e));
+                }
+            })
+        });
+
+        this._processNext().catch(e => console.error(e));
+
+        return out;
+    }
+
+    async _processNext() {
+        if (this._processing) {
+            return;
+        }
+        this._processing = true;
+        try {
+            while (this._installQueue.length > 0) {
+                const item = this._installQueue.shift();
+
+                await item();
+            }
+        } finally {
+            this._processing = false;
+        }
+    }
+
+    async ensureAsset(handle, name, version) {
+        const fullName = `${handle}/${name}`;
+        const ref = `${fullName}:${version}`;
+
+        if (version === 'local') {
+            //TODO: Get dependencies for local asset
+            return null;
+        }
+
+        const installedAsset = ClusterConfiguration.getDefinitions().find(d =>
+                        d.definition.metadata.name === fullName &&
+                        d.version === version);
+
+
+        if (installedAsset && this._cache[ref] === true) {
+            return;
+        }
+
+        if (!installedAsset && this._cache[ref] === false) {
+            return;
+        }
+
+        const assetVersion = await this._registryService.getVersion(fullName, version);
+        if (!assetVersion) {
+            this._cache[ref] = false;
+            return;
+        }
+
+        this._cache[ref] = true;
+        if (!installedAsset) {
+            await this._install([ref]);
+        } else {
+            //Ensure dependencies are installed
+            const refs = assetVersion.dependencies.map((dep) => dep.name);
+            await this._install(refs);
+        }
+
+
+    }
+}
+
+module.exports = new RepositoryManager();