@kapeta/local-cluster-service 0.12.1 → 0.13.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [0.13.0](https://github.com/kapetacom/local-cluster-service/compare/v0.12.1...v0.13.0) (2023-08-03)
+
+
+### Features
+
+* Adds background task concept ([#55](https://github.com/kapetacom/local-cluster-service/issues/55)) ([71cc63c](https://github.com/kapetacom/local-cluster-service/commit/71cc63c9c3eb8bec1dec8b31d3340694e93bd3e5))
+
 ## [0.12.1](https://github.com/kapetacom/local-cluster-service/compare/v0.12.0...v0.12.1) (2023-08-02)
 
 

package/dist/cjs/index.js

@@ -20,6 +20,7 @@ const routes_6 = __importDefault(require("./src/filesystem/routes"));
 const routes_7 = __importDefault(require("./src/assets/routes"));
 const routes_8 = __importDefault(require("./src/providers/routes"));
 const routes_9 = __importDefault(require("./src/attachments/routes"));
+const routes_10 = __importDefault(require("./src/tasks/routes"));
 const utils_1 = require("./src/utils/utils");
 const request_1 = __importDefault(require("request"));
 let currentServer = null;
@@ -34,6 +35,7 @@ function createServer() {
     app.use('/assets', routes_7.default);
     app.use('/providers', routes_8.default);
     app.use('/attachments', routes_9.default);
+    app.use('/tasks', routes_10.default);
     app.get('/status', async (req, res) => {
         res.send({
             ok: true,

package/dist/cjs/src/assetManager.d.ts

@@ -13,6 +13,7 @@ export interface EnrichedAsset {
 declare class AssetManager {
     private cache;
     constructor();
+    clearCache(): void;
     /**
      *
      * @param {string[]} [assetKinds]
@@ -21,11 +22,12 @@ declare class AssetManager {
     getAssets(assetKinds?: string[]): EnrichedAsset[];
     getPlans(): EnrichedAsset[];
     getPlan(ref: string, noCache?: boolean): Promise<Definition>;
-    getAsset(ref: string, noCache?: boolean): Promise<EnrichedAsset | undefined>;
+    getAsset(ref: string, noCache?: boolean, autoFetch?: boolean): Promise<EnrichedAsset | undefined>;
     createAsset(path: string, yaml: BlockDefinition): Promise<EnrichedAsset[]>;
     updateAsset(ref: string, yaml: BlockDefinition): Promise<void>;
     importFile(filePath: string): Promise<EnrichedAsset[]>;
     unregisterAsset(ref: string): Promise<void>;
+    installAsset(ref: string): Promise<import("./taskManager").Task<void>[] | undefined>;
 }
 export declare const assetManager: AssetManager;
 export {};

package/dist/cjs/src/assetManager.js

@@ -47,6 +47,9 @@ class AssetManager {
             stdTTL: 60 * 60, // 1 hour
         });
     }
+    clearCache() {
+        this.cache.flushAll();
+    }
     /**
      *
      * @param {string[]} [assetKinds]
@@ -76,22 +79,26 @@ class AssetManager {
         }
         return asset.data;
     }
-    async getAsset(ref, noCache = false) {
+    async getAsset(ref, noCache = false, autoFetch = true) {
         ref = (0, utils_1.normalizeKapetaUri)(ref);
         const cacheKey = `getAsset:${ref}`;
         if (!noCache && this.cache.has(cacheKey)) {
             return this.cache.get(cacheKey);
         }
         const uri = (0, nodejs_utils_1.parseKapetaUri)(ref);
-        await repositoryManager_1.repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
+        if (autoFetch) {
+            await repositoryManager_1.repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, true);
+        }
         let asset = definitionsManager_1.definitionsManager
             .getDefinitions()
             .map(enrichAsset)
             .find((a) => (0, nodejs_utils_1.parseKapetaUri)(a.ref).equals(uri));
-        if (!asset) {
+        if (autoFetch && !asset) {
             throw new Error('Asset not found: ' + ref);
         }
-        this.cache.set(cacheKey, asset);
+        if (asset) {
+            this.cache.set(cacheKey, asset);
+        }
         return asset;
     }
     async createAsset(path, yaml) {
@@ -152,5 +159,14 @@ class AssetManager {
         this.cache.flushAll();
         await nodejs_registry_utils_1.Actions.uninstall(progressListener_1.progressListener, [asset.ref]);
     }
+    async installAsset(ref) {
+        const asset = await this.getAsset(ref, true, false);
+        if (asset) {
+            throw new Error('Asset already installed: ' + ref);
+        }
+        const uri = (0, nodejs_utils_1.parseKapetaUri)(ref);
+        console.log('Installing %s', ref);
+        return await repositoryManager_1.repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
+    }
 }
 exports.assetManager = new AssetManager();

package/dist/cjs/src/assets/routes.js

@@ -39,8 +39,15 @@ router.get('/read', async (req, res) => {
         res.status(400).send({ error: 'Query parameter "ref" is missing' });
         return;
     }
+    const ensure = req.query.ensure !== 'false';
     try {
-        res.send(await assetManager_1.assetManager.getAsset(req.query.ref, true));
+        const asset = await assetManager_1.assetManager.getAsset(req.query.ref, true, ensure);
+        if (asset) {
+            res.send(asset);
+        }
+        else {
+            res.status(404).send({ error: 'Asset not found' });
+        }
     }
     catch (err) {
         res.status(400).send({ error: err.message });
@@ -114,4 +121,18 @@ router.put('/import', async (req, res) => {
         res.status(400).send({ error: err.message });
     }
 });
+router.put('/install', async (req, res) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+    try {
+        const tasks = await assetManager_1.assetManager.installAsset(req.query.ref);
+        const taskIds = tasks?.map((t) => t.id) ?? [];
+        res.status(200).send(taskIds);
+    }
+    catch (err) {
+        res.status(400).send({ error: err.message });
+    }
+});
 exports.default = router;

package/dist/cjs/src/containerManager.js

@@ -15,8 +15,8 @@ const local_cluster_config_1 = __importDefault(require("@kapeta/local-cluster-co
 const node_uuid_1 = __importDefault(require("node-uuid"));
 const md5_1 = __importDefault(require("md5"));
 const utils_1 = require("./utils/utils");
-const socketManager_1 = require("./socketManager");
 const nodejs_api_client_1 = require("@kapeta/nodejs-api-client");
+const taskManager_1 = require("./taskManager");
 const EVENT_IMAGE_PULL = 'docker-image-pull';
 exports.CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
@@ -162,120 +162,141 @@ class ContainerManager {
             console.log('Image found: %s', image);
             return false;
         }
-        const timeStarted = Date.now();
-        socketManager_1.socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: -1 });
-        const api = new nodejs_api_client_1.KapetaAPI();
-        const accessToken = await api.getAccessToken();
-        const auth = image.startsWith('docker.kapeta.com/')
-            ? {
-                username: 'kapeta',
-                password: accessToken,
-                serveraddress: 'docker.kapeta.com',
-            }
-            : {};
-        const stream = (await this.docker().image.create(auth, {
-            fromImage: imageName,
-            tag: tag,
-        }));
-        const chunks = {};
-        let lastEmitted = Date.now();
-        await promisifyStream(stream, (rawData) => {
-            const lines = rawData.toString().trim().split('\n');
-            lines.forEach((line) => {
-                const data = JSON.parse(line);
-                if (![
-                    'Waiting',
-                    'Downloading',
-                    'Extracting',
-                    'Download complete',
-                    'Pull complete',
-                    'Already exists',
-                ].includes(data.status)) {
-                    return;
-                }
-                if (!chunks[data.id]) {
-                    chunks[data.id] = {
-                        downloading: {
-                            total: 0,
-                            current: 0,
-                        },
-                        extracting: {
-                            total: 0,
-                            current: 0,
-                        },
-                        done: false,
-                    };
-                }
-                const chunk = chunks[data.id];
-                switch (data.status) {
-                    case 'Downloading':
-                        chunk.downloading = data.progressDetail;
-                        break;
-                    case 'Extracting':
-                        chunk.extracting = data.progressDetail;
-                        break;
-                    case 'Download complete':
-                        chunk.downloading.current = chunks[data.id].downloading.total;
-                        break;
-                    case 'Pull complete':
-                        chunk.extracting.current = chunks[data.id].extracting.total;
-                        chunk.done = true;
-                        break;
-                    case 'Already exists':
-                        // Force layer to be done
-                        chunk.downloading.current = 1;
-                        chunk.downloading.total = 1;
-                        chunk.extracting.current = 1;
-                        chunk.extracting.total = 1;
-                        chunk.done = true;
-                        break;
-                }
-            });
-            if (Date.now() - lastEmitted < 1000) {
-                return;
-            }
-            const chunkList = Object.values(chunks);
-            let totals = {
-                downloading: {
-                    total: 0,
-                    current: 0,
-                },
-                extracting: {
-                    total: 0,
-                    current: 0,
-                },
-                total: chunkList.length,
-                done: 0,
-            };
-            chunkList.forEach((chunk) => {
-                if (chunk.downloading.current > 0) {
-                    totals.downloading.current += chunk.downloading.current;
-                }
-                if (chunk.downloading.total > 0) {
-                    totals.downloading.total += chunk.downloading.total;
-                }
-                if (chunk.extracting.current > 0) {
-                    totals.extracting.current += chunk.extracting.current;
+        let friendlyImageName = image;
+        const imageParts = imageName.split('/');
+        if (imageParts.length > 2) {
+            //Strip the registry to make the name shorter
+            friendlyImageName = `${imageParts.slice(1).join('/')}:${tag}`;
+        }
+        const taskName = `Pulling image ${friendlyImageName}`;
+        const processor = async (task) => {
+            const timeStarted = Date.now();
+            const api = new nodejs_api_client_1.KapetaAPI();
+            const accessToken = await api.getAccessToken();
+            const auth = image.startsWith('docker.kapeta.com/')
+                ? {
+                    username: 'kapeta',
+                    password: accessToken,
+                    serveraddress: 'docker.kapeta.com',
                 }
-                if (chunk.extracting.total > 0) {
-                    totals.extracting.total += chunk.extracting.total;
-                }
-                if (chunk.done) {
-                    totals.done++;
+                : {};
+            const stream = (await this.docker().image.create(auth, {
+                fromImage: imageName,
+                tag: tag,
+            }));
+            const chunks = {};
+            let lastEmitted = Date.now();
+            await promisifyStream(stream, (rawData) => {
+                const lines = rawData.toString().trim().split('\n');
+                lines.forEach((line) => {
+                    const data = JSON.parse(line);
+                    if (![
+                        'Waiting',
+                        'Downloading',
+                        'Extracting',
+                        'Download complete',
+                        'Pull complete',
+                        'Already exists',
+                    ].includes(data.status)) {
+                        return;
+                    }
+                    if (!chunks[data.id]) {
+                        chunks[data.id] = {
+                            downloading: {
+                                total: 0,
+                                current: 0,
+                            },
+                            extracting: {
+                                total: 0,
+                                current: 0,
+                            },
+                            done: false,
+                        };
+                    }
+                    const chunk = chunks[data.id];
+                    switch (data.status) {
+                        case 'Downloading':
+                            chunk.downloading = data.progressDetail;
+                            break;
+                        case 'Extracting':
+                            chunk.extracting = data.progressDetail;
+                            break;
+                        case 'Download complete':
+                            chunk.downloading.current = chunks[data.id].downloading.total;
+                            break;
+                        case 'Pull complete':
+                            chunk.extracting.current = chunks[data.id].extracting.total;
+                            chunk.done = true;
+                            break;
+                        case 'Already exists':
+                            // Force layer to be done
+                            chunk.downloading.current = 1;
+                            chunk.downloading.total = 1;
+                            chunk.extracting.current = 1;
+                            chunk.extracting.total = 1;
+                            chunk.done = true;
+                            break;
+                    }
+                });
+                if (Date.now() - lastEmitted < 1000) {
+                    return;
                 }
+                const chunkList = Object.values(chunks);
+                let totals = {
+                    downloading: {
+                        total: 0,
+                        current: 0,
+                    },
+                    extracting: {
+                        total: 0,
+                        current: 0,
+                    },
+                    total: chunkList.length,
+                    done: 0,
+                };
+                chunkList.forEach((chunk) => {
+                    if (chunk.downloading.current > 0) {
+                        totals.downloading.current += chunk.downloading.current;
+                    }
+                    if (chunk.downloading.total > 0) {
+                        totals.downloading.total += chunk.downloading.total;
+                    }
+                    if (chunk.extracting.current > 0) {
+                        totals.extracting.current += chunk.extracting.current;
+                    }
+                    if (chunk.extracting.total > 0) {
+                        totals.extracting.total += chunk.extracting.total;
+                    }
+                    if (chunk.done) {
+                        totals.done++;
+                    }
+                });
+                const progress = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
+                task.metadata = {
+                    ...task.metadata,
+                    image,
+                    progress,
+                    status: totals,
+                    timeTaken: Date.now() - timeStarted,
+                };
+                task.emitUpdate();
+                lastEmitted = Date.now();
+                //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
             });
-            const percent = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
-            //We emit at most every second to not spam the client
-            socketManager_1.socketManager.emitGlobal(EVENT_IMAGE_PULL, {
+            task.metadata = {
+                ...task.metadata,
                 image,
-                percent,
-                status: totals,
+                progress: 100,
                 timeTaken: Date.now() - timeStarted,
-            });
-            lastEmitted = Date.now();
-            //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
+            };
+            task.emitUpdate();
+        };
+        const task = taskManager_1.taskManager.add(`docker:image:pull:${image}`, processor, {
+            name: taskName,
+            image,
+            progress: -1,
         });
-        socketManager_1.socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 100, timeTaken: Date.now() - timeStarted });
+        await task.wait();
         return true;
     }
     toDockerMounts(mounts) {

package/dist/cjs/src/instanceManager.d.ts

@@ -1,4 +1,5 @@
 import { InstanceInfo, LogEntry } from './types';
+import { Task } from './taskManager';
 export declare class InstanceManager {
     private _interval;
     private readonly _instances;
@@ -18,11 +19,11 @@ export declare class InstanceManager {
     registerInstanceFromSDK(systemId: string, instanceId: string, info: Omit<InstanceInfo, 'systemId' | 'instanceId'>): Promise<InstanceInfo | undefined>;
     private getHealthUrl;
     markAsStopped(systemId: string, instanceId: string): Promise<void>;
-    startAllForPlan(systemId: string): Promise<InstanceInfo[]>;
+    startAllForPlan(systemId: string): Promise<Task<InstanceInfo[]>>;
     stop(systemId: string, instanceId: string): Promise<void>;
     private stopInner;
-    stopAllForPlan(systemId: string): Promise<void>;
-    start(systemId: string, instanceId: string): Promise<InstanceInfo>;
+    stopAllForPlan(systemId: string): Task<void>;
+    start(systemId: string, instanceId: string): Promise<InstanceInfo | Task<InstanceInfo>>;
     /**
      * Stops an instance but does not remove it from the list of active instances
      *

package/dist/cjs/src/instanceManager.js

@@ -19,6 +19,7 @@ const utils_1 = require("./utils/utils");
 const operatorManager_1 = require("./operatorManager");
 const nodejs_utils_1 = require("@kapeta/nodejs-utils");
 const definitionsManager_1 = require("./definitionsManager");
+const taskManager_1 = require("./taskManager");
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
 const EVENT_STATUS_CHANGED = 'status-changed';
@@ -217,27 +218,37 @@ class InstanceManager {
         systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         const plan = await assetManager_1.assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + systemId);
+            throw new Error(`Plan not found: ${systemId}`);
         }
         if (!plan.spec.blocks) {
-            console.warn('No blocks found in plan', systemId);
-            return [];
+            throw new Error(`No blocks found in plan: ${systemId}`);
         }
-        let promises = [];
-        let errors = [];
-        for (let blockInstance of Object.values(plan.spec.blocks)) {
-            try {
-                promises.push(this.start(systemId, blockInstance.id));
+        return taskManager_1.taskManager.add(`plan:start:${systemId}`, async () => {
+            let promises = [];
+            let errors = [];
+            for (let blockInstance of Object.values(plan.spec.blocks)) {
+                try {
+                    promises.push(this.start(systemId, blockInstance.id).then((taskOrInstance) => {
+                        if (taskOrInstance instanceof taskManager_1.Task) {
+                            return taskOrInstance.wait();
+                        }
+                        return taskOrInstance;
+                    }));
+                }
+                catch (e) {
+                    errors.push(e);
+                }
             }
-            catch (e) {
-                errors.push(e);
+            const settled = await Promise.allSettled(promises);
+            if (errors.length > 0) {
+                throw errors[0];
             }
-        }
-        const settled = await Promise.allSettled(promises);
-        if (errors.length > 0) {
-            throw errors[0];
-        }
-        return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p);
+            return settled
+                .map((p) => (p.status === 'fulfilled' ? p.value : null))
+                .filter((p) => !!p);
+        }, {
+            name: `Starting plan ${systemId}`,
+        });
     }
     async stop(systemId, instanceId) {
         return this.stopInner(systemId, instanceId, true);
@@ -294,10 +305,14 @@ class InstanceManager {
             }
         });
     }
-    async stopAllForPlan(systemId) {
+    stopAllForPlan(systemId) {
         systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
-        return this.stopInstances(instancesForPlan);
+        return taskManager_1.taskManager.add(`plan:stop:${systemId}`, async () => {
+            return this.stopInstances(instancesForPlan);
+        }, {
+            name: `Stopping plan ${systemId}`,
+        });
     }
     async start(systemId, instanceId) {
         return this.exclusive(systemId, instanceId, async () => {
@@ -373,47 +388,53 @@ class InstanceManager {
                 }
             }
             const instanceConfig = await configManager_1.configManager.getConfigForSection(systemId, instanceId);
-            const runner = new BlockInstanceRunner_1.BlockInstanceRunner(systemId);
-            const startTime = Date.now();
-            try {
-                const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
-                instance.status = types_1.InstanceStatus.READY;
-                return this.saveInternalInstance({
-                    ...instance,
-                    type: processInfo.type,
-                    pid: processInfo.pid ?? -1,
-                    health: null,
-                    portType: processInfo.portType,
-                    status: types_1.InstanceStatus.READY,
-                });
-            }
-            catch (e) {
-                console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
-                const logs = [
-                    {
-                        source: 'stdout',
-                        level: 'ERROR',
-                        message: e.message,
-                        time: Date.now(),
-                    },
-                ];
-                const out = await this.saveInternalInstance({
-                    ...instance,
-                    type: types_1.InstanceType.UNKNOWN,
-                    pid: null,
-                    health: null,
-                    portType: DEFAULT_HEALTH_PORT_TYPE,
-                    status: types_1.InstanceStatus.FAILED,
-                    errorMessage: e.message ?? 'Failed to start - Check logs for details.',
-                });
-                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
-                this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
-                    error: `Failed to start instance: ${e.message}`,
-                    status: EVENT_INSTANCE_EXITED,
-                    instanceId: blockInstance.id,
-                });
-                return out;
-            }
+            const task = taskManager_1.taskManager.add(`instance:start:${systemId}:${instanceId}`, async () => {
+                const runner = new BlockInstanceRunner_1.BlockInstanceRunner(systemId);
+                const startTime = Date.now();
+                try {
+                    const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
+                    instance.status = types_1.InstanceStatus.READY;
+                    return this.saveInternalInstance({
+                        ...instance,
+                        type: processInfo.type,
+                        pid: processInfo.pid ?? -1,
+                        health: null,
+                        portType: processInfo.portType,
+                        status: types_1.InstanceStatus.READY,
+                    });
+                }
+                catch (e) {
+                    console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
+                    const logs = [
+                        {
+                            source: 'stdout',
+                            level: 'ERROR',
+                            message: e.message,
+                            time: Date.now(),
+                        },
+                    ];
+                    const out = await this.saveInternalInstance({
+                        ...instance,
+                        type: types_1.InstanceType.UNKNOWN,
+                        pid: null,
+                        health: null,
+                        portType: DEFAULT_HEALTH_PORT_TYPE,
+                        status: types_1.InstanceStatus.FAILED,
+                        errorMessage: e.message ?? 'Failed to start - Check logs for details.',
+                    });
+                    this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
+                    this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
+                        error: `Failed to start instance: ${e.message}`,
+                        status: EVENT_INSTANCE_EXITED,
+                        instanceId: blockInstance.id,
+                    });
+                    return out;
+                }
+            }, {
+                name: `Starting instance: ${instance.name}`,
+                systemId,
+            });
+            return task;
         });
     }
     /**

package/dist/cjs/src/instances/routes.js

@@ -10,6 +10,7 @@ const cors_1 = require("../middleware/cors");
 const kapeta_1 = require("../middleware/kapeta");
 const stringBody_1 = require("../middleware/stringBody");
 const types_1 = require("../types");
+const taskManager_1 = require("../taskManager");
 const router = (0, express_promise_router_1.default)();
 router.use('/', cors_1.corsHandler);
 router.use('/', kapeta_1.kapetaHeaders);
@@ -29,33 +30,40 @@ router.get('/:systemId/instances', (req, res) => {
  * Start all instances in a plan
  */
 router.post('/:systemId/start', async (req, res) => {
-    const instances = await instanceManager_1.instanceManager.startAllForPlan(req.params.systemId);
+    const task = await instanceManager_1.instanceManager.startAllForPlan(req.params.systemId);
     res.status(202).send({
         ok: true,
-        processes: instances.map((p) => {
-            return { pid: p.pid, type: p.type };
-        }),
+        taskId: task.id,
     });
 });
 /**
  * Stop all instances in plan
  */
 router.post('/:systemId/stop', async (req, res) => {
-    await instanceManager_1.instanceManager.stopAllForPlan(req.params.systemId);
+    const task = instanceManager_1.instanceManager.stopAllForPlan(req.params.systemId);
     res.status(202).send({
         ok: true,
+        taskId: task.id,
     });
 });
 /**
  * Start single instance in a plan
  */
 router.post('/:systemId/:instanceId/start', async (req, res) => {
-    const process = await instanceManager_1.instanceManager.start(req.params.systemId, req.params.instanceId);
-    res.status(202).send({
-        ok: true,
-        pid: process.pid,
-        type: process.type,
-    });
+    const result = await instanceManager_1.instanceManager.start(req.params.systemId, req.params.instanceId);
+    if (result instanceof taskManager_1.Task) {
+        res.status(202).send({
+            ok: true,
+            taskId: result.id,
+        });
+    }
+    else {
+        res.status(202).send({
+            ok: true,
+            pid: result.pid,
+            type: result.type,
+        });
+    }
 });
 /**
  * Stop single instance in a plan