@kapeta/local-cluster-service 0.12.1 → 0.13.0

package/src/assets/routes.ts

@@ -44,8 +44,15 @@ router.get('/read', async (req: Request, res: Response) => {
         return;
     }
 
+    const ensure = req.query.ensure !== 'false';
+
     try {
-        res.send(await assetManager.getAsset(req.query.ref as string, true));
+        const asset = await assetManager.getAsset(req.query.ref as string, true, ensure);
+        if (asset) {
+            res.send(asset);
+        } else {
+            res.status(404).send({ error: 'Asset not found' });
+        }
     } catch (err: any) {
         res.status(400).send({ error: err.message });
     }
@@ -129,4 +136,19 @@ router.put('/import', async (req: Request, res: Response) => {
     }
 });
 
+router.put('/install', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        const tasks = await assetManager.installAsset(req.query.ref as string);
+        const taskIds = tasks?.map((t) => t.id) ?? [];
+        res.status(200).send(taskIds);
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
 export default router;

package/src/containerManager.ts

@@ -15,6 +15,7 @@ import { socketManager } from './socketManager';
 import { handlers as ArtifactHandlers } from '@kapeta/nodejs-registry-utils';
 import { progressListener } from './progressListener';
 import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { taskManager, Task } from './taskManager';
 
 const EVENT_IMAGE_PULL = 'docker-image-pull';
 
@@ -67,7 +68,6 @@ const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
 const HEALTH_CHECK_MAX = 20;
 
-
 export const HEALTH_CHECK_TIMEOUT = HEALTH_CHECK_INTERVAL * HEALTH_CHECK_MAX * 2;
 
 const promisifyStream = (stream: ReadStream, handler: (d: string | Buffer) => void) =>
@@ -238,151 +238,176 @@ class ContainerManager {
             return false;
         }
 
-        const timeStarted = Date.now();
-        socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: -1 });
+        let friendlyImageName = image;
+        const imageParts = imageName.split('/');
+        if (imageParts.length > 2) {
+            //Strip the registry to make the name shorter
+            friendlyImageName = `${imageParts.slice(1).join('/')}:${tag}`;
+        }
 
-        const api = new KapetaAPI();
-        const accessToken = await api.getAccessToken();
+        const taskName = `Pulling image ${friendlyImageName}`;
+
+        const processor = async (task: Task) => {
+            const timeStarted = Date.now();
+            const api = new KapetaAPI();
+            const accessToken = await api.getAccessToken();
+
+            const auth = image.startsWith('docker.kapeta.com/')
+                ? {
+                      username: 'kapeta',
+                      password: accessToken,
+                      serveraddress: 'docker.kapeta.com',
+                  }
+                : {};
+
+            const stream = (await this.docker().image.create(auth, {
+                fromImage: imageName,
+                tag: tag,
+            })) as ReadStream;
+
+            const chunks: {
+                [p: string]: {
+                    downloading: {
+                        total: number;
+                        current: number;
+                    };
+                    extracting: {
+                        total: number;
+                        current: number;
+                    };
+                    done: boolean;
+                };
+            } = {};
+
+            let lastEmitted = Date.now();
+            await promisifyStream(stream, (rawData) => {
+                const lines = rawData.toString().trim().split('\n');
+                lines.forEach((line) => {
+                    const data = JSON.parse(line);
+                    if (
+                        ![
+                            'Waiting',
+                            'Downloading',
+                            'Extracting',
+                            'Download complete',
+                            'Pull complete',
+                            'Already exists',
+                        ].includes(data.status)
+                    ) {
+                        return;
+                    }
 
-        const auth = image.startsWith('docker.kapeta.com/')
-            ? {
-                  username: 'kapeta',
-                  password: accessToken,
-                  serveraddress: 'docker.kapeta.com',
-              }
-            : {};
+                    if (!chunks[data.id]) {
+                        chunks[data.id] = {
+                            downloading: {
+                                total: 0,
+                                current: 0,
+                            },
+                            extracting: {
+                                total: 0,
+                                current: 0,
+                            },
+                            done: false,
+                        };
+                    }
 
-        const stream = (await this.docker().image.create(auth, {
-            fromImage: imageName,
-            tag: tag,
-        })) as ReadStream;
+                    const chunk = chunks[data.id];
+
+                    switch (data.status) {
+                        case 'Downloading':
+                            chunk.downloading = data.progressDetail;
+                            break;
+                        case 'Extracting':
+                            chunk.extracting = data.progressDetail;
+                            break;
+                        case 'Download complete':
+                            chunk.downloading.current = chunks[data.id].downloading.total;
+                            break;
+                        case 'Pull complete':
+                            chunk.extracting.current = chunks[data.id].extracting.total;
+                            chunk.done = true;
+                            break;
+                        case 'Already exists':
+                            // Force layer to be done
+                            chunk.downloading.current = 1;
+                            chunk.downloading.total = 1;
+                            chunk.extracting.current = 1;
+                            chunk.extracting.total = 1;
+                            chunk.done = true;
+                            break;
+                    }
+                });
 
-        const chunks: {
-            [p: string]: {
-                downloading: {
-                    total: number;
-                    current: number;
-                };
-                extracting: {
-                    total: number;
-                    current: number;
-                };
-                done: boolean;
-            };
-        } = {};
-
-        let lastEmitted = Date.now();
-        await promisifyStream(stream, (rawData) => {
-            const lines = rawData.toString().trim().split('\n');
-            lines.forEach((line) => {
-                const data = JSON.parse(line);
-                if (
-                    ![
-                        'Waiting',
-                        'Downloading',
-                        'Extracting',
-                        'Download complete',
-                        'Pull complete',
-                        'Already exists',
-                    ].includes(data.status)
-                ) {
+                if (Date.now() - lastEmitted < 1000) {
                     return;
                 }
 
-                if (!chunks[data.id]) {
-                    chunks[data.id] = {
-                        downloading: {
-                            total: 0,
-                            current: 0,
-                        },
-                        extracting: {
-                            total: 0,
-                            current: 0,
-                        },
-                        done: false,
-                    };
-                }
-
-                const chunk = chunks[data.id];
-
-                switch (data.status) {
-                    case 'Downloading':
-                        chunk.downloading = data.progressDetail;
-                        break;
-                    case 'Extracting':
-                        chunk.extracting = data.progressDetail;
-                        break;
-                    case 'Download complete':
-                        chunk.downloading.current = chunks[data.id].downloading.total;
-                        break;
-                    case 'Pull complete':
-                        chunk.extracting.current = chunks[data.id].extracting.total;
-                        chunk.done = true;
-                        break;
-                    case 'Already exists':
-                        // Force layer to be done
-                        chunk.downloading.current = 1;
-                        chunk.downloading.total = 1;
-                        chunk.extracting.current = 1;
-                        chunk.extracting.total = 1;
-                        chunk.done = true;
-                        break;
-                }
-            });
+                const chunkList = Object.values(chunks);
+                let totals = {
+                    downloading: {
+                        total: 0,
+                        current: 0,
+                    },
+                    extracting: {
+                        total: 0,
+                        current: 0,
+                    },
+                    total: chunkList.length,
+                    done: 0,
+                };
 
-            if (Date.now() - lastEmitted < 1000) {
-                return;
-            }
+                chunkList.forEach((chunk) => {
+                    if (chunk.downloading.current > 0) {
+                        totals.downloading.current += chunk.downloading.current;
+                    }
 
-            const chunkList = Object.values(chunks);
-            let totals = {
-                downloading: {
-                    total: 0,
-                    current: 0,
-                },
-                extracting: {
-                    total: 0,
-                    current: 0,
-                },
-                total: chunkList.length,
-                done: 0,
-            };
+                    if (chunk.downloading.total > 0) {
+                        totals.downloading.total += chunk.downloading.total;
+                    }
 
-            chunkList.forEach((chunk) => {
-                if (chunk.downloading.current > 0) {
-                    totals.downloading.current += chunk.downloading.current;
-                }
+                    if (chunk.extracting.current > 0) {
+                        totals.extracting.current += chunk.extracting.current;
+                    }
 
-                if (chunk.downloading.total > 0) {
-                    totals.downloading.total += chunk.downloading.total;
-                }
+                    if (chunk.extracting.total > 0) {
+                        totals.extracting.total += chunk.extracting.total;
+                    }
 
-                if (chunk.extracting.current > 0) {
-                    totals.extracting.current += chunk.extracting.current;
-                }
+                    if (chunk.done) {
+                        totals.done++;
+                    }
+                });
 
-                if (chunk.extracting.total > 0) {
-                    totals.extracting.total += chunk.extracting.total;
-                }
+                const progress = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
 
-                if (chunk.done) {
-                    totals.done++;
-                }
+                task.metadata = {
+                    ...task.metadata,
+                    image,
+                    progress,
+                    status: totals,
+                    timeTaken: Date.now() - timeStarted,
+                };
+                task.emitUpdate();
+                lastEmitted = Date.now();
+                //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
             });
 
-            const percent = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
-            //We emit at most every second to not spam the client
-            socketManager.emitGlobal(EVENT_IMAGE_PULL, {
+            task.metadata = {
+                ...task.metadata,
                 image,
-                percent,
-                status: totals,
+                progress: 100,
                 timeTaken: Date.now() - timeStarted,
-            });
-            lastEmitted = Date.now();
-            //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
+            };
+            task.emitUpdate();
+        };
+
+        const task = taskManager.add(`docker:image:pull:${image}`, processor, {
+            name: taskName,
+            image,
+            progress: -1,
         });
 
-        socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 100, timeTaken: Date.now() - timeStarted });
+        await task.wait();
 
         return true;
     }

package/src/instanceManager.ts

@@ -9,11 +9,12 @@ import { assetManager } from './assetManager';
 import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
 import { configManager } from './configManager';
 import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceStatus, InstanceType, LogEntry } from './types';
-import {BlockDefinitionSpec, BlockInstance, Plan} from '@kapeta/schemas';
+import { BlockDefinitionSpec, BlockInstance, Plan } from '@kapeta/schemas';
 import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
 import { KIND_OPERATOR, operatorManager } from './operatorManager';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { definitionsManager } from './definitionsManager';
+import { Task, taskManager } from './taskManager';
 
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
@@ -74,7 +75,9 @@ export class InstanceManager {
 
         const instanceIds = plan.spec.blocks.map((block) => block.id);
 
-        return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
+        return this._instances.filter(
+            (instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId)
+        );
     }
 
     public getInstance(systemId: string, instanceId: string) {
@@ -271,35 +274,51 @@ export class InstanceManager {
         });
     }
 
-    public async startAllForPlan(systemId: string): Promise<InstanceInfo[]> {
+    public async startAllForPlan(systemId: string): Promise<Task<InstanceInfo[]>> {
         systemId = normalizeKapetaUri(systemId);
         const plan = await assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + systemId);
+            throw new Error(`Plan not found: ${systemId}`);
         }
 
         if (!plan.spec.blocks) {
-            console.warn('No blocks found in plan', systemId);
-            return [];
+            throw new Error(`No blocks found in plan: ${systemId}`);
         }
 
-        let promises: Promise<InstanceInfo>[] = [];
-        let errors = [];
-        for (let blockInstance of Object.values(plan.spec.blocks as BlockInstance[])) {
-            try {
-                promises.push(this.start(systemId, blockInstance.id));
-            } catch (e) {
-                errors.push(e);
-            }
-        }
+        return taskManager.add(
+            `plan:start:${systemId}`,
+            async () => {
+                let promises: Promise<InstanceInfo>[] = [];
+                let errors = [];
+                for (let blockInstance of Object.values(plan.spec.blocks as BlockInstance[])) {
+                    try {
+                        promises.push(
+                            this.start(systemId, blockInstance.id).then((taskOrInstance) => {
+                                if (taskOrInstance instanceof Task) {
+                                    return taskOrInstance.wait();
+                                }
+                                return taskOrInstance;
+                            })
+                        );
+                    } catch (e) {
+                        errors.push(e);
+                    }
+                }
 
-        const settled = await Promise.allSettled(promises);
+                const settled = await Promise.allSettled(promises);
 
-        if (errors.length > 0) {
-            throw errors[0];
-        }
+                if (errors.length > 0) {
+                    throw errors[0];
+                }
 
-        return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p) as InstanceInfo[];
+                return settled
+                    .map((p) => (p.status === 'fulfilled' ? p.value : null))
+                    .filter((p) => !!p) as InstanceInfo[];
+            },
+            {
+                name: `Starting plan ${systemId}`,
+            }
+        );
     }
 
     public async stop(systemId: string, instanceId: string) {
@@ -363,14 +382,21 @@ export class InstanceManager {
         });
     }
 
-    public async stopAllForPlan(systemId: string) {
+    public stopAllForPlan(systemId: string) {
         systemId = normalizeKapetaUri(systemId);
         const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
-
-        return this.stopInstances(instancesForPlan);
+        return taskManager.add(
+            `plan:stop:${systemId}`,
+            async () => {
+                return this.stopInstances(instancesForPlan);
+            },
+            {
+                name: `Stopping plan ${systemId}`,
+            }
+        );
     }
 
-    public async start(systemId: string, instanceId: string): Promise<InstanceInfo> {
+    public async start(systemId: string, instanceId: string): Promise<InstanceInfo | Task<InstanceInfo>> {
         return this.exclusive(systemId, instanceId, async () => {
             systemId = normalizeKapetaUri(systemId);
             const plan = await assetManager.getPlan(systemId, true);
@@ -463,53 +489,63 @@ export class InstanceManager {
             }
 
             const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
-            const runner = new BlockInstanceRunner(systemId);
-
-            const startTime = Date.now();
-            try {
-                const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
-
-                instance.status = InstanceStatus.READY;
-
-                return this.saveInternalInstance({
-                    ...instance,
-                    type: processInfo.type,
-                    pid: processInfo.pid ?? -1,
-                    health: null,
-                    portType: processInfo.portType,
-                    status: InstanceStatus.READY,
-                });
-            } catch (e: any) {
-                console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
-                const logs: LogEntry[] = [
-                    {
-                        source: 'stdout',
-                        level: 'ERROR',
-                        message: e.message,
-                        time: Date.now(),
-                    },
-                ];
-
-                const out = await this.saveInternalInstance({
-                    ...instance,
-                    type: InstanceType.UNKNOWN,
-                    pid: null,
-                    health: null,
-                    portType: DEFAULT_HEALTH_PORT_TYPE,
-                    status: InstanceStatus.FAILED,
-                    errorMessage: e.message ?? 'Failed to start - Check logs for details.',
-                });
-
-                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
-
-                this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
-                    error: `Failed to start instance: ${e.message}`,
-                    status: EVENT_INSTANCE_EXITED,
-                    instanceId: blockInstance.id,
-                });
+            const task = taskManager.add(
+                `instance:start:${systemId}:${instanceId}`,
+                async () => {
+                    const runner = new BlockInstanceRunner(systemId);
+                    const startTime = Date.now();
+                    try {
+                        const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
+
+                        instance.status = InstanceStatus.READY;
+
+                        return this.saveInternalInstance({
+                            ...instance,
+                            type: processInfo.type,
+                            pid: processInfo.pid ?? -1,
+                            health: null,
+                            portType: processInfo.portType,
+                            status: InstanceStatus.READY,
+                        });
+                    } catch (e: any) {
+                        console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
+                        const logs: LogEntry[] = [
+                            {
+                                source: 'stdout',
+                                level: 'ERROR',
+                                message: e.message,
+                                time: Date.now(),
+                            },
+                        ];
+
+                        const out = await this.saveInternalInstance({
+                            ...instance,
+                            type: InstanceType.UNKNOWN,
+                            pid: null,
+                            health: null,
+                            portType: DEFAULT_HEALTH_PORT_TYPE,
+                            status: InstanceStatus.FAILED,
+                            errorMessage: e.message ?? 'Failed to start - Check logs for details.',
+                        });
+
+                        this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
+
+                        this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
+                            error: `Failed to start instance: ${e.message}`,
+                            status: EVENT_INSTANCE_EXITED,
+                            instanceId: blockInstance.id,
+                        });
+
+                        return out;
+                    }
+                },
+                {
+                    name: `Starting instance: ${instance.name}`,
+                    systemId,
+                }
+            );
 
-                return out;
-            }
+            return task;
         });
     }
 

package/src/instances/routes.ts

@@ -6,6 +6,7 @@ import { NextFunction, Request, Response } from 'express';
 import { kapetaHeaders, KapetaRequest } from '../middleware/kapeta';
 import { stringBody } from '../middleware/stringBody';
 import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceType, KapetaBodyRequest } from '../types';
+import { Task } from '../taskManager';
 
 const router = Router();
 router.use('/', corsHandler);
@@ -28,13 +29,11 @@ router.get('/:systemId/instances', (req: Request, res: Response) => {
  * Start all instances in a plan
  */
 router.post('/:systemId/start', async (req: Request, res: Response) => {
-    const instances = await instanceManager.startAllForPlan(req.params.systemId);
+    const task = await instanceManager.startAllForPlan(req.params.systemId);
 
     res.status(202).send({
         ok: true,
-        processes: instances.map((p) => {
-            return { pid: p.pid, type: p.type };
-        }),
+        taskId: task.id,
     });
 });
 
@@ -42,10 +41,11 @@ router.post('/:systemId/start', async (req: Request, res: Response) => {
  * Stop all instances in plan
  */
 router.post('/:systemId/stop', async (req: Request, res: Response) => {
-    await instanceManager.stopAllForPlan(req.params.systemId);
+    const task = instanceManager.stopAllForPlan(req.params.systemId);
 
     res.status(202).send({
         ok: true,
+        taskId: task.id,
     });
 });
 
@@ -53,13 +53,19 @@ router.post('/:systemId/stop', async (req: Request, res: Response) => {
  * Start single instance in a plan
  */
 router.post('/:systemId/:instanceId/start', async (req: Request, res: Response) => {
-    const process = await instanceManager.start(req.params.systemId, req.params.instanceId);
-
-    res.status(202).send({
-        ok: true,
-        pid: process.pid,
-        type: process.type,
-    });
+    const result = await instanceManager.start(req.params.systemId, req.params.instanceId);
+    if (result instanceof Task) {
+        res.status(202).send({
+            ok: true,
+            taskId: result.id,
+        });
+    } else {
+        res.status(202).send({
+            ok: true,
+            pid: result.pid,
+            type: result.type,
+        });
+    }
 });
 
 /**