@kapeta/local-cluster-service 0.12.0 → 0.13.0

package/dist/esm/src/taskManager.js

@@ -0,0 +1,159 @@
+/**
+ * Class that handles processing background tasks.
+ */
+import { socketManager } from './socketManager';
+const EVENT_TASK_UPDATED = 'task-updated';
+const EVENT_TASK_ADDED = 'task-added';
+const EVENT_TASK_REMOVED = 'task-removed';
+export var TaskStatus;
+(function (TaskStatus) {
+    TaskStatus["PENDING"] = "PENDING";
+    TaskStatus["RUNNING"] = "RUNNING";
+    TaskStatus["COMPLETED"] = "COMPLETED";
+    TaskStatus["FAILED"] = "FAILED";
+})(TaskStatus || (TaskStatus = {}));
+export class Task {
+    data;
+    constructor(task) {
+        this.data = task;
+    }
+    get id() {
+        return this.data.id;
+    }
+    get status() {
+        return this.data.status;
+    }
+    get errorMessage() {
+        return this.data.errorMessage;
+    }
+    get metadata() {
+        return this.data.metadata;
+    }
+    get future() {
+        return this.data.future;
+    }
+    get run() {
+        return this.data.run;
+    }
+    set status(status) {
+        this.data.status = status;
+    }
+    set errorMessage(errorMessage) {
+        this.data.errorMessage = errorMessage;
+    }
+    set metadata(metadata) {
+        this.data.metadata = metadata;
+    }
+    emitUpdate() {
+        socketManager.emitGlobal(EVENT_TASK_UPDATED, this.toData());
+    }
+    async wait() {
+        return this.future.promise;
+    }
+    toData() {
+        return { ...this.data };
+    }
+}
+function createFuture() {
+    let resolve = () => { };
+    let reject = () => { };
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return {
+        promise,
+        resolve,
+        reject,
+    };
+}
+class TaskManager {
+    _tasks = [];
+    add(id, runner, metadata) {
+        const existingTask = this.get(id);
+        if (existingTask) {
+            return existingTask;
+        }
+        const future = createFuture();
+        const task = new Task({
+            id,
+            status: TaskStatus.PENDING,
+            metadata,
+            future,
+            run: runner,
+        });
+        this._tasks.push(task);
+        socketManager.emitGlobal(EVENT_TASK_ADDED, task.toData());
+        this.invokeTask(task).catch(() => { });
+        return task;
+    }
+    async waitFor(filter) {
+        const tasks = this._tasks.filter(filter);
+        while (tasks.length > 0) {
+            const task = tasks.shift();
+            if (!task) {
+                continue;
+            }
+            try {
+                await task.wait();
+            }
+            catch (e) {
+                //Ignore
+            }
+        }
+    }
+    get(taskId) {
+        return this._tasks.find((t) => t.id === taskId);
+    }
+    exists(taskId) {
+        return !!this.get(taskId);
+    }
+    remove(taskId) {
+        const task = this.get(taskId);
+        if (!task) {
+            return;
+        }
+        if (task.status === TaskStatus.RUNNING) {
+            throw new Error('Cannot remove a running task');
+        }
+        this._tasks = this._tasks.filter((t) => t.id !== taskId);
+        socketManager.emitGlobal(EVENT_TASK_REMOVED, task.toData());
+    }
+    list() {
+        return this._tasks.map((t) => t.toData());
+    }
+    async invokeTask(task) {
+        if (task.metadata.group) {
+            const existingTaskInGroup = this._tasks.find((t) => t.id !== task.id && t.metadata.group === task.metadata.group && t.status === TaskStatus.RUNNING);
+            if (existingTaskInGroup) {
+                console.log('existingTaskInGroup', existingTaskInGroup.toData());
+                return;
+            }
+        }
+        try {
+            task.status = TaskStatus.RUNNING;
+            task.emitUpdate();
+            const result = await task.run(task);
+            task.status = TaskStatus.COMPLETED;
+            task.future.resolve(result);
+            task.emitUpdate();
+        }
+        catch (e) {
+            task.errorMessage = e.message;
+            task.status = TaskStatus.FAILED;
+            task.future.reject(e);
+            task.emitUpdate();
+        }
+        finally {
+            this.remove(task.id);
+        }
+        if (task.metadata.group) {
+            const nextTaskInGroup = this._tasks.find((t) => t.id !== task.id && t.metadata.group === task.metadata.group && t.status === TaskStatus.PENDING);
+            if (nextTaskInGroup) {
+                console.log('nextTaskInGroup', nextTaskInGroup.toData());
+                return this.invokeTask(nextTaskInGroup);
+            }
+        }
+    }
+}
+export const taskManager = new TaskManager();

package/dist/esm/src/tasks/routes.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="express" />
+declare const router: import("express").Router;
+export default router;

package/dist/esm/src/tasks/routes.js

@@ -0,0 +1,30 @@
+import Router from 'express-promise-router';
+import { corsHandler } from '../middleware/cors';
+import { taskManager } from '../taskManager';
+const router = Router();
+router.use('/', corsHandler);
+/**
+ * Get all current tasks
+ */
+router.get('/', (req, res) => {
+    res.send(taskManager.list());
+});
+router.get('/:taskId', (req, res) => {
+    const task = taskManager.get(req.params.taskId);
+    if (!task) {
+        res.status(404).send({ error: 'Task not found' });
+        return;
+    }
+    res.send(task.toData());
+});
+router.delete('/:taskId', (req, res) => {
+    try {
+        taskManager.remove(req.params.taskId);
+        res.send({ ok: true });
+    }
+    catch (e) {
+        res.status(400).send({ error: e.message });
+        return;
+    }
+});
+export default router;

package/dist/esm/src/utils/BlockInstanceRunner.js

@@ -318,7 +318,6 @@ export class BlockInstanceRunner {
     }
     async ensureContainer(opts) {
         const container = await containerManager.ensureContainer(opts);
-        await containerManager.waitForReady(container);
         return this._handleContainer(container);
     }
     async _handleContainer(container) {

package/index.ts

@@ -16,6 +16,7 @@ import FilesystemRoutes from './src/filesystem/routes';
 import AssetsRoutes from './src/assets/routes';
 import ProviderRoutes from './src/providers/routes';
 import AttachmentRoutes from './src/attachments/routes';
+import TaskRoutes from './src/tasks/routes';
 import { getBindHost } from './src/utils/utils';
 import request from 'request';
 
@@ -36,6 +37,7 @@ function createServer() {
     app.use('/assets', AssetsRoutes);
     app.use('/providers', ProviderRoutes);
     app.use('/attachments', AttachmentRoutes);
+    app.use('/tasks', TaskRoutes);
     app.get('/status', async (req, res) => {
         res.send({
             ok: true,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.12.0",
+    "version": "0.13.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {

package/src/assetManager.ts

@@ -62,6 +62,10 @@ class AssetManager {
         });
     }
 
+    public clearCache() {
+        this.cache.flushAll();
+    }
+
     /**
      *
      * @param {string[]} [assetKinds]
@@ -98,23 +102,32 @@ class AssetManager {
         return asset.data;
     }
 
-    async getAsset(ref: string, noCache: boolean = false): Promise<EnrichedAsset | undefined> {
+    async getAsset(
+        ref: string,
+        noCache: boolean = false,
+        autoFetch: boolean = true
+    ): Promise<EnrichedAsset | undefined> {
         ref = normalizeKapetaUri(ref);
         const cacheKey = `getAsset:${ref}`;
         if (!noCache && this.cache.has(cacheKey)) {
             return this.cache.get(cacheKey);
         }
         const uri = parseKapetaUri(ref);
-        await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
+        if (autoFetch) {
+            await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, true);
+        }
 
         let asset = definitionsManager
             .getDefinitions()
             .map(enrichAsset)
             .find((a) => parseKapetaUri(a.ref).equals(uri));
-        if (!asset) {
+        if (autoFetch && !asset) {
             throw new Error('Asset not found: ' + ref);
         }
-        this.cache.set(cacheKey, asset);
+        if (asset) {
+            this.cache.set(cacheKey, asset);
+        }
+
         return asset;
     }
 
@@ -189,6 +202,17 @@ class AssetManager {
         this.cache.flushAll();
         await Actions.uninstall(progressListener, [asset.ref]);
     }
+
+    async installAsset(ref: string) {
+        const asset = await this.getAsset(ref, true, false);
+        if (asset) {
+            throw new Error('Asset already installed: ' + ref);
+        }
+        const uri = parseKapetaUri(ref);
+        console.log('Installing %s', ref);
+
+        return await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
+    }
 }
 
 export const assetManager = new AssetManager();

package/src/assets/routes.ts

@@ -44,8 +44,15 @@ router.get('/read', async (req: Request, res: Response) => {
         return;
     }
 
+    const ensure = req.query.ensure !== 'false';
+
     try {
-        res.send(await assetManager.getAsset(req.query.ref as string, true));
+        const asset = await assetManager.getAsset(req.query.ref as string, true, ensure);
+        if (asset) {
+            res.send(asset);
+        } else {
+            res.status(404).send({ error: 'Asset not found' });
+        }
     } catch (err: any) {
         res.status(400).send({ error: err.message });
     }
@@ -129,4 +136,19 @@ router.put('/import', async (req: Request, res: Response) => {
     }
 });
 
+router.put('/install', async (req: Request, res: Response) => {
+    if (!req.query.ref) {
+        res.status(400).send({ error: 'Query parameter "ref" is missing' });
+        return;
+    }
+
+    try {
+        const tasks = await assetManager.installAsset(req.query.ref as string);
+        const taskIds = tasks?.map((t) => t.id) ?? [];
+        res.status(200).send(taskIds);
+    } catch (err: any) {
+        res.status(400).send({ error: err.message });
+    }
+});
+
 export default router;

package/src/containerManager.ts

@@ -15,6 +15,7 @@ import { socketManager } from './socketManager';
 import { handlers as ArtifactHandlers } from '@kapeta/nodejs-registry-utils';
 import { progressListener } from './progressListener';
 import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import { taskManager, Task } from './taskManager';
 
 const EVENT_IMAGE_PULL = 'docker-image-pull';
 
@@ -66,8 +67,6 @@ export const CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
 const HEALTH_CHECK_MAX = 20;
-const IMAGE_PULL_CACHE_TTL = 30 * 60 * 1000;
-const IMAGE_PULL_CACHE: { [key: string]: number } = {};
 
 export const HEALTH_CHECK_TIMEOUT = HEALTH_CHECK_INTERVAL * HEALTH_CHECK_MAX * 2;
 
@@ -223,19 +222,12 @@ class ContainerManager {
         return undefined;
     }
 
-    async pull(image: string, cacheForMS: number = IMAGE_PULL_CACHE_TTL) {
+    async pull(image: string) {
         let [imageName, tag] = image.split(/:/);
         if (!tag) {
             tag = 'latest';
         }
 
-        if (IMAGE_PULL_CACHE[image]) {
-            const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
-            if (timeSince < cacheForMS) {
-                return false;
-            }
-        }
-
         const imageTagList = (await this.docker().image.list())
             .map((image) => image.data as any)
             .filter((imageData) => !!imageData.RepoTags)
@@ -246,153 +238,176 @@ class ContainerManager {
             return false;
         }
 
-        const timeStarted = Date.now();
-        socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 0 });
+        let friendlyImageName = image;
+        const imageParts = imageName.split('/');
+        if (imageParts.length > 2) {
+            //Strip the registry to make the name shorter
+            friendlyImageName = `${imageParts.slice(1).join('/')}:${tag}`;
+        }
 
-        const api = new KapetaAPI();
-        const accessToken = await api.getAccessToken();
+        const taskName = `Pulling image ${friendlyImageName}`;
+
+        const processor = async (task: Task) => {
+            const timeStarted = Date.now();
+            const api = new KapetaAPI();
+            const accessToken = await api.getAccessToken();
+
+            const auth = image.startsWith('docker.kapeta.com/')
+                ? {
+                      username: 'kapeta',
+                      password: accessToken,
+                      serveraddress: 'docker.kapeta.com',
+                  }
+                : {};
+
+            const stream = (await this.docker().image.create(auth, {
+                fromImage: imageName,
+                tag: tag,
+            })) as ReadStream;
+
+            const chunks: {
+                [p: string]: {
+                    downloading: {
+                        total: number;
+                        current: number;
+                    };
+                    extracting: {
+                        total: number;
+                        current: number;
+                    };
+                    done: boolean;
+                };
+            } = {};
+
+            let lastEmitted = Date.now();
+            await promisifyStream(stream, (rawData) => {
+                const lines = rawData.toString().trim().split('\n');
+                lines.forEach((line) => {
+                    const data = JSON.parse(line);
+                    if (
+                        ![
+                            'Waiting',
+                            'Downloading',
+                            'Extracting',
+                            'Download complete',
+                            'Pull complete',
+                            'Already exists',
+                        ].includes(data.status)
+                    ) {
+                        return;
+                    }
 
-        const auth = image.startsWith('docker.kapeta.com/')
-            ? {
-                  username: 'kapeta',
-                  password: accessToken,
-                  serveraddress: 'docker.kapeta.com',
-              }
-            : {};
+                    if (!chunks[data.id]) {
+                        chunks[data.id] = {
+                            downloading: {
+                                total: 0,
+                                current: 0,
+                            },
+                            extracting: {
+                                total: 0,
+                                current: 0,
+                            },
+                            done: false,
+                        };
+                    }
 
-        const stream = (await this.docker().image.create(auth, {
-            fromImage: imageName,
-            tag: tag,
-        })) as ReadStream;
+                    const chunk = chunks[data.id];
+
+                    switch (data.status) {
+                        case 'Downloading':
+                            chunk.downloading = data.progressDetail;
+                            break;
+                        case 'Extracting':
+                            chunk.extracting = data.progressDetail;
+                            break;
+                        case 'Download complete':
+                            chunk.downloading.current = chunks[data.id].downloading.total;
+                            break;
+                        case 'Pull complete':
+                            chunk.extracting.current = chunks[data.id].extracting.total;
+                            chunk.done = true;
+                            break;
+                        case 'Already exists':
+                            // Force layer to be done
+                            chunk.downloading.current = 1;
+                            chunk.downloading.total = 1;
+                            chunk.extracting.current = 1;
+                            chunk.extracting.total = 1;
+                            chunk.done = true;
+                            break;
+                    }
+                });
 
-        const chunks: {
-            [p: string]: {
-                downloading: {
-                    total: number;
-                    current: number;
-                };
-                extracting: {
-                    total: number;
-                    current: number;
-                };
-                done: boolean;
-            };
-        } = {};
-
-        let lastEmitted = Date.now();
-        await promisifyStream(stream, (rawData) => {
-            const lines = rawData.toString().trim().split('\n');
-            lines.forEach((line) => {
-                const data = JSON.parse(line);
-                if (
-                    ![
-                        'Waiting',
-                        'Downloading',
-                        'Extracting',
-                        'Download complete',
-                        'Pull complete',
-                        'Already exists',
-                    ].includes(data.status)
-                ) {
+                if (Date.now() - lastEmitted < 1000) {
                     return;
                 }
 
-                if (!chunks[data.id]) {
-                    chunks[data.id] = {
-                        downloading: {
-                            total: 0,
-                            current: 0,
-                        },
-                        extracting: {
-                            total: 0,
-                            current: 0,
-                        },
-                        done: false,
-                    };
-                }
-
-                const chunk = chunks[data.id];
-
-                switch (data.status) {
-                    case 'Downloading':
-                        chunk.downloading = data.progressDetail;
-                        break;
-                    case 'Extracting':
-                        chunk.extracting = data.progressDetail;
-                        break;
-                    case 'Download complete':
-                        chunk.downloading.current = chunks[data.id].downloading.total;
-                        break;
-                    case 'Pull complete':
-                        chunk.extracting.current = chunks[data.id].extracting.total;
-                        chunk.done = true;
-                        break;
-                    case 'Already exists':
-                        // Force layer to be done
-                        chunk.downloading.current = 1;
-                        chunk.downloading.total = 1;
-                        chunk.extracting.current = 1;
-                        chunk.extracting.total = 1;
-                        chunk.done = true;
-                        break;
-                }
-            });
+                const chunkList = Object.values(chunks);
+                let totals = {
+                    downloading: {
+                        total: 0,
+                        current: 0,
+                    },
+                    extracting: {
+                        total: 0,
+                        current: 0,
+                    },
+                    total: chunkList.length,
+                    done: 0,
+                };
 
-            if (Date.now() - lastEmitted < 1000) {
-                return;
-            }
+                chunkList.forEach((chunk) => {
+                    if (chunk.downloading.current > 0) {
+                        totals.downloading.current += chunk.downloading.current;
+                    }
 
-            const chunkList = Object.values(chunks);
-            let totals = {
-                downloading: {
-                    total: 0,
-                    current: 0,
-                },
-                extracting: {
-                    total: 0,
-                    current: 0,
-                },
-                total: chunkList.length,
-                done: 0,
-            };
+                    if (chunk.downloading.total > 0) {
+                        totals.downloading.total += chunk.downloading.total;
+                    }
 
-            chunkList.forEach((chunk) => {
-                if (chunk.downloading.current > 0) {
-                    totals.downloading.current += chunk.downloading.current;
-                }
+                    if (chunk.extracting.current > 0) {
+                        totals.extracting.current += chunk.extracting.current;
+                    }
 
-                if (chunk.downloading.total > 0) {
-                    totals.downloading.total += chunk.downloading.total;
-                }
+                    if (chunk.extracting.total > 0) {
+                        totals.extracting.total += chunk.extracting.total;
+                    }
 
-                if (chunk.extracting.current > 0) {
-                    totals.extracting.current += chunk.extracting.current;
-                }
+                    if (chunk.done) {
+                        totals.done++;
+                    }
+                });
 
-                if (chunk.extracting.total > 0) {
-                    totals.extracting.total += chunk.extracting.total;
-                }
+                const progress = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
 
-                if (chunk.done) {
-                    totals.done++;
-                }
+                task.metadata = {
+                    ...task.metadata,
+                    image,
+                    progress,
+                    status: totals,
+                    timeTaken: Date.now() - timeStarted,
+                };
+                task.emitUpdate();
+                lastEmitted = Date.now();
+                //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
             });
 
-            const percent = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
-            //We emit at most every second to not spam the client
-            socketManager.emitGlobal(EVENT_IMAGE_PULL, {
+            task.metadata = {
+                ...task.metadata,
                 image,
-                percent,
-                status: totals,
+                progress: 100,
                 timeTaken: Date.now() - timeStarted,
-            });
-            lastEmitted = Date.now();
-            //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
-        });
+            };
+            task.emitUpdate();
+        };
 
-        IMAGE_PULL_CACHE[image] = Date.now();
+        const task = taskManager.add(`docker:image:pull:${image}`, processor, {
+            name: taskName,
+            image,
+            progress: -1,
+        });
 
-        socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 100, timeTaken: Date.now() - timeStarted });
+        await task.wait();
 
         return true;
     }
@@ -435,11 +450,7 @@ class ContainerManager {
     }
 
     public async ensureContainer(opts: any) {
-        const container = await this.createOrUpdateContainer(opts);
-
-        await this.waitForReady(container);
-
-        return container;
+        return await this.createOrUpdateContainer(opts);
     }
 
     private async createOrUpdateContainer(opts: any) {