@kapeta/local-cluster-service 0.12.1 → 0.13.0

package/dist/esm/src/operatorManager.js

@@ -9,17 +9,21 @@ import { definitionsManager } from './definitionsManager';
 import { getBindHost, normalizeKapetaUri } from './utils/utils';
 import _ from 'lodash';
 import AsyncLock from 'async-lock';
+import { taskManager } from './taskManager';
 export const KIND_OPERATOR = 'core/resource-type-operator';
 class Operator {
     _data;
     constructor(data) {
         this._data = data;
     }
-    getData() {
+    getLocalData() {
+        return this._data.definition.spec.local;
+    }
+    getDefinitionInfo() {
         return this._data;
     }
     getCredentials() {
-        return this._data.credentials;
+        return this._data.definition.spec.local.credentials;
     }
 }
 class OperatorManager {
@@ -52,7 +56,7 @@ class OperatorManager {
         if (!operator.definition.spec || !operator.definition.spec.local) {
             throw new Error(`Operator missing local definition: ${resourceType}:${version}`);
         }
-        return new Operator(operator.definition.spec.local);
+        return new Operator(operator);
     }
     /**
      * Get information about a specific consumed resource
@@ -113,7 +117,7 @@ class OperatorManager {
         const key = `${systemId}#${resourceType}:${version}`;
         return await this.operatorLock.acquire(key, async () => {
             const operator = this.getOperator(resourceType, version);
-            const operatorData = operator.getData();
+            const operatorData = operator.getLocalData();
             const portTypes = Object.keys(operatorData.ports);
             portTypes.sort();
             const ports = {};
@@ -141,6 +145,7 @@ class OperatorManager {
             const Labels = {
                 kapeta: 'true',
             };
+            const operatorMetadata = operator.getDefinitionInfo().definition.metadata;
             const bindHost = getBindHost();
             const ExposedPorts = {};
             _.forEach(ports, (portInfo, containerPort) => {
@@ -157,26 +162,32 @@ class OperatorManager {
             _.forEach(operatorData.env, (value, name) => {
                 Env.push(name + '=' + value);
             });
-            let HealthCheck = undefined;
-            if (operatorData.health) {
-                HealthCheck = containerManager.toDockerHealth(operatorData.health);
-            }
-            const container = await containerManager.ensureContainer({
-                name: containerName,
-                Image: operatorData.image,
-                Hostname: containerName + '.kapeta',
-                Labels,
-                Cmd: operatorData.cmd,
-                ExposedPorts,
-                Env,
-                HealthCheck,
-                HostConfig: {
-                    PortBindings,
-                    Mounts,
-                },
+            const task = taskManager.add(`operator:ensure:${key}`, async () => {
+                let HealthCheck = undefined;
+                if (operatorData.health) {
+                    HealthCheck = containerManager.toDockerHealth(operatorData.health);
+                }
+                const container = await containerManager.ensureContainer({
+                    name: containerName,
+                    Image: operatorData.image,
+                    Hostname: containerName + '.kapeta',
+                    Labels,
+                    Cmd: operatorData.cmd,
+                    ExposedPorts,
+                    Env,
+                    HealthCheck,
+                    HostConfig: {
+                        PortBindings,
+                        Mounts,
+                    },
+                });
+                await containerManager.waitForReady(container);
+                return new ContainerInfo(container);
+            }, {
+                name: `Ensuring ${operatorMetadata.title ?? operatorMetadata.name}`,
+                systemId,
             });
-            await containerManager.waitForReady(container);
-            return new ContainerInfo(container);
+            return task.wait();
         });
     }
 }

package/dist/esm/src/providerManager.js

@@ -16,7 +16,7 @@ class ProviderManager {
         if (this._webAssetCache[id] && (await FSExtra.pathExists(this._webAssetCache[id]))) {
             return FSExtra.readFile(this._webAssetCache[id], 'utf8');
         }
-        await repositoryManager.ensureAsset(handle, name, version);
+        await repositoryManager.ensureAsset(handle, name, version, true);
         const installedProvider = this.getWebProviders().find((providerDefinition) => {
             return providerDefinition.definition.metadata.name === fullName && providerDefinition.version === version;
         });

package/dist/esm/src/repositoryManager.d.ts

@@ -1,17 +1,15 @@
+import { Task } from './taskManager';
 declare class RepositoryManager {
     private changeEventsEnabled;
     private _registryService;
     private _cache;
     private watcher?;
-    private _installQueue;
-    private _processing;
     constructor();
     setChangeEventsEnabled(enabled: boolean): void;
     listenForChanges(): void;
     stopListening(): void;
     private _install;
-    _processNext(): Promise<void>;
-    ensureAsset(handle: string, name: string, version: string): Promise<void>;
+    ensureAsset(handle: string, name: string, version: string, wait?: boolean): Promise<undefined | Task[]>;
 }
 export declare const repositoryManager: RepositoryManager;
 export {};

package/dist/esm/src/repositoryManager.js

@@ -9,20 +9,20 @@ import { socketManager } from './socketManager';
 import { progressListener } from './progressListener';
 import { Actions, Config, RegistryService } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
+import { taskManager } from './taskManager';
+import { normalizeKapetaUri } from './utils/utils';
+import { assetManager } from './assetManager';
 const INSTALL_ATTEMPTED = {};
 class RepositoryManager {
     changeEventsEnabled;
     _registryService;
     _cache;
     watcher;
-    _installQueue;
-    _processing = false;
     constructor() {
         this.changeEventsEnabled = true;
         this.listenForChanges();
         this._registryService = new RegistryService(Config.data.registry.url);
         this._cache = {};
-        this._installQueue = [];
     }
     setChangeEventsEnabled(enabled) {
         this.changeEventsEnabled = enabled;
@@ -103,74 +103,54 @@ class RepositoryManager {
     }
     async _install(refs) {
         //We make sure to only install one asset at a time - otherwise unexpected things might happen
-        const out = new Promise((resolve, reject) => {
-            this._installQueue.push(async () => {
-                try {
-                    const normalizedRefs = refs.map((ref) => parseKapetaUri(ref).id);
-                    const filteredRefs = normalizedRefs
-                        .filter((ref) => !INSTALL_ATTEMPTED[ref])
-                        .filter((ref) => !definitionsManager.exists(ref));
-                    if (filteredRefs.length > 0) {
-                        console.log(`Auto-installing dependencies: ${filteredRefs.join(', ')}`);
-                        filteredRefs.forEach((ref) => (INSTALL_ATTEMPTED[ref] = true));
-                        //Auto-install missing asset
-                        try {
-                            //We change to a temp dir to avoid issues with the current working directory
-                            process.chdir(os.tmpdir());
-                            //Disable change events while installing
-                            this.setChangeEventsEnabled(false);
-                            socketManager.emit(`install`, 'install:action', {
-                                type: 'start',
-                                refs,
-                            });
-                            await Actions.install(progressListener, normalizedRefs, {});
-                            socketManager.emit(`install`, 'install:action', {
-                                type: 'done',
-                                refs,
-                            });
-                        }
-                        catch (e) {
-                            socketManager.emit(`install`, 'install:action', {
-                                type: 'failed',
-                                refs,
-                                error: e.message,
-                            });
-                        }
-                        finally {
-                            this.setChangeEventsEnabled(true);
-                        }
-                    }
-                    resolve();
+        const createInstaller = (ref) => {
+            return async () => {
+                if (INSTALL_ATTEMPTED[ref]) {
+                    return;
                 }
-                catch (e) {
-                    reject(e);
+                if (definitionsManager.exists(ref)) {
+                    return;
                 }
-                finally {
-                    this._processNext().catch((e) => console.error(e));
+                console.log(`Installing asset: ${ref}`);
+                INSTALL_ATTEMPTED[ref] = true;
+                //Auto-install missing asset
+                try {
+                    //We change to a temp dir to avoid issues with the current working directory
+                    process.chdir(os.tmpdir());
+                    //Disable change events while installing
+                    this.setChangeEventsEnabled(false);
+                    await Actions.install(progressListener, [ref], {});
                 }
-            });
-        });
-        this._processNext().catch((e) => console.error(e));
-        return out;
-    }
-    async _processNext() {
-        if (this._processing) {
-            return;
-        }
-        this._processing = true;
-        try {
-            while (this._installQueue.length > 0) {
-                const item = this._installQueue.shift();
-                if (item) {
-                    await item();
+                finally {
+                    this.setChangeEventsEnabled(true);
                 }
+                definitionsManager.clearCache();
+                assetManager.clearCache();
+                console.log(`Asset installed: ${ref}`);
+            };
+        };
+        const tasks = [];
+        while (refs.length > 0) {
+            let ref = refs.shift();
+            if (!ref) {
+                continue;
             }
+            ref = normalizeKapetaUri(ref);
+            if (INSTALL_ATTEMPTED[ref]) {
+                continue;
+            }
+            if (definitionsManager.exists(ref)) {
+                continue;
+            }
+            const task = taskManager.add(`asset:install:${ref}`, createInstaller(ref), {
+                name: `Installing ${ref}`,
+                group: 'asset:install:',
+            });
+            tasks.push(task);
         }
-        finally {
-            this._processing = false;
-        }
+        return tasks;
     }
-    async ensureAsset(handle, name, version) {
+    async ensureAsset(handle, name, version, wait = true) {
         const fullName = `${handle}/${name}`;
         const ref = `${fullName}:${version}`;
         if (version === 'local') {
@@ -201,16 +181,21 @@ class RepositoryManager {
             throw e;
         }
         this._cache[ref] = true;
+        let tasks = undefined;
         if (!installedAsset) {
-            await this._install([ref]);
+            tasks = await this._install([ref]);
         }
         else {
             //Ensure dependencies are installed
             const refs = assetVersion.dependencies.map((dep) => dep.name);
             if (refs.length > 0) {
-                await this._install(refs);
+                tasks = await this._install(refs);
             }
         }
+        if (tasks && wait) {
+            await Promise.all(tasks.map((t) => t.future.promise));
+        }
+        return tasks;
     }
 }
 export const repositoryManager = new RepositoryManager();

package/dist/esm/src/socketManager.js

@@ -24,7 +24,7 @@ export class SocketManager {
         this.io.to(context).emit(type, { context, payload });
     }
     emitGlobal(type, payload) {
-        this.io.emit(type, { payload });
+        this.io.emit(type, payload);
     }
     _bindIO() {
         this.io.on('connection', (socket) => this._handleSocketCreated(socket));

package/dist/esm/src/taskManager.d.ts

@@ -0,0 +1,64 @@
+export type TaskRunner<T> = (task: Task<T>) => Promise<T>;
+export declare enum TaskStatus {
+    PENDING = "PENDING",
+    RUNNING = "RUNNING",
+    COMPLETED = "COMPLETED",
+    FAILED = "FAILED"
+}
+interface Future<T = void> {
+    promise: Promise<T>;
+    resolve: (result: T) => void;
+    reject: (e: any) => void;
+}
+interface TaskMetadata {
+    name: string;
+    /**
+     * A unique prefix for the task. If defined only 1 task with this ID prefix will be executed at a time
+     */
+    group?: string;
+    progress?: number;
+    [key: string]: any;
+}
+interface TaskData<T = void> {
+    id: string;
+    status: TaskStatus;
+    errorMessage?: string;
+    metadata: TaskMetadata;
+    future: Future<T>;
+    run: TaskRunner<T>;
+}
+export declare class Task<T = void> implements TaskData<T> {
+    private data;
+    constructor(task: TaskData<T>);
+    get id(): string;
+    get status(): TaskStatus;
+    get errorMessage(): string | undefined;
+    get metadata(): TaskMetadata;
+    get future(): Future<T>;
+    get run(): TaskRunner<T>;
+    set status(status: TaskStatus);
+    set errorMessage(errorMessage: string | undefined);
+    set metadata(metadata: TaskMetadata);
+    emitUpdate(): void;
+    wait(): Promise<T>;
+    toData(): {
+        id: string;
+        status: TaskStatus;
+        errorMessage?: string | undefined;
+        metadata: TaskMetadata;
+        future: Future<T>;
+        run: TaskRunner<T>;
+    };
+}
+declare class TaskManager {
+    private _tasks;
+    add<T>(id: string, runner: TaskRunner<T>, metadata: TaskMetadata): Task<T>;
+    waitFor(filter: (task: Task<any>) => boolean): Promise<void>;
+    get(taskId: string): Task<any> | undefined;
+    exists(taskId: string): boolean;
+    remove(taskId: string): void;
+    list(): TaskData[];
+    private invokeTask;
+}
+export declare const taskManager: TaskManager;
+export {};

package/dist/esm/src/taskManager.js

@@ -0,0 +1,159 @@
+/**
+ * Class that handles processing background tasks.
+ */
+import { socketManager } from './socketManager';
+const EVENT_TASK_UPDATED = 'task-updated';
+const EVENT_TASK_ADDED = 'task-added';
+const EVENT_TASK_REMOVED = 'task-removed';
+export var TaskStatus;
+(function (TaskStatus) {
+    TaskStatus["PENDING"] = "PENDING";
+    TaskStatus["RUNNING"] = "RUNNING";
+    TaskStatus["COMPLETED"] = "COMPLETED";
+    TaskStatus["FAILED"] = "FAILED";
+})(TaskStatus || (TaskStatus = {}));
+export class Task {
+    data;
+    constructor(task) {
+        this.data = task;
+    }
+    get id() {
+        return this.data.id;
+    }
+    get status() {
+        return this.data.status;
+    }
+    get errorMessage() {
+        return this.data.errorMessage;
+    }
+    get metadata() {
+        return this.data.metadata;
+    }
+    get future() {
+        return this.data.future;
+    }
+    get run() {
+        return this.data.run;
+    }
+    set status(status) {
+        this.data.status = status;
+    }
+    set errorMessage(errorMessage) {
+        this.data.errorMessage = errorMessage;
+    }
+    set metadata(metadata) {
+        this.data.metadata = metadata;
+    }
+    emitUpdate() {
+        socketManager.emitGlobal(EVENT_TASK_UPDATED, this.toData());
+    }
+    async wait() {
+        return this.future.promise;
+    }
+    toData() {
+        return { ...this.data };
+    }
+}
+function createFuture() {
+    let resolve = () => { };
+    let reject = () => { };
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return {
+        promise,
+        resolve,
+        reject,
+    };
+}
+class TaskManager {
+    _tasks = [];
+    add(id, runner, metadata) {
+        const existingTask = this.get(id);
+        if (existingTask) {
+            return existingTask;
+        }
+        const future = createFuture();
+        const task = new Task({
+            id,
+            status: TaskStatus.PENDING,
+            metadata,
+            future,
+            run: runner,
+        });
+        this._tasks.push(task);
+        socketManager.emitGlobal(EVENT_TASK_ADDED, task.toData());
+        this.invokeTask(task).catch(() => { });
+        return task;
+    }
+    async waitFor(filter) {
+        const tasks = this._tasks.filter(filter);
+        while (tasks.length > 0) {
+            const task = tasks.shift();
+            if (!task) {
+                continue;
+            }
+            try {
+                await task.wait();
+            }
+            catch (e) {
+                //Ignore
+            }
+        }
+    }
+    get(taskId) {
+        return this._tasks.find((t) => t.id === taskId);
+    }
+    exists(taskId) {
+        return !!this.get(taskId);
+    }
+    remove(taskId) {
+        const task = this.get(taskId);
+        if (!task) {
+            return;
+        }
+        if (task.status === TaskStatus.RUNNING) {
+            throw new Error('Cannot remove a running task');
+        }
+        this._tasks = this._tasks.filter((t) => t.id !== taskId);
+        socketManager.emitGlobal(EVENT_TASK_REMOVED, task.toData());
+    }
+    list() {
+        return this._tasks.map((t) => t.toData());
+    }
+    async invokeTask(task) {
+        if (task.metadata.group) {
+            const existingTaskInGroup = this._tasks.find((t) => t.id !== task.id && t.metadata.group === task.metadata.group && t.status === TaskStatus.RUNNING);
+            if (existingTaskInGroup) {
+                console.log('existingTaskInGroup', existingTaskInGroup.toData());
+                return;
+            }
+        }
+        try {
+            task.status = TaskStatus.RUNNING;
+            task.emitUpdate();
+            const result = await task.run(task);
+            task.status = TaskStatus.COMPLETED;
+            task.future.resolve(result);
+            task.emitUpdate();
+        }
+        catch (e) {
+            task.errorMessage = e.message;
+            task.status = TaskStatus.FAILED;
+            task.future.reject(e);
+            task.emitUpdate();
+        }
+        finally {
+            this.remove(task.id);
+        }
+        if (task.metadata.group) {
+            const nextTaskInGroup = this._tasks.find((t) => t.id !== task.id && t.metadata.group === task.metadata.group && t.status === TaskStatus.PENDING);
+            if (nextTaskInGroup) {
+                console.log('nextTaskInGroup', nextTaskInGroup.toData());
+                return this.invokeTask(nextTaskInGroup);
+            }
+        }
+    }
+}
+export const taskManager = new TaskManager();

package/dist/esm/src/tasks/routes.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="express" />
+declare const router: import("express").Router;
+export default router;

package/dist/esm/src/tasks/routes.js

@@ -0,0 +1,30 @@
+import Router from 'express-promise-router';
+import { corsHandler } from '../middleware/cors';
+import { taskManager } from '../taskManager';
+const router = Router();
+router.use('/', corsHandler);
+/**
+ * Get all current tasks
+ */
+router.get('/', (req, res) => {
+    res.send(taskManager.list());
+});
+router.get('/:taskId', (req, res) => {
+    const task = taskManager.get(req.params.taskId);
+    if (!task) {
+        res.status(404).send({ error: 'Task not found' });
+        return;
+    }
+    res.send(task.toData());
+});
+router.delete('/:taskId', (req, res) => {
+    try {
+        taskManager.remove(req.params.taskId);
+        res.send({ ok: true });
+    }
+    catch (e) {
+        res.status(400).send({ error: e.message });
+        return;
+    }
+});
+export default router;

package/index.ts

@@ -16,6 +16,7 @@ import FilesystemRoutes from './src/filesystem/routes';
 import AssetsRoutes from './src/assets/routes';
 import ProviderRoutes from './src/providers/routes';
 import AttachmentRoutes from './src/attachments/routes';
+import TaskRoutes from './src/tasks/routes';
 import { getBindHost } from './src/utils/utils';
 import request from 'request';
 
@@ -36,6 +37,7 @@ function createServer() {
     app.use('/assets', AssetsRoutes);
     app.use('/providers', ProviderRoutes);
     app.use('/attachments', AttachmentRoutes);
+    app.use('/tasks', TaskRoutes);
     app.get('/status', async (req, res) => {
         res.send({
             ok: true,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.12.1",
+    "version": "0.13.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {

package/src/assetManager.ts

@@ -62,6 +62,10 @@ class AssetManager {
         });
     }
 
+    public clearCache() {
+        this.cache.flushAll();
+    }
+
     /**
      *
      * @param {string[]} [assetKinds]
@@ -98,23 +102,32 @@ class AssetManager {
         return asset.data;
     }
 
-    async getAsset(ref: string, noCache: boolean = false): Promise<EnrichedAsset | undefined> {
+    async getAsset(
+        ref: string,
+        noCache: boolean = false,
+        autoFetch: boolean = true
+    ): Promise<EnrichedAsset | undefined> {
         ref = normalizeKapetaUri(ref);
         const cacheKey = `getAsset:${ref}`;
         if (!noCache && this.cache.has(cacheKey)) {
             return this.cache.get(cacheKey);
         }
         const uri = parseKapetaUri(ref);
-        await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
+        if (autoFetch) {
+            await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, true);
+        }
 
         let asset = definitionsManager
             .getDefinitions()
             .map(enrichAsset)
             .find((a) => parseKapetaUri(a.ref).equals(uri));
-        if (!asset) {
+        if (autoFetch && !asset) {
             throw new Error('Asset not found: ' + ref);
         }
-        this.cache.set(cacheKey, asset);
+        if (asset) {
+            this.cache.set(cacheKey, asset);
+        }
+
         return asset;
     }
 
@@ -189,6 +202,17 @@ class AssetManager {
         this.cache.flushAll();
         await Actions.uninstall(progressListener, [asset.ref]);
     }
+
+    async installAsset(ref: string) {
+        const asset = await this.getAsset(ref, true, false);
+        if (asset) {
+            throw new Error('Asset already installed: ' + ref);
+        }
+        const uri = parseKapetaUri(ref);
+        console.log('Installing %s', ref);
+
+        return await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version, false);
+    }
 }
 
 export const assetManager = new AssetManager();