@kapeta/local-cluster-service 0.12.0 → 0.13.0

package/src/instanceManager.ts

@@ -9,11 +9,12 @@ import { assetManager } from './assetManager';
 import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
 import { configManager } from './configManager';
 import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceStatus, InstanceType, LogEntry } from './types';
-import { BlockDefinitionSpec, BlockInstance } from '@kapeta/schemas';
+import { BlockDefinitionSpec, BlockInstance, Plan } from '@kapeta/schemas';
 import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
 import { KIND_OPERATOR, operatorManager } from './operatorManager';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { definitionsManager } from './definitionsManager';
+import { Task, taskManager } from './taskManager';
 
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
@@ -64,7 +65,19 @@ export class InstanceManager {
 
         systemId = normalizeKapetaUri(systemId);
 
-        return this._instances.filter((instance) => instance.systemId === systemId);
+        const planInfo = definitionsManager.getDefinition(systemId);
+
+        if (!planInfo) {
+            return [];
+        }
+
+        const plan = planInfo.definition as Plan;
+
+        const instanceIds = plan.spec.blocks.map((block) => block.id);
+
+        return this._instances.filter(
+            (instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId)
+        );
     }
 
     public getInstance(systemId: string, instanceId: string) {
@@ -261,35 +274,51 @@ export class InstanceManager {
         });
     }
 
-    public async startAllForPlan(systemId: string): Promise<InstanceInfo[]> {
+    public async startAllForPlan(systemId: string): Promise<Task<InstanceInfo[]>> {
         systemId = normalizeKapetaUri(systemId);
         const plan = await assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + systemId);
+            throw new Error(`Plan not found: ${systemId}`);
         }
 
         if (!plan.spec.blocks) {
-            console.warn('No blocks found in plan', systemId);
-            return [];
+            throw new Error(`No blocks found in plan: ${systemId}`);
         }
 
-        let promises: Promise<InstanceInfo>[] = [];
-        let errors = [];
-        for (let blockInstance of Object.values(plan.spec.blocks as BlockInstance[])) {
-            try {
-                promises.push(this.start(systemId, blockInstance.id));
-            } catch (e) {
-                errors.push(e);
-            }
-        }
+        return taskManager.add(
+            `plan:start:${systemId}`,
+            async () => {
+                let promises: Promise<InstanceInfo>[] = [];
+                let errors = [];
+                for (let blockInstance of Object.values(plan.spec.blocks as BlockInstance[])) {
+                    try {
+                        promises.push(
+                            this.start(systemId, blockInstance.id).then((taskOrInstance) => {
+                                if (taskOrInstance instanceof Task) {
+                                    return taskOrInstance.wait();
+                                }
+                                return taskOrInstance;
+                            })
+                        );
+                    } catch (e) {
+                        errors.push(e);
+                    }
+                }
 
-        const settled = await Promise.allSettled(promises);
+                const settled = await Promise.allSettled(promises);
 
-        if (errors.length > 0) {
-            throw errors[0];
-        }
+                if (errors.length > 0) {
+                    throw errors[0];
+                }
 
-        return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p) as InstanceInfo[];
+                return settled
+                    .map((p) => (p.status === 'fulfilled' ? p.value : null))
+                    .filter((p) => !!p) as InstanceInfo[];
+            },
+            {
+                name: `Starting plan ${systemId}`,
+            }
+        );
     }
 
     public async stop(systemId: string, instanceId: string) {
@@ -353,14 +382,21 @@ export class InstanceManager {
         });
     }
 
-    public async stopAllForPlan(systemId: string) {
+    public stopAllForPlan(systemId: string) {
         systemId = normalizeKapetaUri(systemId);
         const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
-
-        return this.stopInstances(instancesForPlan);
+        return taskManager.add(
+            `plan:stop:${systemId}`,
+            async () => {
+                return this.stopInstances(instancesForPlan);
+            },
+            {
+                name: `Stopping plan ${systemId}`,
+            }
+        );
     }
 
-    public async start(systemId: string, instanceId: string): Promise<InstanceInfo> {
+    public async start(systemId: string, instanceId: string): Promise<InstanceInfo | Task<InstanceInfo>> {
         return this.exclusive(systemId, instanceId, async () => {
             systemId = normalizeKapetaUri(systemId);
             const plan = await assetManager.getPlan(systemId, true);
@@ -453,53 +489,63 @@ export class InstanceManager {
             }
 
             const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
-            const runner = new BlockInstanceRunner(systemId);
-
-            const startTime = Date.now();
-            try {
-                const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
-
-                instance.status = InstanceStatus.READY;
-
-                return this.saveInternalInstance({
-                    ...instance,
-                    type: processInfo.type,
-                    pid: processInfo.pid ?? -1,
-                    health: null,
-                    portType: processInfo.portType,
-                    status: InstanceStatus.READY,
-                });
-            } catch (e: any) {
-                console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
-                const logs: LogEntry[] = [
-                    {
-                        source: 'stdout',
-                        level: 'ERROR',
-                        message: e.message,
-                        time: Date.now(),
-                    },
-                ];
-
-                const out = await this.saveInternalInstance({
-                    ...instance,
-                    type: InstanceType.LOCAL,
-                    pid: null,
-                    health: null,
-                    portType: DEFAULT_HEALTH_PORT_TYPE,
-                    status: InstanceStatus.FAILED,
-                    errorMessage: e.message ?? 'Failed to start - Check logs for details.',
-                });
-
-                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
-
-                this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
-                    error: `Failed to start instance: ${e.message}`,
-                    status: EVENT_INSTANCE_EXITED,
-                    instanceId: blockInstance.id,
-                });
+            const task = taskManager.add(
+                `instance:start:${systemId}:${instanceId}`,
+                async () => {
+                    const runner = new BlockInstanceRunner(systemId);
+                    const startTime = Date.now();
+                    try {
+                        const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
+
+                        instance.status = InstanceStatus.READY;
+
+                        return this.saveInternalInstance({
+                            ...instance,
+                            type: processInfo.type,
+                            pid: processInfo.pid ?? -1,
+                            health: null,
+                            portType: processInfo.portType,
+                            status: InstanceStatus.READY,
+                        });
+                    } catch (e: any) {
+                        console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
+                        const logs: LogEntry[] = [
+                            {
+                                source: 'stdout',
+                                level: 'ERROR',
+                                message: e.message,
+                                time: Date.now(),
+                            },
+                        ];
+
+                        const out = await this.saveInternalInstance({
+                            ...instance,
+                            type: InstanceType.UNKNOWN,
+                            pid: null,
+                            health: null,
+                            portType: DEFAULT_HEALTH_PORT_TYPE,
+                            status: InstanceStatus.FAILED,
+                            errorMessage: e.message ?? 'Failed to start - Check logs for details.',
+                        });
+
+                        this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
+
+                        this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
+                            error: `Failed to start instance: ${e.message}`,
+                            status: EVENT_INSTANCE_EXITED,
+                            instanceId: blockInstance.id,
+                        });
+
+                        return out;
+                    }
+                },
+                {
+                    name: `Starting instance: ${instance.name}`,
+                    systemId,
+                }
+            );
 
-                return out;
-            }
+            return task;
         });
     }
 

package/src/instances/routes.ts

@@ -6,6 +6,7 @@ import { NextFunction, Request, Response } from 'express';
 import { kapetaHeaders, KapetaRequest } from '../middleware/kapeta';
 import { stringBody } from '../middleware/stringBody';
 import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceType, KapetaBodyRequest } from '../types';
+import { Task } from '../taskManager';
 
 const router = Router();
 router.use('/', corsHandler);
@@ -28,13 +29,11 @@ router.get('/:systemId/instances', (req: Request, res: Response) => {
  * Start all instances in a plan
  */
 router.post('/:systemId/start', async (req: Request, res: Response) => {
-    const instances = await instanceManager.startAllForPlan(req.params.systemId);
+    const task = await instanceManager.startAllForPlan(req.params.systemId);
 
     res.status(202).send({
         ok: true,
-        processes: instances.map((p) => {
-            return { pid: p.pid, type: p.type };
-        }),
+        taskId: task.id,
     });
 });
 
@@ -42,10 +41,11 @@ router.post('/:systemId/start', async (req: Request, res: Response) => {
  * Stop all instances in plan
  */
 router.post('/:systemId/stop', async (req: Request, res: Response) => {
-    await instanceManager.stopAllForPlan(req.params.systemId);
+    const task = instanceManager.stopAllForPlan(req.params.systemId);
 
     res.status(202).send({
         ok: true,
+        taskId: task.id,
     });
 });
 
@@ -53,13 +53,19 @@ router.post('/:systemId/stop', async (req: Request, res: Response) => {
  * Start single instance in a plan
  */
 router.post('/:systemId/:instanceId/start', async (req: Request, res: Response) => {
-    const process = await instanceManager.start(req.params.systemId, req.params.instanceId);
-
-    res.status(202).send({
-        ok: true,
-        pid: process.pid,
-        type: process.type,
-    });
+    const result = await instanceManager.start(req.params.systemId, req.params.instanceId);
+    if (result instanceof Task) {
+        res.status(202).send({
+            ok: true,
+            taskId: result.id,
+        });
+    } else {
+        res.status(202).send({
+            ok: true,
+            pid: result.pid,
+            type: result.type,
+        });
+    }
 });
 
 /**
@@ -146,8 +152,10 @@ router.put('/', async (req: KapetaBodyRequest, res: Response) => {
         const oldInstance = instanceManager.getInstance(req.kapeta!.systemId, req.kapeta!.instanceId);
         if (oldInstance) {
             instance.pid = oldInstance.pid;
+            instance.desiredStatus = oldInstance.desiredStatus;
         }
         instance.type = InstanceType.DOCKER;
+        instance.owner = InstanceOwner.INTERNAL;
     } else {
         // Coming from user starting the instance outside of kapeta
         instance.type = InstanceType.LOCAL;

package/src/operatorManager.ts

@@ -12,21 +12,26 @@ import { definitionsManager } from './definitionsManager';
 import { getBindHost, normalizeKapetaUri } from './utils/utils';
 import _ from 'lodash';
 import AsyncLock from 'async-lock';
+import { taskManager } from './taskManager';
 
 export const KIND_OPERATOR = 'core/resource-type-operator';
 
 class Operator {
-    private _data: any;
-    constructor(data: any) {
+    private readonly _data: DefinitionInfo;
+    constructor(data: DefinitionInfo) {
         this._data = data;
     }
 
-    getData() {
+    getLocalData() {
+        return this._data.definition.spec.local;
+    }
+
+    getDefinitionInfo() {
         return this._data;
     }
 
     getCredentials() {
-        return this._data.credentials;
+        return this._data.definition.spec.local.credentials;
     }
 }
 
@@ -72,7 +77,7 @@ class OperatorManager {
             throw new Error(`Operator missing local definition: ${resourceType}:${version}`);
         }
 
-        return new Operator(operator.definition.spec.local);
+        return new Operator(operator);
     }
 
     /**
@@ -161,7 +166,7 @@ class OperatorManager {
         return await this.operatorLock.acquire(key, async () => {
             const operator = this.getOperator(resourceType, version);
 
-            const operatorData = operator.getData();
+            const operatorData = operator.getLocalData();
 
             const portTypes = Object.keys(operatorData.ports);
 
@@ -203,6 +208,8 @@ class OperatorManager {
                 kapeta: 'true',
             };
 
+            const operatorMetadata = operator.getDefinitionInfo().definition.metadata;
+
             const bindHost = getBindHost();
 
             const ExposedPorts: { [key: string]: any } = {};
@@ -225,28 +232,41 @@ class OperatorManager {
                 Env.push(name + '=' + value);
             });
 
-            let HealthCheck = undefined;
-
-            if (operatorData.health) {
-                HealthCheck = containerManager.toDockerHealth(operatorData.health);
-            }
-
-            const container = await containerManager.ensureContainer({
-                name: containerName,
-                Image: operatorData.image,
-                Hostname: containerName + '.kapeta',
-                Labels,
-                Cmd: operatorData.cmd,
-                ExposedPorts,
-                Env,
-                HealthCheck,
-                HostConfig: {
-                    PortBindings,
-                    Mounts,
+            const task = taskManager.add(
+                `operator:ensure:${key}`,
+                async () => {
+                    let HealthCheck = undefined;
+
+                    if (operatorData.health) {
+                        HealthCheck = containerManager.toDockerHealth(operatorData.health);
+                    }
+
+                    const container = await containerManager.ensureContainer({
+                        name: containerName,
+                        Image: operatorData.image,
+                        Hostname: containerName + '.kapeta',
+                        Labels,
+                        Cmd: operatorData.cmd,
+                        ExposedPorts,
+                        Env,
+                        HealthCheck,
+                        HostConfig: {
+                            PortBindings,
+                            Mounts,
+                        },
+                    });
+
+                    await containerManager.waitForReady(container);
+
+                    return new ContainerInfo(container);
                 },
-            });
+                {
+                    name: `Ensuring ${operatorMetadata.title ?? operatorMetadata.name}`,
+                    systemId,
+                }
+            );
 
-            return new ContainerInfo(container);
+            return task.wait();
         });
     }
 }

package/src/providerManager.ts

@@ -23,7 +23,7 @@ class ProviderManager {
             return FSExtra.readFile(this._webAssetCache[id], 'utf8');
         }
 
-        await repositoryManager.ensureAsset(handle, name, version);
+        await repositoryManager.ensureAsset(handle, name, version, true);
 
         const installedProvider = this.getWebProviders().find((providerDefinition) => {
             return providerDefinition.definition.metadata.name === fullName && providerDefinition.version === version;

package/src/repositoryManager.ts

@@ -10,6 +10,9 @@ import { progressListener } from './progressListener';
 import { Dependency } from '@kapeta/schemas';
 import { Actions, Config, RegistryService } from '@kapeta/nodejs-registry-utils';
 import { definitionsManager } from './definitionsManager';
+import { Task, taskManager } from './taskManager';
+import { normalizeKapetaUri } from './utils/utils';
+import { assetManager } from './assetManager';
 
 const INSTALL_ATTEMPTED: { [p: string]: boolean } = {};
 
@@ -18,14 +21,12 @@ class RepositoryManager {
     private _registryService: RegistryService;
     private _cache: { [key: string]: boolean };
     private watcher?: () => void;
-    private _installQueue: (() => Promise<void>)[];
-    private _processing: boolean = false;
+
     constructor() {
         this.changeEventsEnabled = true;
         this.listenForChanges();
         this._registryService = new RegistryService(Config.data.registry.url);
         this._cache = {};
-        this._installQueue = [];
     }
 
     setChangeEventsEnabled(enabled: boolean) {
@@ -113,75 +114,69 @@ class RepositoryManager {
         this.watcher = undefined;
     }
 
-    private async _install(refs: string[]): Promise<void> {
+    private async _install(refs: string[]): Promise<Task[]> {
         //We make sure to only install one asset at a time - otherwise unexpected things might happen
-        const out = new Promise<void>((resolve, reject) => {
-            this._installQueue.push(async () => {
+        const createInstaller = (ref: string) => {
+            return async () => {
+                if (INSTALL_ATTEMPTED[ref]) {
+                    return;
+                }
+
+                if (definitionsManager.exists(ref)) {
+                    return;
+                }
+                console.log(`Installing asset: ${ref}`);
+                INSTALL_ATTEMPTED[ref] = true;
+                //Auto-install missing asset
                 try {
-                    const normalizedRefs = refs.map((ref) => parseKapetaUri(ref).id);
-                    const filteredRefs = normalizedRefs
-                        .filter((ref) => !INSTALL_ATTEMPTED[ref])
-                        .filter((ref) => !definitionsManager.exists(ref));
-                    if (filteredRefs.length > 0) {
-                        console.log(`Auto-installing dependencies: ${filteredRefs.join(', ')}`);
-                        filteredRefs.forEach((ref) => (INSTALL_ATTEMPTED[ref] = true));
-                        //Auto-install missing asset
-                        try {
-                            //We change to a temp dir to avoid issues with the current working directory
-                            process.chdir(os.tmpdir());
-                            //Disable change events while installing
-                            this.setChangeEventsEnabled(false);
-                            socketManager.emit(`install`, 'install:action', {
-                                type: 'start',
-                                refs,
-                            });
-                            await Actions.install(progressListener, normalizedRefs, {});
-                            socketManager.emit(`install`, 'install:action', {
-                                type: 'done',
-                                refs,
-                            });
-                        } catch (e: any) {
-                            socketManager.emit(`install`, 'install:action', {
-                                type: 'failed',
-                                refs,
-                                error: e.message,
-                            });
-                        } finally {
-                            this.setChangeEventsEnabled(true);
-                        }
-                    }
-                    resolve();
-                } catch (e) {
-                    reject(e);
+                    //We change to a temp dir to avoid issues with the current working directory
+                    process.chdir(os.tmpdir());
+                    //Disable change events while installing
+                    this.setChangeEventsEnabled(false);
+                    await Actions.install(progressListener, [ref], {});
                 } finally {
-                    this._processNext().catch((e) => console.error(e));
+                    this.setChangeEventsEnabled(true);
                 }
-            });
-        });
+                definitionsManager.clearCache();
+                assetManager.clearCache();
+                console.log(`Asset installed: ${ref}`);
+            };
+        };
 
-        this._processNext().catch((e) => console.error(e));
+        const tasks: Task[] = [];
 
-        return out;
-    }
+        while (refs.length > 0) {
+            let ref = refs.shift();
+            if (!ref) {
+                continue;
+            }
+            ref = normalizeKapetaUri(ref);
 
-    async _processNext() {
-        if (this._processing) {
-            return;
-        }
-        this._processing = true;
-        try {
-            while (this._installQueue.length > 0) {
-                const item = this._installQueue.shift();
-                if (item) {
-                    await item();
-                }
+            if (INSTALL_ATTEMPTED[ref]) {
+                continue;
             }
-        } finally {
-            this._processing = false;
+
+            if (definitionsManager.exists(ref)) {
+                continue;
+            }
+
+            const task = taskManager.add(`asset:install:${ref}`, createInstaller(ref), {
+                name: `Installing ${ref}`,
+                group: 'asset:install:',
+            });
+
+            tasks.push(task);
         }
+
+        return tasks;
     }
 
-    async ensureAsset(handle: string, name: string, version: string) {
+    async ensureAsset(
+        handle: string,
+        name: string,
+        version: string,
+        wait: boolean = true
+    ): Promise<undefined | Task[]> {
         const fullName = `${handle}/${name}`;
         const ref = `${fullName}:${version}`;
 
@@ -219,15 +214,22 @@ class RepositoryManager {
         }
 
         this._cache[ref] = true;
+        let tasks: Task[] | undefined = undefined;
         if (!installedAsset) {
-            await this._install([ref]);
+            tasks = await this._install([ref]);
         } else {
             //Ensure dependencies are installed
             const refs = assetVersion.dependencies.map((dep: Dependency) => dep.name);
             if (refs.length > 0) {
-                await this._install(refs);
+                tasks = await this._install(refs);
             }
         }
+
+        if (tasks && wait) {
+            await Promise.all(tasks.map((t) => t.future.promise));
+        }
+
+        return tasks;
     }
 }
 

package/src/socketManager.ts

@@ -32,7 +32,7 @@ export class SocketManager {
     }
 
     emitGlobal(type: string, payload: any) {
-        this.io.emit(type, { payload });
+        this.io.emit(type, payload);
     }
 
     _bindIO() {