@kapeta/local-cluster-service 0.11.0 → 0.12.0

package/src/instanceManager.ts

@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import request from 'request';
+import AsyncLock from 'async-lock';
 import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
 import { storageService } from './storageService';
 import { socketManager } from './socketManager';
@@ -8,8 +9,11 @@ import { assetManager } from './assetManager';
 import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
 import { configManager } from './configManager';
 import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceStatus, InstanceType, LogEntry } from './types';
-import { BlockInstance } from '@kapeta/schemas';
+import { BlockDefinitionSpec, BlockInstance } from '@kapeta/schemas';
 import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
+import { KIND_OPERATOR, operatorManager } from './operatorManager';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import { definitionsManager } from './definitionsManager';
 
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
@@ -25,6 +29,8 @@ export class InstanceManager {
 
     private readonly _instances: InstanceInfo[] = [];
 
+    private readonly instanceLocks: AsyncLock = new AsyncLock();
+
     constructor() {
         this._instances = storageService.section('instances', []);
 
@@ -67,8 +73,16 @@ export class InstanceManager {
         return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
     }
 
+    private async exclusive<T = any>(systemId: string, instanceId: string, fn: () => Promise<T>) {
+        systemId = normalizeKapetaUri(systemId);
+        const key = `${systemId}/${instanceId}`;
+        //console.log(`Acquiring lock for ${key}`, this.instanceLocks.isBusy(key));
+        const result = await this.instanceLocks.acquire(key, fn);
+        //console.log(`Releasing lock for ${key}`, this.instanceLocks.isBusy(key));
+        return result;
+    }
 
-    public async getLogs(systemId: string, instanceId: string):Promise<LogEntry[]> {
+    public async getLogs(systemId: string, instanceId: string): Promise<LogEntry[]> {
         const instance = this.getInstance(systemId, instanceId);
         if (!instance) {
             throw new Error(`Instance ${systemId}/${instanceId} not found`);
@@ -76,23 +90,27 @@ export class InstanceManager {
 
         switch (instance.type) {
             case InstanceType.DOCKER:
-                 return await containerManager.getLogs(instance);
+                return await containerManager.getLogs(instance);
 
             case InstanceType.UNKNOWN:
-                return [{
-                    level: 'INFO',
-                    message: 'Instance is starting...',
-                    time: Date.now(),
-                    source: 'stdout',
-                }];
+                return [
+                    {
+                        level: 'INFO',
+                        message: 'Instance is starting...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    },
+                ];
 
             case InstanceType.LOCAL:
-                return [{
-                    level: 'INFO',
-                    message: 'Instance started outside Kapeta - logs not available...',
-                    time: Date.now(),
-                    source: 'stdout',
-                }];
+                return [
+                    {
+                        level: 'INFO',
+                        message: 'Instance started outside Kapeta - logs not available...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    },
+                ];
         }
 
         return [];
@@ -142,74 +160,79 @@ export class InstanceManager {
         instanceId: string,
         info: Omit<InstanceInfo, 'systemId' | 'instanceId'>
     ) {
-        systemId = normalizeKapetaUri(systemId);
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = normalizeKapetaUri(systemId);
 
-        let instance = this.getInstance(systemId, instanceId);
+            let instance = this.getInstance(systemId, instanceId);
 
-        //Get target address
-        const address = await serviceManager.getProviderAddress(
-            systemId,
-            instanceId,
-            info.portType ?? DEFAULT_HEALTH_PORT_TYPE
-        );
-
-        const healthUrl = this.getHealthUrl(info, address);
+            //Get target address
+            const address = await serviceManager.getProviderAddress(
+                systemId,
+                instanceId,
+                info.portType ?? DEFAULT_HEALTH_PORT_TYPE
+            );
 
-        if (instance) {
-            if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
-                //If instance is stopping do not interfere
-                return;
-            }
+            const healthUrl = this.getHealthUrl(info, address);
 
-            if (info.owner === InstanceOwner.EXTERNAL) {
-                //If instance was started externally - then we want to replace the internal instance with that
+            if (instance) {
                 if (
-                    instance.owner === InstanceOwner.INTERNAL &&
-                    (instance.status === InstanceStatus.READY ||
-                        instance.status === InstanceStatus.STARTING ||
-                        instance.status === InstanceStatus.UNHEALTHY)
+                    instance.status === InstanceStatus.STOPPING &&
+                    instance.desiredStatus === DesiredInstanceStatus.STOP
                 ) {
-                    throw new Error(`Instance ${instanceId} is already running`);
+                    //If instance is stopping do not interfere
+                    return;
                 }
 
-                instance.desiredStatus = info.desiredStatus;
-                instance.owner = info.owner;
-                instance.status = InstanceStatus.STARTING;
-                instance.startedAt = Date.now();
-            }
-
-            instance.pid = info.pid;
-            instance.address = address;
-            if (info.type) {
-                instance.type = info.type;
-            }
-            if (healthUrl) {
-                instance.health = healthUrl;
-            }
+                if (info.owner === InstanceOwner.EXTERNAL) {
+                    //If instance was started externally - then we want to replace the internal instance with that
+                    if (
+                        instance.owner === InstanceOwner.INTERNAL &&
+                        (instance.status === InstanceStatus.READY ||
+                            instance.status === InstanceStatus.STARTING ||
+                            instance.status === InstanceStatus.UNHEALTHY)
+                    ) {
+                        throw new Error(`Instance ${instanceId} is already running`);
+                    }
 
-            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
-        } else {
-            //If instance was not found - then we're receiving an externally started instance
-            instance = {
-                ...info,
-                systemId,
-                instanceId,
-                status: InstanceStatus.STARTING,
-                startedAt: Date.now(),
-                desiredStatus: DesiredInstanceStatus.EXTERNAL,
-                owner: InstanceOwner.EXTERNAL,
-                health: healthUrl,
-                address,
-            };
+                    instance.desiredStatus = info.desiredStatus;
+                    instance.owner = info.owner;
+                    instance.status = InstanceStatus.STARTING;
+                    instance.startedAt = Date.now();
+                }
 
-            this._instances.push(instance);
+                instance.pid = info.pid;
+                instance.address = address;
+                if (info.type) {
+                    instance.type = info.type;
+                }
+                if (healthUrl) {
+                    instance.health = healthUrl;
+                }
 
-            this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
-        }
+                this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            } else {
+                //If instance was not found - then we're receiving an externally started instance
+                instance = {
+                    ...info,
+                    systemId,
+                    instanceId,
+                    status: InstanceStatus.STARTING,
+                    startedAt: Date.now(),
+                    desiredStatus: DesiredInstanceStatus.EXTERNAL,
+                    owner: InstanceOwner.EXTERNAL,
+                    health: healthUrl,
+                    address,
+                };
+
+                this._instances.push(instance);
+
+                this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
+            }
 
-        this.save();
+            this.save();
 
-        return instance;
+            return instance;
+        });
     }
 
     private getHealthUrl(info: Omit<InstanceInfo, 'systemId' | 'instanceId'>, address: string) {
@@ -225,15 +248,17 @@ export class InstanceManager {
     }
 
     public markAsStopped(systemId: string, instanceId: string) {
-        systemId = normalizeKapetaUri(systemId);
-        const instance = _.find(this._instances, { systemId, instanceId });
-        if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
-            instance.status = InstanceStatus.STOPPED;
-            instance.pid = null;
-            instance.health = null;
-            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
-            this.save();
-        }
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = normalizeKapetaUri(systemId);
+            const instance = _.find(this._instances, { systemId, instanceId });
+            if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
+                instance.status = InstanceStatus.STOPPED;
+                instance.pid = null;
+                instance.health = null;
+                this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                this.save();
+            }
+        });
     }
 
     public async startAllForPlan(systemId: string): Promise<InstanceInfo[]> {
@@ -268,58 +293,64 @@ export class InstanceManager {
     }
 
     public async stop(systemId: string, instanceId: string) {
-        systemId = normalizeKapetaUri(systemId);
-        const instance = this.getInstance(systemId, instanceId);
-        if (!instance) {
-            return;
-        }
+        return this.stopInner(systemId, instanceId, true);
+    }
 
-        if (instance.status === InstanceStatus.STOPPED) {
-            return;
-        }
+    private async stopInner(systemId: string, instanceId: string, changeDesired: boolean = false) {
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = normalizeKapetaUri(systemId);
+            const instance = this.getInstance(systemId, instanceId);
+            if (!instance) {
+                return;
+            }
 
-        if (instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
-            instance.desiredStatus = DesiredInstanceStatus.STOP;
-        }
+            if (instance.status === InstanceStatus.STOPPED) {
+                return;
+            }
 
-        instance.status = InstanceStatus.STOPPING;
+            if (changeDesired && instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
+                instance.desiredStatus = DesiredInstanceStatus.STOP;
+            }
 
-        this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
-        console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
-        this.save();
+            instance.status = InstanceStatus.STOPPING;
 
-        try {
-            if (instance.type === 'docker') {
-                const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
-                const container = await containerManager.getContainerByName(containerName);
-                if (container) {
-                    try {
-                        await container.stop();
-                        instance.status = InstanceStatus.STOPPED;
-                        this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
-                        this.save();
-                    } catch (e) {
-                        console.error('Failed to stop container', e);
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+            this.save();
+
+            try {
+                if (instance.type === 'docker') {
+                    const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
+                    const container = await containerManager.getContainerByName(containerName);
+                    if (container) {
+                        try {
+                            await container.stop();
+                            instance.status = InstanceStatus.STOPPED;
+                            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                            this.save();
+                        } catch (e) {
+                            console.error('Failed to stop container', e);
+                        }
+                    } else {
+                        console.warn('Container not found', containerName);
                     }
-                } else {
-                    console.warn('Container not found', containerName);
+                    return;
                 }
-                return;
-            }
 
-            if (!instance.pid) {
+                if (!instance.pid) {
+                    instance.status = InstanceStatus.STOPPED;
+                    this.save();
+                    return;
+                }
+
+                process.kill(instance.pid as number, 'SIGTERM');
                 instance.status = InstanceStatus.STOPPED;
+                this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
                 this.save();
-                return;
+            } catch (e) {
+                console.error('Failed to stop process', e);
             }
-
-            process.kill(instance.pid as number, 'SIGTERM');
-            instance.status = InstanceStatus.STOPPED;
-            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
-            this.save();
-        } catch (e) {
-            console.error('Failed to stop process', e);
-        }
+        });
     }
 
     public async stopAllForPlan(systemId: string) {
@@ -330,128 +361,158 @@ export class InstanceManager {
     }
 
     public async start(systemId: string, instanceId: string): Promise<InstanceInfo> {
-        systemId = normalizeKapetaUri(systemId);
-        const plan = await assetManager.getPlan(systemId, true);
-        if (!plan) {
-            throw new Error('Plan not found: ' + systemId);
-        }
-
-        const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
-        if (!blockInstance) {
-            throw new Error('Block instance not found: ' + instanceId);
-        }
-
-        const blockRef = normalizeKapetaUri(blockInstance.block.ref);
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = normalizeKapetaUri(systemId);
+            const plan = await assetManager.getPlan(systemId, true);
+            if (!plan) {
+                throw new Error('Plan not found: ' + systemId);
+            }
 
-        const blockAsset = await assetManager.getAsset(blockRef, true);
-        if (!blockAsset) {
-            throw new Error('Block not found: ' + blockRef);
-        }
+            const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
+            if (!blockInstance) {
+                throw new Error('Block instance not found: ' + instanceId);
+            }
 
-        const existingInstance = this.getInstance(systemId, instanceId);
+            const blockRef = normalizeKapetaUri(blockInstance.block.ref);
 
-        if (existingInstance) {
-            if (existingInstance.status === InstanceStatus.READY) {
-                // Instance is already running
-                return existingInstance;
+            const blockAsset = await assetManager.getAsset(blockRef, true);
+            if (!blockAsset) {
+                throw new Error('Block not found: ' + blockRef);
             }
 
-            if (
-                existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
-                existingInstance.status === InstanceStatus.STARTING
-            ) {
-                // Internal instance is already starting - don't start it again
-                return existingInstance;
-            }
+            const existingInstance = this.getInstance(systemId, instanceId);
 
-            if (
-                existingInstance.owner === InstanceOwner.EXTERNAL &&
-                existingInstance.status === InstanceStatus.STARTING
-            ) {
-                // External instance is already starting - don't start it again
-                return existingInstance;
-            }
-        }
+            if (existingInstance) {
+                if (existingInstance.status === InstanceStatus.READY) {
+                    // Instance is already running
+                    return existingInstance;
+                }
 
-        let instance: InstanceInfo = {
-            systemId,
-            instanceId,
-            ref: blockRef,
-            name: blockAsset.data.metadata.name,
-            desiredStatus: DesiredInstanceStatus.RUN,
-            owner: InstanceOwner.INTERNAL,
-            type: existingInstance?.type ?? InstanceType.UNKNOWN,
-            status: InstanceStatus.STARTING,
-            startedAt: Date.now(),
-        };
-
-        console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
-        // Save the instance before starting it, so that we can track the status
-        await this.saveInternalInstance(instance);
+                if (
+                    existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
+                    existingInstance.status === InstanceStatus.STARTING
+                ) {
+                    // Internal instance is already starting - don't start it again
+                    return existingInstance;
+                }
 
-        if (existingInstance) {
-            // Check if the instance is already running - but after we've commmuicated the desired status
-            const currentStatus = await this.requestInstanceStatus(existingInstance);
-            if (currentStatus === InstanceStatus.READY) {
-                // Instance is already running
-                return existingInstance;
+                if (
+                    existingInstance.owner === InstanceOwner.EXTERNAL &&
+                    existingInstance.status === InstanceStatus.STARTING
+                ) {
+                    // External instance is already starting - don't start it again
+                    return existingInstance;
+                }
             }
-        }
 
-        const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
-        const runner = new BlockInstanceRunner(systemId);
+            let instance: InstanceInfo = {
+                systemId,
+                instanceId,
+                ref: blockRef,
+                name: blockAsset.data.metadata.name,
+                desiredStatus: DesiredInstanceStatus.RUN,
+                owner: InstanceOwner.INTERNAL,
+                type: existingInstance?.type ?? InstanceType.UNKNOWN,
+                status: InstanceStatus.STARTING,
+                startedAt: Date.now(),
+            };
 
-        const startTime = Date.now();
-        try {
-            const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
+            console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+            // Save the instance before starting it, so that we can track the status
+            await this.saveInternalInstance(instance);
+
+            const blockSpec = blockAsset.data.spec as BlockDefinitionSpec;
+            if (blockSpec.consumers) {
+                const promises = blockSpec.consumers.map((consumer) => {
+                    const consumerUri = parseKapetaUri(consumer.kind);
+                    const asset = definitionsManager.getDefinition(consumer.kind);
+                    if (!asset) {
+                        // Definition not found
+                        return Promise.resolve();
+                    }
 
-            instance.status = InstanceStatus.READY;
+                    if (KIND_OPERATOR.toLowerCase() !== asset.definition.kind.toLowerCase()) {
+                        // Not an operator
+                        return Promise.resolve();
+                    }
 
-            return this.saveInternalInstance({
-                ...instance,
-                type: processInfo.type,
-                pid: processInfo.pid ?? -1,
-                health: null,
-                portType: processInfo.portType,
-                status: InstanceStatus.READY,
-            });
-        } catch (e: any) {
-            console.warn('Failed to start instance', e);
-            const logs: LogEntry[] = [
-                {
-                    source: 'stdout',
-                    level: 'ERROR',
-                    message: e.message,
-                    time: Date.now(),
-                },
-            ];
-
-            const out = await this.saveInternalInstance({
-                ...instance,
-                type: InstanceType.LOCAL,
-                pid: null,
-                health: null,
-                portType: DEFAULT_HEALTH_PORT_TYPE,
-                status: InstanceStatus.FAILED,
-            });
+                    console.log('Ensuring resource: %s in %s', consumerUri.id, systemId);
+                    return operatorManager.ensureResource(systemId, consumerUri.fullName, consumerUri.version);
+                });
 
-            this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
+                await Promise.all(promises);
+            }
 
-            this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
-                error: `Failed to start instance: ${e.message}`,
-                status: EVENT_INSTANCE_EXITED,
-                instanceId: blockInstance.id,
-            });
+            if (existingInstance) {
+                // Check if the instance is already running - but after we've commmuicated the desired status
+                const currentStatus = await this.requestInstanceStatus(existingInstance);
+                if (currentStatus === InstanceStatus.READY) {
+                    // Instance is already running
+                    return existingInstance;
+                }
+            }
 
-            return out;
-        }
+            const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
+            const runner = new BlockInstanceRunner(systemId);
+
+            const startTime = Date.now();
+            try {
+                const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
+
+                instance.status = InstanceStatus.READY;
+
+                return this.saveInternalInstance({
+                    ...instance,
+                    type: processInfo.type,
+                    pid: processInfo.pid ?? -1,
+                    health: null,
+                    portType: processInfo.portType,
+                    status: InstanceStatus.READY,
+                });
+            } catch (e: any) {
+                console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
+                const logs: LogEntry[] = [
+                    {
+                        source: 'stdout',
+                        level: 'ERROR',
+                        message: e.message,
+                        time: Date.now(),
+                    },
+                ];
+
+                const out = await this.saveInternalInstance({
+                    ...instance,
+                    type: InstanceType.LOCAL,
+                    pid: null,
+                    health: null,
+                    portType: DEFAULT_HEALTH_PORT_TYPE,
+                    status: InstanceStatus.FAILED,
+                    errorMessage: e.message ?? 'Failed to start - Check logs for details.',
+                });
+
+                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
+
+                this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
+                    error: `Failed to start instance: ${e.message}`,
+                    status: EVENT_INSTANCE_EXITED,
+                    instanceId: blockInstance.id,
+                });
+
+                return out;
+            }
+        });
     }
 
-    public async restart(systemId: string, instanceId: string) {
+    /**
+     * Stops an instance but does not remove it from the list of active instances
+     *
+     * It will be started again next time the system checks the status of the instance
+     *
+     * We do it this way to not cause the user to wait for the instance to start again
+     */
+    public async prepareForRestart(systemId: string, instanceId: string) {
         systemId = normalizeKapetaUri(systemId);
-        await this.stop(systemId, instanceId);
-
-        return this.start(systemId, instanceId);
+        await this.stopInner(systemId, instanceId);
     }
 
     public async stopAll() {
@@ -483,7 +544,7 @@ export class InstanceManager {
         const all = [...this._instances];
         while (all.length > 0) {
             // Check a few instances at a time - docker doesn't like too many concurrent requests
-            const chunk = all.splice(0, 20);
+            const chunk = all.splice(0, 30);
             const promises = chunk.map(async (instance) => {
                 if (!instance.systemId) {
                     return;
@@ -523,8 +584,7 @@ export class InstanceManager {
                     const skipUpdate =
                         (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
                         ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
-                            instance.status === InstanceStatus.STOPPING &&
-                            instance.desiredStatus === DesiredInstanceStatus.STOP) ||
+                            instance.status === InstanceStatus.STOPPING) ||
                         (newStatus === InstanceStatus.STOPPED &&
                             instance.status === InstanceStatus.STARTING &&
                             instance.desiredStatus === DesiredInstanceStatus.RUN);
@@ -563,7 +623,7 @@ export class InstanceManager {
                 ) {
                     //If the instance is running but we want it to stop, stop it
                     try {
-                        await this.stop(instance.systemId, instance.instanceId);
+                        await this.stopInner(instance.systemId, instance.instanceId);
                     } catch (e) {
                         console.warn('Failed to stop instance', instance.systemId, instance.instanceId, e);
                     }
@@ -578,7 +638,7 @@ export class InstanceManager {
                     //If the instance is unhealthy, try to restart it
                     console.log('Restarting unhealthy instance', instance);
                     try {
-                        await this.restart(instance.systemId, instance.instanceId);
+                        await this.prepareForRestart(instance.systemId, instance.instanceId);
                     } catch (e) {
                         console.warn('Failed to restart instance', instance.systemId, instance.instanceId, e);
                     }

package/src/instances/routes.ts

@@ -86,7 +86,7 @@ router.get('/:systemId/:instanceId/logs', async (req: Request, res: Response) =>
         res.status(200).send({
             logs,
         });
-    } catch (e:any) {
+    } catch (e: any) {
         res.status(500).send({ ok: false, error: e.message });
     }
 });