@kapeta/local-cluster-service 0.8.3 → 0.9.1

package/dist/esm/src/instanceManager.js

@@ -1,117 +1,37 @@
 import _ from 'lodash';
 import request from 'request';
-import EventEmitter from 'events';
 import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
 import { storageService } from './storageService';
 import { socketManager } from './socketManager';
 import { serviceManager } from './serviceManager';
 import { assetManager } from './assetManager';
-import { containerManager } from './containerManager';
+import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
 import { configManager } from './configManager';
-const CHECK_INTERVAL = 10000;
+import { DesiredInstanceStatus, InstanceOwner, InstanceStatus, InstanceType } from './types';
+import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
+const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
 const EVENT_STATUS_CHANGED = 'status-changed';
 const EVENT_INSTANCE_CREATED = 'instance-created';
 const EVENT_INSTANCE_EXITED = 'instance-exited';
 const EVENT_INSTANCE_LOG = 'instance-log';
-const STATUS_STARTING = 'starting';
-const STATUS_READY = 'ready';
-const STATUS_UNHEALTHY = 'unhealthy';
-const STATUS_STOPPED = 'stopped';
 const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
-class InstanceManager {
-    _interval;
-    /**
-     * Contains an array of running instances that have self-registered with this
-     * cluster service. This is done by the Kapeta SDKs
-     */
+export class InstanceManager {
+    _interval = undefined;
     _instances = [];
-    /**
-     * Contains the process info for the instances started by this manager. In memory only
-     * so can't be relied on for knowing everything that's running.
-     *
-     */
-    _processes = {};
     constructor() {
-        this._interval = setInterval(() => this._checkInstances(), CHECK_INTERVAL);
         this._instances = storageService.section('instances', []);
-        this._processes = {};
-        this._checkInstances();
-    }
-    _save() {
-        storageService.put('instances', this._instances);
-    }
-    async _checkInstances() {
-        let changed = false;
-        for (let i = 0; i < this._instances.length; i++) {
-            const instance = this._instances[i];
-            const newStatus = await this._getInstanceStatus(instance);
-            if (newStatus === STATUS_UNHEALTHY && instance.status === STATUS_STARTING) {
-                // If instance is starting we consider unhealthy an indication
-                // that it is still starting
-                continue;
-            }
-            if (instance.status !== newStatus) {
-                instance.status = newStatus;
-                console.log('Instance status changed: %s %s -> %s', instance.systemId, instance.instanceId, instance.status);
-                this._emit(instance.systemId, EVENT_STATUS_CHANGED, instance);
-                changed = true;
-            }
-        }
-        if (changed) {
-            this._save();
-        }
+        // We need to wait a bit before running the first check
+        this.checkInstancesLater(1000);
     }
-    async _isRunning(instance) {
-        if (!instance.pid) {
-            return;
-        }
-        if (instance.type === 'docker') {
-            const container = await containerManager.get(instance.pid);
-            if (!container) {
-                console.warn('Container not found: %s', instance.pid);
-                return false;
-            }
-            return await container.isRunning();
-        }
-        //Otherwise its just a normal process.
-        //TODO: Handle for Windows
-        try {
-            return process.kill(instance.pid, 0);
-        }
-        catch (err) {
-            return err.code === 'EPERM';
-        }
-    }
-    async _getInstanceStatus(instance) {
-        if (instance.status === STATUS_STOPPED) {
-            //Will only change when it reregisters
-            return STATUS_STOPPED;
-        }
-        if (!(await this._isRunning(instance))) {
-            return STATUS_STOPPED;
-        }
-        if (!instance.health) {
-            //No health url means we assume it's healthy as soon as it's running
-            return STATUS_READY;
-        }
-        return new Promise((resolve) => {
-            if (!instance.health) {
-                resolve(STATUS_READY);
-                return;
-            }
-            request(instance.health, (err, response) => {
-                if (err) {
-                    resolve(STATUS_UNHEALTHY);
-                    return;
-                }
-                if (response.statusCode > 399) {
-                    resolve(STATUS_UNHEALTHY);
-                    return;
-                }
-                resolve(STATUS_READY);
-            });
-        });
+    checkInstancesLater(time = CHECK_INTERVAL) {
+        if (this._interval) {
+            clearTimeout(this._interval);
+        }
+        this._interval = setTimeout(async () => {
+            await this.checkInstances();
+            this.checkInstancesLater();
+        }, time);
     }
     getInstances() {
         if (!this._instances) {
@@ -123,39 +43,67 @@ class InstanceManager {
         if (!this._instances) {
             return [];
         }
+        systemId = normalizeKapetaUri(systemId);
         return this._instances.filter((instance) => instance.systemId === systemId);
     }
-    /**
-     * Get instance information
-     *
-     * @param {string} systemId
-     * @param {string} instanceId
-     * @return {*}
-     */
     getInstance(systemId, instanceId) {
-        return _.find(this._instances, { systemId, instanceId });
+        systemId = normalizeKapetaUri(systemId);
+        return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
+    }
+    async saveInternalInstance(instance) {
+        instance.systemId = normalizeKapetaUri(instance.systemId);
+        if (instance.ref) {
+            instance.ref = normalizeKapetaUri(instance.ref);
+        }
+        //Get target address
+        let address = await serviceManager.getProviderAddress(instance.systemId, instance.instanceId, instance.portType ?? DEFAULT_HEALTH_PORT_TYPE);
+        const healthUrl = this.getHealthUrl(instance, address);
+        instance.address = address;
+        if (healthUrl) {
+            instance.health = healthUrl;
+        }
+        let existingInstance = this.getInstance(instance.systemId, instance.instanceId);
+        if (existingInstance) {
+            const ix = this._instances.indexOf(existingInstance);
+            this._instances.splice(ix, 1, instance);
+            this.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+        }
+        else {
+            this._instances.push(instance);
+            this.emitSystemEvent(instance.systemId, EVENT_INSTANCE_CREATED, instance);
+        }
+        this.save();
+        return instance;
     }
     /**
-     *
-     * @param {string} systemId
-     * @param {string} instanceId
-     * @param {InstanceInfo} info
-     * @return {Promise<void>}
+     * Method is called when instance is started from the Kapeta SDKs (e.g. NodeJS SDK)
+     * which self-registers with the cluster service locally on startup.
      */
-    async registerInstance(systemId, instanceId, info) {
+    async registerInstanceFromSDK(systemId, instanceId, info) {
+        systemId = normalizeKapetaUri(systemId);
         let instance = this.getInstance(systemId, instanceId);
         //Get target address
-        let address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
-        let healthUrl = null;
-        let health = info.health;
-        if (health) {
-            if (health.startsWith('/')) {
-                health = health.substring(1);
-            }
-            healthUrl = address + health;
-        }
+        const address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
+        const healthUrl = this.getHealthUrl(info, address);
         if (instance) {
-            instance.status = STATUS_STARTING;
+            if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
+                //If instance is stopping do not interfere
+                return;
+            }
+            if (info.owner === InstanceOwner.EXTERNAL) {
+                //If instance was started externally - then we want to replace the internal instance with that
+                if (instance.owner === InstanceOwner.INTERNAL &&
+                    (instance.status === InstanceStatus.READY ||
+                        instance.status === InstanceStatus.STARTING ||
+                        instance.status === InstanceStatus.UNHEALTHY)) {
+                    throw new Error(`Instance ${instanceId} is already running`);
+                }
+                instance.desiredStatus = info.desiredStatus;
+                instance.owner = info.owner;
+                instance.internal = undefined;
+                instance.status = InstanceStatus.STARTING;
+                instance.startedAt = Date.now();
+            }
             instance.pid = info.pid;
             instance.address = address;
             if (info.type) {
@@ -164,56 +112,64 @@ class InstanceManager {
             if (healthUrl) {
                 instance.health = healthUrl;
             }
-            this._emit(systemId, EVENT_STATUS_CHANGED, instance);
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
         }
         else {
+            //If instance was not found - then we're receiving an externally started instance
             instance = {
+                ...info,
                 systemId,
                 instanceId,
-                status: STATUS_STARTING,
-                pid: info.pid,
-                type: info.type,
+                status: InstanceStatus.STARTING,
+                startedAt: Date.now(),
+                desiredStatus: DesiredInstanceStatus.EXTERNAL,
+                owner: InstanceOwner.EXTERNAL,
                 health: healthUrl,
                 address,
             };
             this._instances.push(instance);
-            this._emit(systemId, EVENT_INSTANCE_CREATED, instance);
+            this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
+        }
+        this.save();
+        return instance;
+    }
+    getHealthUrl(info, address) {
+        let healthUrl = null;
+        let health = info.health;
+        if (health) {
+            if (health.startsWith('/')) {
+                health = health.substring(1);
+            }
+            healthUrl = address + health;
         }
-        this._save();
+        return healthUrl;
     }
-    setInstanceAsStopped(systemId, instanceId) {
+    markAsStopped(systemId, instanceId) {
+        systemId = normalizeKapetaUri(systemId);
         const instance = _.find(this._instances, { systemId, instanceId });
-        if (instance) {
-            instance.status = STATUS_STOPPED;
+        if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
+            instance.status = InstanceStatus.STOPPED;
             instance.pid = null;
             instance.health = null;
-            this._emit(systemId, EVENT_STATUS_CHANGED, instance);
-            this._save();
-        }
-    }
-    _emit(systemId, type, payload) {
-        try {
-            socketManager.emit(`${systemId}/instances`, type, payload);
-        }
-        catch (e) {
-            console.warn('Failed to emit instance event: %s', e.message);
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            this.save();
         }
     }
-    async createProcessesForPlan(planRef) {
-        await this.stopAllForPlan(planRef);
-        const plan = await assetManager.getPlan(planRef, true);
+    async startAllForPlan(systemId) {
+        systemId = normalizeKapetaUri(systemId);
+        const plan = await assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + planRef);
+            throw new Error('Plan not found: ' + systemId);
         }
         if (!plan.spec.blocks) {
-            console.warn('No blocks found in plan', planRef);
+            console.warn('No blocks found in plan', systemId);
             return [];
         }
         let promises = [];
         let errors = [];
         for (let blockInstance of Object.values(plan.spec.blocks)) {
             try {
-                promises.push(this.createProcess(planRef, blockInstance.id));
+                promises.push(this.start(systemId, blockInstance.id));
             }
             catch (e) {
                 errors.push(e);
@@ -225,88 +181,134 @@ class InstanceManager {
         }
         return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p);
     }
-    async _stopInstance(instance) {
-        if (!instance.pid) {
+    async stop(systemId, instanceId) {
+        systemId = normalizeKapetaUri(systemId);
+        const instance = this.getInstance(systemId, instanceId);
+        if (!instance) {
             return;
         }
-        if (instance.status === 'stopped') {
+        if (instance.status === InstanceStatus.STOPPED) {
             return;
         }
+        if (instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
+            instance.desiredStatus = DesiredInstanceStatus.STOP;
+        }
+        instance.status = InstanceStatus.STOPPING;
+        this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+        console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+        this.save();
         try {
             if (instance.type === 'docker') {
-                const container = await containerManager.get(instance.pid);
+                const containerName = getBlockInstanceContainerName(instance.instanceId);
+                const container = await containerManager.getContainerByName(containerName);
                 if (container) {
                     try {
                         await container.stop();
+                        instance.status = InstanceStatus.STOPPED;
+                        this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                        this.save();
                     }
                     catch (e) {
                         console.error('Failed to stop container', e);
                     }
                 }
+                else {
+                    console.warn('Container not found', containerName);
+                }
+                return;
+            }
+            if (!instance.pid) {
+                instance.status = InstanceStatus.STOPPED;
+                this.save();
                 return;
             }
             process.kill(instance.pid, 'SIGTERM');
+            instance.status = InstanceStatus.STOPPED;
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            this.save();
         }
         catch (e) {
             console.error('Failed to stop process', e);
         }
     }
-    async stopAllForPlan(planRef) {
-        if (this._processes[planRef]) {
-            const promises = [];
-            console.log('Stopping all processes for plan', planRef);
-            for (let instance of Object.values(this._processes[planRef])) {
-                promises.push(instance.stop());
-            }
-            await Promise.all(promises);
-            this._processes[planRef] = {};
-        }
-        //Also stop instances not being maintained by the cluster service
-        const instancesForPlan = this._instances.filter((instance) => instance.systemId === planRef);
-        const promises = [];
-        for (let instance of instancesForPlan) {
-            promises.push(this._stopInstance(instance));
-        }
-        await Promise.all(promises);
+    async stopAllForPlan(systemId) {
+        systemId = normalizeKapetaUri(systemId);
+        const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
+        return this.stopInstances(instancesForPlan);
     }
-    async createProcess(planRef, instanceId) {
-        const plan = await assetManager.getPlan(planRef, true);
+    async start(systemId, instanceId) {
+        systemId = normalizeKapetaUri(systemId);
+        const plan = await assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + planRef);
+            throw new Error('Plan not found: ' + systemId);
         }
         const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
         if (!blockInstance) {
             throw new Error('Block instance not found: ' + instanceId);
         }
-        const blockRef = blockInstance.block.ref;
+        const blockRef = normalizeKapetaUri(blockInstance.block.ref);
         const blockAsset = await assetManager.getAsset(blockRef, true);
-        const instanceConfig = await configManager.getConfigForSection(planRef, instanceId);
         if (!blockAsset) {
             throw new Error('Block not found: ' + blockRef);
         }
-        if (!this._processes[planRef]) {
-            this._processes[planRef] = {};
+        const existingInstance = this.getInstance(systemId, instanceId);
+        if (existingInstance) {
+            if (existingInstance.status === InstanceStatus.READY) {
+                // Instance is already running
+                return existingInstance;
+            }
+            if (existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
+                existingInstance.status === InstanceStatus.STARTING) {
+                // Internal instance is already starting - don't start it again
+                return existingInstance;
+            }
+            if (existingInstance.owner === InstanceOwner.EXTERNAL &&
+                existingInstance.status === InstanceStatus.STARTING) {
+                // External instance is already starting - don't start it again
+                return existingInstance;
+            }
         }
-        await this.stopProcess(planRef, instanceId);
-        const type = blockAsset.version === 'local' ? 'local' : 'docker';
-        const runner = new BlockInstanceRunner(planRef);
+        let instance = {
+            systemId,
+            instanceId,
+            ref: blockRef,
+            name: blockAsset.data.metadata.name,
+            desiredStatus: DesiredInstanceStatus.RUN,
+            owner: InstanceOwner.INTERNAL,
+            type: InstanceType.UNKNOWN,
+            status: InstanceStatus.STARTING,
+            startedAt: Date.now(),
+        };
+        console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+        // Save the instance before starting it, so that we can track the status
+        await this.saveInternalInstance(instance);
+        if (existingInstance) {
+            // Check if the instance is already running - but after we've commmuicated the desired status
+            const currentStatus = await this.requestInstanceStatus(existingInstance);
+            if (currentStatus === InstanceStatus.READY) {
+                // Instance is already running
+                return existingInstance;
+            }
+        }
+        const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
+        const runner = new BlockInstanceRunner(systemId);
         const startTime = Date.now();
         try {
-            const process = await runner.start(blockRef, instanceId, instanceConfig);
+            const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
             //emit stdout/stderr via sockets
-            process.output.on('data', (data) => {
+            processInfo.output.on('data', (data) => {
                 const payload = {
                     source: 'stdout',
                     level: 'INFO',
                     message: data.toString(),
                     time: Date.now(),
                 };
-                this._emit(instanceId, EVENT_INSTANCE_LOG, payload);
+                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, payload);
             });
-            process.output.on('exit', (exitCode) => {
+            processInfo.output.on('exit', (exitCode) => {
                 const timeRunning = Date.now() - startTime;
-                const instance = this.getInstance(planRef, instanceId);
-                if (instance?.status === STATUS_READY) {
+                const instance = this.getInstance(systemId, instanceId);
+                if (instance?.status === InstanceStatus.READY) {
                     //It's already been running
                     return;
                 }
@@ -316,21 +318,31 @@ class InstanceManager {
                     return;
                 }
                 if (exitCode !== 0 || timeRunning < MIN_TIME_RUNNING) {
-                    this._emit(blockInstance.id, EVENT_INSTANCE_EXITED, {
+                    const instance = this.getInstance(systemId, instanceId);
+                    if (instance) {
+                        instance.status = InstanceStatus.FAILED;
+                        this.save();
+                    }
+                    this.emitSystemEvent(systemId, EVENT_INSTANCE_EXITED, {
                         error: 'Failed to start instance',
                         status: EVENT_INSTANCE_EXITED,
                         instanceId: blockInstance.id,
                     });
                 }
             });
-            await this.registerInstance(planRef, instanceId, {
-                type: process.type,
-                pid: process.pid ?? -1,
+            instance.status = InstanceStatus.READY;
+            return this.saveInternalInstance({
+                ...instance,
+                type: processInfo.type,
+                pid: processInfo.pid ?? -1,
                 health: null,
-                portType: process.portType,
-                status: STATUS_STARTING,
+                portType: processInfo.portType,
+                status: InstanceStatus.READY,
+                internal: {
+                    logs: processInfo.logs,
+                    output: processInfo.output,
+                },
             });
-            return (this._processes[planRef][instanceId] = process);
         }
         catch (e) {
             console.warn('Failed to start instance', e);
@@ -342,77 +354,245 @@ class InstanceManager {
                     time: Date.now(),
                 },
             ];
-            await this.registerInstance(planRef, instanceId, {
-                type: 'local',
+            const out = await this.saveInternalInstance({
+                ...instance,
+                type: InstanceType.LOCAL,
                 pid: null,
                 health: null,
                 portType: DEFAULT_HEALTH_PORT_TYPE,
-                status: STATUS_UNHEALTHY,
+                status: InstanceStatus.FAILED,
             });
-            this._emit(instanceId, EVENT_INSTANCE_LOG, logs[0]);
-            this._emit(blockInstance.id, EVENT_INSTANCE_EXITED, {
+            this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
+            this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
                 error: `Failed to start instance: ${e.message}`,
                 status: EVENT_INSTANCE_EXITED,
                 instanceId: blockInstance.id,
             });
-            return (this._processes[planRef][instanceId] = {
-                pid: -1,
-                type,
-                logs: () => logs,
-                stop: () => Promise.resolve(),
-                ref: blockRef,
-                id: instanceId,
-                name: blockInstance.name,
-                output: new EventEmitter(),
-            });
+            return out;
         }
     }
-    /**
-     *
-     * @param {string} planRef
-     * @param {string} instanceId
-     * @return {ProcessInfo|null}
-     */
-    getProcessForInstance(planRef, instanceId) {
-        if (!this._processes[planRef]) {
-            return null;
+    async restart(systemId, instanceId) {
+        systemId = normalizeKapetaUri(systemId);
+        await this.stop(systemId, instanceId);
+        return this.start(systemId, instanceId);
+    }
+    async stopAll() {
+        return this.stopInstances(this._instances);
+    }
+    async stopInstances(instances) {
+        const promises = instances.map((instance) => this.stop(instance.systemId, instance.instanceId));
+        await Promise.allSettled(promises);
+        this.save();
+    }
+    save() {
+        try {
+            storageService.put('instances', this._instances.map((instance) => {
+                const copy = { ...instance };
+                delete copy.internal;
+                return copy;
+            }));
+        }
+        catch (e) {
+            console.error('Failed to save instances', this._instances, e);
         }
-        return this._processes[planRef][instanceId];
     }
-    async restartIfRunning(planRef, instanceId) {
-        if (!this._processes[planRef] || !this._processes[planRef][instanceId]) {
-            return;
+    async checkInstances() {
+        //console.log('\n## Checking instances:');
+        let changed = false;
+        const all = [...this._instances];
+        while (all.length > 0) {
+            // Check a few instances at a time - docker doesn't like too many concurrent requests
+            const chunk = all.splice(0, 20);
+            const promises = chunk.map(async (instance) => {
+                if (!instance.systemId) {
+                    return;
+                }
+                instance.systemId = normalizeKapetaUri(instance.systemId);
+                if (instance.ref) {
+                    instance.ref = normalizeKapetaUri(instance.ref);
+                }
+                const newStatus = await this.requestInstanceStatus(instance);
+                /*
+                console.log('Check instance %s %s: [current: %s, new: %s, desired: %s]',
+                    instance.systemId, instance.instanceId, instance.status, newStatus, instance.desiredStatus);
+                */
+                if (newStatus === InstanceStatus.BUSY) {
+                    // If instance is busy we skip it
+                    //console.log('Instance %s %s is busy', instance.systemId, instance.instanceId);
+                    return;
+                }
+                if (instance.startedAt !== undefined &&
+                    newStatus === InstanceStatus.UNHEALTHY &&
+                    instance.startedAt + HEALTH_CHECK_TIMEOUT < Date.now() &&
+                    instance.status === InstanceStatus.STARTING) {
+                    // If instance is starting we consider unhealthy an indication
+                    // that it is still starting
+                    //console.log('Instance %s %s is still starting', instance.systemId, instance.instanceId);
+                    return;
+                }
+                if (instance.status !== newStatus) {
+                    const oldStatus = instance.status;
+                    const skipUpdate = (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
+                        ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
+                            instance.status === InstanceStatus.STOPPING &&
+                            instance.desiredStatus === DesiredInstanceStatus.STOP) ||
+                        (newStatus === InstanceStatus.STOPPED &&
+                            instance.status === InstanceStatus.STARTING &&
+                            instance.desiredStatus === DesiredInstanceStatus.RUN);
+                    if (!skipUpdate) {
+                        const oldStatus = instance.status;
+                        instance.status = newStatus;
+                        console.log('Instance status changed: %s %s: %s -> %s', instance.systemId, instance.instanceId, oldStatus, instance.status);
+                        this.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+                        changed = true;
+                    }
+                }
+                if (instance.desiredStatus === DesiredInstanceStatus.RUN && newStatus === InstanceStatus.STOPPED) {
+                    //If the instance is stopped but we want it to run, start it
+                    try {
+                        await this.start(instance.systemId, instance.instanceId);
+                    }
+                    catch (e) {
+                        console.warn('Failed to start instance', instance.systemId, instance.instanceId, e);
+                    }
+                    return;
+                }
+                if (instance.desiredStatus === DesiredInstanceStatus.STOP && newStatus === InstanceStatus.READY) {
+                    //If the instance is running but we want it to stop, stop it
+                    try {
+                        await this.stop(instance.systemId, instance.instanceId);
+                    }
+                    catch (e) {
+                        console.warn('Failed to stop instance', instance.systemId, instance.instanceId, e);
+                    }
+                    return;
+                }
+                if (instance.desiredStatus === DesiredInstanceStatus.RUN &&
+                    instance.status !== newStatus &&
+                    newStatus === InstanceStatus.UNHEALTHY) {
+                    //If the instance is unhealthy, try to restart it
+                    console.log('Restarting unhealthy instance', instance);
+                    try {
+                        await this.restart(instance.systemId, instance.instanceId);
+                    }
+                    catch (e) {
+                        console.warn('Failed to restart instance', instance.systemId, instance.instanceId, e);
+                    }
+                }
+            });
+            await Promise.allSettled(promises);
         }
-        // createProcess will stop the process first if it's running
-        return this.createProcess(planRef, instanceId);
+        if (changed) {
+            this.save();
+        }
+        //console.log('\n##\n');
     }
-    async stopProcess(planRef, instanceId) {
-        if (!this._processes[planRef]) {
-            return;
+    async getExternalStatus(instance) {
+        if (instance.type === InstanceType.DOCKER) {
+            const containerName = getBlockInstanceContainerName(instance.instanceId);
+            const container = await containerManager.getContainerByName(containerName);
+            if (!container) {
+                // If the container doesn't exist, we consider the instance stopped
+                return InstanceStatus.STOPPED;
+            }
+            const state = await container.status();
+            if (state.Status === 'running') {
+                if (state.Health?.Status === 'healthy') {
+                    return InstanceStatus.READY;
+                }
+                if (state.Health?.Status === 'starting') {
+                    return InstanceStatus.STARTING;
+                }
+                if (state.Health?.Status === 'unhealthy') {
+                    return InstanceStatus.UNHEALTHY;
+                }
+                return InstanceStatus.READY;
+            }
+            if (state.Status === 'created') {
+                return InstanceStatus.STARTING;
+            }
+            if (state.Status === 'exited' || state.Status === 'dead') {
+                return InstanceStatus.STOPPED;
+            }
+            if (state.Status === 'removing') {
+                return InstanceStatus.BUSY;
+            }
+            if (state.Status === 'restarting') {
+                return InstanceStatus.BUSY;
+            }
+            if (state.Status === 'paused') {
+                return InstanceStatus.BUSY;
+            }
+            return InstanceStatus.STOPPED;
         }
-        if (this._processes[planRef][instanceId]) {
-            try {
-                await this._processes[planRef][instanceId].stop();
+        if (!instance.pid) {
+            return InstanceStatus.STOPPED;
+        }
+        //Otherwise its just a normal process.
+        //TODO: Handle for Windows
+        try {
+            if (process.kill(instance.pid, 0)) {
+                return InstanceStatus.READY;
             }
-            catch (e) {
-                console.error('Failed to stop process for instance: %s -> %s', planRef, instanceId, e);
+        }
+        catch (err) {
+            if (err.code === 'EPERM') {
+                return InstanceStatus.READY;
             }
-            delete this._processes[planRef][instanceId];
         }
+        return InstanceStatus.STOPPED;
     }
-    async stopAllProcesses() {
-        for (let processesForPlan of Object.values(this._processes)) {
-            for (let processInfo of Object.values(processesForPlan)) {
-                await processInfo.stop();
+    async requestInstanceStatus(instance) {
+        const externalStatus = await this.getExternalStatus(instance);
+        if (instance.type === InstanceType.DOCKER) {
+            // For docker instances we can rely on docker status
+            return externalStatus;
+        }
+        if (externalStatus === InstanceStatus.STOPPED) {
+            return externalStatus;
+        }
+        if (!instance.health) {
+            //No health url means we assume it's healthy as soon as it's running
+            return InstanceStatus.READY;
+        }
+        return new Promise((resolve) => {
+            if (!instance.health) {
+                resolve(InstanceStatus.READY);
+                return;
             }
+            request(instance.health, (err, response) => {
+                if (err) {
+                    resolve(InstanceStatus.UNHEALTHY);
+                    return;
+                }
+                if (response.statusCode > 399) {
+                    resolve(InstanceStatus.UNHEALTHY);
+                    return;
+                }
+                resolve(InstanceStatus.READY);
+            });
+        });
+    }
+    emitSystemEvent(systemId, type, payload) {
+        systemId = normalizeKapetaUri(systemId);
+        try {
+            socketManager.emit(`${systemId}/instances`, type, payload);
+        }
+        catch (e) {
+            console.warn('Failed to emit instance event: %s', e.message);
         }
-        this._processes = {};
-        for (let instance of this._instances) {
-            await this._stopInstance(instance);
+    }
+    emitInstanceEvent(systemId, instanceId, type, payload) {
+        systemId = normalizeKapetaUri(systemId);
+        try {
+            socketManager.emit(`${systemId}/instances/${instanceId}`, type, payload);
+        }
+        catch (e) {
+            console.warn('Failed to emit instance event: %s', e.message);
         }
     }
 }
 export const instanceManager = new InstanceManager();
 process.on('exit', async () => {
-    await instanceManager.stopAllProcesses();
+    await instanceManager.stopAll();
 });