@kapeta/local-cluster-service 0.8.2 → 0.9.0

package/src/instanceManager.ts

@@ -1,204 +1,153 @@
 import _ from 'lodash';
 import request from 'request';
-import EventEmitter from 'events';
 import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
 import { storageService } from './storageService';
 import { socketManager } from './socketManager';
 import { serviceManager } from './serviceManager';
 import { assetManager } from './assetManager';
-import { containerManager } from './containerManager';
+import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
 import { configManager } from './configManager';
-import { InstanceInfo, LogEntry, ProcessInfo } from './types';
+import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceStatus, InstanceType, LogEntry } from './types';
 import { BlockInstance } from '@kapeta/schemas';
+import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
 
-const CHECK_INTERVAL = 10000;
+const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
 
 const EVENT_STATUS_CHANGED = 'status-changed';
 const EVENT_INSTANCE_CREATED = 'instance-created';
 const EVENT_INSTANCE_EXITED = 'instance-exited';
 const EVENT_INSTANCE_LOG = 'instance-log';
-
-const STATUS_STARTING = 'starting';
-const STATUS_READY = 'ready';
-const STATUS_UNHEALTHY = 'unhealthy';
-const STATUS_STOPPED = 'stopped';
-
 const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
 
-class InstanceManager {
-    private _interval: NodeJS.Timer;
-
-    /**
-     * Contains an array of running instances that have self-registered with this
-     * cluster service. This is done by the Kapeta SDKs
-     */
-    private _instances: InstanceInfo[] = [];
+export class InstanceManager {
+    private _interval: NodeJS.Timer | undefined = undefined;
 
-    /**
-     * Contains the process info for the instances started by this manager. In memory only
-     * so can't be relied on for knowing everything that's running.
-     *
-     */
-    private _processes: { [systemId: string]: { [instanceId: string]: ProcessInfo } } = {};
+    private readonly _instances: InstanceInfo[] = [];
 
     constructor() {
-        this._interval = setInterval(() => this._checkInstances(), CHECK_INTERVAL);
         this._instances = storageService.section('instances', []);
-        this._processes = {};
-
-        this._checkInstances();
-    }
 
-    _save() {
-        storageService.put('instances', this._instances);
+        // We need to wait a bit before running the first check
+        this.checkInstancesLater(1000);
     }
 
-    async _checkInstances() {
-        let changed = false;
-        for (let i = 0; i < this._instances.length; i++) {
-            const instance = this._instances[i];
-
-            const newStatus = await this._getInstanceStatus(instance);
+    private checkInstancesLater(time = CHECK_INTERVAL) {
+        if (this._interval) {
+            clearTimeout(this._interval);
+        }
 
-            if (newStatus === STATUS_UNHEALTHY && instance.status === STATUS_STARTING) {
-                // If instance is starting we consider unhealthy an indication
-                // that it is still starting
-                continue;
-            }
+        this._interval = setTimeout(async () => {
+            await this.checkInstances();
+            this.checkInstancesLater();
+        }, time);
+    }
 
-            if (instance.status !== newStatus) {
-                instance.status = newStatus;
-                console.log(
-                    'Instance status changed: %s %s -> %s',
-                    instance.systemId,
-                    instance.instanceId,
-                    instance.status
-                );
-                this._emit(instance.systemId, EVENT_STATUS_CHANGED, instance);
-                changed = true;
-            }
+    public getInstances() {
+        if (!this._instances) {
+            return [];
         }
 
-        if (changed) {
-            this._save();
-        }
+        return [...this._instances];
     }
 
-    async _isRunning(instance: InstanceInfo) {
-        if (!instance.pid) {
-            return;
+    public getInstancesForPlan(systemId: string) {
+        if (!this._instances) {
+            return [];
         }
 
-        if (instance.type === 'docker') {
-            const container = await containerManager.get(instance.pid as string);
-            if (!container) {
-                console.warn('Container not found: %s', instance.pid);
-                return false;
-            }
-            return await container.isRunning();
-        }
+        systemId = normalizeKapetaUri(systemId);
 
-        //Otherwise its just a normal process.
-        //TODO: Handle for Windows
-        try {
-            return process.kill(instance.pid as number, 0);
-        } catch (err: any) {
-            return err.code === 'EPERM';
-        }
+        return this._instances.filter((instance) => instance.systemId === systemId);
     }
 
-    async _getInstanceStatus(instance: InstanceInfo): Promise<string> {
-        if (instance.status === STATUS_STOPPED) {
-            //Will only change when it reregisters
-            return STATUS_STOPPED;
-        }
+    public getInstance(systemId: string, instanceId: string) {
+        systemId = normalizeKapetaUri(systemId);
 
-        if (!(await this._isRunning(instance))) {
-            return STATUS_STOPPED;
-        }
+        return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
+    }
 
-        if (!instance.health) {
-            //No health url means we assume it's healthy as soon as it's running
-            return STATUS_READY;
+    public async saveInternalInstance(instance: InstanceInfo) {
+        instance.systemId = normalizeKapetaUri(instance.systemId);
+        if (instance.ref) {
+            instance.ref = normalizeKapetaUri(instance.ref);
         }
 
-        return new Promise((resolve) => {
-            if (!instance.health) {
-                resolve(STATUS_READY);
-                return;
-            }
-            request(instance.health, (err, response) => {
-                if (err) {
-                    resolve(STATUS_UNHEALTHY);
-                    return;
-                }
-
-                if (response.statusCode > 399) {
-                    resolve(STATUS_UNHEALTHY);
-                    return;
-                }
+        //Get target address
+        let address = await serviceManager.getProviderAddress(
+            instance.systemId,
+            instance.instanceId,
+            instance.portType ?? DEFAULT_HEALTH_PORT_TYPE
+        );
 
-                resolve(STATUS_READY);
-            });
-        });
-    }
+        const healthUrl = this.getHealthUrl(instance, address);
 
-    getInstances() {
-        if (!this._instances) {
-            return [];
+        instance.address = address;
+        if (healthUrl) {
+            instance.health = healthUrl;
         }
 
-        return [...this._instances];
-    }
-
-    getInstancesForPlan(systemId: string) {
-        if (!this._instances) {
-            return [];
+        let existingInstance = this.getInstance(instance.systemId, instance.instanceId);
+        if (existingInstance) {
+            const ix = this._instances.indexOf(existingInstance);
+            this._instances.splice(ix, 1, instance);
+            this.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+        } else {
+            this._instances.push(instance);
+            this.emitSystemEvent(instance.systemId, EVENT_INSTANCE_CREATED, instance);
         }
 
-        return this._instances.filter((instance) => instance.systemId === systemId);
-    }
+        this.save();
 
-    /**
-     * Get instance information
-     *
-     * @param {string} systemId
-     * @param {string} instanceId
-     * @return {*}
-     */
-    getInstance(systemId: string, instanceId: string) {
-        return _.find(this._instances, { systemId, instanceId });
+        return instance;
     }
 
     /**
-     *
-     * @param {string} systemId
-     * @param {string} instanceId
-     * @param {InstanceInfo} info
-     * @return {Promise<void>}
+     * Method is called when instance is started from the Kapeta SDKs (e.g. NodeJS SDK)
+     * which self-registers with the cluster service locally on startup.
      */
-    async registerInstance(systemId: string, instanceId: string, info: Omit<InstanceInfo, 'systemId' | 'instanceId'>) {
+    public async registerInstanceFromSDK(
+        systemId: string,
+        instanceId: string,
+        info: Omit<InstanceInfo, 'systemId' | 'instanceId'>
+    ) {
+        systemId = normalizeKapetaUri(systemId);
+
         let instance = this.getInstance(systemId, instanceId);
 
         //Get target address
-        let address = await serviceManager.getProviderAddress(
+        const address = await serviceManager.getProviderAddress(
             systemId,
             instanceId,
             info.portType ?? DEFAULT_HEALTH_PORT_TYPE
         );
 
-        let healthUrl = null;
-        let health = info.health;
-        if (health) {
-            if (health.startsWith('/')) {
-                health = health.substring(1);
-            }
-            healthUrl = address + health;
-        }
+        const healthUrl = this.getHealthUrl(info, address);
 
         if (instance) {
-            instance.status = STATUS_STARTING;
+            if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
+                //If instance is stopping do not interfere
+                return;
+            }
+
+            if (info.owner === InstanceOwner.EXTERNAL) {
+                //If instance was started externally - then we want to replace the internal instance with that
+                if (
+                    instance.owner === InstanceOwner.INTERNAL &&
+                    (instance.status === InstanceStatus.READY ||
+                        instance.status === InstanceStatus.STARTING ||
+                        instance.status === InstanceStatus.UNHEALTHY)
+                ) {
+                    throw new Error(`Instance ${instanceId} is already running`);
+                }
+
+                instance.desiredStatus = info.desiredStatus;
+                instance.owner = info.owner;
+                instance.internal = undefined;
+                instance.status = InstanceStatus.STARTING;
+                instance.startedAt = Date.now();
+            }
+
             instance.pid = info.pid;
             instance.address = address;
             if (info.type) {
@@ -207,63 +156,73 @@ class InstanceManager {
             if (healthUrl) {
                 instance.health = healthUrl;
             }
-            this._emit(systemId, EVENT_STATUS_CHANGED, instance);
+
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
         } else {
+            //If instance was not found - then we're receiving an externally started instance
             instance = {
+                ...info,
                 systemId,
                 instanceId,
-                status: STATUS_STARTING,
-                pid: info.pid,
-                type: info.type,
+                status: InstanceStatus.STARTING,
+                startedAt: Date.now(),
+                desiredStatus: DesiredInstanceStatus.EXTERNAL,
+                owner: InstanceOwner.EXTERNAL,
                 health: healthUrl,
                 address,
             };
 
             this._instances.push(instance);
 
-            this._emit(systemId, EVENT_INSTANCE_CREATED, instance);
+            this.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
         }
 
-        this._save();
+        this.save();
+
+        return instance;
     }
 
-    setInstanceAsStopped(systemId: string, instanceId: string) {
-        const instance = _.find(this._instances, { systemId, instanceId });
-        if (instance) {
-            instance.status = STATUS_STOPPED;
-            instance.pid = null;
-            instance.health = null;
-            this._emit(systemId, EVENT_STATUS_CHANGED, instance);
-            this._save();
+    private getHealthUrl(info: Omit<InstanceInfo, 'systemId' | 'instanceId'>, address: string) {
+        let healthUrl = null;
+        let health = info.health;
+        if (health) {
+            if (health.startsWith('/')) {
+                health = health.substring(1);
+            }
+            healthUrl = address + health;
         }
+        return healthUrl;
     }
 
-    _emit(systemId: string, type: string, payload: any) {
-        try {
-            socketManager.emit(`${systemId}/instances`, type, payload);
-        } catch (e: any) {
-            console.warn('Failed to emit instance event: %s', e.message);
+    public markAsStopped(systemId: string, instanceId: string) {
+        systemId = normalizeKapetaUri(systemId);
+        const instance = _.find(this._instances, { systemId, instanceId });
+        if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
+            instance.status = InstanceStatus.STOPPED;
+            instance.pid = null;
+            instance.health = null;
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            this.save();
         }
     }
 
-    async createProcessesForPlan(planRef: string): Promise<ProcessInfo[]> {
-        await this.stopAllForPlan(planRef);
-
-        const plan = await assetManager.getPlan(planRef, true);
+    public async startAllForPlan(systemId: string): Promise<InstanceInfo[]> {
+        systemId = normalizeKapetaUri(systemId);
+        const plan = await assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + planRef);
+            throw new Error('Plan not found: ' + systemId);
         }
 
         if (!plan.spec.blocks) {
-            console.warn('No blocks found in plan', planRef);
+            console.warn('No blocks found in plan', systemId);
             return [];
         }
 
-        let promises: Promise<ProcessInfo>[] = [];
+        let promises: Promise<InstanceInfo>[] = [];
         let errors = [];
         for (let blockInstance of Object.values(plan.spec.blocks as BlockInstance[])) {
             try {
-                promises.push(this.createProcess(planRef, blockInstance.id));
+                promises.push(this.start(systemId, blockInstance.id));
             } catch (e) {
                 errors.push(e);
             }
@@ -275,64 +234,76 @@ class InstanceManager {
             throw errors[0];
         }
 
-        return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p) as ProcessInfo[];
+        return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p) as InstanceInfo[];
     }
 
-    async _stopInstance(instance: InstanceInfo) {
-        if (!instance.pid) {
+    public async stop(systemId: string, instanceId: string) {
+        systemId = normalizeKapetaUri(systemId);
+        const instance = this.getInstance(systemId, instanceId);
+        if (!instance) {
             return;
         }
 
-        if (instance.status === 'stopped') {
+        if (instance.status === InstanceStatus.STOPPED) {
             return;
         }
 
+        if (instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
+            instance.desiredStatus = DesiredInstanceStatus.STOP;
+        }
+
+        instance.status = InstanceStatus.STOPPING;
+
+        this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+        console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+        this.save();
+
         try {
             if (instance.type === 'docker') {
-                const container = await containerManager.get(instance.pid as string);
+                const containerName = getBlockInstanceContainerName(instance.instanceId);
+                const container = await containerManager.getContainerByName(containerName);
                 if (container) {
                     try {
                         await container.stop();
+                        instance.status = InstanceStatus.STOPPED;
+                        this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                        this.save();
                     } catch (e) {
                         console.error('Failed to stop container', e);
                     }
+                } else {
+                    console.warn('Container not found', containerName);
                 }
                 return;
             }
+
+            if (!instance.pid) {
+                instance.status = InstanceStatus.STOPPED;
+                this.save();
+                return;
+            }
+
             process.kill(instance.pid as number, 'SIGTERM');
+            instance.status = InstanceStatus.STOPPED;
+            this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            this.save();
         } catch (e) {
             console.error('Failed to stop process', e);
         }
     }
 
-    async stopAllForPlan(planRef: string) {
-        if (this._processes[planRef]) {
-            const promises = [];
-            console.log('Stopping all processes for plan', planRef);
-            for (let instance of Object.values(this._processes[planRef])) {
-                promises.push(instance.stop());
-            }
-
-            await Promise.all(promises);
+    public async stopAllForPlan(systemId: string) {
+        systemId = normalizeKapetaUri(systemId);
+        const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
 
-            this._processes[planRef] = {};
-        }
-
-        //Also stop instances not being maintained by the cluster service
-        const instancesForPlan = this._instances.filter((instance) => instance.systemId === planRef);
-
-        const promises = [];
-        for (let instance of instancesForPlan) {
-            promises.push(this._stopInstance(instance));
-        }
-
-        await Promise.all(promises);
+        return this.stopInstances(instancesForPlan);
     }
 
-    async createProcess(planRef: string, instanceId: string): Promise<ProcessInfo> {
-        const plan = await assetManager.getPlan(planRef, true);
+    public async start(systemId: string, instanceId: string): Promise<InstanceInfo> {
+        systemId = normalizeKapetaUri(systemId);
+        const plan = await assetManager.getPlan(systemId, true);
         if (!plan) {
-            throw new Error('Plan not found: ' + planRef);
+            throw new Error('Plan not found: ' + systemId);
         }
 
         const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
@@ -340,42 +311,84 @@ class InstanceManager {
             throw new Error('Block instance not found: ' + instanceId);
         }
 
-        const blockRef = blockInstance.block.ref;
+        const blockRef = normalizeKapetaUri(blockInstance.block.ref);
 
         const blockAsset = await assetManager.getAsset(blockRef, true);
-        const instanceConfig = await configManager.getConfigForSection(planRef, instanceId);
-
         if (!blockAsset) {
             throw new Error('Block not found: ' + blockRef);
         }
 
-        if (!this._processes[planRef]) {
-            this._processes[planRef] = {};
+        const existingInstance = this.getInstance(systemId, instanceId);
+
+        if (existingInstance) {
+            if (existingInstance.status === InstanceStatus.READY) {
+                // Instance is already running
+                return existingInstance;
+            }
+
+            if (
+                existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
+                existingInstance.status === InstanceStatus.STARTING
+            ) {
+                // Internal instance is already starting - don't start it again
+                return existingInstance;
+            }
+
+            if (
+                existingInstance.owner === InstanceOwner.EXTERNAL &&
+                existingInstance.status === InstanceStatus.STARTING
+            ) {
+                // External instance is already starting - don't start it again
+                return existingInstance;
+            }
         }
 
-        await this.stopProcess(planRef, instanceId);
-        const type = blockAsset.version === 'local' ? 'local' : 'docker';
+        let instance: InstanceInfo = {
+            systemId,
+            instanceId,
+            ref: blockRef,
+            name: blockAsset.data.metadata.name,
+            desiredStatus: DesiredInstanceStatus.RUN,
+            owner: InstanceOwner.INTERNAL,
+            type: InstanceType.UNKNOWN,
+            status: InstanceStatus.STARTING,
+            startedAt: Date.now(),
+        };
+
+        console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+        // Save the instance before starting it, so that we can track the status
+        await this.saveInternalInstance(instance);
+
+        if (existingInstance) {
+            // Check if the instance is already running - but after we've commmuicated the desired status
+            const currentStatus = await this.requestInstanceStatus(existingInstance);
+            if (currentStatus === InstanceStatus.READY) {
+                // Instance is already running
+                return existingInstance;
+            }
+        }
 
-        const runner = new BlockInstanceRunner(planRef);
+        const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
+        const runner = new BlockInstanceRunner(systemId);
 
         const startTime = Date.now();
         try {
-            const process = await runner.start(blockRef, instanceId, instanceConfig);
+            const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
             //emit stdout/stderr via sockets
-            process.output.on('data', (data: Buffer) => {
+            processInfo.output.on('data', (data: Buffer) => {
                 const payload = {
                     source: 'stdout',
                     level: 'INFO',
                     message: data.toString(),
                     time: Date.now(),
                 };
-                this._emit(instanceId, EVENT_INSTANCE_LOG, payload);
+                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, payload);
             });
 
-            process.output.on('exit', (exitCode: number) => {
+            processInfo.output.on('exit', (exitCode: number) => {
                 const timeRunning = Date.now() - startTime;
-                const instance = this.getInstance(planRef, instanceId);
-                if (instance?.status === STATUS_READY) {
+                const instance = this.getInstance(systemId, instanceId);
+                if (instance?.status === InstanceStatus.READY) {
                     //It's already been running
                     return;
                 }
@@ -387,7 +400,13 @@ class InstanceManager {
                 }
 
                 if (exitCode !== 0 || timeRunning < MIN_TIME_RUNNING) {
-                    this._emit(blockInstance.id, EVENT_INSTANCE_EXITED, {
+                    const instance = this.getInstance(systemId, instanceId);
+                    if (instance) {
+                        instance.status = InstanceStatus.FAILED;
+                        this.save();
+                    }
+
+                    this.emitSystemEvent(systemId, EVENT_INSTANCE_EXITED, {
                         error: 'Failed to start instance',
                         status: EVENT_INSTANCE_EXITED,
                         instanceId: blockInstance.id,
@@ -395,15 +414,20 @@ class InstanceManager {
                 }
             });
 
-            await this.registerInstance(planRef, instanceId, {
-                type: process.type,
-                pid: process.pid ?? -1,
+            instance.status = InstanceStatus.READY;
+
+            return this.saveInternalInstance({
+                ...instance,
+                type: processInfo.type,
+                pid: processInfo.pid ?? -1,
                 health: null,
-                portType: process.portType,
-                status: STATUS_STARTING,
+                portType: processInfo.portType,
+                status: InstanceStatus.READY,
+                internal: {
+                    logs: processInfo.logs,
+                    output: processInfo.output,
+                },
             });
-
-            return (this._processes[planRef][instanceId] = process);
         } catch (e: any) {
             console.warn('Failed to start instance', e);
             const logs: LogEntry[] = [
@@ -415,83 +439,288 @@ class InstanceManager {
                 },
             ];
 
-            await this.registerInstance(planRef, instanceId, {
-                type: 'local',
+            const out = await this.saveInternalInstance({
+                ...instance,
+                type: InstanceType.LOCAL,
                 pid: null,
                 health: null,
                 portType: DEFAULT_HEALTH_PORT_TYPE,
-                status: STATUS_UNHEALTHY,
+                status: InstanceStatus.FAILED,
             });
 
-            this._emit(instanceId, EVENT_INSTANCE_LOG, logs[0]);
+            this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
 
-            this._emit(blockInstance.id, EVENT_INSTANCE_EXITED, {
+            this.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
                 error: `Failed to start instance: ${e.message}`,
                 status: EVENT_INSTANCE_EXITED,
                 instanceId: blockInstance.id,
             });
 
-            return (this._processes[planRef][instanceId] = {
-                pid: -1,
-                type,
-                logs: () => logs,
-                stop: () => Promise.resolve(),
-                ref: blockRef,
-                id: instanceId,
-                name: blockInstance.name,
-                output: new EventEmitter(),
-            });
+            return out;
         }
     }
 
-    /**
-     *
-     * @param {string} planRef
-     * @param {string} instanceId
-     * @return {ProcessInfo|null}
-     */
-    getProcessForInstance(planRef: string, instanceId: string) {
-        if (!this._processes[planRef]) {
-            return null;
-        }
+    public async restart(systemId: string, instanceId: string) {
+        systemId = normalizeKapetaUri(systemId);
+        await this.stop(systemId, instanceId);
 
-        return this._processes[planRef][instanceId];
+        return this.start(systemId, instanceId);
     }
 
-    async restartIfRunning(planRef: string, instanceId: string) {
-        if (!this._processes[planRef] || !this._processes[planRef][instanceId]) {
-            return;
+    public async stopAll() {
+        return this.stopInstances(this._instances);
+    }
+
+    private async stopInstances(instances: InstanceInfo[]) {
+        const promises = instances.map((instance) => this.stop(instance.systemId, instance.instanceId));
+        await Promise.allSettled(promises);
+        this.save();
+    }
+
+    private save() {
+        try {
+            storageService.put(
+                'instances',
+                this._instances.map((instance) => {
+                    const copy = { ...instance };
+                    delete copy.internal;
+                    return copy;
+                })
+            );
+        } catch (e) {
+            console.error('Failed to save instances', this._instances, e);
+        }
+    }
+
+    private async checkInstances() {
+        //console.log('\n## Checking instances:');
+        let changed = false;
+        const all = [...this._instances];
+        while (all.length > 0) {
+            // Check a few instances at a time - docker doesn't like too many concurrent requests
+            const chunk = all.splice(0, 20);
+            const promises = chunk.map(async (instance) => {
+                if (!instance.systemId) {
+                    return;
+                }
+
+                instance.systemId = normalizeKapetaUri(instance.systemId);
+                if (instance.ref) {
+                    instance.ref = normalizeKapetaUri(instance.ref);
+                }
+
+                const newStatus = await this.requestInstanceStatus(instance);
+                /*
+                console.log('Check instance %s %s: [current: %s, new: %s, desired: %s]',
+                    instance.systemId, instance.instanceId, instance.status, newStatus, instance.desiredStatus);
+                */
+
+                if (newStatus === InstanceStatus.BUSY) {
+                    // If instance is busy we skip it
+                    //console.log('Instance %s %s is busy', instance.systemId, instance.instanceId);
+                    return;
+                }
+
+                if (
+                    instance.startedAt !== undefined &&
+                    newStatus === InstanceStatus.UNHEALTHY &&
+                    instance.startedAt + HEALTH_CHECK_TIMEOUT < Date.now() &&
+                    instance.status === InstanceStatus.STARTING
+                ) {
+                    // If instance is starting we consider unhealthy an indication
+                    // that it is still starting
+                    //console.log('Instance %s %s is still starting', instance.systemId, instance.instanceId);
+                    return;
+                }
+
+                if (instance.status !== newStatus) {
+                    const oldStatus = instance.status;
+                    const skipUpdate =
+                        (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
+                        ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
+                            instance.status === InstanceStatus.STOPPING &&
+                            instance.desiredStatus === DesiredInstanceStatus.STOP) ||
+                        (newStatus === InstanceStatus.STOPPED &&
+                            instance.status === InstanceStatus.STARTING &&
+                            instance.desiredStatus === DesiredInstanceStatus.RUN);
+
+                    if (!skipUpdate) {
+                        const oldStatus = instance.status;
+                        instance.status = newStatus;
+                        console.log(
+                            'Instance status changed: %s %s: %s -> %s',
+                            instance.systemId,
+                            instance.instanceId,
+                            oldStatus,
+                            instance.status
+                        );
+                        this.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+                        changed = true;
+                    }
+                }
+
+                if (instance.desiredStatus === DesiredInstanceStatus.RUN && newStatus === InstanceStatus.STOPPED) {
+                    //If the instance is stopped but we want it to run, start it
+                    try {
+                        await this.start(instance.systemId, instance.instanceId);
+                    } catch (e: any) {
+                        console.warn('Failed to start instance', instance.systemId, instance.instanceId, e);
+                    }
+                    return;
+                }
+
+                if (instance.desiredStatus === DesiredInstanceStatus.STOP && newStatus === InstanceStatus.READY) {
+                    //If the instance is running but we want it to stop, stop it
+                    try {
+                        await this.stop(instance.systemId, instance.instanceId);
+                    } catch (e) {
+                        console.warn('Failed to stop instance', instance.systemId, instance.instanceId, e);
+                    }
+                    return;
+                }
+
+                if (
+                    instance.desiredStatus === DesiredInstanceStatus.RUN &&
+                    instance.status !== newStatus &&
+                    newStatus === InstanceStatus.UNHEALTHY
+                ) {
+                    //If the instance is unhealthy, try to restart it
+                    console.log('Restarting unhealthy instance', instance);
+                    try {
+                        await this.restart(instance.systemId, instance.instanceId);
+                    } catch (e) {
+                        console.warn('Failed to restart instance', instance.systemId, instance.instanceId, e);
+                    }
+                }
+            });
+
+            await Promise.allSettled(promises);
+        }
+
+        if (changed) {
+            this.save();
         }
 
-        // createProcess will stop the process first if it's running
-        return this.createProcess(planRef, instanceId);
+        //console.log('\n##\n');
     }
 
-    async stopProcess(planRef: string, instanceId: string) {
-        if (!this._processes[planRef]) {
-            return;
+    private async getExternalStatus(instance: InstanceInfo): Promise<InstanceStatus> {
+        if (instance.type === InstanceType.DOCKER) {
+            const containerName = getBlockInstanceContainerName(instance.instanceId);
+            const container = await containerManager.getContainerByName(containerName);
+            if (!container) {
+                // If the container doesn't exist, we consider the instance stopped
+                return InstanceStatus.STOPPED;
+            }
+            const state = await container.status();
+
+            if (state.Status === 'running') {
+                if (state.Health?.Status === 'healthy') {
+                    return InstanceStatus.READY;
+                }
+                if (state.Health?.Status === 'starting') {
+                    return InstanceStatus.STARTING;
+                }
+                if (state.Health?.Status === 'unhealthy') {
+                    return InstanceStatus.UNHEALTHY;
+                }
+
+                return InstanceStatus.READY;
+            }
+            if (state.Status === 'created') {
+                return InstanceStatus.STARTING;
+            }
+
+            if (state.Status === 'exited' || state.Status === 'dead') {
+                return InstanceStatus.STOPPED;
+            }
+
+            if (state.Status === 'removing') {
+                return InstanceStatus.BUSY;
+            }
+
+            if (state.Status === 'restarting') {
+                return InstanceStatus.BUSY;
+            }
+
+            if (state.Status === 'paused') {
+                return InstanceStatus.BUSY;
+            }
+
+            return InstanceStatus.STOPPED;
         }
 
-        if (this._processes[planRef][instanceId]) {
-            try {
-                await this._processes[planRef][instanceId].stop();
-            } catch (e) {
-                console.error('Failed to stop process for instance: %s -> %s', planRef, instanceId, e);
+        if (!instance.pid) {
+            return InstanceStatus.STOPPED;
+        }
+
+        //Otherwise its just a normal process.
+        //TODO: Handle for Windows
+        try {
+            if (process.kill(instance.pid as number, 0)) {
+                return InstanceStatus.READY;
+            }
+        } catch (err: any) {
+            if (err.code === 'EPERM') {
+                return InstanceStatus.READY;
             }
-            delete this._processes[planRef][instanceId];
         }
+
+        return InstanceStatus.STOPPED;
     }
 
-    async stopAllProcesses() {
-        for (let processesForPlan of Object.values(this._processes)) {
-            for (let processInfo of Object.values(processesForPlan)) {
-                await processInfo.stop();
+    private async requestInstanceStatus(instance: InstanceInfo): Promise<InstanceStatus> {
+        const externalStatus = await this.getExternalStatus(instance);
+        if (instance.type === InstanceType.DOCKER) {
+            // For docker instances we can rely on docker status
+            return externalStatus;
+        }
+
+        if (externalStatus === InstanceStatus.STOPPED) {
+            return externalStatus;
+        }
+
+        if (!instance.health) {
+            //No health url means we assume it's healthy as soon as it's running
+            return InstanceStatus.READY;
+        }
+
+        return new Promise((resolve) => {
+            if (!instance.health) {
+                resolve(InstanceStatus.READY);
+                return;
             }
+            request(instance.health, (err, response) => {
+                if (err) {
+                    resolve(InstanceStatus.UNHEALTHY);
+                    return;
+                }
+
+                if (response.statusCode > 399) {
+                    resolve(InstanceStatus.UNHEALTHY);
+                    return;
+                }
+
+                resolve(InstanceStatus.READY);
+            });
+        });
+    }
+
+    private emitSystemEvent(systemId: string, type: string, payload: any) {
+        systemId = normalizeKapetaUri(systemId);
+        try {
+            socketManager.emit(`${systemId}/instances`, type, payload);
+        } catch (e: any) {
+            console.warn('Failed to emit instance event: %s', e.message);
         }
-        this._processes = {};
+    }
 
-        for (let instance of this._instances) {
-            await this._stopInstance(instance);
+    private emitInstanceEvent(systemId: string, instanceId: string, type: string, payload: any) {
+        systemId = normalizeKapetaUri(systemId);
+        try {
+            socketManager.emit(`${systemId}/instances/${instanceId}`, type, payload);
+        } catch (e: any) {
+            console.warn('Failed to emit instance event: %s', e.message);
         }
     }
 }
@@ -499,5 +728,5 @@ class InstanceManager {
 export const instanceManager = new InstanceManager();
 
 process.on('exit', async () => {
-    await instanceManager.stopAllProcesses();
+    await instanceManager.stopAll();
 });