@kapeta/local-cluster-service 0.0.0-96f91ef

package/dist/esm/src/instanceManager.js

@@ -0,0 +1,910 @@
+"use strict";
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instanceManager = exports.InstanceManager = void 0;
+const lodash_1 = __importDefault(require("lodash"));
+const request_1 = __importDefault(require("request"));
+const async_lock_1 = __importDefault(require("async-lock"));
+const BlockInstanceRunner_1 = require("./utils/BlockInstanceRunner");
+const storageService_1 = require("./storageService");
+const socketManager_1 = require("./socketManager");
+const serviceManager_1 = require("./serviceManager");
+const assetManager_1 = require("./assetManager");
+const containerManager_1 = require("./containerManager");
+const configManager_1 = require("./configManager");
+const types_1 = require("./types");
+const utils_1 = require("./utils/utils");
+const operatorManager_1 = require("./operatorManager");
+const nodejs_utils_1 = require("@kapeta/nodejs-utils");
+const definitionsManager_1 = require("./definitionsManager");
+const taskManager_1 = require("./taskManager");
+const CHECK_INTERVAL = 5000;
+const DEFAULT_HEALTH_PORT_TYPE = 'http';
+const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
+class InstanceManager {
+    _interval = undefined;
+    _instances = [];
+    instanceLocks = new async_lock_1.default();
+    constructor() {
+        this._instances = storageService_1.storageService.section('instances', []);
+        // We need to wait a bit before running the first check
+        this.checkInstancesLater(1000);
+    }
+    checkInstancesLater(time = CHECK_INTERVAL) {
+        if (this._interval) {
+            clearTimeout(this._interval);
+        }
+        this._interval = setTimeout(async () => {
+            await this.checkInstances();
+            this.checkInstancesLater();
+        }, time);
+    }
+    getInstances() {
+        if (!this._instances) {
+            return [];
+        }
+        return [...this._instances];
+    }
+    async getInstancesForPlan(systemId) {
+        if (!this._instances) {
+            return [];
+        }
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        const planInfo = await definitionsManager_1.definitionsManager.getDefinition(systemId);
+        if (!planInfo) {
+            return [];
+        }
+        const plan = planInfo.definition;
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const instanceIds = plan.spec?.blocks?.map((block) => block.id) || [];
+        return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
+    }
+    getInstance(systemId, instanceId) {
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
+    }
+    async exclusive(systemId, instanceId, fn) {
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        const key = `${systemId}/${instanceId}`;
+        //console.log(`Acquiring lock for ${key}`, this.instanceLocks.isBusy(key));
+        const result = await this.instanceLocks.acquire(key, fn);
+        //console.log(`Releasing lock for ${key}`, this.instanceLocks.isBusy(key));
+        return result;
+    }
+    isLocked(systemId, instanceId) {
+        return this.instanceLocks.isBusy(`${systemId}/${instanceId}`);
+    }
+    async getLogs(systemId, instanceId) {
+        const instance = this.getInstance(systemId, instanceId);
+        if (!instance) {
+            throw new Error(`Instance ${systemId}/${instanceId} not found`);
+        }
+        switch (instance.type) {
+            case types_1.InstanceType.DOCKER:
+                return await containerManager_1.containerManager.getLogs(instance);
+            case types_1.InstanceType.UNKNOWN:
+                return [
+                    {
+                        level: 'INFO',
+                        message: 'Instance is starting...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    },
+                ];
+            case types_1.InstanceType.LOCAL:
+                return [
+                    {
+                        level: 'INFO',
+                        message: 'Instance started outside Kapeta - logs not available...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    },
+                ];
+        }
+        return [];
+    }
+    async saveInternalInstance(instance) {
+        instance.systemId = (0, nodejs_utils_1.normalizeKapetaUri)(instance.systemId);
+        if (instance.ref) {
+            instance.ref = (0, nodejs_utils_1.normalizeKapetaUri)(instance.ref);
+        }
+        //Get target address
+        let address = await serviceManager_1.serviceManager.getProviderAddress(instance.systemId, instance.instanceId, instance.portType ?? DEFAULT_HEALTH_PORT_TYPE);
+        const healthUrl = this.getHealthUrl(instance, address);
+        instance.address = address;
+        if (healthUrl) {
+            instance.health = healthUrl;
+        }
+        let existingInstance = this.getInstance(instance.systemId, instance.instanceId);
+        if (existingInstance) {
+            const ix = this._instances.indexOf(existingInstance);
+            this._instances.splice(ix, 1, instance);
+            socketManager_1.socketManager.emitSystemEvent(instance.systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+        }
+        else {
+            this._instances.push(instance);
+            socketManager_1.socketManager.emitSystemEvent(instance.systemId, socketManager_1.EVENT_INSTANCE_CREATED, instance);
+        }
+        this.save();
+        return instance;
+    }
+    /**
+     * Method is called when instance is started from the Kapeta SDKs (e.g. NodeJS SDK)
+     * which self-registers with the cluster service locally on startup.
+     */
+    async registerInstanceFromSDK(systemId, instanceId, info) {
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+            let instance = this.getInstance(systemId, instanceId);
+            //Get target address
+            const address = await serviceManager_1.serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
+            const healthUrl = this.getHealthUrl(info, address);
+            if (instance) {
+                if (instance.status === types_1.InstanceStatus.STOPPING &&
+                    instance.desiredStatus === types_1.DesiredInstanceStatus.STOP) {
+                    //If instance is stopping do not interfere
+                    return;
+                }
+                if (info.owner === types_1.InstanceOwner.EXTERNAL) {
+                    //If instance was started externally - then we want to replace the internal instance with that
+                    if (instance.owner === types_1.InstanceOwner.INTERNAL &&
+                        (instance.status === types_1.InstanceStatus.READY ||
+                            instance.status === types_1.InstanceStatus.STARTING ||
+                            instance.status === types_1.InstanceStatus.UNHEALTHY)) {
+                        throw new Error(`Instance ${instanceId} is already running`);
+                    }
+                    instance.desiredStatus = info.desiredStatus;
+                    instance.owner = info.owner;
+                    instance.status = types_1.InstanceStatus.STARTING;
+                    instance.startedAt = Date.now();
+                }
+                instance.pid = info.pid;
+                instance.address = address;
+                if (info.type) {
+                    instance.type = info.type;
+                }
+                if (healthUrl) {
+                    instance.health = healthUrl;
+                }
+                if (info.portType) {
+                    instance.portType = info.portType;
+                }
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+            }
+            else {
+                //If instance was not found - then we're receiving an externally started instance
+                instance = {
+                    ...info,
+                    systemId,
+                    instanceId,
+                    status: types_1.InstanceStatus.STARTING,
+                    startedAt: Date.now(),
+                    desiredStatus: types_1.DesiredInstanceStatus.EXTERNAL,
+                    owner: types_1.InstanceOwner.EXTERNAL,
+                    health: healthUrl,
+                    address,
+                };
+                this._instances.push(instance);
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_INSTANCE_CREATED, instance);
+            }
+            this.save();
+            return instance;
+        });
+    }
+    getHealthUrl(info, address) {
+        let healthUrl = null;
+        let health = info.health ?? '/.kapeta/health';
+        if (health) {
+            if (health.startsWith('/')) {
+                health = health.substring(1);
+            }
+            healthUrl = address + health;
+        }
+        return healthUrl;
+    }
+    markAsStopped(systemId, instanceId) {
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+            const instance = lodash_1.default.find(this._instances, { systemId, instanceId });
+            if (instance && instance.owner === types_1.InstanceOwner.EXTERNAL && instance.status !== types_1.InstanceStatus.STOPPED) {
+                if (instance.status != types_1.InstanceStatus.FAILED) {
+                    instance.status = types_1.InstanceStatus.STOPPED;
+                }
+                instance.pid = null;
+                instance.health = null;
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+                this.save();
+            }
+        });
+    }
+    async startAllForPlan(systemId) {
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        const plan = await assetManager_1.assetManager.getPlan(systemId, true);
+        if (!plan) {
+            throw new Error(`Plan not found: ${systemId}`);
+        }
+        if (!plan.spec.blocks) {
+            throw new Error(`No blocks found in plan: ${systemId}`);
+        }
+        return taskManager_1.taskManager.add(`plan:start:${systemId}`, async () => {
+            const promises = [];
+            const errors = [];
+            const instanceIds = await this.getAllInstancesExceptKind(systemId, types_1.KIND_BLOCK_TYPE_EXECUTABLE);
+            for (const instanceId of instanceIds) {
+                try {
+                    promises.push(this.start(systemId, instanceId).then((taskOrInstance) => {
+                        if (taskOrInstance instanceof taskManager_1.Task) {
+                            return taskOrInstance.wait();
+                        }
+                        return taskOrInstance;
+                    }));
+                }
+                catch (e) {
+                    errors.push(e);
+                }
+            }
+            const settled = await Promise.allSettled(promises);
+            if (errors.length > 0) {
+                throw errors[0];
+            }
+            return settled
+                .map((p) => (p.status === 'fulfilled' ? p.value : null))
+                .filter((p) => !!p);
+        }, {
+            name: `Starting plan ${systemId}`,
+        });
+    }
+    stopAllForPlan(systemId) {
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
+        return taskManager_1.taskManager.add(`plan:stop:${systemId}`, async () => {
+            return this.stopInstances(instancesForPlan);
+        }, {
+            name: `Stopping plan ${systemId}`,
+        });
+    }
+    async getInstanceOperator(systemId, instanceId, environment, ensureContainer = true) {
+        const blockInstance = await assetManager_1.assetManager.getBlockInstance(systemId, instanceId);
+        if (!blockInstance) {
+            throw new Error(`Instance not found: ${systemId}/${instanceId}`);
+        }
+        const blockRef = (0, nodejs_utils_1.normalizeKapetaUri)(blockInstance.block.ref);
+        const block = await assetManager_1.assetManager.getAsset(blockRef, true);
+        if (!block) {
+            throw new Error(`Block not found: ${blockRef}`);
+        }
+        const operatorDefinition = await definitionsManager_1.definitionsManager.getDefinition(block.kind);
+        if (!operatorDefinition) {
+            throw new Error(`Operator not found: ${block.kind}`);
+        }
+        if (operatorDefinition.definition.kind !== types_1.KIND_BLOCK_TYPE_OPERATOR) {
+            throw new Error(`Block is not an operator: ${blockRef}`);
+        }
+        if (!operatorDefinition.definition.spec.local) {
+            throw new Error(`Operator block has no local definition: ${blockRef}`);
+        }
+        const localConfig = operatorDefinition.definition.spec.local;
+        const ports = {};
+        if (ensureContainer) {
+            let instance = await this.start(systemId, instanceId);
+            if (instance instanceof taskManager_1.Task) {
+                instance = await instance.wait();
+            }
+            const container = await containerManager_1.containerManager.get(instance.pid);
+            if (!container) {
+                throw new Error(`Container not found: ${instance.pid}`);
+            }
+            const portInfo = await container.getPorts();
+            if (!portInfo) {
+                throw new Error(`No ports found for instance: ${instanceId}`);
+            }
+            Object.entries(portInfo).forEach(([key, value]) => {
+                ports[key] = {
+                    protocol: value.protocol,
+                    port: parseInt(value.hostPort),
+                };
+            });
+        }
+        else {
+            // If we're not ensuring the container is running we just get the ports from the local config
+            const instancePorts = await (0, utils_1.getOperatorInstancePorts)(systemId, instanceId, localConfig);
+            instancePorts.forEach((port) => {
+                ports[port.portType] = {
+                    protocol: port.protocol,
+                    port: port.hostPort,
+                };
+            });
+        }
+        const hostname = (0, utils_1.getRemoteHostForEnvironment)(environment);
+        return {
+            hostname,
+            ports,
+            credentials: localConfig.credentials,
+            options: localConfig.options,
+        };
+    }
+    async stop(systemId, instanceId) {
+        return this.stopInner(systemId, instanceId, true);
+    }
+    async stopInner(systemId, instanceId, changeDesired = false, checkForSingleton = true) {
+        if (checkForSingleton) {
+            const blockInstance = await assetManager_1.assetManager.getBlockInstance(systemId, instanceId);
+            const blockRef = (0, nodejs_utils_1.normalizeKapetaUri)(blockInstance.block.ref);
+            const blockAsset = await assetManager_1.assetManager.getAsset(blockRef, true);
+            if (!blockAsset) {
+                throw new Error('Block not found: ' + blockRef);
+            }
+            if (await this.isSingletonOperator(blockAsset)) {
+                const instances = await this.getAllInstancesForKind(systemId, blockAsset.data.kind);
+                if (instances.length > 1) {
+                    const promises = instances.map((id) => {
+                        return this.stopInner(systemId, id, changeDesired, false);
+                    });
+                    await Promise.all(promises);
+                    return;
+                }
+            }
+        }
+        return this.exclusive(systemId, instanceId, async () => {
+            systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+            const instance = this.getInstance(systemId, instanceId);
+            if (!instance) {
+                return;
+            }
+            if (instance.status === types_1.InstanceStatus.STOPPED) {
+                return;
+            }
+            if (instance.status === types_1.InstanceStatus.STOPPING) {
+                return;
+            }
+            if (changeDesired && instance.desiredStatus !== types_1.DesiredInstanceStatus.EXTERNAL) {
+                instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+            }
+            const wasFailed = instance.status === types_1.InstanceStatus.FAILED;
+            instance.status = types_1.InstanceStatus.STOPPING;
+            socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+            console.log('Stopping instance: %s::%s [desired: %s] [intentional: %s]', systemId, instanceId, instance.desiredStatus, changeDesired);
+            this.save();
+            try {
+                if (instance.type === 'docker') {
+                    const containerName = await (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
+                    const container = await containerManager_1.containerManager.getContainerByName(containerName);
+                    if (container) {
+                        try {
+                            if (wasFailed) {
+                                await container.remove();
+                            }
+                            else {
+                                await container.stop();
+                            }
+                            instance.status = types_1.InstanceStatus.STOPPED;
+                            socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+                            this.save();
+                        }
+                        catch (e) {
+                            console.error('Failed to stop container', e);
+                        }
+                    }
+                    else {
+                        console.warn('Container not found', containerName);
+                    }
+                    return;
+                }
+                if (!instance.pid) {
+                    instance.status = types_1.InstanceStatus.STOPPED;
+                    this.save();
+                    return;
+                }
+                process.kill(instance.pid, 'SIGTERM');
+                instance.status = types_1.InstanceStatus.STOPPED;
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+                this.save();
+            }
+            catch (e) {
+                console.error('Failed to stop process', e);
+            }
+        });
+    }
+    async start(systemId, instanceId, checkForSingleton = true) {
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        const blockInstance = await assetManager_1.assetManager.getBlockInstance(systemId, instanceId);
+        const blockRef = (0, nodejs_utils_1.normalizeKapetaUri)(blockInstance.block.ref);
+        const blockAsset = await assetManager_1.assetManager.getAsset(blockRef, true);
+        if (!blockAsset) {
+            throw new Error('Block not found: ' + blockRef);
+        }
+        if (checkForSingleton && (await this.isSingletonOperator(blockAsset))) {
+            const instances = await this.getAllInstancesForKind(systemId, blockAsset.data.kind);
+            if (instances.length > 1) {
+                const promises = instances.map((id) => {
+                    return this.start(systemId, id, false);
+                });
+                await Promise.all(promises);
+                return promises[0];
+            }
+        }
+        return this.exclusive(systemId, instanceId, async () => {
+            let existingInstance = this.getInstance(systemId, instanceId);
+            if (existingInstance && existingInstance.pid) {
+                const container = await containerManager_1.containerManager.get(existingInstance.pid);
+                if (!container) {
+                    // The container is not running
+                    existingInstance = undefined;
+                }
+            }
+            if (existingInstance && existingInstance.pid) {
+                if (existingInstance.status === types_1.InstanceStatus.READY) {
+                    // Instance is already running
+                    return existingInstance;
+                }
+                if (existingInstance.desiredStatus === types_1.DesiredInstanceStatus.RUN &&
+                    existingInstance.status === types_1.InstanceStatus.STARTING) {
+                    // Internal instance is already starting - don't start it again
+                    return existingInstance;
+                }
+                if (existingInstance.owner === types_1.InstanceOwner.EXTERNAL &&
+                    existingInstance.status === types_1.InstanceStatus.STARTING) {
+                    // External instance is already starting - don't start it again
+                    return existingInstance;
+                }
+            }
+            let instance = {
+                systemId,
+                instanceId,
+                ref: blockRef,
+                name: blockAsset.data.metadata.name,
+                desiredStatus: types_1.DesiredInstanceStatus.RUN,
+                owner: types_1.InstanceOwner.INTERNAL,
+                type: existingInstance?.type ?? types_1.InstanceType.UNKNOWN,
+                status: types_1.InstanceStatus.STARTING,
+                startedAt: Date.now(),
+            };
+            console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
+            // Save the instance before starting it, so that we can track the status
+            await this.saveInternalInstance(instance);
+            const blockSpec = blockAsset.data.spec;
+            if (blockSpec.consumers) {
+                const promises = blockSpec.consumers.map(async (consumer) => {
+                    const consumerUri = (0, nodejs_utils_1.parseKapetaUri)(consumer.kind);
+                    const asset = await definitionsManager_1.definitionsManager.getDefinition(consumer.kind);
+                    if (!asset) {
+                        // Definition not found
+                        return Promise.resolve();
+                    }
+                    if (types_1.KIND_RESOURCE_OPERATOR.toLowerCase() !== asset.definition.kind.toLowerCase()) {
+                        // Not an operator
+                        return Promise.resolve();
+                    }
+                    // Check if the operator has a local definition, if not we skip it since we can't start it
+                    if (!asset.definition.spec.local) {
+                        console.log('Skipping operator since it as no local definition: %s', consumer.kind);
+                        return Promise.resolve();
+                    }
+                    console.log('Ensuring resource: %s in %s', consumerUri.id, systemId);
+                    return operatorManager_1.operatorManager.ensureOperator(systemId, consumerUri.fullName, consumerUri.version);
+                });
+                await Promise.all(promises);
+            }
+            if (existingInstance) {
+                // Check if the instance is already running - but after we've commmuicated the desired status
+                const currentStatus = await this.requestInstanceStatus(existingInstance);
+                if (currentStatus === types_1.InstanceStatus.READY) {
+                    // Instance is already running
+                    return existingInstance;
+                }
+            }
+            const resolvedConfig = await configManager_1.configManager.getConfigForBlockInstance(systemId, instanceId);
+            const task = taskManager_1.taskManager.add(`instance:start:${systemId}:${instanceId}`, async () => {
+                const runner = new BlockInstanceRunner_1.BlockInstanceRunner(systemId);
+                const startTime = Date.now();
+                try {
+                    const processInfo = await runner.start(blockRef, instanceId, resolvedConfig);
+                    instance.status = types_1.InstanceStatus.STARTING;
+                    return this.saveInternalInstance({
+                        ...instance,
+                        type: processInfo.type,
+                        pid: processInfo.pid ?? -1,
+                        health: null,
+                        portType: processInfo.portType,
+                        status: types_1.InstanceStatus.STARTING,
+                    });
+                }
+                catch (e) {
+                    console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e);
+                    const logs = [
+                        {
+                            source: 'stdout',
+                            level: 'ERROR',
+                            message: e.message,
+                            time: Date.now(),
+                        },
+                    ];
+                    const out = await this.saveInternalInstance({
+                        ...instance,
+                        type: types_1.InstanceType.DOCKER,
+                        health: null,
+                        portType: DEFAULT_HEALTH_PORT_TYPE,
+                        status: types_1.InstanceStatus.FAILED,
+                        errorMessage: e.message ?? 'Failed to start - Check logs for details.',
+                    });
+                    socketManager_1.socketManager.emitInstanceLog(systemId, instanceId, logs[0]);
+                    return out;
+                }
+            }, {
+                name: `Starting instance: ${instance.name}`,
+                systemId,
+            });
+            return task;
+        });
+    }
+    /**
+     * Stops an instance but does not remove it from the list of active instances
+     *
+     * It will be started again next time the system checks the status of the instance
+     *
+     * We do it this way to not cause the user to wait for the instance to start again
+     */
+    async prepareForRestart(systemId, instanceId) {
+        systemId = (0, nodejs_utils_1.normalizeKapetaUri)(systemId);
+        console.log('Stopping instance during restart...', systemId, instanceId);
+        await this.stopInner(systemId, instanceId);
+    }
+    async stopAll() {
+        return this.stopInstances(this._instances);
+    }
+    async stopInstances(instances) {
+        const promises = instances.map((instance) => this.stop(instance.systemId, instance.instanceId));
+        await Promise.allSettled(promises);
+        this.save();
+    }
+    save() {
+        try {
+            storageService_1.storageService.put('instances', this._instances.map((instance) => {
+                return { ...instance };
+            }));
+        }
+        catch (e) {
+            console.error('Failed to save instances', this._instances, e);
+        }
+    }
+    async checkInstances() {
+        //console.log('\n## Checking instances:');
+        let changed = false;
+        const all = [...this._instances];
+        while (all.length > 0) {
+            // Check a few instances at a time - docker doesn't like too many concurrent requests
+            const chunk = all.splice(0, 30);
+            const promises = chunk.map(async (oldInstance) => {
+                if (!oldInstance.systemId) {
+                    return;
+                }
+                // Grab the latest here
+                const instance = this.getInstance(oldInstance.systemId, oldInstance.instanceId);
+                if (!instance) {
+                    return;
+                }
+                instance.systemId = (0, nodejs_utils_1.normalizeKapetaUri)(instance.systemId);
+                if (instance.ref) {
+                    instance.ref = (0, nodejs_utils_1.normalizeKapetaUri)(instance.ref);
+                }
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.RUN) {
+                    // Check if the plan still exists and the instance is still in the plan
+                    // - and that the block definition exists
+                    try {
+                        const plan = await assetManager_1.assetManager.getAsset(instance.systemId, true, false);
+                        if (!plan) {
+                            console.log('Plan not found - reset to stop', instance.ref, instance.systemId);
+                            instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                            changed = true;
+                            return;
+                        }
+                        const planData = plan.data;
+                        const planInstance = planData?.spec?.blocks?.find((b) => b.id === instance.instanceId);
+                        if (!planInstance || !planInstance?.block?.ref) {
+                            console.log('Plan instance not found - reset to stop', instance.ref, instance.systemId);
+                            instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                            changed = true;
+                            return;
+                        }
+                        const blockDef = await assetManager_1.assetManager.getAsset(instance.ref, true, false);
+                        if (!blockDef) {
+                            console.log('Block definition not found - reset to stop', instance.ref, instance.systemId);
+                            instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                            changed = true;
+                            return;
+                        }
+                    }
+                    catch (e) {
+                        console.warn('Failed to check assets', instance.systemId, e);
+                        instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                        return;
+                    }
+                }
+                const newStatus = await this.requestInstanceStatus(instance);
+                /*
+                console.log('Check instance %s %s: [current: %s, new: %s, desired: %s]',
+                    instance.systemId, instance.instanceId, instance.status, newStatus, instance.desiredStatus);
+                */
+                if (newStatus === types_1.InstanceStatus.BUSY) {
+                    // If instance is busy we skip it
+                    //console.log('Instance %s %s is busy', instance.systemId, instance.instanceId);
+                    return;
+                }
+                if (instance.startedAt !== undefined &&
+                    newStatus === types_1.InstanceStatus.UNHEALTHY &&
+                    instance.startedAt + containerManager_1.HEALTH_CHECK_TIMEOUT < Date.now() &&
+                    instance.status === types_1.InstanceStatus.STARTING) {
+                    // If instance is starting we consider unhealthy an indication
+                    // that it is still starting
+                    //console.log('Instance %s %s is still starting', instance.systemId, instance.instanceId);
+                    return;
+                }
+                if (instance.status !== newStatus) {
+                    const oldStatus = instance.status;
+                    const skipUpdate = ([types_1.InstanceStatus.READY, types_1.InstanceStatus.UNHEALTHY].includes(newStatus) &&
+                        instance.status === types_1.InstanceStatus.STOPPING) ||
+                        (newStatus === types_1.InstanceStatus.STOPPED &&
+                            instance.status === types_1.InstanceStatus.STARTING &&
+                            instance.desiredStatus === types_1.DesiredInstanceStatus.RUN);
+                    if (!skipUpdate) {
+                        const oldStatus = instance.status;
+                        instance.status = newStatus;
+                        console.log('Instance status changed: %s %s: %s -> %s', instance.systemId, instance.instanceId, oldStatus, instance.status);
+                        socketManager_1.socketManager.emitSystemEvent(instance.systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
+                        changed = true;
+                    }
+                }
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.RUN &&
+                    [types_1.InstanceStatus.STOPPED, types_1.InstanceStatus.STOPPING].includes(newStatus)) {
+                    //If the instance is stopped but we want it to run, start it
+                    try {
+                        await this.start(instance.systemId, instance.instanceId);
+                    }
+                    catch (e) {
+                        console.warn('Failed to start previously stopped instance', instance.systemId, instance.instanceId, e);
+                    }
+                    return;
+                }
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.STOP &&
+                    [types_1.InstanceStatus.READY, types_1.InstanceStatus.STARTING, types_1.InstanceStatus.UNHEALTHY].includes(newStatus)) {
+                    //If the instance is running but we want it to stop, stop it
+                    try {
+                        console.log('Stopping instance since it is its desired state', instance.systemId, instance.instanceId);
+                        await this.stopInner(instance.systemId, instance.instanceId);
+                    }
+                    catch (e) {
+                        console.warn('Failed to stop instance', instance.systemId, instance.instanceId, e);
+                    }
+                    return;
+                }
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.RUN &&
+                    instance.status !== newStatus &&
+                    newStatus === types_1.InstanceStatus.UNHEALTHY) {
+                    //If the instance is unhealthy, try to restart it
+                    console.log('Restarting unhealthy instance', instance);
+                    try {
+                        await this.prepareForRestart(instance.systemId, instance.instanceId);
+                    }
+                    catch (e) {
+                        console.warn('Failed to restart instance', instance.systemId, instance.instanceId, e);
+                    }
+                }
+            });
+            await Promise.allSettled(promises);
+        }
+        if (changed) {
+            this.save();
+        }
+        //console.log('\n##\n');
+    }
+    async getExternalStatus(instance) {
+        if (instance.type === types_1.InstanceType.DOCKER) {
+            const containerName = await (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
+            const container = await containerManager_1.containerManager.getContainerByName(containerName);
+            if (!container) {
+                // If the container doesn't exist, we consider the instance stopped
+                return types_1.InstanceStatus.STOPPED;
+            }
+            const state = await container.status();
+            if (!state) {
+                return types_1.InstanceStatus.STOPPED;
+            }
+            const statusType = state.Status;
+            if (statusType === 'running') {
+                if (state.Health?.Status) {
+                    const healthStatusType = state.Health.Status;
+                    if (healthStatusType === 'healthy' || healthStatusType === 'none') {
+                        return types_1.InstanceStatus.READY;
+                    }
+                    if (healthStatusType === 'starting') {
+                        return types_1.InstanceStatus.STARTING;
+                    }
+                    if (healthStatusType === 'unhealthy') {
+                        return types_1.InstanceStatus.UNHEALTHY;
+                    }
+                }
+                return types_1.InstanceStatus.READY;
+            }
+            if (statusType === 'created') {
+                if (state.ExitCode !== undefined && state.ExitCode !== 0) {
+                    // Failed during creation. Exit code is not always reliable though
+                    if (state.Error) {
+                        return types_1.InstanceStatus.FAILED;
+                    }
+                    else {
+                        return types_1.InstanceStatus.STOPPED;
+                    }
+                }
+                return types_1.InstanceStatus.STARTING;
+            }
+            if (statusType === 'exited' || statusType === 'dead') {
+                if (!state.Error) {
+                    // Exit code is not always reliable - if there is no error we assume it's stopped
+                    return types_1.InstanceStatus.STOPPED;
+                }
+                return types_1.InstanceStatus.FAILED;
+            }
+            if (statusType === 'removing') {
+                return types_1.InstanceStatus.BUSY;
+            }
+            if (statusType === 'restarting') {
+                return types_1.InstanceStatus.BUSY;
+            }
+            if (statusType === 'paused') {
+                return types_1.InstanceStatus.BUSY;
+            }
+            return types_1.InstanceStatus.STOPPED;
+        }
+        if (!instance.pid) {
+            return types_1.InstanceStatus.STOPPED;
+        }
+        //Otherwise its just a normal process.
+        //TODO: Handle for Windows
+        try {
+            if (process.kill(instance.pid, 0)) {
+                return types_1.InstanceStatus.READY;
+            }
+        }
+        catch (err) {
+            if (err.code === 'EPERM') {
+                return types_1.InstanceStatus.READY;
+            }
+        }
+        return types_1.InstanceStatus.STOPPED;
+    }
+    async requestInstanceStatus(instance) {
+        const externalStatus = await this.getExternalStatus(instance);
+        if (instance.type === types_1.InstanceType.DOCKER) {
+            // For docker instances we can rely on docker status
+            return externalStatus;
+        }
+        if (externalStatus === types_1.InstanceStatus.STOPPED) {
+            return externalStatus;
+        }
+        if (!instance.health) {
+            //No health url means we assume it's healthy as soon as it's running
+            return types_1.InstanceStatus.READY;
+        }
+        return new Promise((resolve) => {
+            if (!instance.health) {
+                resolve(types_1.InstanceStatus.READY);
+                return;
+            }
+            (0, request_1.default)(instance.health, (err, response) => {
+                if (err) {
+                    resolve(types_1.InstanceStatus.UNHEALTHY);
+                    return;
+                }
+                if (response.statusCode > 399) {
+                    resolve(types_1.InstanceStatus.UNHEALTHY);
+                    return;
+                }
+                resolve(types_1.InstanceStatus.READY);
+            });
+        });
+    }
+    async isSingletonOperator(blockAsset) {
+        const provider = await assetManager_1.assetManager.getAsset(blockAsset.data.kind);
+        if (!provider) {
+            return false;
+        }
+        if ((0, nodejs_utils_1.parseKapetaUri)(provider.kind).fullName === types_1.KIND_BLOCK_TYPE_OPERATOR) {
+            const localConfig = provider.data.spec.local;
+            return localConfig.singleton ?? false;
+        }
+        return false;
+    }
+    async getKindForAssetRef(assetRef) {
+        const block = await assetManager_1.assetManager.getAsset(assetRef);
+        if (!block) {
+            return null;
+        }
+        return block.data.kind;
+    }
+    /**
+     * Get the kind of an asset. Use the maxDepth parameter to specify how deep to look for the
+     * kind. For example, if maxDepth is 2, the method will look for the kind of the asset and then
+     * the kind of the kind.
+     * @param assetRef The asset reference
+     * @param maxDepth The maximum depth to look for the kind
+     * @returns The kind of the asset or null if not found
+     */
+    async getDeepKindForAssetRef(assetRef, maxDepth) {
+        if (maxDepth <= 0) {
+            return null;
+        }
+        try {
+            const asset = await assetManager_1.assetManager.getAsset(assetRef);
+            if (!asset || !asset.data.kind) {
+                return null;
+            }
+            if (maxDepth === 1) {
+                return asset.data.kind;
+            }
+            else {
+                // Recurse with the kind of the current block and one less depth
+                return await this.getDeepKindForAssetRef(asset.data.kind, maxDepth - 1);
+            }
+        }
+        catch (error) {
+            console.error('Error fetching kind for assetRef:', assetRef, error);
+            return null;
+        }
+    }
+    async isUsingKind(ref, kind) {
+        const assetKind = await this.getKindForAssetRef(ref);
+        if (!assetKind) {
+            return false;
+        }
+        return (0, nodejs_utils_1.parseKapetaUri)(assetKind).fullName === (0, nodejs_utils_1.parseKapetaUri)(kind).fullName;
+    }
+    async getAllInstancesForKind(systemId, kind) {
+        const plan = await assetManager_1.assetManager.getPlan(systemId);
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const out = [];
+        for (const block of plan.spec.blocks) {
+            if (await this.isUsingKind(block.block.ref, kind)) {
+                out.push(block.id);
+            }
+        }
+        return out;
+    }
+    /**
+     * Get the ids for all block instances except the ones of the specified kind
+     * @param systemId The plan reference id
+     * @param kind The kind to exclude. Can be a string or an array of strings
+     * @returns An array of block instance ids
+     */
+    async getAllInstancesExceptKind(systemId, kind) {
+        const plan = await assetManager_1.assetManager.getPlan(systemId);
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const out = [];
+        const excludedKinds = kind instanceof Array ? kind : [kind];
+        for (const block of plan.spec.blocks) {
+            const blockKindOfKind = await this.getDeepKindForAssetRef(block.block.ref, 2);
+            if (!blockKindOfKind) {
+                continue;
+            }
+            const shouldIncludeBlock = excludedKinds.some((excludedKind) => excludedKind === (0, nodejs_utils_1.parseKapetaUri)(blockKindOfKind).fullName) ===
+                false;
+            if (shouldIncludeBlock) {
+                out.push(block.id);
+            }
+        }
+        return out;
+    }
+}
+exports.InstanceManager = InstanceManager;
+exports.instanceManager = new InstanceManager();