@kapeta/local-cluster-service 0.16.7 → 0.17.0

package/dist/esm/src/instanceManager.js

@@ -1,28 +1,34 @@
-import _ from 'lodash';
-import request from 'request';
-import AsyncLock from 'async-lock';
-import { BlockInstanceRunner } from './utils/BlockInstanceRunner';
-import { storageService } from './storageService';
-import { EVENT_INSTANCE_CREATED, EVENT_INSTANCE_EXITED, EVENT_STATUS_CHANGED, socketManager, } from './socketManager';
-import { serviceManager } from './serviceManager';
-import { assetManager } from './assetManager';
-import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
-import { configManager } from './configManager';
-import { DesiredInstanceStatus, InstanceOwner, InstanceStatus, InstanceType } from './types';
-import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
-import { KIND_OPERATOR, operatorManager } from './operatorManager';
-import { parseKapetaUri } from '@kapeta/nodejs-utils';
-import { definitionsManager } from './definitionsManager';
-import { Task, taskManager } from './taskManager';
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instanceManager = exports.InstanceManager = void 0;
+const lodash_1 = __importDefault(require("lodash"));
+const request_1 = __importDefault(require("request"));
+const async_lock_1 = __importDefault(require("async-lock"));
+const BlockInstanceRunner_1 = require("./utils/BlockInstanceRunner");
+const storageService_1 = require("./storageService");
+const socketManager_1 = require("./socketManager");
+const serviceManager_1 = require("./serviceManager");
+const assetManager_1 = require("./assetManager");
+const containerManager_1 = require("./containerManager");
+const configManager_1 = require("./configManager");
+const types_1 = require("./types");
+const utils_1 = require("./utils/utils");
+const operatorManager_1 = require("./operatorManager");
+const nodejs_utils_1 = require("@kapeta/nodejs-utils");
+const definitionsManager_1 = require("./definitionsManager");
+const taskManager_1 = require("./taskManager");
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
 const MIN_TIME_RUNNING = 30000; //If something didnt run for more than 30 secs - it failed
-export class InstanceManager {
+class InstanceManager {
     _interval = undefined;
     _instances = [];
-    instanceLocks = new AsyncLock();
+    instanceLocks = new async_lock_1.default();
     constructor() {
-        this._instances = storageService.section('instances', []);
+        this._instances = storageService_1.storageService.section('instances', []);
         // We need to wait a bit before running the first check
         this.checkInstancesLater(1000);
     }
@@ -45,8 +51,8 @@ export class InstanceManager {
         if (!this._instances) {
             return [];
         }
-        systemId = normalizeKapetaUri(systemId);
-        const planInfo = definitionsManager.getDefinition(systemId);
+        systemId = (0, utils_1.normalizeKapetaUri)(systemId);
+        const planInfo = definitionsManager_1.definitionsManager.getDefinition(systemId);
         if (!planInfo) {
             return [];
         }
@@ -58,11 +64,11 @@ export class InstanceManager {
         return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
     }
     getInstance(systemId, instanceId) {
-        systemId = normalizeKapetaUri(systemId);
+        systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
     }
     async exclusive(systemId, instanceId, fn) {
-        systemId = normalizeKapetaUri(systemId);
+        systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         const key = `${systemId}/${instanceId}`;
         //console.log(`Acquiring lock for ${key}`, this.instanceLocks.isBusy(key));
         const result = await this.instanceLocks.acquire(key, fn);
@@ -75,9 +81,9 @@ export class InstanceManager {
             throw new Error(`Instance ${systemId}/${instanceId} not found`);
         }
         switch (instance.type) {
-            case InstanceType.DOCKER:
-                return await containerManager.getLogs(instance);
-            case InstanceType.UNKNOWN:
+            case types_1.InstanceType.DOCKER:
+                return await containerManager_1.containerManager.getLogs(instance);
+            case types_1.InstanceType.UNKNOWN:
                 return [
                     {
                         level: 'INFO',
@@ -86,7 +92,7 @@ export class InstanceManager {
                         source: 'stdout',
                     },
                 ];
-            case InstanceType.LOCAL:
+            case types_1.InstanceType.LOCAL:
                 return [
                     {
                         level: 'INFO',
@@ -99,12 +105,12 @@ export class InstanceManager {
         return [];
     }
     async saveInternalInstance(instance) {
-        instance.systemId = normalizeKapetaUri(instance.systemId);
+        instance.systemId = (0, utils_1.normalizeKapetaUri)(instance.systemId);
         if (instance.ref) {
-            instance.ref = normalizeKapetaUri(instance.ref);
+            instance.ref = (0, utils_1.normalizeKapetaUri)(instance.ref);
         }
         //Get target address
-        let address = await serviceManager.getProviderAddress(instance.systemId, instance.instanceId, instance.portType ?? DEFAULT_HEALTH_PORT_TYPE);
+        let address = await serviceManager_1.serviceManager.getProviderAddress(instance.systemId, instance.instanceId, instance.portType ?? DEFAULT_HEALTH_PORT_TYPE);
         const healthUrl = this.getHealthUrl(instance, address);
         instance.address = address;
         if (healthUrl) {
@@ -114,11 +120,11 @@ export class InstanceManager {
         if (existingInstance) {
             const ix = this._instances.indexOf(existingInstance);
             this._instances.splice(ix, 1, instance);
-            socketManager.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+            socketManager_1.socketManager.emitSystemEvent(instance.systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
         }
         else {
             this._instances.push(instance);
-            socketManager.emitSystemEvent(instance.systemId, EVENT_INSTANCE_CREATED, instance);
+            socketManager_1.socketManager.emitSystemEvent(instance.systemId, socketManager_1.EVENT_INSTANCE_CREATED, instance);
         }
         this.save();
         return instance;
@@ -129,28 +135,28 @@ export class InstanceManager {
      */
     async registerInstanceFromSDK(systemId, instanceId, info) {
         return this.exclusive(systemId, instanceId, async () => {
-            systemId = normalizeKapetaUri(systemId);
+            systemId = (0, utils_1.normalizeKapetaUri)(systemId);
             let instance = this.getInstance(systemId, instanceId);
             //Get target address
-            const address = await serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
+            const address = await serviceManager_1.serviceManager.getProviderAddress(systemId, instanceId, info.portType ?? DEFAULT_HEALTH_PORT_TYPE);
             const healthUrl = this.getHealthUrl(info, address);
             if (instance) {
-                if (instance.status === InstanceStatus.STOPPING &&
-                    instance.desiredStatus === DesiredInstanceStatus.STOP) {
+                if (instance.status === types_1.InstanceStatus.STOPPING &&
+                    instance.desiredStatus === types_1.DesiredInstanceStatus.STOP) {
                     //If instance is stopping do not interfere
                     return;
                 }
-                if (info.owner === InstanceOwner.EXTERNAL) {
+                if (info.owner === types_1.InstanceOwner.EXTERNAL) {
                     //If instance was started externally - then we want to replace the internal instance with that
-                    if (instance.owner === InstanceOwner.INTERNAL &&
-                        (instance.status === InstanceStatus.READY ||
-                            instance.status === InstanceStatus.STARTING ||
-                            instance.status === InstanceStatus.UNHEALTHY)) {
+                    if (instance.owner === types_1.InstanceOwner.INTERNAL &&
+                        (instance.status === types_1.InstanceStatus.READY ||
+                            instance.status === types_1.InstanceStatus.STARTING ||
+                            instance.status === types_1.InstanceStatus.UNHEALTHY)) {
                         throw new Error(`Instance ${instanceId} is already running`);
                     }
                     instance.desiredStatus = info.desiredStatus;
                     instance.owner = info.owner;
-                    instance.status = InstanceStatus.STARTING;
+                    instance.status = types_1.InstanceStatus.STARTING;
                     instance.startedAt = Date.now();
                 }
                 instance.pid = info.pid;
@@ -161,7 +167,7 @@ export class InstanceManager {
                 if (healthUrl) {
                     instance.health = healthUrl;
                 }
-                socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
             }
             else {
                 //If instance was not found - then we're receiving an externally started instance
@@ -169,15 +175,15 @@ export class InstanceManager {
                     ...info,
                     systemId,
                     instanceId,
-                    status: InstanceStatus.STARTING,
+                    status: types_1.InstanceStatus.STARTING,
                     startedAt: Date.now(),
-                    desiredStatus: DesiredInstanceStatus.EXTERNAL,
-                    owner: InstanceOwner.EXTERNAL,
+                    desiredStatus: types_1.DesiredInstanceStatus.EXTERNAL,
+                    owner: types_1.InstanceOwner.EXTERNAL,
                     health: healthUrl,
                     address,
                 };
                 this._instances.push(instance);
-                socketManager.emitSystemEvent(systemId, EVENT_INSTANCE_CREATED, instance);
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_INSTANCE_CREATED, instance);
             }
             this.save();
             return instance;
@@ -196,33 +202,33 @@ export class InstanceManager {
     }
     markAsStopped(systemId, instanceId) {
         return this.exclusive(systemId, instanceId, async () => {
-            systemId = normalizeKapetaUri(systemId);
-            const instance = _.find(this._instances, { systemId, instanceId });
-            if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
-                instance.status = InstanceStatus.STOPPED;
+            systemId = (0, utils_1.normalizeKapetaUri)(systemId);
+            const instance = lodash_1.default.find(this._instances, { systemId, instanceId });
+            if (instance && instance.owner === types_1.InstanceOwner.EXTERNAL && instance.status !== types_1.InstanceStatus.STOPPED) {
+                instance.status = types_1.InstanceStatus.STOPPED;
                 instance.pid = null;
                 instance.health = null;
-                socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
                 this.save();
             }
         });
     }
     async startAllForPlan(systemId) {
-        systemId = normalizeKapetaUri(systemId);
-        const plan = await assetManager.getPlan(systemId, true);
+        systemId = (0, utils_1.normalizeKapetaUri)(systemId);
+        const plan = await assetManager_1.assetManager.getPlan(systemId, true);
         if (!plan) {
             throw new Error(`Plan not found: ${systemId}`);
         }
         if (!plan.spec.blocks) {
             throw new Error(`No blocks found in plan: ${systemId}`);
         }
-        return taskManager.add(`plan:start:${systemId}`, async () => {
+        return taskManager_1.taskManager.add(`plan:start:${systemId}`, async () => {
             let promises = [];
             let errors = [];
             for (let blockInstance of Object.values(plan.spec.blocks)) {
                 try {
                     promises.push(this.start(systemId, blockInstance.id).then((taskOrInstance) => {
-                        if (taskOrInstance instanceof Task) {
+                        if (taskOrInstance instanceof taskManager_1.Task) {
                             return taskOrInstance.wait();
                         }
                         return taskOrInstance;
@@ -248,30 +254,30 @@ export class InstanceManager {
     }
     async stopInner(systemId, instanceId, changeDesired = false) {
         return this.exclusive(systemId, instanceId, async () => {
-            systemId = normalizeKapetaUri(systemId);
+            systemId = (0, utils_1.normalizeKapetaUri)(systemId);
             const instance = this.getInstance(systemId, instanceId);
             if (!instance) {
                 return;
             }
-            if (instance.status === InstanceStatus.STOPPED) {
+            if (instance.status === types_1.InstanceStatus.STOPPED) {
                 return;
             }
-            if (changeDesired && instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
-                instance.desiredStatus = DesiredInstanceStatus.STOP;
+            if (changeDesired && instance.desiredStatus !== types_1.DesiredInstanceStatus.EXTERNAL) {
+                instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
             }
-            instance.status = InstanceStatus.STOPPING;
-            socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+            instance.status = types_1.InstanceStatus.STOPPING;
+            socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
             console.log('Stopping instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
             this.save();
             try {
                 if (instance.type === 'docker') {
-                    const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
-                    const container = await containerManager.getContainerByName(containerName);
+                    const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
+                    const container = await containerManager_1.containerManager.getContainerByName(containerName);
                     if (container) {
                         try {
                             await container.stop();
-                            instance.status = InstanceStatus.STOPPED;
-                            socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                            instance.status = types_1.InstanceStatus.STOPPED;
+                            socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
                             this.save();
                         }
                         catch (e) {
@@ -284,13 +290,13 @@ export class InstanceManager {
                     return;
                 }
                 if (!instance.pid) {
-                    instance.status = InstanceStatus.STOPPED;
+                    instance.status = types_1.InstanceStatus.STOPPED;
                     this.save();
                     return;
                 }
                 process.kill(instance.pid, 'SIGTERM');
-                instance.status = InstanceStatus.STOPPED;
-                socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
+                instance.status = types_1.InstanceStatus.STOPPED;
+                socketManager_1.socketManager.emitSystemEvent(systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
                 this.save();
             }
             catch (e) {
@@ -299,9 +305,9 @@ export class InstanceManager {
         });
     }
     stopAllForPlan(systemId) {
-        systemId = normalizeKapetaUri(systemId);
+        systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         const instancesForPlan = this._instances.filter((instance) => instance.systemId === systemId);
-        return taskManager.add(`plan:stop:${systemId}`, async () => {
+        return taskManager_1.taskManager.add(`plan:stop:${systemId}`, async () => {
             return this.stopInstances(instancesForPlan);
         }, {
             name: `Stopping plan ${systemId}`,
@@ -309,33 +315,33 @@ export class InstanceManager {
     }
     async start(systemId, instanceId) {
         return this.exclusive(systemId, instanceId, async () => {
-            systemId = normalizeKapetaUri(systemId);
-            const plan = await assetManager.getPlan(systemId, true);
+            systemId = (0, utils_1.normalizeKapetaUri)(systemId);
+            const plan = await assetManager_1.assetManager.getPlan(systemId, true);
             if (!plan) {
                 throw new Error('Plan not found: ' + systemId);
             }
-            const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
+            const blockInstance = plan.spec && plan.spec.blocks ? lodash_1.default.find(plan.spec.blocks, { id: instanceId }) : null;
             if (!blockInstance) {
                 throw new Error('Block instance not found: ' + instanceId);
             }
-            const blockRef = normalizeKapetaUri(blockInstance.block.ref);
-            const blockAsset = await assetManager.getAsset(blockRef, true);
+            const blockRef = (0, utils_1.normalizeKapetaUri)(blockInstance.block.ref);
+            const blockAsset = await assetManager_1.assetManager.getAsset(blockRef, true);
             if (!blockAsset) {
                 throw new Error('Block not found: ' + blockRef);
             }
             const existingInstance = this.getInstance(systemId, instanceId);
             if (existingInstance) {
-                if (existingInstance.status === InstanceStatus.READY) {
+                if (existingInstance.status === types_1.InstanceStatus.READY) {
                     // Instance is already running
                     return existingInstance;
                 }
-                if (existingInstance.desiredStatus === DesiredInstanceStatus.RUN &&
-                    existingInstance.status === InstanceStatus.STARTING) {
+                if (existingInstance.desiredStatus === types_1.DesiredInstanceStatus.RUN &&
+                    existingInstance.status === types_1.InstanceStatus.STARTING) {
                     // Internal instance is already starting - don't start it again
                     return existingInstance;
                 }
-                if (existingInstance.owner === InstanceOwner.EXTERNAL &&
-                    existingInstance.status === InstanceStatus.STARTING) {
+                if (existingInstance.owner === types_1.InstanceOwner.EXTERNAL &&
+                    existingInstance.status === types_1.InstanceStatus.STARTING) {
                     // External instance is already starting - don't start it again
                     return existingInstance;
                 }
@@ -345,10 +351,10 @@ export class InstanceManager {
                 instanceId,
                 ref: blockRef,
                 name: blockAsset.data.metadata.name,
-                desiredStatus: DesiredInstanceStatus.RUN,
-                owner: InstanceOwner.INTERNAL,
-                type: existingInstance?.type ?? InstanceType.UNKNOWN,
-                status: InstanceStatus.STARTING,
+                desiredStatus: types_1.DesiredInstanceStatus.RUN,
+                owner: types_1.InstanceOwner.INTERNAL,
+                type: existingInstance?.type ?? types_1.InstanceType.UNKNOWN,
+                status: types_1.InstanceStatus.STARTING,
                 startedAt: Date.now(),
             };
             console.log('Starting instance: %s::%s [desired: %s]', systemId, instanceId, instance.desiredStatus);
@@ -357,43 +363,43 @@ export class InstanceManager {
             const blockSpec = blockAsset.data.spec;
             if (blockSpec.consumers) {
                 const promises = blockSpec.consumers.map((consumer) => {
-                    const consumerUri = parseKapetaUri(consumer.kind);
-                    const asset = definitionsManager.getDefinition(consumer.kind);
+                    const consumerUri = (0, nodejs_utils_1.parseKapetaUri)(consumer.kind);
+                    const asset = definitionsManager_1.definitionsManager.getDefinition(consumer.kind);
                     if (!asset) {
                         // Definition not found
                         return Promise.resolve();
                     }
-                    if (KIND_OPERATOR.toLowerCase() !== asset.definition.kind.toLowerCase()) {
+                    if (operatorManager_1.KIND_OPERATOR.toLowerCase() !== asset.definition.kind.toLowerCase()) {
                         // Not an operator
                         return Promise.resolve();
                     }
                     console.log('Ensuring resource: %s in %s', consumerUri.id, systemId);
-                    return operatorManager.ensureResource(systemId, consumerUri.fullName, consumerUri.version);
+                    return operatorManager_1.operatorManager.ensureResource(systemId, consumerUri.fullName, consumerUri.version);
                 });
                 await Promise.all(promises);
             }
             if (existingInstance) {
                 // Check if the instance is already running - but after we've commmuicated the desired status
                 const currentStatus = await this.requestInstanceStatus(existingInstance);
-                if (currentStatus === InstanceStatus.READY) {
+                if (currentStatus === types_1.InstanceStatus.READY) {
                     // Instance is already running
                     return existingInstance;
                 }
             }
-            const instanceConfig = await configManager.getConfigForSection(systemId, instanceId);
-            const task = taskManager.add(`instance:start:${systemId}:${instanceId}`, async () => {
-                const runner = new BlockInstanceRunner(systemId);
+            const instanceConfig = await configManager_1.configManager.getConfigForSection(systemId, instanceId);
+            const task = taskManager_1.taskManager.add(`instance:start:${systemId}:${instanceId}`, async () => {
+                const runner = new BlockInstanceRunner_1.BlockInstanceRunner(systemId);
                 const startTime = Date.now();
                 try {
                     const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
-                    instance.status = InstanceStatus.READY;
+                    instance.status = types_1.InstanceStatus.READY;
                     return this.saveInternalInstance({
                         ...instance,
                         type: processInfo.type,
                         pid: processInfo.pid ?? -1,
                         health: null,
                         portType: processInfo.portType,
-                        status: InstanceStatus.READY,
+                        status: types_1.InstanceStatus.READY,
                     });
                 }
                 catch (e) {
@@ -408,17 +414,17 @@ export class InstanceManager {
                     ];
                     const out = await this.saveInternalInstance({
                         ...instance,
-                        type: InstanceType.UNKNOWN,
+                        type: types_1.InstanceType.UNKNOWN,
                         pid: null,
                         health: null,
                         portType: DEFAULT_HEALTH_PORT_TYPE,
-                        status: InstanceStatus.FAILED,
+                        status: types_1.InstanceStatus.FAILED,
                         errorMessage: e.message ?? 'Failed to start - Check logs for details.',
                     });
-                    socketManager.emitInstanceLog(systemId, instanceId, logs[0]);
-                    socketManager.emitInstanceEvent(systemId, blockInstance.id, EVENT_INSTANCE_EXITED, {
+                    socketManager_1.socketManager.emitInstanceLog(systemId, instanceId, logs[0]);
+                    socketManager_1.socketManager.emitInstanceEvent(systemId, blockInstance.id, socketManager_1.EVENT_INSTANCE_EXITED, {
                         error: `Failed to start instance: ${e.message}`,
-                        status: EVENT_INSTANCE_EXITED,
+                        status: socketManager_1.EVENT_INSTANCE_EXITED,
                         instanceId: blockInstance.id,
                     });
                     return out;
@@ -438,7 +444,7 @@ export class InstanceManager {
      * We do it this way to not cause the user to wait for the instance to start again
      */
     async prepareForRestart(systemId, instanceId) {
-        systemId = normalizeKapetaUri(systemId);
+        systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         await this.stopInner(systemId, instanceId);
     }
     async stopAll() {
@@ -451,7 +457,7 @@ export class InstanceManager {
     }
     save() {
         try {
-            storageService.put('instances', this._instances.map((instance) => {
+            storageService_1.storageService.put('instances', this._instances.map((instance) => {
                 return { ...instance };
             }));
         }
@@ -470,24 +476,54 @@ export class InstanceManager {
                 if (!instance.systemId) {
                     return;
                 }
-                instance.systemId = normalizeKapetaUri(instance.systemId);
+                instance.systemId = (0, utils_1.normalizeKapetaUri)(instance.systemId);
                 if (instance.ref) {
-                    instance.ref = normalizeKapetaUri(instance.ref);
+                    instance.ref = (0, utils_1.normalizeKapetaUri)(instance.ref);
+                }
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.RUN) {
+                    // Check if the plan still exists and the instance is still in the plan
+                    // - and that the block definition exists
+                    try {
+                        const plan = await assetManager_1.assetManager.getAsset(instance.systemId, true, false);
+                        if (!plan) {
+                            instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                            changed = true;
+                            return;
+                        }
+                        const planData = plan.data;
+                        const planInstance = planData?.spec?.blocks?.find((b) => b.id === instance.instanceId);
+                        if (!planInstance || !planInstance?.block?.ref) {
+                            instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                            changed = true;
+                            return;
+                        }
+                        const blockDef = await assetManager_1.assetManager.getAsset(instance.ref, true, false);
+                        if (!blockDef) {
+                            instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                            changed = true;
+                            return;
+                        }
+                    }
+                    catch (e) {
+                        console.warn('Failed to check assets', instance.systemId, e);
+                        instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
+                        return;
+                    }
                 }
                 const newStatus = await this.requestInstanceStatus(instance);
                 /*
                 console.log('Check instance %s %s: [current: %s, new: %s, desired: %s]',
                     instance.systemId, instance.instanceId, instance.status, newStatus, instance.desiredStatus);
                 */
-                if (newStatus === InstanceStatus.BUSY) {
+                if (newStatus === types_1.InstanceStatus.BUSY) {
                     // If instance is busy we skip it
                     //console.log('Instance %s %s is busy', instance.systemId, instance.instanceId);
                     return;
                 }
                 if (instance.startedAt !== undefined &&
-                    newStatus === InstanceStatus.UNHEALTHY &&
-                    instance.startedAt + HEALTH_CHECK_TIMEOUT < Date.now() &&
-                    instance.status === InstanceStatus.STARTING) {
+                    newStatus === types_1.InstanceStatus.UNHEALTHY &&
+                    instance.startedAt + containerManager_1.HEALTH_CHECK_TIMEOUT < Date.now() &&
+                    instance.status === types_1.InstanceStatus.STARTING) {
                     // If instance is starting we consider unhealthy an indication
                     // that it is still starting
                     //console.log('Instance %s %s is still starting', instance.systemId, instance.instanceId);
@@ -495,22 +531,22 @@ export class InstanceManager {
                 }
                 if (instance.status !== newStatus) {
                     const oldStatus = instance.status;
-                    const skipUpdate = (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
-                        ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
-                            instance.status === InstanceStatus.STOPPING) ||
-                        (newStatus === InstanceStatus.STOPPED &&
-                            instance.status === InstanceStatus.STARTING &&
-                            instance.desiredStatus === DesiredInstanceStatus.RUN);
+                    const skipUpdate = (newStatus === types_1.InstanceStatus.STOPPED && instance.status === types_1.InstanceStatus.FAILED) ||
+                        ([types_1.InstanceStatus.READY, types_1.InstanceStatus.UNHEALTHY].includes(newStatus) &&
+                            instance.status === types_1.InstanceStatus.STOPPING) ||
+                        (newStatus === types_1.InstanceStatus.STOPPED &&
+                            instance.status === types_1.InstanceStatus.STARTING &&
+                            instance.desiredStatus === types_1.DesiredInstanceStatus.RUN);
                     if (!skipUpdate) {
                         const oldStatus = instance.status;
                         instance.status = newStatus;
                         console.log('Instance status changed: %s %s: %s -> %s', instance.systemId, instance.instanceId, oldStatus, instance.status);
-                        socketManager.emitSystemEvent(instance.systemId, EVENT_STATUS_CHANGED, instance);
+                        socketManager_1.socketManager.emitSystemEvent(instance.systemId, socketManager_1.EVENT_STATUS_CHANGED, instance);
                         changed = true;
                     }
                 }
-                if (instance.desiredStatus === DesiredInstanceStatus.RUN &&
-                    [InstanceStatus.STOPPED, InstanceStatus.FAILED, InstanceStatus.STOPPING].includes(newStatus)) {
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.RUN &&
+                    [types_1.InstanceStatus.STOPPED, types_1.InstanceStatus.FAILED, types_1.InstanceStatus.STOPPING].includes(newStatus)) {
                     //If the instance is stopped but we want it to run, start it
                     try {
                         await this.start(instance.systemId, instance.instanceId);
@@ -520,8 +556,8 @@ export class InstanceManager {
                     }
                     return;
                 }
-                if (instance.desiredStatus === DesiredInstanceStatus.STOP &&
-                    [InstanceStatus.READY, InstanceStatus.STARTING, InstanceStatus.UNHEALTHY].includes(newStatus)) {
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.STOP &&
+                    [types_1.InstanceStatus.READY, types_1.InstanceStatus.STARTING, types_1.InstanceStatus.UNHEALTHY].includes(newStatus)) {
                     //If the instance is running but we want it to stop, stop it
                     try {
                         await this.stopInner(instance.systemId, instance.instanceId);
@@ -531,9 +567,9 @@ export class InstanceManager {
                     }
                     return;
                 }
-                if (instance.desiredStatus === DesiredInstanceStatus.RUN &&
+                if (instance.desiredStatus === types_1.DesiredInstanceStatus.RUN &&
                     instance.status !== newStatus &&
-                    newStatus === InstanceStatus.UNHEALTHY) {
+                    newStatus === types_1.InstanceStatus.UNHEALTHY) {
                     //If the instance is unhealthy, try to restart it
                     console.log('Restarting unhealthy instance', instance);
                     try {
@@ -552,93 +588,94 @@ export class InstanceManager {
         //console.log('\n##\n');
     }
     async getExternalStatus(instance) {
-        if (instance.type === InstanceType.DOCKER) {
-            const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
-            const container = await containerManager.getContainerByName(containerName);
+        if (instance.type === types_1.InstanceType.DOCKER) {
+            const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
+            const container = await containerManager_1.containerManager.getContainerByName(containerName);
             if (!container) {
                 // If the container doesn't exist, we consider the instance stopped
-                return InstanceStatus.STOPPED;
+                return types_1.InstanceStatus.STOPPED;
             }
             const state = await container.status();
             if (state.Status === 'running') {
                 if (state.Health?.Status === 'healthy') {
-                    return InstanceStatus.READY;
+                    return types_1.InstanceStatus.READY;
                 }
                 if (state.Health?.Status === 'starting') {
-                    return InstanceStatus.STARTING;
+                    return types_1.InstanceStatus.STARTING;
                 }
                 if (state.Health?.Status === 'unhealthy') {
-                    return InstanceStatus.UNHEALTHY;
+                    return types_1.InstanceStatus.UNHEALTHY;
                 }
-                return InstanceStatus.READY;
+                return types_1.InstanceStatus.READY;
             }
             if (state.Status === 'created') {
-                return InstanceStatus.STARTING;
+                return types_1.InstanceStatus.STARTING;
             }
             if (state.Status === 'exited' || state.Status === 'dead') {
-                return InstanceStatus.STOPPED;
+                return types_1.InstanceStatus.STOPPED;
             }
             if (state.Status === 'removing') {
-                return InstanceStatus.BUSY;
+                return types_1.InstanceStatus.BUSY;
             }
             if (state.Status === 'restarting') {
-                return InstanceStatus.BUSY;
+                return types_1.InstanceStatus.BUSY;
             }
             if (state.Status === 'paused') {
-                return InstanceStatus.BUSY;
+                return types_1.InstanceStatus.BUSY;
             }
-            return InstanceStatus.STOPPED;
+            return types_1.InstanceStatus.STOPPED;
         }
         if (!instance.pid) {
-            return InstanceStatus.STOPPED;
+            return types_1.InstanceStatus.STOPPED;
         }
         //Otherwise its just a normal process.
         //TODO: Handle for Windows
         try {
             if (process.kill(instance.pid, 0)) {
-                return InstanceStatus.READY;
+                return types_1.InstanceStatus.READY;
             }
         }
         catch (err) {
             if (err.code === 'EPERM') {
-                return InstanceStatus.READY;
+                return types_1.InstanceStatus.READY;
             }
         }
-        return InstanceStatus.STOPPED;
+        return types_1.InstanceStatus.STOPPED;
     }
     async requestInstanceStatus(instance) {
         const externalStatus = await this.getExternalStatus(instance);
-        if (instance.type === InstanceType.DOCKER) {
+        if (instance.type === types_1.InstanceType.DOCKER) {
             // For docker instances we can rely on docker status
             return externalStatus;
         }
-        if (externalStatus === InstanceStatus.STOPPED) {
+        if (externalStatus === types_1.InstanceStatus.STOPPED) {
             return externalStatus;
         }
         if (!instance.health) {
             //No health url means we assume it's healthy as soon as it's running
-            return InstanceStatus.READY;
+            return types_1.InstanceStatus.READY;
         }
         return new Promise((resolve) => {
             if (!instance.health) {
-                resolve(InstanceStatus.READY);
+                resolve(types_1.InstanceStatus.READY);
                 return;
             }
-            request(instance.health, (err, response) => {
+            (0, request_1.default)(instance.health, (err, response) => {
                 if (err) {
-                    resolve(InstanceStatus.UNHEALTHY);
+                    resolve(types_1.InstanceStatus.UNHEALTHY);
                     return;
                 }
                 if (response.statusCode > 399) {
-                    resolve(InstanceStatus.UNHEALTHY);
+                    resolve(types_1.InstanceStatus.UNHEALTHY);
                     return;
                 }
-                resolve(InstanceStatus.READY);
+                resolve(types_1.InstanceStatus.READY);
             });
         });
     }
 }
-export const instanceManager = new InstanceManager();
+exports.InstanceManager = InstanceManager;
+exports.instanceManager = new InstanceManager();
 process.on('exit', async () => {
-    await instanceManager.stopAll();
+    await exports.instanceManager.stopAll();
 });