@phystack/device-phyos 4.3.40-dev

package/dist/utilities/docker.js

@@ -0,0 +1,819 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const dockerode_1 = __importDefault(require("dockerode"));
+const lodash_1 = require("lodash");
+const phy_logger_1 = require("@phystack/phy-logger");
+const twin_types_1 = require("../types/twin.types");
+const async_mutex_1 = require("async-mutex");
+const twin_manager_1 = require("./twin-manager");
+const docker_progress_tracker_1 = require("./docker-progress-tracker");
+const uuid_1 = require("uuid");
+const hub_client_1 = require("@phystack/hub-client");
+class DockerManager {
+    docker;
+    logger;
+    twinInstancesRunning;
+    imagesInProgress;
+    instances;
+    firstRunCompleted = false;
+    hubDevice = false;
+    phyHubClient = null;
+    signalsClient;
+    sessionId = null;
+    operationLock = new async_mutex_1.Mutex();
+    imagePullsInProgress = new Map();
+    constructor() {
+        this.docker = new dockerode_1.default();
+        this.logger = new phy_logger_1.PhyLogger({
+            logToFile: false,
+            logToConsole: true,
+            includeTrace: true,
+            namespace: 'DockerManager',
+        });
+        this.twinInstancesRunning = new Map();
+        this.imagesInProgress = new Map();
+        this.instances = new Map();
+        this.listenToDockerEvents();
+    }
+    async initializeHubSignals(initSignalsParams, dataResidency) {
+        try {
+            this.phyHubClient = await (0, hub_client_1.connectPhyClient)({ moduleName: 'docker-crash', dataResidency });
+            this.signalsClient = await this.phyHubClient.initializeSignals(initSignalsParams);
+        }
+        catch (error) {
+            this.logger.error('Failed to initialize signals client for docker-crash signals:', error);
+        }
+    }
+    async listenToDockerEvents() {
+        try {
+            const eventStream = await this.docker.getEvents({
+                filters: {
+                    event: ['die', 'stop', 'kill'],
+                    type: ['container'],
+                },
+            });
+            eventStream.on('data', async (buffer) => {
+                try {
+                    const event = JSON.parse(buffer.toString());
+                    if (event.status === 'die' && event.exitCode !== 0) {
+                        await this.onContainerCrash(event.id);
+                    }
+                }
+                catch (error) {
+                }
+            });
+            eventStream.on('error', (error) => {
+                setTimeout(() => this.listenToDockerEvents(), 5000);
+            });
+            eventStream.on('end', () => {
+                setTimeout(() => this.listenToDockerEvents(), 5000);
+            });
+        }
+        catch (error) {
+            setTimeout(() => this.listenToDockerEvents(), 5000);
+        }
+    }
+    getDefaultPlatform() {
+        const arch = process.arch;
+        if (arch === 'x64') {
+            return 'linux/amd64';
+        }
+        else if (arch === 'arm64') {
+            return 'linux/arm64';
+        }
+        else if (arch === 'arm') {
+            return 'linux/arm/v7';
+        }
+        else if (arch === 'ia32') {
+            return 'linux/386';
+        }
+        return 'linux/amd64';
+    }
+    extractNameFromImage = (image) => {
+        try {
+            const regex = /[^\/]+\.([^\/:]+:[^\/]+)$/;
+            const match = image.match(regex);
+            return match && match.length > 0 ? match[1].split(':')[0] : null;
+        }
+        catch (error) {
+            return null;
+        }
+    };
+    async onContainerCrash(containerId) {
+        try {
+            const twinManager = await (0, twin_manager_1.getTwinManagerInstance)();
+            if (!twinManager) {
+                throw new Error('Error getting twin manager instance');
+            }
+            this.hubDevice = await twinManager.getHubDevice();
+            let twinId;
+            for (const [id, instance] of this.instances.entries()) {
+                if (instance.container.id === containerId) {
+                    twinId = id;
+                    break;
+                }
+            }
+            const container = this.docker.getContainer(containerId);
+            const logs = await container.logs({ stdout: true, stderr: true, tail: 50 });
+            const logString = logs.toString();
+            if (twinId) {
+                const reportedProperties = {
+                    lastError: {
+                        message: `container crashed`,
+                        timestamp: new Date(),
+                        errorObject: {
+                            containerLogs: logString,
+                        },
+                    },
+                };
+                this.hubDevice.reportEdgeTwinProperties(twinId, reportedProperties);
+            }
+            const deviceStatus = await this.hubDevice.getDeviceStatus();
+            const twinResponse = await this.hubDevice.getEdgeInstance({ twinId });
+            let twin = undefined;
+            if (twinResponse && twinResponse.twin) {
+                twin = twinResponse.twin;
+            }
+            let ip = '127.0.0.1';
+            if (deviceStatus.ip && deviceStatus.ip.length > 0) {
+                ip = deviceStatus.ip[0].ipv4;
+            }
+            if (!this.phyHubClient) {
+                const initSignalsParams = {
+                    deviceId: deviceStatus.deviceId,
+                    installationId: twin?.properties.desired.installationId ?? 'XXXXXXXXXXXXXXXXXXXXXXXX',
+                    spaceId: deviceStatus.spaceId,
+                    tenantId: deviceStatus.tenantId,
+                    appVersion: 'XXXXXXXXXXXXXXXXXXXXXXXX',
+                    appId: 'XXXXXXXXXXXXXXXXXXXXXXXX',
+                    installationVersion: 'XXXXXXXXXXXXXXXXXXXXXXXX',
+                    environment: deviceStatus.gridEnv,
+                    dataResidency: deviceStatus.dataResidency.toUpperCase(),
+                    country: 'SE',
+                    accessToken: deviceStatus.accessKey,
+                    clientUserAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
+                    ip,
+                };
+                await this.initializeHubSignals(initSignalsParams, deviceStatus.dataResidency.toUpperCase());
+            }
+            if (this.phyHubClient) {
+                if (!this.sessionId) {
+                    const instance = this.signalsClient.getInstanceProps();
+                    this.sessionId = instance.sessionId;
+                    const clientId = instance.clientId ?? (0, uuid_1.v4)();
+                    await this.hubDevice.sendSessionSignal({
+                        deviceId: this.hubDevice.deviceId,
+                        data: {
+                            c: instance.tenantId,
+                            b: this.sessionId,
+                            d: instance.sessionCreated,
+                            f: instance.environment,
+                            g: instance.dataResidency,
+                            m: instance.country,
+                            n: instance.locationAccuracy,
+                            o: instance.latitude,
+                            p: instance.longitude,
+                            h: instance.spaceId,
+                            i: instance.appId,
+                            j: instance.appVersion,
+                            k: instance.installationId,
+                            l: instance.installationVersion,
+                            q: instance.deviceId,
+                            r: clientId,
+                            e: instance.clientIp ?? instance.ip,
+                        },
+                    });
+                    await this.hubDevice.sendClientSignal({
+                        deviceId: this.hubDevice.deviceId,
+                        data: {
+                            c: instance.tenantId,
+                            r: clientId,
+                            ah: instance.clientCreated,
+                            aj: instance.clientUserAgent,
+                            g: instance.dataResidency,
+                            m: instance.country,
+                            o: instance.latitude,
+                            p: instance.longitude,
+                            n: instance.locationAccuracy,
+                            e: instance.ip ?? instance.clientIp ?? undefined,
+                        },
+                    });
+                }
+                const timeWithoutZ = new Date().toISOString().replace(/Z$/, '');
+                const eventPayload = {
+                    g: deviceStatus.dataResidency.toUpperCase(),
+                    b: this.sessionId,
+                    t: timeWithoutZ,
+                    s: 'MONITOR_CONTAINER_CRASH',
+                    u: false,
+                    h: deviceStatus.spaceId,
+                    ac: twinId,
+                    ad: deviceStatus.deviceId,
+                    c: deviceStatus.tenantId,
+                };
+                await this.hubDevice.sendEventSignal({
+                    deviceId: this.hubDevice.deviceId,
+                    data: eventPayload,
+                });
+            }
+        }
+        catch (error) {
+            this.logger.error(`onContainerCrash(): Error in reporting the container crash`, error);
+        }
+    }
+    async imageExists(image) {
+        this.logger.info(`imageExists(): Checking if docker image ${image} exists`);
+        try {
+            await this.docker.getImage(image).inspect();
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async pullImage(image, authConfig, abortController) {
+        this.logger.info(`pullImage(): Pulling docker image ${image}`);
+        return new Promise((resolve, reject) => {
+            const platform = this.getDefaultPlatform();
+            this.logger.info(`pullImage(): Using platform ${platform} for image ${image}`);
+            if (abortController.signal.aborted) {
+                this.logger.info(`pullImage(): Pull for image ${image} was aborted before starting`);
+                return resolve();
+            }
+            let isCompleted = false;
+            const abortHandler = () => {
+                if (isCompleted)
+                    return;
+                this.logger.info(`pullImage(): Pull for image ${image} was aborted`);
+                isCompleted = true;
+                const imageProgress = this.imagesInProgress.get(image);
+                if (imageProgress) {
+                    imageProgress.inProgress = false;
+                    this.imagesInProgress.delete(image);
+                }
+                resolve();
+            };
+            abortController.signal.addEventListener('abort', abortHandler);
+            const timeoutId = setTimeout(() => {
+                if (!isCompleted) {
+                    this.logger.error(`pullImage(): Timeout waiting for image ${image} to pull`);
+                    abortController.abort();
+                }
+            }, 7200000);
+            const progressTracker = new docker_progress_tracker_1.DockerProgressTracker(image, 'DockerManager');
+            const imageProgress = {
+                inProgress: true,
+                layersProgress: new Map(),
+            };
+            this.imagesInProgress.set(image, imageProgress);
+            this.docker.pull(image, {
+                authconfig: authConfig,
+                platform,
+                architecture: process.arch,
+                os: 'linux',
+            }, (err, stream) => {
+                if (err) {
+                    clearTimeout(timeoutId);
+                    this.logger.error(`pullImage(): Error initiating pull for image ${image}`, err);
+                    const isCredentialError = err.message &&
+                        (err.message.includes('unauthorized') ||
+                            err.message.includes('authentication required') ||
+                            err.message.includes('access denied') ||
+                            err.message.includes('credential') ||
+                            err.statusCode === 401);
+                    if (isCredentialError) {
+                        this.logger.error(`pullImage(): Credential error detected for image ${image}, releasing lock immediately`);
+                        const imageProgress = this.imagesInProgress.get(image);
+                        if (imageProgress) {
+                            imageProgress.inProgress = false;
+                            this.imagesInProgress.delete(image);
+                        }
+                    }
+                    isCompleted = true;
+                    progressTracker.markFailed(err);
+                    return reject(err);
+                }
+                if (abortController.signal.aborted) {
+                    clearTimeout(timeoutId);
+                    abortHandler();
+                    return;
+                }
+                this.docker.modem.followProgress(stream, (err, output) => {
+                    clearTimeout(timeoutId);
+                    if (isCompleted)
+                        return;
+                    if (err) {
+                        this.logger.error(`pullImage(): Error pulling image ${image}`, err);
+                        imageProgress.inProgress = false;
+                        this.imagesInProgress.delete(image);
+                        isCompleted = true;
+                        progressTracker.markFailed(err);
+                        reject(err);
+                        return;
+                    }
+                    this.logger.info(`pullImage(): Image ${image} download complete`);
+                    imageProgress.inProgress = false;
+                    this.imagesInProgress.delete(image);
+                    isCompleted = true;
+                    progressTracker.markComplete();
+                    resolve();
+                }, (event) => {
+                    if (abortController.signal.aborted || isCompleted) {
+                        return;
+                    }
+                    progressTracker.processEvent(event);
+                    if (event.id && event.progressDetail) {
+                        imageProgress.layersProgress.set(event.id, event.progressDetail);
+                    }
+                });
+            });
+        });
+    }
+    async createContainer(twin, skipGpu = false) {
+        const { image, createOptions } = twin.properties.desired;
+        const parsedOptions = createOptions;
+        this.logger.info(`createContainer(): Twin object for ${twin.id}:`, JSON.stringify(twin, null, 2));
+        this.logger.info(`createContainer(): Parsed options for ${twin.id}:`, JSON.stringify(parsedOptions, null, 2));
+        this.logger.info(`createContainer(): Device requests for ${twin.id}:`, JSON.stringify(parsedOptions?.HostConfig?.DeviceRequests, null, 2));
+        const isHostNetwork = parsedOptions?.HostConfig?.NetworkMode === 'host';
+        const containerConfig = {
+            Image: image,
+            Env: [...(parsedOptions?.Env ?? []), `TWIN_ID=${twin.id}`],
+            HostConfig: {
+                ...parsedOptions?.HostConfig,
+                NetworkMode: isHostNetwork ? 'host' : 'default',
+                ExtraHosts: [isHostNetwork ? `phyos:127.0.0.1` : `phyos:172.26.128.1`],
+                RestartPolicy: parsedOptions?.HostConfig?.RestartPolicy ?? { Name: 'always' },
+                Privileged: parsedOptions?.HostConfig?.Privileged ?? false,
+                Memory: parsedOptions?.HostConfig?.Memory ?? 0,
+                CpuShares: parsedOptions?.HostConfig?.CpuShares ?? 0,
+                StorageOpt: parsedOptions?.HostConfig?.StorageOpt ?? {},
+                DiskQuota: parsedOptions?.HostConfig?.DiskQuota ?? 0,
+                ReadonlyRootfs: parsedOptions?.HostConfig?.ReadonlyRootfs ?? false,
+                BlkioWeight: parsedOptions?.HostConfig?.BlkioWeight ?? 0,
+                BlkioDeviceReadBps: parsedOptions?.HostConfig?.BlkioDeviceReadBps ?? [],
+                BlkioDeviceWriteBps: parsedOptions?.HostConfig?.BlkioDeviceWriteBps ?? [],
+                Devices: parsedOptions?.HostConfig?.Devices ?? [],
+                DeviceRequests: parsedOptions?.HostConfig?.DeviceRequests ?? [],
+                Binds: parsedOptions?.HostConfig?.Binds ?? [],
+            },
+        };
+        if (skipGpu && parsedOptions?.HostConfig?.DeviceRequests) {
+            this.logger.info(`createContainer(): Original DeviceRequests before filtering:`, JSON.stringify(parsedOptions.HostConfig.DeviceRequests, null, 2));
+            const filteredDeviceRequests = parsedOptions.HostConfig.DeviceRequests.filter((request) => {
+                const isGpuRequest = request.Capabilities &&
+                    request.Capabilities.some((cap) => cap.includes('gpu'));
+                return !isGpuRequest;
+            });
+            if (containerConfig.HostConfig) {
+                containerConfig.HostConfig.DeviceRequests = filteredDeviceRequests;
+                this.logger.info(`createContainer(): Creating container for twin ${twin.id} without GPU support. Filtered DeviceRequests:`, JSON.stringify(containerConfig.HostConfig.DeviceRequests, null, 2));
+            }
+        }
+        else if (parsedOptions?.HostConfig?.DeviceRequests) {
+            this.logger.info(`createContainer(): Original DeviceRequests:`, JSON.stringify(parsedOptions.HostConfig.DeviceRequests, null, 2));
+            if (containerConfig.HostConfig) {
+                containerConfig.HostConfig.DeviceRequests = parsedOptions.HostConfig.DeviceRequests;
+                const hasGpuRequest = parsedOptions.HostConfig.DeviceRequests.some((req) => req.Capabilities && req.Capabilities.some((cap) => cap.includes('gpu')));
+                if (hasGpuRequest) {
+                    try {
+                        const info = await this.docker.info();
+                        const availableRuntimes = info.Runtimes ? Object.keys(info.Runtimes) : [];
+                        this.logger.info(`Available Docker runtimes:`, JSON.stringify(availableRuntimes, null, 2));
+                        if (availableRuntimes.includes('nvidia')) {
+                            containerConfig.HostConfig.Runtime = 'nvidia';
+                            this.logger.info(`createContainer(): Setting nvidia runtime for GPU support`);
+                        }
+                        else {
+                            this.logger.warn(`createContainer(): NVIDIA runtime not available, GPU may not work`);
+                        }
+                    }
+                    catch (error) {
+                        this.logger.warn(`createContainer(): Error checking for available runtimes:`, error);
+                        containerConfig.HostConfig.Runtime = 'nvidia';
+                    }
+                }
+                this.logger.info(`createContainer(): Applied DeviceRequests to container config:`, JSON.stringify(containerConfig.HostConfig.DeviceRequests, null, 2));
+            }
+            if (parsedOptions.HostConfig.DeviceRequests.some((req) => req.Capabilities && req.Capabilities.some((cap) => cap.includes('gpu')))) {
+                this.logger.info(`createContainer(): Creating container for twin ${twin.id} with GPU support`);
+            }
+        }
+        if (!isHostNetwork) {
+            containerConfig.NetworkingConfig = {
+                EndpointsConfig: {
+                    'phyos-edge-net': {
+                        Aliases: [twin.id],
+                    },
+                },
+            };
+        }
+        this.logger.info(`createContainer(): Creating container for twin ${twin.id}`, {
+            containerConfig,
+        });
+        this.logger.info(`createContainer(): Final container config for twin ${twin.id}:`, JSON.stringify(containerConfig, null, 2));
+        const container = await this.docker.createContainer(containerConfig);
+        try {
+            const containerDetails = await container.inspect();
+            this.logger.info(`createContainer(): Created container details for twin ${twin.id}:`, JSON.stringify(containerDetails, null, 2));
+        }
+        catch (error) {
+            this.logger.error(`createContainer(): Failed to inspect container for twin ${twin.id}:`, error);
+        }
+        return container;
+    }
+    async _startContainer(twin) {
+        if (this.twinInstancesRunning.get(twin.id)) {
+            this.logger.info(`startContainer(): Container for twin ${twin.id} is already running or starting`);
+            return;
+        }
+        this.twinInstancesRunning.set(twin.id, true);
+        try {
+            this.logger.info(`startContainer(): Attempting to create container for twin ${twin.id} with GPU support`);
+            try {
+                const container = await this.createContainer(twin, false);
+                this.logger.info(`startContainer(): Container created for twin ${twin.id}, preparing to start it`);
+                this.instances.set(twin.id, { container, twin });
+                await container.start();
+                this.logger.info(`startContainer(): Started container for twin ${twin.id}`);
+                const logs = await container.logs({ stdout: true, stderr: true });
+                this.logger.info(`Container logs for twin ${twin.id}:`, logs.toString());
+            }
+            catch (error) {
+                this.logger.error(`startContainer(): Error starting container with GPU for twin ${twin.id}:`, error.message);
+                const isGpuError = error.message &&
+                    ((error.message.includes('could not select device driver') &&
+                        error.message.includes('capabilities: [[gpu]]')) ||
+                        error.message.includes('nvidia-container-cli: initialization error: nvml error: driver not loaded') ||
+                        error.message.includes('nvidia-container-cli') ||
+                        error.message.includes('unknown or invalid runtime name: nvidia') ||
+                        error.message.includes('nvidia runtime') ||
+                        error.message.includes('GPU') ||
+                        error.message.includes('gpu'));
+                if (isGpuError) {
+                    this.logger.warn(`startContainer(): GPU support not available for twin ${twin.id}, retrying without GPU capabilities. Error: ${error.message}`, { error: error.message });
+                    try {
+                        this.instances.delete(twin.id);
+                        const containerId = error.json?.message?.match(/^.*container: (.*?)(?:\s|$)/)?.[1];
+                        if (containerId) {
+                            try {
+                                await this.docker.getContainer(containerId).remove({ force: true });
+                                this.logger.info(`startContainer(): Removed failed GPU container ${containerId}`);
+                            }
+                            catch (removeError) {
+                                this.logger.info(`startContainer(): Could not remove container ${containerId}, may not exist`);
+                            }
+                        }
+                    }
+                    catch (cleanupError) {
+                        this.logger.warn(`startContainer(): Error during cleanup: ${cleanupError.message}`);
+                    }
+                    const container = await this.createContainer(twin, true);
+                    this.instances.set(twin.id, { container, twin });
+                    await container.start();
+                    this.logger.info(`startContainer(): Started container for twin ${twin.id} without GPU support`);
+                    const logs = await container.logs({ stdout: true, stderr: true });
+                    this.logger.info(`Container logs for twin ${twin.id}:`, logs.toString());
+                }
+                else {
+                    throw error;
+                }
+            }
+        }
+        catch (error) {
+            this.logger.error(`startContainer(): Failed to start container for twin ${twin.id}`, error);
+            if (error.message.includes('port is already allocated')) {
+                this.logger.error(`startContainer(): Port conflict detected for twin ${twin.id}`);
+            }
+            this.instances.delete(twin.id);
+            this.twinInstancesRunning.set(twin.id, false);
+            throw error;
+        }
+    }
+    abortImagePull(image) {
+        const pullInfo = this.imagePullsInProgress.get(image);
+        if (pullInfo) {
+            this.logger.info(`abortImagePull(): Aborting pull for image ${image}`);
+            pullInfo.abortController.abort();
+            const imageProgress = this.imagesInProgress.get(image);
+            if (imageProgress) {
+                imageProgress.inProgress = false;
+                this.imagesInProgress.delete(image);
+            }
+            setTimeout(() => {
+                if (this.imagePullsInProgress.has(image)) {
+                    this.logger.warn(`abortImagePull(): Cleaning up stale pull info for ${image}`);
+                    this.imagePullsInProgress.delete(image);
+                }
+            }, 5000);
+        }
+    }
+    isImagePullInProgress(image) {
+        return this.imagePullsInProgress.has(image);
+    }
+    async ensureImageAvailable(image, authConfig) {
+        const imageExists = await this.imageExists(image);
+        if (imageExists)
+            return true;
+        const existingPull = this.imagePullsInProgress.get(image);
+        if (existingPull) {
+            this.logger.info(`ensureImageAvailable(): Pull already in progress for ${image}, waiting for it to complete`);
+            try {
+                await existingPull.promise;
+                this.logger.info(`ensureImageAvailable(): Existing pull for ${image} completed successfully`);
+                return true;
+            }
+            catch (error) {
+                this.logger.error(`ensureImageAvailable(): Existing pull for ${image} failed`, error);
+                const isCredentialError = error.message &&
+                    (error.message.includes('unauthorized') ||
+                        error.message.includes('authentication required') ||
+                        error.message.includes('access denied') ||
+                        error.message.includes('credential') ||
+                        error.statusCode === 401);
+                if (isCredentialError) {
+                    this.logger.error(`ensureImageAvailable(): Credential error detected for existing pull of ${image}`);
+                }
+                this.imagePullsInProgress.delete(image);
+                return false;
+            }
+        }
+        this.logger.info(`ensureImageAvailable(): Starting new pull for ${image}`);
+        const abortController = new AbortController();
+        const pullPromise = this.pullImage(image, authConfig, abortController).finally(() => {
+            this.imagePullsInProgress.delete(image);
+        });
+        this.imagePullsInProgress.set(image, {
+            promise: pullPromise,
+            abortController,
+        });
+        try {
+            await pullPromise;
+            this.logger.info(`ensureImageAvailable(): Pull for ${image} completed successfully`);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`ensureImageAvailable(): Pull failed for ${image}`, error);
+            const isCredentialError = error.message &&
+                (error.message.includes('unauthorized') ||
+                    error.message.includes('authentication required') ||
+                    error.message.includes('access denied') ||
+                    error.message.includes('credential') ||
+                    error.statusCode === 401);
+            if (isCredentialError) {
+                this.logger.error(`ensureImageAvailable(): Credential error detected for pull of ${image}`);
+            }
+            this.imagePullsInProgress.delete(image);
+            return false;
+        }
+    }
+    async startEdgeApp(twin) {
+        const { image } = twin.properties.desired;
+        if (!image) {
+            return {
+                status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                error: {
+                    message: 'No image found in desired properties',
+                    timestamp: new Date(),
+                },
+            };
+        }
+        const hasRegistry = image.includes('/');
+        let authConfig;
+        let pullError = null;
+        if (hasRegistry) {
+            const [hostname] = image.split('/');
+            authConfig = twin.properties.desired.credentials;
+            if (!authConfig) {
+                this.logger.error(`startEdgeApp(): No credentials found for registry: ${hostname}`);
+                return {
+                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                    error: {
+                        message: `No credentials found for registry: ${hostname}`,
+                        timestamp: new Date(),
+                    },
+                };
+            }
+            try {
+                const pullSuccess = await this.ensureImageAvailable(image, {
+                    username: authConfig.username,
+                    password: authConfig.password,
+                    serveraddress: authConfig.address,
+                });
+                if (!pullSuccess) {
+                    this.logger.error(`startEdgeApp(): Failed to pull image ${image}`);
+                    return {
+                        status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                        error: {
+                            message: `Failed to pull image ${image}`,
+                            timestamp: new Date(),
+                        },
+                    };
+                }
+            }
+            catch (error) {
+                this.logger.error(`startEdgeApp(): Error pulling image ${image}`, error);
+                const errorMsg = error.message || '';
+                const isCredentialError = errorMsg.includes('unauthorized') ||
+                    errorMsg.includes('authentication required') ||
+                    errorMsg.includes('access denied') ||
+                    errorMsg.includes('credential') ||
+                    error.statusCode === 401;
+                return {
+                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                    error: {
+                        message: isCredentialError
+                            ? `Authentication failed for image ${image}. Please check your credentials.`
+                            : `Failed to pull image ${image}: ${errorMsg}`,
+                        timestamp: new Date(),
+                        errorObject: error,
+                    },
+                };
+            }
+        }
+        else {
+            try {
+                const pullSuccess = await this.ensureImageAvailable(image, {});
+                if (!pullSuccess) {
+                    this.logger.error(`startEdgeApp(): Failed to pull local image ${image}`);
+                    return {
+                        status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                        error: {
+                            message: `Failed to pull local image ${image}`,
+                            timestamp: new Date(),
+                        },
+                    };
+                }
+            }
+            catch (error) {
+                this.logger.error(`startEdgeApp(): Error pulling local image ${image}`, error);
+                return {
+                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                    error: {
+                        message: `Failed to pull local image ${image}: ${error.message || ''}`,
+                        timestamp: new Date(),
+                        errorObject: error,
+                    },
+                };
+            }
+        }
+        return this.operationLock.runExclusive(() => this._startEdgeApp(twin));
+    }
+    async _startEdgeApp(twin) {
+        const { image } = twin.properties.desired;
+        try {
+            const existingTwin = this.instances.get(twin.id);
+            const hasImageChanged = existingTwin && existingTwin.twin.properties.desired.image !== image;
+            if (existingTwin && hasImageChanged) {
+                const oldImage = existingTwin.twin.properties.desired.image;
+                this.logger.info(`startEdgeApp(): Twin ${twin.id} image has changed from ${oldImage} to ${image}, aborting old pull if in progress`);
+                this.abortImagePull(oldImage);
+            }
+            const hasConfigChanged = existingTwin &&
+                !(0, lodash_1.isEqual)(existingTwin.twin.properties.desired.containerCreateOptions, twin.properties.desired.containerCreateOptions);
+            if (existingTwin && (hasConfigChanged || hasImageChanged)) {
+                this.logger.info(`startEdgeApp(): Twin ${twin.id} configuration or image has changed, recreating container...`);
+                await this._stopAndRemoveContainer(twin.id);
+            }
+            const finalImageExists = await this.imageExists(image);
+            if (!finalImageExists) {
+                this.logger.error(`startEdgeApp(): Image ${image} not available locally`);
+                return {
+                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
+                    error: {
+                        message: `Image ${image} not available locally`,
+                        timestamp: new Date(),
+                    },
+                };
+            }
+            if (!this.instances.get(twin.id)) {
+                await this._startContainer(twin);
+            }
+            return {
+                status: twin_types_1.TwinStatusEnum.Online,
+                error: undefined,
+            };
+        }
+        catch (error) {
+            this.logger.error(`startEdgeApp(): Failed to start edge app for twin ${twin.id}`, error);
+            return {
+                status: twin_types_1.TwinStatusEnum.Exited,
+                error: {
+                    message: `Failed to start edge app`,
+                    timestamp: new Date(),
+                    errorObject: error,
+                },
+            };
+        }
+    }
+    async _stopAndRemoveContainer(twinId) {
+        const containerInfo = this.instances.get(twinId);
+        if (!containerInfo) {
+            return;
+        }
+        try {
+            const containerDetails = await containerInfo.container.inspect();
+            if (containerDetails.State.Running) {
+                await containerInfo.container.stop();
+                this.logger.info(`stopAndRemoveContainer(): Stopped container ${containerDetails.Id}`);
+            }
+            await containerInfo.container.remove({ force: true });
+            this.logger.info(`stopAndRemoveContainer(): Removed container ${containerDetails.Id} for twin ${twinId}`);
+            this.instances.delete(twinId);
+        }
+        catch (error) {
+            const isNoSuchContainerError = error.statusCode === 404 &&
+                (error.reason === 'no such container' ||
+                    (error.json && error.json.message && error.json.message.includes('No such container')));
+            if (isNoSuchContainerError) {
+                this.logger.warn(`stopAndRemoveContainer(): Container for twin ${twinId} no longer exists, cleaning up references`, { error: error.message });
+                this.instances.delete(twinId);
+            }
+            else {
+                this.logger.error(`stopAndRemoveContainer(): Failed to stop/remove container for twin ${twinId}`, error);
+                throw error;
+            }
+        }
+        finally {
+            this.twinInstancesRunning.set(twinId, false);
+        }
+    }
+    async _stopAndRemoveAllContainers() {
+        if (this.firstRunCompleted) {
+            this.logger.info('stopAndRemoveAllContainers(): First run completed, skipping');
+            return;
+        }
+        this.firstRunCompleted = true;
+        this.logger.info('stopAndRemoveAllContainers(): Stopping and removing all running containers');
+        try {
+            const containers = await this.docker.listContainers({ all: true });
+            await Promise.allSettled(containers.map(async (containerInfo) => {
+                const container = this.docker.getContainer(containerInfo.Id);
+                try {
+                    const containerDetails = await container.inspect();
+                    if (containerDetails.State.Running) {
+                        await container.stop();
+                        this.logger.info(`stopAndRemoveAllContainers(): Stopped container ${containerInfo.Id}`);
+                    }
+                    await container.remove();
+                    this.logger.info(`stopAndRemoveAllContainers(): Removed container ${containerInfo.Id}`);
+                }
+                catch (error) {
+                    this.logger.error(`stopAndRemoveAllContainers(): Failed to stop/remove container ${containerInfo.Id}`, error);
+                }
+            }));
+        }
+        catch (error) {
+            this.logger.error('stopAndRemoveAllContainers(): Failed to list running containers', error);
+            throw error;
+        }
+    }
+    async stopAndRemoveContainer(twinId) {
+        return this.operationLock.runExclusive(() => this._stopAndRemoveContainer(twinId));
+    }
+    async stopAndRemoveAllContainers() {
+        return this.operationLock.runExclusive(() => this._stopAndRemoveAllContainers());
+    }
+    getInstances() {
+        return this.instances;
+    }
+    async manageDockerImages(image) {
+        try {
+            const images = await this.docker.listImages({
+                filters: {
+                    reference: [image.split(':')[0]],
+                },
+            });
+            images.sort((a, b) => b.Created - a.Created);
+            if (images.length > 2) {
+                for (const img of images.slice(2)) {
+                    try {
+                        await this.docker.getImage(img.Id).remove({ force: true });
+                        this.logger.info(`manageDockerImages(): Removed old image ${img.Id}`);
+                    }
+                    catch (error) {
+                        this.logger.warn(`manageDockerImages(): Failed to remove image ${img.Id}`, error);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            this.logger.error(`manageDockerImages(): Error managing images for ${image}`, error);
+        }
+    }
+    async isContainerRunning(twinId) {
+        try {
+            const containers = await this.docker.listContainers();
+            return containers.some(container => container.Names.some(name => name.includes(twinId)) && container.State === 'running');
+        }
+        catch (error) {
+            this.logger.error(`isContainerRunning(): Error checking container status for ${twinId}:`, error);
+            return false;
+        }
+    }
+}
+exports.default = DockerManager;
+//# sourceMappingURL=docker.js.map