@phystack/device-phyos 4.5.55-dev → 4.5.57-dev

package/dist/utilities/docker.js

@@ -1,1090 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const dockerode_1 = __importDefault(require("dockerode"));
-const lodash_1 = require("lodash");
-const phy_logger_1 = require("@phystack/phy-logger");
-const twin_types_1 = require("../types/twin.types");
-const async_mutex_1 = require("async-mutex");
-const twin_manager_1 = require("./twin-manager");
-const docker_progress_tracker_1 = require("./docker-progress-tracker");
-const uuid_1 = require("uuid");
-const hub_client_1 = require("@phystack/hub-client");
-const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
-const IMAGE_PULL_TIMES_FILE = '/data/settings/phyhub/docker-image-pull-times.json';
-class DockerManager {
-    docker;
-    logger;
-    twinInstancesRunning;
-    imagesInProgress;
-    instances;
-    firstRunCompleted = false;
-    hubDevice = false;
-    phyHubClient = null;
-    signalsClient;
-    sessionId = null;
-    operationLock = new async_mutex_1.Mutex();
-    imagePullsInProgress = new Map();
-    imagePullTimes = new Map();
-    constructor() {
-        this.docker = new dockerode_1.default();
-        this.logger = new phy_logger_1.PhyLogger({
-            logToFile: false,
-            logToConsole: true,
-            includeTrace: true,
-            namespace: 'DockerManager',
-        });
-        this.twinInstancesRunning = new Map();
-        this.imagesInProgress = new Map();
-        this.instances = new Map();
-        this.loadImagePullTimes();
-        this.cleanupStaleImagePullTimes().catch((error) => {
-            this.logger.warn('cleanupStaleImagePullTimes(): Failed to cleanup stale entries', error);
-        });
-        this.listenToDockerEvents();
-    }
-    async initializeHubSignals(initSignalsParams, dataResidency) {
-        try {
-            this.phyHubClient = await (0, hub_client_1.connectPhyClient)({ moduleName: 'docker-crash', dataResidency });
-            this.signalsClient = await this.phyHubClient.initializeSignals(initSignalsParams);
-        }
-        catch (error) {
-            this.logger.error('Failed to initialize signals client for docker-crash signals:', error);
-        }
-    }
-    async listenToDockerEvents() {
-        try {
-            const eventStream = await this.docker.getEvents({
-                filters: {
-                    event: ['die', 'stop', 'kill'],
-                    type: ['container'],
-                },
-            });
-            eventStream.on('data', async (buffer) => {
-                try {
-                    const event = JSON.parse(buffer.toString());
-                    if (event.status === 'die') {
-                        const exitCode = parseInt(event.Actor?.Attributes?.exitCode || '0', 10);
-                        if (exitCode !== 0) {
-                            await this.onContainerCrash(event.id);
-                        }
-                    }
-                }
-                catch (error) {
-                    this.logger.error('listenToDockerEvents(): Error processing Docker event:', error);
-                }
-            });
-            eventStream.on('error', (error) => {
-                this.logger.error('listenToDockerEvents(): Docker event stream error:', error);
-                setTimeout(() => this.listenToDockerEvents(), 5000);
-            });
-            eventStream.on('end', () => {
-                this.logger.warn('listenToDockerEvents(): Docker event stream ended, attempting to reconnect...');
-                setTimeout(() => this.listenToDockerEvents(), 5000);
-            });
-        }
-        catch (error) {
-            this.logger.error('listenToDockerEvents(): Failed to set up Docker event listener:', error);
-            setTimeout(() => this.listenToDockerEvents(), 5000);
-        }
-    }
-    isCredentialError(error) {
-        const msg = error?.message || '';
-        return (msg.includes('unauthorized') ||
-            msg.includes('authentication required') ||
-            msg.includes('access denied') ||
-            msg.includes('credential') ||
-            error?.statusCode === 401);
-    }
-    getDefaultPlatform() {
-        const arch = process.arch;
-        if (arch === 'x64') {
-            return 'linux/amd64';
-        }
-        else if (arch === 'arm64') {
-            return 'linux/arm64';
-        }
-        else if (arch === 'arm') {
-            return 'linux/arm/v7';
-        }
-        else if (arch === 'ia32') {
-            return 'linux/386';
-        }
-        return 'linux/amd64';
-    }
-    extractNameFromImage = (image) => {
-        try {
-            const regex = /[^\/]+\.([^\/:]+:[^\/]+)$/;
-            const match = image.match(regex);
-            return match && match.length > 0 ? match[1].split(':')[0] : null;
-        }
-        catch (error) {
-            return null;
-        }
-    };
-    async onContainerCrash(containerId) {
-        try {
-            const twinManager = await (0, twin_manager_1.getTwinManagerInstance)();
-            if (!twinManager) {
-                throw new Error('Error getting twin manager instance');
-            }
-            this.hubDevice = await twinManager.getHubDevice();
-            let twinId;
-            for (const [id, instance] of this.instances.entries()) {
-                if (instance.container.id === containerId) {
-                    twinId = id;
-                    break;
-                }
-            }
-            const container = this.docker.getContainer(containerId);
-            const logs = await container.logs({ stdout: true, stderr: true, tail: 50 });
-            const logString = logs.toString();
-            const coreDumpDir = '/var/coredumps';
-            let coreDumpInfo;
-            try {
-                if (fs_1.default.existsSync(coreDumpDir)) {
-                    const containerInspect = await container.inspect();
-                    const crashTime = containerInspect.State.FinishedAt
-                        ? new Date(containerInspect.State.FinishedAt).getTime() - 5000
-                        : 0;
-                    const files = fs_1.default.readdirSync(coreDumpDir);
-                    const coreFiles = files
-                        .filter(fileName => fileName.startsWith('core.'))
-                        .map(fileName => ({
-                        name: fileName,
-                        mtime: fs_1.default.statSync(`${coreDumpDir}/${fileName}`).mtime.getTime(),
-                    }))
-                        .filter(coreFile => crashTime === 0 || coreFile.mtime >= crashTime)
-                        .sort((firstFile, secondFile) => secondFile.mtime - firstFile.mtime);
-                    if (coreFiles.length > 0) {
-                        coreDumpInfo = {
-                            files: [coreFiles[0].name],
-                            count: coreFiles.length,
-                        };
-                        this.logger.info(`onContainerCrash(): Found core dump for container ${containerId}: ${coreFiles[0].name}`);
-                    }
-                }
-            }
-            catch (error) {
-                this.logger.warn(`onContainerCrash(): Error checking for core dumps`, error);
-            }
-            if (twinId) {
-                const reportedProperties = {
-                    lastError: {
-                        message: `container crashed`,
-                        timestamp: new Date(),
-                        errorObject: {
-                            containerLogs: logString,
-                            coreDumps: coreDumpInfo,
-                        },
-                    },
-                };
-                this.hubDevice.reportEdgeTwinProperties(twinId, reportedProperties);
-            }
-            const deviceStatus = await this.hubDevice.getDeviceStatus();
-            const twinResponse = await this.hubDevice.getEdgeInstance({ twinId });
-            let twin = undefined;
-            if (twinResponse && twinResponse.twin) {
-                twin = twinResponse.twin;
-            }
-            let ip = '127.0.0.1';
-            if (deviceStatus.ip && deviceStatus.ip.length > 0) {
-                ip = deviceStatus.ip[0].ipv4;
-            }
-            if (!this.phyHubClient) {
-                const initSignalsParams = {
-                    deviceId: deviceStatus.deviceId,
-                    installationId: twin?.properties.desired.installationId ?? 'XXXXXXXXXXXXXXXXXXXXXXXX',
-                    spaceId: deviceStatus.spaceId,
-                    tenantId: deviceStatus.tenantId,
-                    appVersion: 'XXXXXXXXXXXXXXXXXXXXXXXX',
-                    appId: 'XXXXXXXXXXXXXXXXXXXXXXXX',
-                    installationVersion: 'XXXXXXXXXXXXXXXXXXXXXXXX',
-                    environment: deviceStatus.gridEnv,
-                    dataResidency: deviceStatus.dataResidency.toUpperCase(),
-                    country: 'SE',
-                    accessToken: deviceStatus.accessKey,
-                    clientUserAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
-                    ip,
-                };
-                await this.initializeHubSignals(initSignalsParams, deviceStatus.dataResidency.toUpperCase());
-            }
-            if (this.phyHubClient && this.signalsClient) {
-                if (!this.sessionId) {
-                    const instance = this.signalsClient.getInstanceProps();
-                    this.sessionId = instance.sessionId;
-                    const clientId = instance.clientId ?? (0, uuid_1.v4)();
-                    await this.hubDevice.sendSessionSignal({
-                        deviceId: this.hubDevice.deviceId,
-                        data: {
-                            c: instance.tenantId,
-                            b: this.sessionId,
-                            d: instance.sessionCreated,
-                            f: instance.environment,
-                            g: instance.dataResidency,
-                            m: instance.country,
-                            n: instance.locationAccuracy,
-                            o: instance.latitude,
-                            p: instance.longitude,
-                            h: instance.spaceId,
-                            i: instance.appId,
-                            j: instance.appVersion,
-                            k: instance.installationId,
-                            l: instance.installationVersion,
-                            q: instance.deviceId,
-                            r: clientId,
-                            e: instance.clientIp ?? instance.ip,
-                        },
-                    });
-                    await this.hubDevice.sendClientSignal({
-                        deviceId: this.hubDevice.deviceId,
-                        data: {
-                            c: instance.tenantId,
-                            r: clientId,
-                            ah: instance.clientCreated,
-                            aj: instance.clientUserAgent,
-                            g: instance.dataResidency,
-                            m: instance.country,
-                            o: instance.latitude,
-                            p: instance.longitude,
-                            n: instance.locationAccuracy,
-                            e: instance.ip ?? instance.clientIp ?? undefined,
-                        },
-                    });
-                }
-                const timeWithoutZ = new Date().toISOString().replace(/Z$/, '');
-                let coreDumpPath;
-                if (coreDumpInfo && coreDumpInfo.files && coreDumpInfo.files.length > 0) {
-                    const fileName = coreDumpInfo.files[0];
-                    coreDumpPath = `${coreDumpDir}/${fileName}`;
-                }
-                const eventPayload = {
-                    g: deviceStatus.dataResidency.toUpperCase(),
-                    b: this.sessionId,
-                    t: timeWithoutZ,
-                    s: 'MONITOR_CONTAINER_CRASH',
-                    u: false,
-                    h: deviceStatus.spaceId,
-                    ac: twinId,
-                    ad: deviceStatus.deviceId,
-                    c: deviceStatus.tenantId,
-                };
-                if (coreDumpPath) {
-                    eventPayload.ae = coreDumpPath;
-                }
-                await this.hubDevice.sendEventSignal({
-                    deviceId: this.hubDevice.deviceId,
-                    data: eventPayload,
-                });
-            }
-        }
-        catch (error) {
-            this.logger.error(`onContainerCrash(): Error in reporting the container crash`, error);
-        }
-    }
-    async imageExists(image) {
-        this.logger.info(`imageExists(): Checking if docker image ${image} exists`);
-        try {
-            await this.docker.getImage(image).inspect();
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    async pullImage(image, authConfig, abortController) {
-        this.logger.info(`pullImage(): Pulling docker image ${image}`);
-        return new Promise((resolve, reject) => {
-            const platform = this.getDefaultPlatform();
-            this.logger.info(`pullImage(): Using platform ${platform} for image ${image}`);
-            if (abortController.signal.aborted) {
-                this.logger.info(`pullImage(): Pull for image ${image} was aborted before starting`);
-                return resolve();
-            }
-            let isCompleted = false;
-            const streamErrors = [];
-            const abortHandler = () => {
-                if (isCompleted)
-                    return;
-                this.logger.info(`pullImage(): Pull for image ${image} was aborted`);
-                isCompleted = true;
-                const imageProgress = this.imagesInProgress.get(image);
-                if (imageProgress) {
-                    imageProgress.inProgress = false;
-                    this.imagesInProgress.delete(image);
-                }
-                resolve();
-            };
-            abortController.signal.addEventListener('abort', abortHandler);
-            const timeoutId = setTimeout(() => {
-                if (!isCompleted) {
-                    this.logger.error(`pullImage(): Timeout waiting for image ${image} to pull`);
-                    abortController.abort();
-                }
-            }, 7200000);
-            const progressTracker = new docker_progress_tracker_1.DockerProgressTracker(image, 'DockerManager');
-            const imageProgress = {
-                inProgress: true,
-                layersProgress: new Map(),
-            };
-            this.imagesInProgress.set(image, imageProgress);
-            this.docker.pull(image, {
-                authconfig: authConfig,
-                platform,
-                architecture: process.arch,
-                os: 'linux',
-            }, (err, stream) => {
-                if (err) {
-                    clearTimeout(timeoutId);
-                    this.logger.error(`pullImage(): Error initiating pull for image ${image}`, err);
-                    if (this.isCredentialError(err)) {
-                        this.logger.error(`pullImage(): Credential error detected for image ${image}, releasing lock immediately`);
-                        const imageProgress = this.imagesInProgress.get(image);
-                        if (imageProgress) {
-                            imageProgress.inProgress = false;
-                            this.imagesInProgress.delete(image);
-                        }
-                    }
-                    isCompleted = true;
-                    progressTracker.markFailed(err);
-                    return reject(err);
-                }
-                if (abortController.signal.aborted) {
-                    clearTimeout(timeoutId);
-                    abortHandler();
-                    return;
-                }
-                this.docker.modem.followProgress(stream, (err, output) => {
-                    clearTimeout(timeoutId);
-                    if (isCompleted)
-                        return;
-                    const actualError = err ||
-                        (streamErrors.length > 0
-                            ? new Error(`Pull failed with ${streamErrors.length} error(s): ${streamErrors.map(streamError => streamError.message).join('; ')}`)
-                            : null);
-                    if (actualError) {
-                        this.logger.error(`pullImage(): Error pulling image ${image}`, actualError);
-                        if (streamErrors.length > 0) {
-                            this.logger.error(`pullImage(): Stream errors captured during pull:`, streamErrors.map(streamError => streamError.message));
-                        }
-                        imageProgress.inProgress = false;
-                        this.imagesInProgress.delete(image);
-                        isCompleted = true;
-                        progressTracker.markFailed(actualError);
-                        reject(actualError);
-                        return;
-                    }
-                    this.logger.info(`pullImage(): Image ${image} download complete`);
-                    (async () => {
-                        try {
-                            const imageExists = await this.imageExists(image);
-                            if (!imageExists) {
-                                const errorMsg = `Image ${image} pull reported success but image is not available locally. ` +
-                                    `This may indicate a network issue, proxy problem, or the image does not exist.`;
-                                this.logger.error(`pullImage(): ${errorMsg}`);
-                                imageProgress.inProgress = false;
-                                this.imagesInProgress.delete(image);
-                                isCompleted = true;
-                                progressTracker.markFailed(new Error(errorMsg));
-                                reject(new Error(errorMsg));
-                                return;
-                            }
-                            imageProgress.inProgress = false;
-                            this.imagesInProgress.delete(image);
-                            isCompleted = true;
-                            progressTracker.markComplete();
-                            (async () => {
-                                try {
-                                    const imageObject = this.docker.getImage(image);
-                                    const imageInspection = await imageObject.inspect();
-                                    const imageId = imageInspection.Id;
-                                    const pullTimestamp = Math.floor(Date.now() / 1000);
-                                    this.imagePullTimes.set(imageId, pullTimestamp);
-                                    const pullTimeFormatted = new Date(pullTimestamp * 1000).toISOString();
-                                    this.logger.info(`pullImage(): Recorded pull time for ${image} ` +
-                                        `(ID: ${imageId.substring(0, 12)}): ${pullTimeFormatted}`);
-                                }
-                                catch (error) {
-                                    this.logger.warn(`pullImage(): Failed to record pull time for ${image}`, error);
-                                }
-                            })();
-                            this.logger.info(`pullImage(): Starting image management for ${image}`);
-                            this.manageDockerImages(image).catch((error) => {
-                                this.logger.error(`pullImage(): Error managing images after pull for ${image}`, error);
-                            });
-                            resolve();
-                        }
-                        catch (verifyError) {
-                            const errorMsg = `Failed to verify image ${image} exists after pull: ` +
-                                `${verifyError.message || verifyError}`;
-                            this.logger.error(`pullImage(): ${errorMsg}`, verifyError);
-                            imageProgress.inProgress = false;
-                            this.imagesInProgress.delete(image);
-                            isCompleted = true;
-                            progressTracker.markFailed(verifyError);
-                            reject(new Error(errorMsg));
-                        }
-                    })();
-                }, (event) => {
-                    if (abortController.signal.aborted || isCompleted) {
-                        return;
-                    }
-                    if (event.error || event.errorDetail) {
-                        const errorMsg = event.errorDetail?.message || event.error || 'Unknown pull error';
-                        this.logger.error(`pullImage(): Error event during pull for ${image}: ${errorMsg}`, { event });
-                        streamErrors.push(new Error(errorMsg));
-                    }
-                    progressTracker.processEvent(event);
-                    if (event.id && event.progressDetail) {
-                        imageProgress.layersProgress.set(event.id, event.progressDetail);
-                    }
-                });
-            });
-        });
-    }
-    async createContainer(twin, skipGpu = false) {
-        const { image, createOptions } = twin.properties.desired;
-        const parsedOptions = createOptions;
-        this.logger.info(`createContainer(): Twin object for ${twin.id}:`, JSON.stringify(twin, null, 2));
-        this.logger.info(`createContainer(): Parsed options for ${twin.id}:`, JSON.stringify(parsedOptions, null, 2));
-        this.logger.info(`createContainer(): Device requests for ${twin.id}:`, JSON.stringify(parsedOptions?.HostConfig?.DeviceRequests, null, 2));
-        const isHostNetwork = parsedOptions?.HostConfig?.NetworkMode === 'host';
-        const existingBinds = parsedOptions?.HostConfig?.Binds ?? [];
-        const coreDumpHostDir = '/var/coredumps';
-        try {
-            if (!fs_1.default.existsSync(coreDumpHostDir)) {
-                fs_1.default.mkdirSync(coreDumpHostDir, { recursive: true, mode: 0o777 });
-                this.logger.info(`createContainer(): Created core dump directory ${coreDumpHostDir}`);
-            }
-            try {
-                fs_1.default.chmodSync(coreDumpHostDir, 0o777);
-                this.logger.info(`createContainer(): Set permissions on ${coreDumpHostDir} to 777`);
-            }
-            catch (chmodError) {
-                this.logger.error(`createContainer(): CRITICAL: Could not set permissions on ${coreDumpHostDir}. ` +
-                    `Container may not be able to write coredumps. Error:`, chmodError);
-            }
-        }
-        catch (error) {
-            this.logger.error(`createContainer(): Failed to create or verify ${coreDumpHostDir}, mount may fail`, error);
-        }
-        const coreDumpBind = '/var/coredumps:/var/coredumps:z';
-        const hasCoreDumpBind = existingBinds.some((bind) => bind.includes('/var/coredumps'));
-        const allBinds = hasCoreDumpBind ? existingBinds : [...existingBinds, coreDumpBind];
-        const existingUlimits = parsedOptions?.HostConfig?.Ulimits ?? [];
-        const hasCoreUlimit = existingUlimits.some((ulimit) => ulimit.Name === 'core');
-        const coreUlimits = hasCoreUlimit
-            ? existingUlimits
-            : [
-                ...existingUlimits,
-                { Name: 'core', Soft: -1, Hard: -1 },
-            ];
-        const containerConfig = {
-            Image: image,
-            Env: [
-                ...(parsedOptions?.Env ?? []),
-                `TWIN_ID=${twin.id}`,
-                ...(process.env.TZ ? [`TZ=${process.env.TZ}`] : []),
-            ],
-            Entrypoint: parsedOptions?.Entrypoint,
-            Cmd: parsedOptions?.Cmd,
-            HostConfig: {
-                ...parsedOptions?.HostConfig,
-                NetworkMode: isHostNetwork ? 'host' : 'default',
-                ExtraHosts: [isHostNetwork ? `phyos:127.0.0.1` : `phyos:172.26.128.1`],
-                RestartPolicy: parsedOptions?.HostConfig?.RestartPolicy ?? { Name: 'always' },
-                Privileged: parsedOptions?.HostConfig?.Privileged ?? false,
-                Memory: parsedOptions?.HostConfig?.Memory ?? 0,
-                CpuShares: parsedOptions?.HostConfig?.CpuShares ?? 0,
-                StorageOpt: parsedOptions?.HostConfig?.StorageOpt ?? {},
-                DiskQuota: parsedOptions?.HostConfig?.DiskQuota ?? 0,
-                ReadonlyRootfs: parsedOptions?.HostConfig?.ReadonlyRootfs ?? false,
-                BlkioWeight: parsedOptions?.HostConfig?.BlkioWeight ?? 0,
-                BlkioDeviceReadBps: parsedOptions?.HostConfig?.BlkioDeviceReadBps ?? [],
-                BlkioDeviceWriteBps: parsedOptions?.HostConfig?.BlkioDeviceWriteBps ?? [],
-                Devices: parsedOptions?.HostConfig?.Devices ?? [],
-                DeviceRequests: parsedOptions?.HostConfig?.DeviceRequests ?? [],
-                Binds: allBinds,
-                Ulimits: coreUlimits,
-            },
-        };
-        if (skipGpu && parsedOptions?.HostConfig?.DeviceRequests) {
-            this.logger.info(`createContainer(): Original DeviceRequests before filtering:`, JSON.stringify(parsedOptions.HostConfig.DeviceRequests, null, 2));
-            const filteredDeviceRequests = parsedOptions.HostConfig.DeviceRequests.filter((request) => {
-                const isGpuRequest = request.Capabilities &&
-                    request.Capabilities.some((cap) => cap.includes('gpu'));
-                return !isGpuRequest;
-            });
-            if (containerConfig.HostConfig) {
-                containerConfig.HostConfig.DeviceRequests = filteredDeviceRequests;
-                this.logger.info(`createContainer(): Creating container for twin ${twin.id} without GPU support. Filtered DeviceRequests:`, JSON.stringify(containerConfig.HostConfig.DeviceRequests, null, 2));
-            }
-        }
-        else if (parsedOptions?.HostConfig?.DeviceRequests) {
-            this.logger.info(`createContainer(): Original DeviceRequests:`, JSON.stringify(parsedOptions.HostConfig.DeviceRequests, null, 2));
-            if (containerConfig.HostConfig) {
-                containerConfig.HostConfig.DeviceRequests = parsedOptions.HostConfig.DeviceRequests;
-                const hasGpuRequest = parsedOptions.HostConfig.DeviceRequests.some((req) => req.Capabilities && req.Capabilities.some((cap) => cap.includes('gpu')));
-                if (hasGpuRequest) {
-                    try {
-                        const info = await this.docker.info();
-                        const availableRuntimes = info.Runtimes ? Object.keys(info.Runtimes) : [];
-                        this.logger.info(`Available Docker runtimes:`, JSON.stringify(availableRuntimes, null, 2));
-                        if (availableRuntimes.includes('nvidia')) {
-                            containerConfig.HostConfig.Runtime = 'nvidia';
-                            this.logger.info(`createContainer(): Setting nvidia runtime for GPU support`);
-                        }
-                        else {
-                            this.logger.warn(`createContainer(): NVIDIA runtime not available, GPU may not work`);
-                        }
-                    }
-                    catch (error) {
-                        this.logger.warn(`createContainer(): Error checking for available runtimes:`, error);
-                        containerConfig.HostConfig.Runtime = 'nvidia';
-                    }
-                }
-                this.logger.info(`createContainer(): Applied DeviceRequests to container config:`, JSON.stringify(containerConfig.HostConfig.DeviceRequests, null, 2));
-            }
-            if (parsedOptions.HostConfig.DeviceRequests.some((req) => req.Capabilities && req.Capabilities.some((cap) => cap.includes('gpu')))) {
-                this.logger.info(`createContainer(): Creating container for twin ${twin.id} with GPU support`);
-            }
-        }
-        if (!isHostNetwork) {
-            containerConfig.NetworkingConfig = {
-                EndpointsConfig: {
-                    'phyos-edge-net': {
-                        Aliases: [twin.id],
-                    },
-                },
-            };
-        }
-        this.logger.info(`createContainer(): Creating container for twin ${twin.id}`, {
-            containerConfig,
-        });
-        this.logger.info(`createContainer(): Final container config for twin ${twin.id}:`, JSON.stringify(containerConfig, null, 2));
-        const container = await this.docker.createContainer(containerConfig);
-        try {
-            const containerDetails = await container.inspect();
-            this.logger.info(`createContainer(): Created container details for twin ${twin.id}:`, JSON.stringify(containerDetails, null, 2));
-        }
-        catch (error) {
-            this.logger.error(`createContainer(): Failed to inspect container for twin ${twin.id}:`, error);
-        }
-        return container;
-    }
-    async _startContainer(twin) {
-        if (this.twinInstancesRunning.get(twin.id)) {
-            this.logger.info(`startContainer(): Container for twin ${twin.id} is already running or starting`);
-            return;
-        }
-        this.twinInstancesRunning.set(twin.id, true);
-        try {
-            this.logger.info(`startContainer(): Attempting to create container for twin ${twin.id} with GPU support`);
-            try {
-                const container = await this.createContainer(twin, false);
-                this.logger.info(`startContainer(): Container created for twin ${twin.id}, preparing to start it`);
-                this.instances.set(twin.id, { container, twin });
-                await container.start();
-                this.logger.info(`startContainer(): Started container for twin ${twin.id}`);
-                const logs = await container.logs({ stdout: true, stderr: true });
-                this.logger.info(`Container logs for twin ${twin.id}:`, logs.toString());
-            }
-            catch (error) {
-                this.logger.error(`startContainer(): Error starting container with GPU for twin ${twin.id}:`, error.message);
-                const isGpuError = error.message &&
-                    ((error.message.includes('could not select device driver') &&
-                        error.message.includes('capabilities: [[gpu]]')) ||
-                        error.message.includes('nvidia-container-cli: initialization error: nvml error: driver not loaded') ||
-                        error.message.includes('nvidia-container-cli') ||
-                        error.message.includes('unknown or invalid runtime name: nvidia') ||
-                        error.message.includes('nvidia runtime') ||
-                        error.message.includes('GPU') ||
-                        error.message.includes('gpu'));
-                if (isGpuError) {
-                    this.logger.warn(`startContainer(): GPU support not available for twin ${twin.id}, retrying without GPU capabilities. Error: ${error.message}`, { error: error.message });
-                    const existingInstance = this.instances.get(twin.id);
-                    if (existingInstance?.container) {
-                        try {
-                            await existingInstance.container.remove({ force: true });
-                            this.logger.info(`startContainer(): Removed failed GPU container for twin ${twin.id}`);
-                        }
-                        catch (removeError) {
-                            this.logger.info(`startContainer(): Could not remove container for twin ${twin.id}, may not exist`);
-                        }
-                    }
-                    this.instances.delete(twin.id);
-                    const container = await this.createContainer(twin, true);
-                    this.instances.set(twin.id, { container, twin });
-                    await container.start();
-                    this.logger.info(`startContainer(): Started container for twin ${twin.id} without GPU support`);
-                    const logs = await container.logs({ stdout: true, stderr: true });
-                    this.logger.info(`Container logs for twin ${twin.id}:`, logs.toString());
-                }
-                else {
-                    throw error;
-                }
-            }
-        }
-        catch (error) {
-            this.logger.error(`startContainer(): Failed to start container for twin ${twin.id}`, error);
-            if (error.message.includes('port is already allocated')) {
-                this.logger.error(`startContainer(): Port conflict detected for twin ${twin.id}`);
-            }
-            this.instances.delete(twin.id);
-            this.twinInstancesRunning.set(twin.id, false);
-            throw error;
-        }
-    }
-    abortImagePull(image) {
-        const pullInfo = this.imagePullsInProgress.get(image);
-        if (pullInfo) {
-            this.logger.info(`abortImagePull(): Aborting pull for image ${image}`);
-            pullInfo.abortController.abort();
-            const imageProgress = this.imagesInProgress.get(image);
-            if (imageProgress) {
-                imageProgress.inProgress = false;
-                this.imagesInProgress.delete(image);
-            }
-            setTimeout(() => {
-                if (this.imagePullsInProgress.has(image)) {
-                    this.logger.warn(`abortImagePull(): Cleaning up stale pull info for ${image}`);
-                    this.imagePullsInProgress.delete(image);
-                }
-            }, 5000);
-        }
-    }
-    isImagePullInProgress(image) {
-        return this.imagePullsInProgress.has(image);
-    }
-    async ensureImageAvailable(image, authConfig) {
-        const imageExists = await this.imageExists(image);
-        if (imageExists)
-            return true;
-        const existingPull = this.imagePullsInProgress.get(image);
-        if (existingPull) {
-            this.logger.info(`ensureImageAvailable(): Pull already in progress for ${image}, waiting for it to complete`);
-            try {
-                await existingPull.promise;
-                const imageExists = await this.imageExists(image);
-                if (!imageExists) {
-                    const errorMsg = `Existing pull for ${image} reported success but image is not available locally`;
-                    this.logger.error(`ensureImageAvailable(): ${errorMsg}`);
-                    this.imagePullsInProgress.delete(image);
-                    return false;
-                }
-                this.logger.info(`ensureImageAvailable(): Existing pull for ${image} completed successfully ` +
-                    `and image verified locally`);
-                return true;
-            }
-            catch (error) {
-                this.logger.error(`ensureImageAvailable(): Existing pull for ${image} failed`, error);
-                if (this.isCredentialError(error)) {
-                    this.logger.error(`ensureImageAvailable(): Credential error detected for existing pull of ${image}`);
-                }
-                this.imagePullsInProgress.delete(image);
-                return false;
-            }
-        }
-        this.logger.info(`ensureImageAvailable(): Starting new pull for ${image}`);
-        const abortController = new AbortController();
-        const pullPromise = this.pullImage(image, authConfig, abortController).finally(() => {
-            this.imagePullsInProgress.delete(image);
-        });
-        this.imagePullsInProgress.set(image, {
-            promise: pullPromise,
-            abortController,
-        });
-        try {
-            await pullPromise;
-            const imageExists = await this.imageExists(image);
-            if (!imageExists) {
-                const errorMsg = `Pull for ${image} reported success but image is not available locally`;
-                this.logger.error(`ensureImageAvailable(): ${errorMsg}`);
-                return false;
-            }
-            this.logger.info(`ensureImageAvailable(): Pull for ${image} completed successfully ` +
-                `and image verified locally`);
-            return true;
-        }
-        catch (error) {
-            this.logger.error(`ensureImageAvailable(): Pull failed for ${image}`, error);
-            if (this.isCredentialError(error)) {
-                this.logger.error(`ensureImageAvailable(): Credential error detected for pull of ${image}`);
-            }
-            return false;
-        }
-    }
-    async startEdgeApp(twin) {
-        const { image } = twin.properties.desired;
-        if (!image) {
-            return {
-                status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                error: {
-                    message: 'No image found in desired properties',
-                    timestamp: new Date(),
-                },
-            };
-        }
-        const hasRegistry = image.includes('/');
-        let authConfig;
-        let pullError = null;
-        if (hasRegistry) {
-            const [hostname] = image.split('/');
-            authConfig = twin.properties.desired.credentials;
-            if (!authConfig) {
-                this.logger.error(`startEdgeApp(): No credentials found for registry: ${hostname}`);
-                return {
-                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                    error: {
-                        message: `No credentials found for registry: ${hostname}`,
-                        timestamp: new Date(),
-                    },
-                };
-            }
-            try {
-                const pullSuccess = await this.ensureImageAvailable(image, {
-                    username: authConfig.username,
-                    password: authConfig.password,
-                    serveraddress: authConfig.address,
-                });
-                if (!pullSuccess) {
-                    this.logger.error(`startEdgeApp(): Failed to pull image ${image}`);
-                    return {
-                        status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                        error: {
-                            message: `Failed to pull image ${image}`,
-                            timestamp: new Date(),
-                        },
-                    };
-                }
-            }
-            catch (error) {
-                this.logger.error(`startEdgeApp(): Error pulling image ${image}`, error);
-                const errorMsg = error.message || '';
-                return {
-                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                    error: {
-                        message: this.isCredentialError(error)
-                            ? `Authentication failed for image ${image}. Please check your credentials.`
-                            : `Failed to pull image ${image}: ${errorMsg}`,
-                        timestamp: new Date(),
-                        errorObject: error,
-                    },
-                };
-            }
-        }
-        else {
-            try {
-                const pullSuccess = await this.ensureImageAvailable(image, {});
-                if (!pullSuccess) {
-                    this.logger.error(`startEdgeApp(): Failed to pull local image ${image}`);
-                    return {
-                        status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                        error: {
-                            message: `Failed to pull local image ${image}`,
-                            timestamp: new Date(),
-                        },
-                    };
-                }
-            }
-            catch (error) {
-                this.logger.error(`startEdgeApp(): Error pulling local image ${image}`, error);
-                return {
-                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                    error: {
-                        message: `Failed to pull local image ${image}: ${error.message || ''}`,
-                        timestamp: new Date(),
-                        errorObject: error,
-                    },
-                };
-            }
-        }
-        return this.operationLock.runExclusive(() => this._startEdgeApp(twin));
-    }
-    async _startEdgeApp(twin) {
-        const { image } = twin.properties.desired;
-        try {
-            const existingTwin = this.instances.get(twin.id);
-            const hasImageChanged = existingTwin && existingTwin.twin.properties.desired.image !== image;
-            if (existingTwin && hasImageChanged) {
-                const oldImage = existingTwin.twin.properties.desired.image;
-                this.logger.info(`startEdgeApp(): Twin ${twin.id} image has changed from ${oldImage} to ${image}, aborting old pull if in progress`);
-                this.abortImagePull(oldImage);
-            }
-            const hasConfigChanged = existingTwin &&
-                !(0, lodash_1.isEqual)(existingTwin.twin.properties.desired.containerCreateOptions, twin.properties.desired.containerCreateOptions);
-            if (existingTwin && (hasConfigChanged || hasImageChanged)) {
-                this.logger.info(`startEdgeApp(): Twin ${twin.id} configuration or image has changed, recreating container...`);
-                await this._stopAndRemoveContainer(twin.id);
-            }
-            const finalImageExists = await this.imageExists(image);
-            if (!finalImageExists) {
-                this.logger.error(`startEdgeApp(): Image ${image} not available locally`);
-                return {
-                    status: twin_types_1.TwinStatusEnum.ImageNotFound,
-                    error: {
-                        message: `Image ${image} not available locally`,
-                        timestamp: new Date(),
-                    },
-                };
-            }
-            if (!this.instances.get(twin.id)) {
-                await this._startContainer(twin);
-            }
-            return {
-                status: twin_types_1.TwinStatusEnum.Online,
-                error: undefined,
-            };
-        }
-        catch (error) {
-            this.logger.error(`startEdgeApp(): Failed to start edge app for twin ${twin.id}`, error);
-            return {
-                status: twin_types_1.TwinStatusEnum.Exited,
-                error: {
-                    message: `Failed to start edge app`,
-                    timestamp: new Date(),
-                    errorObject: error,
-                },
-            };
-        }
-    }
-    async _stopAndRemoveContainer(twinId) {
-        const containerInfo = this.instances.get(twinId);
-        if (!containerInfo) {
-            return;
-        }
-        try {
-            const containerDetails = await containerInfo.container.inspect();
-            if (containerDetails.State.Running) {
-                await containerInfo.container.stop();
-                this.logger.info(`stopAndRemoveContainer(): Stopped container ${containerDetails.Id}`);
-            }
-            await containerInfo.container.remove({ force: true });
-            this.logger.info(`stopAndRemoveContainer(): Removed container ${containerDetails.Id} for twin ${twinId}`);
-            this.instances.delete(twinId);
-        }
-        catch (error) {
-            const isNoSuchContainerError = error.statusCode === 404 &&
-                (error.reason === 'no such container' ||
-                    (error.json && error.json.message && error.json.message.includes('No such container')));
-            if (isNoSuchContainerError) {
-                this.logger.warn(`stopAndRemoveContainer(): Container for twin ${twinId} no longer exists, cleaning up references`, { error: error.message });
-                this.instances.delete(twinId);
-            }
-            else {
-                this.logger.error(`stopAndRemoveContainer(): Failed to stop/remove container for twin ${twinId}`, error);
-                throw error;
-            }
-        }
-        finally {
-            this.twinInstancesRunning.set(twinId, false);
-        }
-    }
-    async _stopAndRemoveAllContainers() {
-        if (this.firstRunCompleted) {
-            this.logger.info('stopAndRemoveAllContainers(): First run completed, skipping');
-            return;
-        }
-        this.firstRunCompleted = true;
-        this.logger.info('stopAndRemoveAllContainers(): Stopping and removing all running containers');
-        try {
-            const containers = await this.docker.listContainers({ all: true });
-            await Promise.allSettled(containers.map(async (containerInfo) => {
-                const container = this.docker.getContainer(containerInfo.Id);
-                try {
-                    const containerDetails = await container.inspect();
-                    if (containerDetails.State.Running) {
-                        await container.stop();
-                        this.logger.info(`stopAndRemoveAllContainers(): Stopped container ${containerInfo.Id}`);
-                    }
-                    await container.remove();
-                    this.logger.info(`stopAndRemoveAllContainers(): Removed container ${containerInfo.Id}`);
-                }
-                catch (error) {
-                    this.logger.error(`stopAndRemoveAllContainers(): Failed to stop/remove container ${containerInfo.Id}`, error);
-                }
-            }));
-        }
-        catch (error) {
-            this.logger.error('stopAndRemoveAllContainers(): Failed to list running containers', error);
-            throw error;
-        }
-    }
-    async stopAndRemoveContainer(twinId) {
-        return this.operationLock.runExclusive(() => this._stopAndRemoveContainer(twinId));
-    }
-    async stopAndRemoveAllContainers() {
-        return this.operationLock.runExclusive(() => this._stopAndRemoveAllContainers());
-    }
-    getInstances() {
-        return this.instances;
-    }
-    extractRepositoryName(image) {
-        this.logger.info(`extractRepositoryName(): Processing image: ${image}`);
-        let normalizedImage = image;
-        if (image.startsWith('docker.io/')) {
-            normalizedImage = image.substring('docker.io/'.length);
-            this.logger.info(`extractRepositoryName(): Removed docker.io/ prefix: ${normalizedImage}`);
-        }
-        const tagSeparatorIndex = normalizedImage.lastIndexOf(':');
-        const pathSeparatorIndex = normalizedImage.lastIndexOf('/');
-        if (tagSeparatorIndex === -1) {
-            this.logger.info(`extractRepositoryName(): No tag found, repository: ${normalizedImage}`);
-            return normalizedImage;
-        }
-        if (pathSeparatorIndex !== -1 && tagSeparatorIndex < pathSeparatorIndex) {
-            this.logger.info(`extractRepositoryName(): Colon is port separator, repository: ${normalizedImage}`);
-            return normalizedImage;
-        }
-        const repositoryName = normalizedImage.substring(0, tagSeparatorIndex);
-        this.logger.info(`extractRepositoryName(): Extracted repository: ${repositoryName}`);
-        return repositoryName;
-    }
-    async manageDockerImages(image) {
-        try {
-            this.logger.info(`manageDockerImages(): Starting management for ${image}`);
-            const repositoryName = this.extractRepositoryName(image);
-            const referenceFilters = [repositoryName];
-            if (!repositoryName.startsWith('docker.io/')) {
-                referenceFilters.push(`docker.io/${repositoryName}`);
-            }
-            this.logger.info(`manageDockerImages(): Querying images for repository: ${repositoryName}`);
-            let repositoryImages = await this.docker.listImages({
-                filters: { reference: referenceFilters },
-            });
-            if (repositoryImages.length === 0) {
-                this.logger.info(`manageDockerImages(): Reference filter returned no results, trying manual filter`);
-                const allImages = await this.docker.listImages();
-                repositoryImages = allImages.filter(imageInfo => {
-                    const imageTags = imageInfo.RepoTags || [];
-                    return imageTags.some((imageTag) => {
-                        let normalizedTag = imageTag;
-                        if (imageTag.startsWith('docker.io/')) {
-                            normalizedTag = imageTag.substring('docker.io/'.length);
-                        }
-                        const tagRepositoryName = this.extractRepositoryName(normalizedTag);
-                        return (tagRepositoryName === repositoryName || normalizedTag.startsWith(`${repositoryName}:`));
-                    });
-                });
-                this.logger.info(`manageDockerImages(): Manual filter found ${repositoryImages.length} image(s)`);
-            }
-            if (repositoryImages.length === 0) {
-                this.logger.warn(`manageDockerImages(): No images found for repository ${repositoryName}`);
-                return;
-            }
-            if (repositoryImages.length <= 2) {
-                this.logger.info(`manageDockerImages(): Repository has ${repositoryImages.length} image(s), no cleanup needed`);
-                return;
-            }
-            this.logger.info(`manageDockerImages(): Determining pull times for ${repositoryImages.length} images`);
-            const imagesWithPullTimes = repositoryImages.map(imageInfo => {
-                const imageTags = imageInfo.RepoTags || imageInfo.RepoDigests || ['<untagged>'];
-                const trackedPullTime = this.imagePullTimes.get(imageInfo.Id);
-                const pullTime = trackedPullTime || imageInfo.Created;
-                const pullTimeFormatted = new Date(pullTime * 1000).toISOString();
-                const imageTagsString = imageTags.join(', ');
-                if (trackedPullTime) {
-                    this.logger.info(`manageDockerImages(): Using tracked pull time for ${imageTagsString}: ` +
-                        `${pullTimeFormatted}`);
-                }
-                else {
-                    this.logger.info(`manageDockerImages(): Using Created time for ${imageTagsString}: ` +
-                        `${pullTimeFormatted}`);
-                }
-                return {
-                    imageInfo,
-                    pullTime,
-                    imageTags,
-                };
-            });
-            imagesWithPullTimes.sort((firstImage, secondImage) => secondImage.pullTime - firstImage.pullTime);
-            this.logger.info(`manageDockerImages(): Sorted ${imagesWithPullTimes.length} images by pull time`);
-            const imagesToRemove = imagesWithPullTimes.slice(2);
-            this.logger.info(`manageDockerImages(): Removing ${imagesToRemove.length} oldest image(s), keeping newest 2`);
-            for (const imageToRemove of imagesToRemove) {
-                try {
-                    const imageTagsString = imageToRemove.imageTags.join(', ');
-                    const pullTimeFormatted = new Date(imageToRemove.pullTime * 1000).toISOString();
-                    this.logger.info(`manageDockerImages(): Removing image: ${imageTagsString} ` +
-                        `(Pulled: ${pullTimeFormatted})`);
-                    await this.docker.getImage(imageToRemove.imageInfo.Id).remove({ force: true });
-                    this.imagePullTimes.delete(imageToRemove.imageInfo.Id);
-                    this.persistImagePullTimes();
-                    this.logger.info(`manageDockerImages(): Successfully removed: ${imageToRemove.imageTags.join(', ')}`);
-                }
-                catch (error) {
-                    this.logger.error(`manageDockerImages(): Failed to remove image ${imageToRemove.imageInfo.Id}`, error);
-                }
-            }
-            this.logger.info(`manageDockerImages(): Completed management for ${image}`);
-        }
-        catch (error) {
-            this.logger.error(`manageDockerImages(): Error managing images for ${image}`, error);
-        }
-    }
-    loadImagePullTimes() {
-        try {
-            if (fs_1.default.existsSync(IMAGE_PULL_TIMES_FILE)) {
-                const data = JSON.parse(fs_1.default.readFileSync(IMAGE_PULL_TIMES_FILE, 'utf-8'));
-                const validEntries = Object.entries(data)
-                    .filter(([_, value]) => typeof value === 'number' && value > 0)
-                    .map(([key, value]) => [key, value]);
-                this.imagePullTimes = new Map(validEntries);
-                this.logger.info(`loadImagePullTimes(): Loaded ${this.imagePullTimes.size} image pull times from disk`);
-            }
-        }
-        catch (error) {
-            this.logger.warn('loadImagePullTimes(): Failed to load image pull times, starting fresh', error);
-        }
-    }
-    async cleanupStaleImagePullTimes() {
-        try {
-            const allImages = await this.docker.listImages();
-            const existingImageIds = new Set(allImages.map(img => img.Id));
-            let cleanedCount = 0;
-            for (const [imageId] of this.imagePullTimes.entries()) {
-                if (!existingImageIds.has(imageId)) {
-                    this.imagePullTimes.delete(imageId);
-                    cleanedCount++;
-                }
-            }
-            if (cleanedCount > 0) {
-                this.logger.info(`cleanupStaleImagePullTimes(): Removed ${cleanedCount} stale image pull time entries`);
-                this.persistImagePullTimes();
-            }
-        }
-        catch (error) {
-            this.logger.warn('cleanupStaleImagePullTimes(): Error during cleanup', error);
-        }
-    }
-    persistImagePullTimes() {
-        try {
-            const dir = path_1.default.dirname(IMAGE_PULL_TIMES_FILE);
-            if (!fs_1.default.existsSync(dir)) {
-                fs_1.default.mkdirSync(dir, { recursive: true });
-            }
-            const data = Object.fromEntries(this.imagePullTimes);
-            const tempFile = `${IMAGE_PULL_TIMES_FILE}.tmp`;
-            fs_1.default.writeFileSync(tempFile, JSON.stringify(data, null, 2));
-            fs_1.default.renameSync(tempFile, IMAGE_PULL_TIMES_FILE);
-        }
-        catch (error) {
-            this.logger.warn('persistImagePullTimes(): Failed to persist image pull times', error);
-        }
-    }
-    async isContainerRunning(twinId) {
-        try {
-            const containers = await this.docker.listContainers();
-            return containers.some(container => container.Names.some(name => name.includes(twinId)) && container.State === 'running');
-        }
-        catch (error) {
-            this.logger.error(`isContainerRunning(): Error checking container status for ${twinId}:`, error);
-            return false;
-        }
-    }
-}
-exports.default = DockerManager;
-//# sourceMappingURL=docker.js.map