@phystack/device-phyos 4.5.18-dev → 4.5.20-dev

package/dist/utilities/docker.js

@@ -13,6 +13,8 @@ const docker_progress_tracker_1 = require("./docker-progress-tracker");
 const uuid_1 = require("uuid");
 const hub_client_1 = require("@phystack/hub-client");
 const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const IMAGE_PULL_TIMES_FILE = '/data/settings/phyhub/docker-image-pull-times.json';
 class DockerManager {
     docker;
     logger;
@@ -26,6 +28,7 @@ class DockerManager {
     sessionId = null;
     operationLock = new async_mutex_1.Mutex();
     imagePullsInProgress = new Map();
+    imagePullTimes = new Map();
     constructor() {
         this.docker = new dockerode_1.default();
         this.logger = new phy_logger_1.PhyLogger({
@@ -37,6 +40,10 @@ class DockerManager {
         this.twinInstancesRunning = new Map();
         this.imagesInProgress = new Map();
         this.instances = new Map();
+        this.loadImagePullTimes();
+        this.cleanupStaleImagePullTimes().catch((error) => {
+            this.logger.warn('cleanupStaleImagePullTimes(): Failed to cleanup stale entries', error);
+        });
         this.listenToDockerEvents();
     }
     async initializeHubSignals(initSignalsParams, dataResidency) {
@@ -59,24 +66,39 @@ class DockerManager {
             eventStream.on('data', async (buffer) => {
                 try {
                     const event = JSON.parse(buffer.toString());
-                    if (event.status === 'die' && event.exitCode !== 0) {
-                        await this.onContainerCrash(event.id);
+                    if (event.status === 'die') {
+                        const exitCode = parseInt(event.Actor?.Attributes?.exitCode || '0', 10);
+                        if (exitCode !== 0) {
+                            await this.onContainerCrash(event.id);
+                        }
                     }
                 }
                 catch (error) {
+                    this.logger.error('listenToDockerEvents(): Error processing Docker event:', error);
                 }
             });
             eventStream.on('error', (error) => {
+                this.logger.error('listenToDockerEvents(): Docker event stream error:', error);
                 setTimeout(() => this.listenToDockerEvents(), 5000);
             });
             eventStream.on('end', () => {
+                this.logger.warn('listenToDockerEvents(): Docker event stream ended, attempting to reconnect...');
                 setTimeout(() => this.listenToDockerEvents(), 5000);
             });
         }
         catch (error) {
+            this.logger.error('listenToDockerEvents(): Failed to set up Docker event listener:', error);
             setTimeout(() => this.listenToDockerEvents(), 5000);
         }
     }
+    isCredentialError(error) {
+        const msg = error?.message || '';
+        return (msg.includes('unauthorized') ||
+            msg.includes('authentication required') ||
+            msg.includes('access denied') ||
+            msg.includes('credential') ||
+            error?.statusCode === 401);
+    }
     getDefaultPlatform() {
         const arch = process.arch;
         if (arch === 'x64') {
@@ -130,12 +152,12 @@ class DockerManager {
                         : 0;
                     const files = fs_1.default.readdirSync(coreDumpDir);
                     const coreFiles = files
-                        .filter((fileName) => fileName.startsWith('core.'))
-                        .map((fileName) => ({
+                        .filter(fileName => fileName.startsWith('core.'))
+                        .map(fileName => ({
                         name: fileName,
                         mtime: fs_1.default.statSync(`${coreDumpDir}/${fileName}`).mtime.getTime(),
                     }))
-                        .filter((coreFile) => crashTime === 0 || coreFile.mtime >= crashTime)
+                        .filter(coreFile => crashTime === 0 || coreFile.mtime >= crashTime)
                         .sort((firstFile, secondFile) => secondFile.mtime - firstFile.mtime);
                     if (coreFiles.length > 0) {
                         coreDumpInfo = {
@@ -190,7 +212,7 @@ class DockerManager {
                 };
                 await this.initializeHubSignals(initSignalsParams, deviceStatus.dataResidency.toUpperCase());
             }
-            if (this.phyHubClient) {
+            if (this.phyHubClient && this.signalsClient) {
                 if (!this.sessionId) {
                     const instance = this.signalsClient.getInstanceProps();
                     this.sessionId = instance.sessionId;
@@ -283,6 +305,7 @@ class DockerManager {
                 return resolve();
             }
             let isCompleted = false;
+            const streamErrors = [];
             const abortHandler = () => {
                 if (isCompleted)
                     return;
@@ -317,13 +340,7 @@ class DockerManager {
                 if (err) {
                     clearTimeout(timeoutId);
                     this.logger.error(`pullImage(): Error initiating pull for image ${image}`, err);
-                    const isCredentialError = err.message &&
-                        (err.message.includes('unauthorized') ||
-                            err.message.includes('authentication required') ||
-                            err.message.includes('access denied') ||
-                            err.message.includes('credential') ||
-                            err.statusCode === 401);
-                    if (isCredentialError) {
+                    if (this.isCredentialError(err)) {
                         this.logger.error(`pullImage(): Credential error detected for image ${image}, releasing lock immediately`);
                         const imageProgress = this.imagesInProgress.get(image);
                         if (imageProgress) {
@@ -344,25 +361,82 @@ class DockerManager {
                     clearTimeout(timeoutId);
                     if (isCompleted)
                         return;
-                    if (err) {
-                        this.logger.error(`pullImage(): Error pulling image ${image}`, err);
+                    const actualError = err ||
+                        (streamErrors.length > 0
+                            ? new Error(`Pull failed with ${streamErrors.length} error(s): ${streamErrors.map(streamError => streamError.message).join('; ')}`)
+                            : null);
+                    if (actualError) {
+                        this.logger.error(`pullImage(): Error pulling image ${image}`, actualError);
+                        if (streamErrors.length > 0) {
+                            this.logger.error(`pullImage(): Stream errors captured during pull:`, streamErrors.map(streamError => streamError.message));
+                        }
                         imageProgress.inProgress = false;
                         this.imagesInProgress.delete(image);
                         isCompleted = true;
-                        progressTracker.markFailed(err);
-                        reject(err);
+                        progressTracker.markFailed(actualError);
+                        reject(actualError);
                         return;
                     }
                     this.logger.info(`pullImage(): Image ${image} download complete`);
-                    imageProgress.inProgress = false;
-                    this.imagesInProgress.delete(image);
-                    isCompleted = true;
-                    progressTracker.markComplete();
-                    resolve();
+                    (async () => {
+                        try {
+                            const imageExists = await this.imageExists(image);
+                            if (!imageExists) {
+                                const errorMsg = `Image ${image} pull reported success but image is not available locally. ` +
+                                    `This may indicate a network issue, proxy problem, or the image does not exist.`;
+                                this.logger.error(`pullImage(): ${errorMsg}`);
+                                imageProgress.inProgress = false;
+                                this.imagesInProgress.delete(image);
+                                isCompleted = true;
+                                progressTracker.markFailed(new Error(errorMsg));
+                                reject(new Error(errorMsg));
+                                return;
+                            }
+                            imageProgress.inProgress = false;
+                            this.imagesInProgress.delete(image);
+                            isCompleted = true;
+                            progressTracker.markComplete();
+                            (async () => {
+                                try {
+                                    const imageObject = this.docker.getImage(image);
+                                    const imageInspection = await imageObject.inspect();
+                                    const imageId = imageInspection.Id;
+                                    const pullTimestamp = Math.floor(Date.now() / 1000);
+                                    this.imagePullTimes.set(imageId, pullTimestamp);
+                                    const pullTimeFormatted = new Date(pullTimestamp * 1000).toISOString();
+                                    this.logger.info(`pullImage(): Recorded pull time for ${image} ` +
+                                        `(ID: ${imageId.substring(0, 12)}): ${pullTimeFormatted}`);
+                                }
+                                catch (error) {
+                                    this.logger.warn(`pullImage(): Failed to record pull time for ${image}`, error);
+                                }
+                            })();
+                            this.logger.info(`pullImage(): Starting image management for ${image}`);
+                            this.manageDockerImages(image).catch((error) => {
+                                this.logger.error(`pullImage(): Error managing images after pull for ${image}`, error);
+                            });
+                            resolve();
+                        }
+                        catch (verifyError) {
+                            const errorMsg = `Failed to verify image ${image} exists after pull: ` +
+                                `${verifyError.message || verifyError}`;
+                            this.logger.error(`pullImage(): ${errorMsg}`, verifyError);
+                            imageProgress.inProgress = false;
+                            this.imagesInProgress.delete(image);
+                            isCompleted = true;
+                            progressTracker.markFailed(verifyError);
+                            reject(new Error(errorMsg));
+                        }
+                    })();
                 }, (event) => {
                     if (abortController.signal.aborted || isCompleted) {
                         return;
                     }
+                    if (event.error || event.errorDetail) {
+                        const errorMsg = event.errorDetail?.message || event.error || 'Unknown pull error';
+                        this.logger.error(`pullImage(): Error event during pull for ${image}: ${errorMsg}`, { event });
+                        streamErrors.push(new Error(errorMsg));
+                    }
                     progressTracker.processEvent(event);
                     if (event.id && event.progressDetail) {
                         imageProgress.layersProgress.set(event.id, event.progressDetail);
@@ -379,6 +453,24 @@ class DockerManager {
         this.logger.info(`createContainer(): Device requests for ${twin.id}:`, JSON.stringify(parsedOptions?.HostConfig?.DeviceRequests, null, 2));
         const isHostNetwork = parsedOptions?.HostConfig?.NetworkMode === 'host';
         const existingBinds = parsedOptions?.HostConfig?.Binds ?? [];
+        const coreDumpHostDir = '/var/coredumps';
+        try {
+            if (!fs_1.default.existsSync(coreDumpHostDir)) {
+                fs_1.default.mkdirSync(coreDumpHostDir, { recursive: true, mode: 0o777 });
+                this.logger.info(`createContainer(): Created core dump directory ${coreDumpHostDir}`);
+            }
+            try {
+                fs_1.default.chmodSync(coreDumpHostDir, 0o777);
+                this.logger.info(`createContainer(): Set permissions on ${coreDumpHostDir} to 777`);
+            }
+            catch (chmodError) {
+                this.logger.error(`createContainer(): CRITICAL: Could not set permissions on ${coreDumpHostDir}. ` +
+                    `Container may not be able to write coredumps. Error:`, chmodError);
+            }
+        }
+        catch (error) {
+            this.logger.error(`createContainer(): Failed to create or verify ${coreDumpHostDir}, mount may fail`, error);
+        }
         const coreDumpBind = '/var/coredumps:/var/coredumps:z';
         const hasCoreDumpBind = existingBinds.some((bind) => bind.includes('/var/coredumps'));
         const allBinds = hasCoreDumpBind ? existingBinds : [...existingBinds, coreDumpBind];
@@ -392,7 +484,11 @@ class DockerManager {
             ];
         const containerConfig = {
             Image: image,
-            Env: [...(parsedOptions?.Env ?? []), `TWIN_ID=${twin.id}`],
+            Env: [
+                ...(parsedOptions?.Env ?? []),
+                `TWIN_ID=${twin.id}`,
+                ...(process.env.TZ ? [`TZ=${process.env.TZ}`] : []),
+            ],
             Entrypoint: parsedOptions?.Entrypoint,
             Cmd: parsedOptions?.Cmd,
             HostConfig: {
@@ -509,22 +605,17 @@ class DockerManager {
                         error.message.includes('gpu'));
                 if (isGpuError) {
                     this.logger.warn(`startContainer(): GPU support not available for twin ${twin.id}, retrying without GPU capabilities. Error: ${error.message}`, { error: error.message });
-                    try {
-                        this.instances.delete(twin.id);
-                        const containerId = error.json?.message?.match(/^.*container: (.*?)(?:\s|$)/)?.[1];
-                        if (containerId) {
-                            try {
-                                await this.docker.getContainer(containerId).remove({ force: true });
-                                this.logger.info(`startContainer(): Removed failed GPU container ${containerId}`);
-                            }
-                            catch (removeError) {
-                                this.logger.info(`startContainer(): Could not remove container ${containerId}, may not exist`);
-                            }
+                    const existingInstance = this.instances.get(twin.id);
+                    if (existingInstance?.container) {
+                        try {
+                            await existingInstance.container.remove({ force: true });
+                            this.logger.info(`startContainer(): Removed failed GPU container for twin ${twin.id}`);
+                        }
+                        catch (removeError) {
+                            this.logger.info(`startContainer(): Could not remove container for twin ${twin.id}, may not exist`);
                         }
                     }
-                    catch (cleanupError) {
-                        this.logger.warn(`startContainer(): Error during cleanup: ${cleanupError.message}`);
-                    }
+                    this.instances.delete(twin.id);
                     const container = await this.createContainer(twin, true);
                     this.instances.set(twin.id, { container, twin });
                     await container.start();
@@ -577,18 +668,20 @@ class DockerManager {
             this.logger.info(`ensureImageAvailable(): Pull already in progress for ${image}, waiting for it to complete`);
             try {
                 await existingPull.promise;
-                this.logger.info(`ensureImageAvailable(): Existing pull for ${image} completed successfully`);
+                const imageExists = await this.imageExists(image);
+                if (!imageExists) {
+                    const errorMsg = `Existing pull for ${image} reported success but image is not available locally`;
+                    this.logger.error(`ensureImageAvailable(): ${errorMsg}`);
+                    this.imagePullsInProgress.delete(image);
+                    return false;
+                }
+                this.logger.info(`ensureImageAvailable(): Existing pull for ${image} completed successfully ` +
+                    `and image verified locally`);
                 return true;
             }
             catch (error) {
                 this.logger.error(`ensureImageAvailable(): Existing pull for ${image} failed`, error);
-                const isCredentialError = error.message &&
-                    (error.message.includes('unauthorized') ||
-                        error.message.includes('authentication required') ||
-                        error.message.includes('access denied') ||
-                        error.message.includes('credential') ||
-                        error.statusCode === 401);
-                if (isCredentialError) {
+                if (this.isCredentialError(error)) {
                     this.logger.error(`ensureImageAvailable(): Credential error detected for existing pull of ${image}`);
                 }
                 this.imagePullsInProgress.delete(image);
@@ -606,21 +699,21 @@ class DockerManager {
         });
         try {
             await pullPromise;
-            this.logger.info(`ensureImageAvailable(): Pull for ${image} completed successfully`);
+            const imageExists = await this.imageExists(image);
+            if (!imageExists) {
+                const errorMsg = `Pull for ${image} reported success but image is not available locally`;
+                this.logger.error(`ensureImageAvailable(): ${errorMsg}`);
+                return false;
+            }
+            this.logger.info(`ensureImageAvailable(): Pull for ${image} completed successfully ` +
+                `and image verified locally`);
             return true;
         }
         catch (error) {
             this.logger.error(`ensureImageAvailable(): Pull failed for ${image}`, error);
-            const isCredentialError = error.message &&
-                (error.message.includes('unauthorized') ||
-                    error.message.includes('authentication required') ||
-                    error.message.includes('access denied') ||
-                    error.message.includes('credential') ||
-                    error.statusCode === 401);
-            if (isCredentialError) {
+            if (this.isCredentialError(error)) {
                 this.logger.error(`ensureImageAvailable(): Credential error detected for pull of ${image}`);
             }
-            this.imagePullsInProgress.delete(image);
             return false;
         }
     }
@@ -671,15 +764,10 @@ class DockerManager {
             catch (error) {
                 this.logger.error(`startEdgeApp(): Error pulling image ${image}`, error);
                 const errorMsg = error.message || '';
-                const isCredentialError = errorMsg.includes('unauthorized') ||
-                    errorMsg.includes('authentication required') ||
-                    errorMsg.includes('access denied') ||
-                    errorMsg.includes('credential') ||
-                    error.statusCode === 401;
                 return {
                     status: twin_types_1.TwinStatusEnum.ImageNotFound,
                     error: {
-                        message: isCredentialError
+                        message: this.isCredentialError(error)
                             ? `Authentication failed for image ${image}. Please check your credentials.`
                             : `Failed to pull image ${image}: ${errorMsg}`,
                         timestamp: new Date(),
@@ -834,30 +922,159 @@ class DockerManager {
     getInstances() {
         return this.instances;
     }
+    extractRepositoryName(image) {
+        this.logger.info(`extractRepositoryName(): Processing image: ${image}`);
+        let normalizedImage = image;
+        if (image.startsWith('docker.io/')) {
+            normalizedImage = image.substring('docker.io/'.length);
+            this.logger.info(`extractRepositoryName(): Removed docker.io/ prefix: ${normalizedImage}`);
+        }
+        const tagSeparatorIndex = normalizedImage.lastIndexOf(':');
+        const pathSeparatorIndex = normalizedImage.lastIndexOf('/');
+        if (tagSeparatorIndex === -1) {
+            this.logger.info(`extractRepositoryName(): No tag found, repository: ${normalizedImage}`);
+            return normalizedImage;
+        }
+        if (pathSeparatorIndex !== -1 && tagSeparatorIndex < pathSeparatorIndex) {
+            this.logger.info(`extractRepositoryName(): Colon is port separator, repository: ${normalizedImage}`);
+            return normalizedImage;
+        }
+        const repositoryName = normalizedImage.substring(0, tagSeparatorIndex);
+        this.logger.info(`extractRepositoryName(): Extracted repository: ${repositoryName}`);
+        return repositoryName;
+    }
     async manageDockerImages(image) {
         try {
-            const images = await this.docker.listImages({
-                filters: {
-                    reference: [image.split(':')[0]],
-                },
+            this.logger.info(`manageDockerImages(): Starting management for ${image}`);
+            const repositoryName = this.extractRepositoryName(image);
+            const referenceFilters = [repositoryName];
+            if (!repositoryName.startsWith('docker.io/')) {
+                referenceFilters.push(`docker.io/${repositoryName}`);
+            }
+            this.logger.info(`manageDockerImages(): Querying images for repository: ${repositoryName}`);
+            let repositoryImages = await this.docker.listImages({
+                filters: { reference: referenceFilters },
             });
-            images.sort((a, b) => b.Created - a.Created);
-            if (images.length > 2) {
-                for (const img of images.slice(2)) {
-                    try {
-                        await this.docker.getImage(img.Id).remove({ force: true });
-                        this.logger.info(`manageDockerImages(): Removed old image ${img.Id}`);
-                    }
-                    catch (error) {
-                        this.logger.warn(`manageDockerImages(): Failed to remove image ${img.Id}`, error);
-                    }
+            if (repositoryImages.length === 0) {
+                this.logger.info(`manageDockerImages(): Reference filter returned no results, trying manual filter`);
+                const allImages = await this.docker.listImages();
+                repositoryImages = allImages.filter(imageInfo => {
+                    const imageTags = imageInfo.RepoTags || [];
+                    return imageTags.some((imageTag) => {
+                        let normalizedTag = imageTag;
+                        if (imageTag.startsWith('docker.io/')) {
+                            normalizedTag = imageTag.substring('docker.io/'.length);
+                        }
+                        const tagRepositoryName = this.extractRepositoryName(normalizedTag);
+                        return (tagRepositoryName === repositoryName || normalizedTag.startsWith(`${repositoryName}:`));
+                    });
+                });
+                this.logger.info(`manageDockerImages(): Manual filter found ${repositoryImages.length} image(s)`);
+            }
+            if (repositoryImages.length === 0) {
+                this.logger.warn(`manageDockerImages(): No images found for repository ${repositoryName}`);
+                return;
+            }
+            if (repositoryImages.length <= 2) {
+                this.logger.info(`manageDockerImages(): Repository has ${repositoryImages.length} image(s), no cleanup needed`);
+                return;
+            }
+            this.logger.info(`manageDockerImages(): Determining pull times for ${repositoryImages.length} images`);
+            const imagesWithPullTimes = repositoryImages.map(imageInfo => {
+                const imageTags = imageInfo.RepoTags || imageInfo.RepoDigests || ['<untagged>'];
+                const trackedPullTime = this.imagePullTimes.get(imageInfo.Id);
+                const pullTime = trackedPullTime || imageInfo.Created;
+                const pullTimeFormatted = new Date(pullTime * 1000).toISOString();
+                const imageTagsString = imageTags.join(', ');
+                if (trackedPullTime) {
+                    this.logger.info(`manageDockerImages(): Using tracked pull time for ${imageTagsString}: ` +
+                        `${pullTimeFormatted}`);
+                }
+                else {
+                    this.logger.info(`manageDockerImages(): Using Created time for ${imageTagsString}: ` +
+                        `${pullTimeFormatted}`);
+                }
+                return {
+                    imageInfo,
+                    pullTime,
+                    imageTags,
+                };
+            });
+            imagesWithPullTimes.sort((firstImage, secondImage) => secondImage.pullTime - firstImage.pullTime);
+            this.logger.info(`manageDockerImages(): Sorted ${imagesWithPullTimes.length} images by pull time`);
+            const imagesToRemove = imagesWithPullTimes.slice(2);
+            this.logger.info(`manageDockerImages(): Removing ${imagesToRemove.length} oldest image(s), keeping newest 2`);
+            for (const imageToRemove of imagesToRemove) {
+                try {
+                    const imageTagsString = imageToRemove.imageTags.join(', ');
+                    const pullTimeFormatted = new Date(imageToRemove.pullTime * 1000).toISOString();
+                    this.logger.info(`manageDockerImages(): Removing image: ${imageTagsString} ` +
+                        `(Pulled: ${pullTimeFormatted})`);
+                    await this.docker.getImage(imageToRemove.imageInfo.Id).remove({ force: true });
+                    this.imagePullTimes.delete(imageToRemove.imageInfo.Id);
+                    this.persistImagePullTimes();
+                    this.logger.info(`manageDockerImages(): Successfully removed: ${imageToRemove.imageTags.join(', ')}`);
+                }
+                catch (error) {
+                    this.logger.error(`manageDockerImages(): Failed to remove image ${imageToRemove.imageInfo.Id}`, error);
                 }
             }
+            this.logger.info(`manageDockerImages(): Completed management for ${image}`);
         }
         catch (error) {
             this.logger.error(`manageDockerImages(): Error managing images for ${image}`, error);
         }
     }
+    loadImagePullTimes() {
+        try {
+            if (fs_1.default.existsSync(IMAGE_PULL_TIMES_FILE)) {
+                const data = JSON.parse(fs_1.default.readFileSync(IMAGE_PULL_TIMES_FILE, 'utf-8'));
+                const validEntries = Object.entries(data)
+                    .filter(([_, value]) => typeof value === 'number' && value > 0)
+                    .map(([key, value]) => [key, value]);
+                this.imagePullTimes = new Map(validEntries);
+                this.logger.info(`loadImagePullTimes(): Loaded ${this.imagePullTimes.size} image pull times from disk`);
+            }
+        }
+        catch (error) {
+            this.logger.warn('loadImagePullTimes(): Failed to load image pull times, starting fresh', error);
+        }
+    }
+    async cleanupStaleImagePullTimes() {
+        try {
+            const allImages = await this.docker.listImages();
+            const existingImageIds = new Set(allImages.map(img => img.Id));
+            let cleanedCount = 0;
+            for (const [imageId] of this.imagePullTimes.entries()) {
+                if (!existingImageIds.has(imageId)) {
+                    this.imagePullTimes.delete(imageId);
+                    cleanedCount++;
+                }
+            }
+            if (cleanedCount > 0) {
+                this.logger.info(`cleanupStaleImagePullTimes(): Removed ${cleanedCount} stale image pull time entries`);
+                this.persistImagePullTimes();
+            }
+        }
+        catch (error) {
+            this.logger.warn('cleanupStaleImagePullTimes(): Error during cleanup', error);
+        }
+    }
+    persistImagePullTimes() {
+        try {
+            const dir = path_1.default.dirname(IMAGE_PULL_TIMES_FILE);
+            if (!fs_1.default.existsSync(dir)) {
+                fs_1.default.mkdirSync(dir, { recursive: true });
+            }
+            const data = Object.fromEntries(this.imagePullTimes);
+            const tempFile = `${IMAGE_PULL_TIMES_FILE}.tmp`;
+            fs_1.default.writeFileSync(tempFile, JSON.stringify(data, null, 2));
+            fs_1.default.renameSync(tempFile, IMAGE_PULL_TIMES_FILE);
+        }
+        catch (error) {
+            this.logger.warn('persistImagePullTimes(): Failed to persist image pull times', error);
+        }
+    }
     async isContainerRunning(twinId) {
         try {
             const containers = await this.docker.listContainers();