@kapeta/local-cluster-service 0.19.6 → 0.19.7

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [0.19.7](https://github.com/kapetacom/local-cluster-service/compare/v0.19.6...v0.19.7) (2023-09-10)
+
+
+### Bug Fixes
+
+* Replaced node-docker-api with dockerode and types ([#72](https://github.com/kapetacom/local-cluster-service/issues/72)) ([3d4894c](https://github.com/kapetacom/local-cluster-service/commit/3d4894c6f78dc6efe5e75587d6463f33a86ded62))
+
 ## [0.19.6](https://github.com/kapetacom/local-cluster-service/compare/v0.19.5...v0.19.6) (2023-09-08)
 
 

package/dist/cjs/src/containerManager.d.ts

@@ -1,7 +1,6 @@
 /// <reference types="node" />
 import FSExtra from 'fs-extra';
-import { Docker } from 'node-docker-api';
-import { Container } from 'node-docker-api/lib/container';
+import Docker from 'dockerode';
 import { InstanceInfo, LogEntry } from './types';
 type StringMap = {
     [key: string]: string;
@@ -20,24 +19,8 @@ export interface DockerMounts {
     ReadOnly: boolean;
     Consistency: string;
 }
-interface DockerState {
-    Status: 'created' | 'running' | 'paused' | 'restarting' | 'removing' | 'exited' | 'dead';
-    Running: boolean;
-    Paused: boolean;
-    Restarting: boolean;
-    OOMKilled: boolean;
-    Dead: boolean;
-    Pid: number;
-    ExitCode: number;
-    Error: string;
-    StartedAt: string;
-    FinishedAt: string;
-    Health?: {
-        Status: 'starting' | 'healthy' | 'unhealthy' | 'none';
-        FailingStreak: number;
-        Log: any[] | null;
-    };
-}
+export type DockerContainerStatus = 'created' | 'running' | 'paused' | 'restarting' | 'removing' | 'exited' | 'dead';
+export type DockerContainerHealth = 'starting' | 'healthy' | 'unhealthy' | 'none';
 interface Health {
     cmd: string;
     interval?: number;
@@ -45,6 +28,8 @@ interface Health {
     retries?: number;
 }
 export declare const CONTAINER_LABEL_PORT_PREFIX = "kapeta_port-";
+export declare const COMPOSE_LABEL_PROJECT = "com.docker.compose.project";
+export declare const COMPOSE_LABEL_SERVICE = "com.docker.compose.service";
 export declare const HEALTH_CHECK_TIMEOUT: number;
 declare class ContainerManager {
     private _docker;
@@ -71,12 +56,12 @@ declare class ContainerManager {
         Retries: number;
     };
     private applyHash;
-    ensureContainer(opts: any): Promise<Container>;
+    ensureContainer(opts: any): Promise<Docker.Container>;
     private createOrUpdateContainer;
-    startContainer(opts: any): Promise<Container>;
-    waitForReady(container: Container, attempt?: number): Promise<void>;
-    _isReady(container: Container): Promise<any>;
-    remove(container: Container, opts?: {
+    private startContainer;
+    waitForReady(container: Docker.Container, attempt?: number): Promise<void>;
+    _isReady(container: Docker.Container): Promise<boolean>;
+    remove(container: Docker.Container, opts?: {
         force?: boolean;
     }): Promise<void>;
     /**
@@ -84,7 +69,7 @@ declare class ContainerManager {
      * @param name
      * @return {Promise<ContainerInfo>}
      */
-    get(name: string): Promise<ContainerInfo | null>;
+    get(name: string): Promise<ContainerInfo | undefined>;
     getLogs(instance: InstanceInfo): Promise<LogEntry[]>;
     stopLogListening(systemId: string, instanceId: string): Promise<void>;
     ensureLogListening(systemId: string, instanceId: string, handler: (log: LogEntry) => void): Promise<void>;
@@ -102,11 +87,11 @@ export declare class ContainerInfo {
     private readonly _container;
     /**
      *
-     * @param {Container} dockerContainer
+     * @param {Docker.Container} dockerContainer
      */
-    constructor(dockerContainer: Container);
-    get native(): Container;
-    isRunning(): Promise<any>;
+    constructor(dockerContainer: Docker.Container);
+    get native(): Docker.Container;
+    isRunning(): Promise<boolean>;
     start(): Promise<void>;
     restart(): Promise<void>;
     stop(): Promise<void>;
@@ -118,8 +103,30 @@ export declare class ContainerInfo {
         protocol: string;
         hostPort: string;
     } | null>;
-    inspect(): Promise<any>;
-    status(): Promise<DockerState>;
+    inspect(): Promise<Docker.ContainerInspectInfo | undefined>;
+    status(): Promise<{
+        Status: string;
+        Running: boolean;
+        Paused: boolean;
+        Restarting: boolean;
+        OOMKilled: boolean;
+        Dead: boolean;
+        Pid: number;
+        ExitCode: number;
+        Error: string;
+        StartedAt: string;
+        FinishedAt: string;
+        Health?: {
+            Status: string;
+            FailingStreak: number;
+            Log: {
+                Start: string;
+                End: string;
+                ExitCode: number;
+                Output: string;
+            }[];
+        } | undefined;
+    } | undefined>;
     getPorts(): Promise<PortMap | false>;
     getLogStream(): Promise<ClosableLogStream>;
     getLogs(): Promise<LogEntry[]>;

package/dist/cjs/src/containerManager.js

@@ -3,13 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.containerManager = exports.toLocalBindVolume = exports.getExtraHosts = exports.ContainerInfo = exports.HEALTH_CHECK_TIMEOUT = exports.CONTAINER_LABEL_PORT_PREFIX = void 0;
+exports.containerManager = exports.toLocalBindVolume = exports.getExtraHosts = exports.ContainerInfo = exports.HEALTH_CHECK_TIMEOUT = exports.COMPOSE_LABEL_SERVICE = exports.COMPOSE_LABEL_PROJECT = exports.CONTAINER_LABEL_PORT_PREFIX = void 0;
 const path_1 = __importDefault(require("path"));
 const storageService_1 = require("./storageService");
 const os_1 = __importDefault(require("os"));
 const lodash_1 = __importDefault(require("lodash"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
-const node_docker_api_1 = require("node-docker-api");
+const dockerode_1 = __importDefault(require("dockerode"));
 const nodejs_utils_1 = require("@kapeta/nodejs-utils");
 const local_cluster_config_1 = __importDefault(require("@kapeta/local-cluster-config"));
 const node_uuid_1 = __importDefault(require("node-uuid"));
@@ -18,15 +18,45 @@ const utils_1 = require("./utils/utils");
 const nodejs_api_client_1 = require("@kapeta/nodejs-api-client");
 const taskManager_1 = require("./taskManager");
 const node_events_1 = require("node:events");
+const StreamValues_1 = __importDefault(require("stream-json/streamers/StreamValues"));
 exports.CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
 const HEALTH_CHECK_MAX = 20;
+exports.COMPOSE_LABEL_PROJECT = 'com.docker.compose.project';
+exports.COMPOSE_LABEL_SERVICE = 'com.docker.compose.service';
 exports.HEALTH_CHECK_TIMEOUT = HEALTH_CHECK_INTERVAL * HEALTH_CHECK_MAX * 2;
-const promisifyStream = (stream, handler) => new Promise((resolve, reject) => {
-    stream.on('data', handler);
-    stream.on('end', resolve);
-    stream.on('error', reject);
+var DockerPullEventTypes;
+(function (DockerPullEventTypes) {
+    DockerPullEventTypes["PreparingPhase"] = "Preparing";
+    DockerPullEventTypes["WaitingPhase"] = "Waiting";
+    DockerPullEventTypes["PullingFsPhase"] = "Pulling fs layer";
+    DockerPullEventTypes["DownloadingPhase"] = "Downloading";
+    DockerPullEventTypes["DownloadCompletePhase"] = "Download complete";
+    DockerPullEventTypes["ExtractingPhase"] = "Extracting";
+    DockerPullEventTypes["VerifyingChecksumPhase"] = "Verifying Checksum";
+    DockerPullEventTypes["AlreadyExistsPhase"] = "Already exists";
+    DockerPullEventTypes["PullCompletePhase"] = "Pull complete";
+})(DockerPullEventTypes || (DockerPullEventTypes = {}));
+const processJsonStream = (purpose, stream, handler) => new Promise((resolve, reject) => {
+    const jsonStream = StreamValues_1.default.withParser();
+    jsonStream.on('data', (data) => {
+        try {
+            handler(data.value);
+        }
+        catch (e) {
+            console.error('Failed while processing data for stream: %s', purpose, e);
+        }
+    });
+    jsonStream.on('end', () => {
+        console.log('Docker stream ended: %s', purpose);
+        resolve();
+    });
+    jsonStream.on('error', (err) => {
+        console.error('Docker stream failed: %s', purpose, err);
+        reject(err);
+    });
+    stream.pipe(jsonStream);
 });
 class ContainerManager {
     _docker;
@@ -64,9 +94,9 @@ class ContainerManager {
             ];
         for (const opts of connectOptions) {
             try {
-                const client = new node_docker_api_1.Docker({
+                const client = new dockerode_1.default({
                     ...opts,
-                    timeout: 10000,
+                    timeout: 15 * 60 * 1000, //15 minutes should be enough for any operation
                 });
                 await client.ping();
                 this._docker = client;
@@ -143,13 +173,12 @@ class ContainerManager {
         return this._docker;
     }
     async getContainerByName(containerName) {
-        const containers = await this.docker().container.list({ all: true });
+        const containers = await this.docker().listContainers({ all: true });
         const out = containers.find((container) => {
-            const containerData = container.data;
-            return containerData.Names.indexOf(`/${containerName}`) > -1;
+            return container.Names.indexOf(`/${containerName}`) > -1;
         });
         if (out) {
-            return new ContainerInfo(out);
+            return this.get(out.Id);
         }
         return undefined;
     }
@@ -158,8 +187,7 @@ class ContainerManager {
         if (!tag) {
             tag = 'latest';
         }
-        const imageTagList = (await this.docker().image.list())
-            .map((image) => image.data)
+        const imageTagList = (await this.docker().listImages({}))
             .filter((imageData) => !!imageData.RepoTags)
             .map((imageData) => imageData.RepoTags);
         if (imageTagList.some((imageTags) => imageTags.indexOf(image) > -1)) {
@@ -184,66 +212,61 @@ class ContainerManager {
                     serveraddress: 'docker.kapeta.com',
                 }
                 : {};
-            const stream = (await this.docker().image.create(auth, {
-                fromImage: imageName,
-                tag: tag,
-            }));
+            const stream = await this.docker().pull(image, {
+                authconfig: auth,
+            });
             const chunks = {};
             let lastEmitted = Date.now();
-            await promisifyStream(stream, (rawData) => {
-                const lines = rawData.toString().trim().split('\n');
-                lines.forEach((line) => {
-                    const data = JSON.parse(line);
-                    if (![
-                        'Waiting',
-                        'Downloading',
-                        'Extracting',
-                        'Download complete',
-                        'Pull complete',
-                        'Already exists',
-                    ].includes(data.status)) {
-                        return;
-                    }
-                    if (!chunks[data.id]) {
-                        chunks[data.id] = {
-                            downloading: {
-                                total: 0,
-                                current: 0,
-                            },
-                            extracting: {
-                                total: 0,
-                                current: 0,
-                            },
-                            done: false,
+            await processJsonStream(`image:pull:${image}`, stream, (data) => {
+                if (!chunks[data.id]) {
+                    chunks[data.id] = {
+                        downloading: {
+                            total: 0,
+                            current: 0,
+                        },
+                        extracting: {
+                            total: 0,
+                            current: 0,
+                        },
+                        done: false,
+                    };
+                }
+                const chunk = chunks[data.id];
+                switch (data.status) {
+                    case DockerPullEventTypes.PreparingPhase:
+                    case DockerPullEventTypes.WaitingPhase:
+                    case DockerPullEventTypes.PullingFsPhase:
+                        //Do nothing
+                        break;
+                    case DockerPullEventTypes.DownloadingPhase:
+                    case DockerPullEventTypes.VerifyingChecksumPhase:
+                        chunk.downloading = {
+                            total: data.progressDetail?.total ?? 0,
+                            current: data.progressDetail?.current ?? 0,
                         };
-                    }
-                    const chunk = chunks[data.id];
-                    switch (data.status) {
-                        case 'Downloading':
-                            chunk.downloading = data.progressDetail;
-                            break;
-                        case 'Extracting':
-                            chunk.extracting = data.progressDetail;
-                            break;
-                        case 'Download complete':
-                            chunk.downloading.current = chunks[data.id].downloading.total;
-                            break;
-                        case 'Pull complete':
-                            chunk.extracting.current = chunks[data.id].extracting.total;
-                            chunk.done = true;
-                            break;
-                        case 'Already exists':
-                            // Force layer to be done
-                            chunk.downloading.current = 1;
-                            chunk.downloading.total = 1;
-                            chunk.extracting.current = 1;
-                            chunk.extracting.total = 1;
-                            chunk.done = true;
-                            break;
-                    }
-                });
-                if (Date.now() - lastEmitted < 1000) {
-                    return;
+                        break;
+                    case DockerPullEventTypes.ExtractingPhase:
+                        chunk.extracting = {
+                            total: data.progressDetail?.total ?? 0,
+                            current: data.progressDetail?.current ?? 0,
+                        };
+                        break;
+                    case DockerPullEventTypes.DownloadCompletePhase:
+                        chunk.downloading.current = chunks[data.id].downloading.total;
+                        break;
+                    case DockerPullEventTypes.PullCompletePhase:
+                        chunk.extracting.current = chunks[data.id].extracting.total;
+                        chunk.done = true;
+                        break;
+                }
+                if (data.status === DockerPullEventTypes.AlreadyExistsPhase ||
+                    data.status.includes('Image is up to date') ||
+                    data.status.includes('Downloaded newer image')) {
+                    chunk.downloading.current = 1;
+                    chunk.downloading.total = 1;
+                    chunk.extracting.current = 1;
+                    chunk.extracting.total = 1;
+                    chunk.done = true;
                 }
                 const chunkList = Object.values(chunks);
                 let totals = {
@@ -255,6 +278,7 @@ class ContainerManager {
                         total: 0,
                         current: 0,
                     },
+                    percent: 0,
                     total: chunkList.length,
                     done: 0,
                 };
@@ -275,14 +299,17 @@ class ContainerManager {
                         totals.done++;
                     }
                 });
-                const progress = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
+                totals.percent = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
                 task.metadata = {
                     ...task.metadata,
                     image,
-                    progress,
+                    progress: totals.percent,
                     status: totals,
                     timeTaken: Date.now() - timeStarted,
                 };
+                if (Date.now() - lastEmitted < 1000) {
+                    return;
+                }
                 task.emitUpdate();
                 lastEmitted = Date.now();
                 //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
@@ -299,6 +326,7 @@ class ContainerManager {
             name: taskName,
             image,
             progress: -1,
+            group: 'docker:pull', //It's faster to pull images one at a time
         });
         await task.wait();
         return true;
@@ -344,32 +372,32 @@ class ContainerManager {
             console.log('Starting unnamed container: %s', opts.Image);
             return this.startContainer(opts);
         }
-        const containerInfo = await this.getContainerByName(opts.name);
+        const container = await this.getContainerByName(opts.name);
         if (imagePulled) {
+            // If image was pulled always recreate
             console.log('New version of image was pulled: %s', opts.Image);
         }
         else {
-            // If image was pulled always recreate
-            if (!containerInfo) {
+            if (!container) {
                 console.log('Starting new container: %s', opts.name);
                 return this.startContainer(opts);
             }
-            const containerData = containerInfo.native.data;
-            if (containerData?.Labels?.HASH === opts.Labels.HASH) {
-                if (!(await containerInfo.isRunning())) {
+            const containerData = await container.inspect();
+            if (containerData?.Config.Labels?.HASH === opts.Labels.HASH) {
+                if (!(await container.isRunning())) {
                     console.log('Starting previously created container: %s', opts.name);
-                    await containerInfo.start();
+                    await container.start();
                 }
                 else {
                     console.log('Previously created container already running: %s', opts.name);
                 }
-                return containerInfo.native;
+                return container.native;
             }
         }
-        if (containerInfo) {
+        if (container) {
             // Remove the container and start a new one
             console.log('Replacing previously created container: %s', opts.name);
-            await containerInfo.remove({ force: true });
+            await container.remove({ force: true });
         }
         console.log('Starting new container: %s', opts.name);
         return this.startContainer(opts);
@@ -385,7 +413,7 @@ class ContainerManager {
             }
             opts.HostConfig.ExtraHosts = opts.HostConfig.ExtraHosts.concat(extraHosts);
         }
-        const dockerContainer = await this.docker().container.create(opts);
+        const dockerContainer = await this.docker().createContainer(opts);
         await dockerContainer.start();
         return dockerContainer;
     }
@@ -414,30 +442,28 @@ class ContainerManager {
     async _isReady(container) {
         let info;
         try {
-            info = await container.status();
+            info = await container.inspect();
         }
         catch (err) {
             return false;
         }
-        const infoData = info?.data;
-        const state = infoData?.State;
-        if (state?.Status === 'exited' || state?.Status === 'removing' || state?.Status === 'dead') {
+        const state = info.State;
+        if (state.Status === 'exited' || state?.Status === 'removing' || state?.Status === 'dead') {
             throw new Error('Container exited unexpectedly');
         }
-        if (infoData?.State?.Health) {
+        if (state.Health) {
             // If container has health info - wait for it to become healthy
-            return infoData.State.Health.Status === 'healthy';
+            return state.Health.Status === 'healthy';
         }
         else {
-            return infoData?.State?.Running ?? false;
+            return state.Running ?? false;
         }
     }
     async remove(container, opts) {
         const newName = 'deleting-' + node_uuid_1.default.v4();
-        const containerData = container.data;
         // Rename the container first to avoid name conflicts if people start the same container
         await container.rename({ name: newName });
-        await container.delete({ force: !!opts?.force });
+        await container.remove({ force: !!opts?.force });
     }
     /**
      *
@@ -447,15 +473,15 @@ class ContainerManager {
     async get(name) {
         let dockerContainer = null;
         try {
-            dockerContainer = await this.docker().container.get(name);
-            await dockerContainer.status();
+            dockerContainer = await this.docker().getContainer(name);
+            await dockerContainer.stats();
         }
         catch (err) {
             //Ignore
             dockerContainer = null;
         }
         if (!dockerContainer) {
-            return null;
+            return undefined;
         }
         return new ContainerInfo(dockerContainer);
     }
@@ -647,12 +673,12 @@ class ContainerInfo {
     _container;
     /**
      *
-     * @param {Container} dockerContainer
+     * @param {Docker.Container} dockerContainer
      */
     constructor(dockerContainer) {
         /**
          *
-         * @type {Container}
+         * @type {Docker.Container}
          * @private
          */
         this._container = dockerContainer;
@@ -668,12 +694,21 @@ class ContainerInfo {
         return inspectResult.State.Running || inspectResult.State.Restarting;
     }
     async start() {
+        if (await this.isRunning()) {
+            return;
+        }
         await this._container.start();
     }
     async restart() {
+        if (!(await this.isRunning())) {
+            return this.start();
+        }
         await this._container.restart();
     }
     async stop() {
+        if (!(await this.isRunning())) {
+            return;
+        }
         await this._container.stop();
     }
     async remove(opts) {
@@ -688,16 +723,15 @@ class ContainerInfo {
     }
     async inspect() {
         try {
-            const result = await this._container.status();
-            return result ? result.data : null;
+            return await this._container.inspect();
         }
         catch (err) {
-            return null;
+            return undefined;
         }
     }
     async status() {
         const result = await this.inspect();
-        return result.State;
+        return result?.State;
     }
     async getPorts() {
         const inspectResult = await this.inspect();
@@ -742,17 +776,13 @@ class ContainerInfo {
         }
     }
     async getLogs() {
-        const logStream = (await this.native.logs({
+        const logs = await this.native.logs({
             stdout: true,
             stderr: true,
             follow: false,
             timestamps: true,
-        }));
-        const chunks = [];
-        await promisifyStream(logStream, (data) => {
-            chunks.push(data);
         });
-        const out = readLogBuffer(Buffer.concat(chunks));
+        const out = readLogBuffer(logs);
         if (out.length === 0) {
             out.push({
                 time: Date.now(),

package/dist/cjs/src/instanceManager.js

@@ -262,6 +262,9 @@ class InstanceManager {
             if (instance.status === types_1.InstanceStatus.STOPPED) {
                 return;
             }
+            if (instance.status === types_1.InstanceStatus.STOPPING) {
+                return;
+            }
             if (changeDesired && instance.desiredStatus !== types_1.DesiredInstanceStatus.EXTERNAL) {
                 instance.desiredStatus = types_1.DesiredInstanceStatus.STOP;
             }
@@ -392,18 +395,18 @@ class InstanceManager {
                 const startTime = Date.now();
                 try {
                     const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
-                    instance.status = types_1.InstanceStatus.READY;
+                    instance.status = types_1.InstanceStatus.STARTING;
                     return this.saveInternalInstance({
                         ...instance,
                         type: processInfo.type,
                         pid: processInfo.pid ?? -1,
                         health: null,
                         portType: processInfo.portType,
-                        status: types_1.InstanceStatus.READY,
+                        status: types_1.InstanceStatus.STARTING,
                     });
                 }
                 catch (e) {
-                    console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
+                    console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e);
                     const logs = [
                         {
                             source: 'stdout',
@@ -550,7 +553,7 @@ class InstanceManager {
                         await this.start(instance.systemId, instance.instanceId);
                     }
                     catch (e) {
-                        console.warn('Failed to start instance', instance.systemId, instance.instanceId, e);
+                        console.warn('Failed to start previously stopped instance', instance.systemId, instance.instanceId, e);
                     }
                     return;
                 }
@@ -594,31 +597,38 @@ class InstanceManager {
                 return types_1.InstanceStatus.STOPPED;
             }
             const state = await container.status();
-            if (state.Status === 'running') {
-                if (state.Health?.Status === 'healthy') {
-                    return types_1.InstanceStatus.READY;
-                }
-                if (state.Health?.Status === 'starting') {
-                    return types_1.InstanceStatus.STARTING;
-                }
-                if (state.Health?.Status === 'unhealthy') {
-                    return types_1.InstanceStatus.UNHEALTHY;
+            if (!state) {
+                return types_1.InstanceStatus.STOPPED;
+            }
+            const statusType = state.Status;
+            if (statusType === 'running') {
+                if (state.Health?.Status) {
+                    const healthStatusType = state.Health.Status;
+                    if (healthStatusType === 'healthy' || healthStatusType === 'none') {
+                        return types_1.InstanceStatus.READY;
+                    }
+                    if (healthStatusType === 'starting') {
+                        return types_1.InstanceStatus.STARTING;
+                    }
+                    if (healthStatusType === 'unhealthy') {
+                        return types_1.InstanceStatus.UNHEALTHY;
+                    }
                 }
                 return types_1.InstanceStatus.READY;
             }
-            if (state.Status === 'created') {
+            if (statusType === 'created') {
                 return types_1.InstanceStatus.STARTING;
             }
-            if (state.Status === 'exited' || state.Status === 'dead') {
+            if (statusType === 'exited' || statusType === 'dead') {
                 return types_1.InstanceStatus.STOPPED;
             }
-            if (state.Status === 'removing') {
+            if (statusType === 'removing') {
                 return types_1.InstanceStatus.BUSY;
             }
-            if (state.Status === 'restarting') {
+            if (statusType === 'restarting') {
                 return types_1.InstanceStatus.BUSY;
             }
-            if (state.Status === 'paused') {
+            if (statusType === 'paused') {
                 return types_1.InstanceStatus.BUSY;
             }
             return types_1.InstanceStatus.STOPPED;