@kapeta/local-cluster-service 0.9.1 → 0.10.0

package/src/containerManager.ts

@@ -7,8 +7,8 @@ import { Docker } from 'node-docker-api';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import ClusterConfiguration from '@kapeta/local-cluster-config';
 import { Container } from 'node-docker-api/lib/container';
-import { getBindHost } from './utils/utils';
-import uuid from "node-uuid";
+import uuid from 'node-uuid';
+import md5 from 'md5';
 
 type StringMap = { [key: string]: string };
 
@@ -54,7 +54,7 @@ interface Health {
     retries?: number;
 }
 
-const LABEL_PORT_PREFIX = 'kapeta_port-';
+export const CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
 const HEALTH_CHECK_MAX = 20;
@@ -206,24 +206,21 @@ class ContainerManager {
             tag = 'latest';
         }
 
-        if (tag !== 'latest') {
-            if (IMAGE_PULL_CACHE[image]) {
-                const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
-                if (timeSince < cacheForMS) {
-                    return;
-                }
+        if (IMAGE_PULL_CACHE[image]) {
+            const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
+            if (timeSince < cacheForMS) {
+                return false;
             }
+        }
 
-            const imageTagList = (await this.docker().image.list())
-                .map((image) => image.data as any)
-                .filter((imageData) => !!imageData.RepoTags)
-                .map((imageData) => imageData.RepoTags as string[]);
+        const imageTagList = (await this.docker().image.list())
+            .map((image) => image.data as any)
+            .filter((imageData) => !!imageData.RepoTags)
+            .map((imageData) => imageData.RepoTags as string[]);
 
-            if (imageTagList.some((imageTags) => imageTags.indexOf(image) > -1)) {
-                console.log('Image found: %s', image);
-                return;
-            }
-            console.log('Image not found: %s', image);
+        if (imageTagList.some((imageTags) => imageTags.indexOf(image) > -1)) {
+            console.log('Image found: %s', image);
+            return false;
         }
 
         console.log('Pulling image: %s', image);
@@ -240,6 +237,8 @@ class ContainerManager {
         IMAGE_PULL_CACHE[image] = Date.now();
 
         console.log('Image pulled: %s', image);
+
+        return true;
     }
 
     toDockerMounts(mounts: StringMap) {
@@ -266,66 +265,63 @@ class ContainerManager {
         };
     }
 
-    async run(
-        image: string,
-        name: string,
-        opts: { ports: {}; mounts: {}; env: {}; cmd: string; health: Health }
-    ): Promise<ContainerInfo> {
-        const PortBindings: { [key: string]: any } = {};
-        const Env: string[] = [];
-        const Labels: StringMap = {
-            kapeta: 'true',
-        };
-
-        await this.pull(image);
-
-        const bindHost = getBindHost();
+    private applyHash(dockerOpts: any) {
+        if (dockerOpts?.Labels?.HASH) {
+            delete dockerOpts.Labels.HASH;
+        }
 
-        const ExposedPorts: { [key: string]: any } = {};
+        const hash = md5(JSON.stringify(dockerOpts));
 
-        _.forEach(opts.ports, (portInfo: any, containerPort) => {
-            ExposedPorts['' + containerPort] = {};
-            PortBindings['' + containerPort] = [
-                {
-                    HostPort: '' + portInfo.hostPort,
-                    HostIp: bindHost,
-                },
-            ];
+        if (!dockerOpts.Labels) {
+            dockerOpts.Labels = {};
+        }
+        dockerOpts.Labels.HASH = hash;
+    }
 
-            Labels[LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
-        });
+    async ensureContainer(opts: any) {
+        let imagePulled = false;
+        try {
+            imagePulled = await this.pull(opts.Image);
+        } catch (e) {
+            console.warn('Failed to pull image. Continuing...', e);
+        }
 
-        const Mounts = this.toDockerMounts(opts.mounts);
+        this.applyHash(opts);
+        if (!opts.name) {
+            console.log('Starting unnamed container: %s', opts.Image);
+            return this.startContainer(opts);
+        }
+        const containerInfo = await this.getContainerByName(opts.name);
+        if (imagePulled) {
+            console.log('New version of image was pulled: %s', opts.Image);
+        } else {
+            // If image was pulled always recreate
+            if (!containerInfo) {
+                console.log('Starting new container: %s', opts.name);
+                return this.startContainer(opts);
+            }
 
-        _.forEach(opts.env, (value, name) => {
-            Env.push(name + '=' + value);
-        });
+            const containerData = containerInfo.native.data as any;
 
-        let HealthCheck = undefined;
-
-        if (opts.health) {
-            HealthCheck = this.toDockerHealth(opts.health);
-        }
-        const dockerContainer = await this.startContainer({
-            name: name,
-            Image: image,
-            Hostname: name + '.kapeta',
-            Labels,
-            Cmd: opts.cmd,
-            ExposedPorts,
-            Env,
-            HealthCheck,
-            HostConfig: {
-                PortBindings,
-                Mounts,
-            },
-        });
+            if (containerData?.Labels?.HASH === opts.Labels.HASH) {
+                if (!(await containerInfo.isRunning())) {
+                    console.log('Starting previously created container: %s', opts.name);
+                    await containerInfo.start();
+                } else {
+                    console.log('Previously created container already running: %s', opts.name);
+                }
+                return containerInfo.native;
+            }
+        }
 
-        if (opts.health) {
-            await this.waitForHealthy(dockerContainer);
+        if (containerInfo) {
+            // Remove the container and start a new one
+            console.log('Replacing previously created container: %s', opts.name);
+            await containerInfo.remove({ force: true });
         }
 
-        return new ContainerInfo(dockerContainer);
+        console.log('Starting new container: %s', opts.name);
+        return this.startContainer(opts);
     }
 
     async startContainer(opts: any) {
@@ -423,8 +419,8 @@ class ContainerManager {
         }
     }
 
-    async remove(container:Container, opts?: { force?: boolean }) {
-        const newName = 'deleting-' + uuid.v4()
+    async remove(container: Container, opts?: { force?: boolean }) {
+        const newName = 'deleting-' + uuid.v4();
         const containerData = container.data as any;
         // Rename the container first to avoid name conflicts if people start the same container
         await container.rename({ name: newName });
@@ -537,11 +533,11 @@ export class ContainerInfo {
         const ports: PortMap = {};
 
         _.forEach(inspectResult.Config.Labels, (portType, name) => {
-            if (!name.startsWith(LABEL_PORT_PREFIX)) {
+            if (!name.startsWith(CONTAINER_LABEL_PORT_PREFIX)) {
                 return;
             }
 
-            const hostPort = name.substr(LABEL_PORT_PREFIX.length);
+            const hostPort = name.substr(CONTAINER_LABEL_PORT_PREFIX.length);
 
             portTypes[hostPort] = portType;
         });

package/src/instanceManager.ts

@@ -559,7 +559,10 @@ export class InstanceManager {
                     }
                 }
 
-                if (instance.desiredStatus === DesiredInstanceStatus.RUN && newStatus === InstanceStatus.STOPPED) {
+                if (
+                    instance.desiredStatus === DesiredInstanceStatus.RUN &&
+                    [InstanceStatus.STOPPED, InstanceStatus.FAILED, InstanceStatus.STOPPING].includes(newStatus)
+                ) {
                     //If the instance is stopped but we want it to run, start it
                     try {
                         await this.start(instance.systemId, instance.instanceId);
@@ -569,7 +572,10 @@ export class InstanceManager {
                     return;
                 }
 
-                if (instance.desiredStatus === DesiredInstanceStatus.STOP && newStatus === InstanceStatus.READY) {
+                if (
+                    instance.desiredStatus === DesiredInstanceStatus.STOP &&
+                    [InstanceStatus.READY, InstanceStatus.STARTING, InstanceStatus.UNHEALTHY].includes(newStatus)
+                ) {
                     //If the instance is running but we want it to stop, stop it
                     try {
                         await this.stop(instance.systemId, instance.instanceId);

package/src/operatorManager.ts

@@ -4,12 +4,14 @@ import md5 from 'md5';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { serviceManager } from './serviceManager';
 import { storageService } from './storageService';
-import { ContainerInfo, containerManager } from './containerManager';
+import { CONTAINER_LABEL_PORT_PREFIX, ContainerInfo, containerManager } from './containerManager';
 import FSExtra from 'fs-extra';
-import { AnyMap, EnvironmentType, OperatorInfo } from './types';
+import { AnyMap, EnvironmentType, OperatorInfo, StringMap } from './types';
 import { BlockInstance, Resource } from '@kapeta/schemas';
 import { definitionsManager } from './definitionsManager';
-import { normalizeKapetaUri } from './utils/utils';
+import { getBindHost, normalizeKapetaUri } from './utils/utils';
+import _ from 'lodash';
+import { Container } from 'node-docker-api/lib/container';
 
 const KIND_OPERATOR = 'core/resource-type-operator';
 
@@ -191,34 +193,58 @@ class OperatorManager {
         const mounts = containerManager.createMounts(resourceType, operatorData.mounts);
 
         const containerName = containerBaseName + '-' + md5(nameParts.join('_'));
-        let container = await containerManager.get(containerName);
 
-        const isRunning = container ? await container.isRunning() : false;
-        if (container && !isRunning) {
-            await container.start();
-        }
+        const PortBindings: { [key: string]: any } = {};
+        const Env: string[] = [];
 
-        if (!container) {
-            container = await containerManager.run(operatorData.image, containerName, {
-                mounts,
-                ports,
-                health: operatorData.health,
-                env: operatorData.env,
-                cmd: operatorData.cmd,
-            });
-        }
+        const Labels: StringMap = {
+            kapeta: 'true',
+        };
 
-        try {
-            if (operatorData.health) {
-                await containerManager.waitForHealthy(container.native);
-            } else {
-                await containerManager.waitForReady(container.native);
-            }
-        } catch (e: any) {
-            console.error(e.message);
+        const bindHost = getBindHost();
+
+        const ExposedPorts: { [key: string]: any } = {};
+
+        _.forEach(ports, (portInfo: any, containerPort) => {
+            ExposedPorts['' + containerPort] = {};
+            PortBindings['' + containerPort] = [
+                {
+                    HostPort: '' + portInfo.hostPort,
+                    HostIp: bindHost,
+                },
+            ];
+
+            Labels[CONTAINER_LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
+        });
+
+        const Mounts = containerManager.toDockerMounts(mounts);
+
+        _.forEach(operatorData.env, (value, name) => {
+            Env.push(name + '=' + value);
+        });
+
+        let HealthCheck = undefined;
+
+        if (operatorData.health) {
+            HealthCheck = containerManager.toDockerHealth(operatorData.health);
         }
 
-        return container;
+        const container = await containerManager.ensureContainer({
+            name: containerName,
+            Image: operatorData.image,
+            Hostname: containerName + '.kapeta',
+            Labels,
+            Cmd: operatorData.cmd,
+            ExposedPorts,
+            Env,
+            HealthCheck,
+            HostConfig: {
+                PortBindings,
+                Mounts,
+            },
+        });
+
+        return new ContainerInfo(container);
     }
 }
 

package/src/utils/BlockInstanceRunner.ts

@@ -7,9 +7,10 @@ import { containerManager, DockerMounts, toLocalBindVolume } from '../containerM
 import { LogData } from './LogData';
 import EventEmitter from 'events';
 import { clusterService } from '../clusterService';
-import { AnyMap, BlockProcessParams, ProcessInfo, InstanceType, StringMap } from '../types';
+import { AnyMap, BlockProcessParams, InstanceType, ProcessInfo, StringMap } from '../types';
 import { Container } from 'node-docker-api/lib/container';
 import { definitionsManager } from '../definitionsManager';
+import md5 from 'md5';
 
 const KIND_BLOCK_TYPE_OPERATOR = 'core/block-type-operator';
 const KAPETA_SYSTEM_ID = 'KAPETA_SYSTEM_ID';
@@ -169,29 +170,6 @@ export class BlockInstanceRunner {
         }
 
         const containerName = getBlockInstanceContainerName(blockInstance.id);
-        const logs = new LogData();
-        logs.addLog(`Starting block ${blockInstance.ref}`);
-        let containerInfo = await containerManager.getContainerByName(containerName);
-        let container = containerInfo?.native;
-
-        console.log('Starting dev container', containerName);
-
-        if (containerInfo) {
-            console.log(`Dev container already exists. Deleting...`);
-            try {
-                await containerInfo.remove({
-                    force: true,
-                });
-            } catch (e: any) {
-                throw new Error('Failed to delete existing container: ' + e.message);
-            }
-            container = undefined;
-            containerInfo = undefined;
-        }
-
-        logs.addLog(`Creating new container for block: ${containerName}`);
-        console.log('Creating new dev container', containerName, dockerImage);
-        await containerManager.pull(dockerImage);
 
         const startCmd = localContainer.handlers?.onCreate ? localContainer.handlers.onCreate : '';
         const dockerOpts = localContainer.options ?? {};
@@ -228,8 +206,7 @@ export class BlockInstanceRunner {
             HealthCheck = containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
 
-        console.log('Starting dev container', containerName, dockerImage);
-        container = await containerManager.startContainer({
+        return this.ensureContainer({
             Image: dockerImage,
             name: containerName,
             WorkingDir: workingDir,
@@ -256,9 +233,15 @@ export class BlockInstanceRunner {
             },
             ...dockerOpts,
         });
+    }
+
+    private async ensureContainer(opts: any) {
+        const logs = new LogData();
+
+        const container = await containerManager.ensureContainer(opts);
 
         try {
-            if (HealthCheck) {
+            if (opts.HealthCheck) {
                 await containerManager.waitForHealthy(container);
             } else {
                 await containerManager.waitForReady(container);
@@ -352,47 +335,25 @@ export class BlockInstanceRunner {
 
         const containerName = getBlockInstanceContainerName(blockInstance.id);
         const logs = new LogData();
-        const containerInfo = await containerManager.getContainerByName(containerName);
-        let container = containerInfo?.native;
 
         // For windows we need to default to root
         const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
 
-        if (container) {
-            const containerData = container.data as any;
-            if (containerData.State === 'running') {
-                logs.addLog(`Found existing running container for block: ${containerName}`);
-            } else {
-                logs.addLog(`Found existing container for block: ${containerName}. Starting now`);
-                await container.start();
-            }
-        } else {
-            logs.addLog(`Creating new container for block: ${containerName}`);
-
-            container = await containerManager.startContainer({
-                Image: dockerImage,
-                name: containerName,
-                Labels: {
-                    instance: blockInstance.id,
-                },
-                Env: [
-                    ...DOCKER_ENV_VARS,
-                    `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
-                    ...Object.entries(env).map(([key, value]) => `${key}=${value}`),
-                ],
-                HostConfig: {
-                    Binds: [`${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`],
-                },
-            });
-
-            try {
-                await containerManager.waitForReady(container);
-            } catch (e: any) {
-                logs.addLog(e.message, 'ERROR');
-            }
-        }
-
-        return this._handleContainer(container, logs);
+        return this.ensureContainer({
+            Image: dockerImage,
+            name: containerName,
+            Labels: {
+                instance: blockInstance.id,
+            },
+            Env: [
+                ...DOCKER_ENV_VARS,
+                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
+                ...Object.entries(env).map(([key, value]) => `${key}=${value}`),
+            ],
+            HostConfig: {
+                Binds: [`${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`],
+            },
+        });
     }
 
     /**
@@ -430,130 +391,79 @@ export class BlockInstanceRunner {
 
         const dockerImage = spec?.local?.image;
 
-        try {
-            await containerManager.pull(dockerImage);
-        } catch (e) {
-            console.warn('Failed to pull image. Continuing...', e);
-        }
-
         const containerName = getBlockInstanceContainerName(blockInstance.id);
         const logs = new LogData();
-        const containerInfo = await containerManager.getContainerByName(containerName);
-        let container = containerInfo?.native;
-
-        if (container) {
-            const containerData = container.data as any;
-            if (containerData.State === 'running') {
-                logs.addLog(`Found existing running container for block: ${containerName}`);
-            } else {
-                if (containerData.State?.ExitCode > 0) {
-                    logs.addLog(`Container exited with code: ${containerData.State.ExitCode}. Deleting...`);
-                    try {
-                        await containerManager.remove(container);
-                    } catch (e) {}
-                    container = undefined;
-                } else {
-                    logs.addLog(`Found existing container for block: ${containerName}. Starting now`);
-                    try {
-                        await container.start();
-                    } catch (e) {
-                        console.warn('Failed to start container. Deleting...', e);
-                        try {
-                            await containerManager.remove(container);
-                        } catch (e) {}
-                        container = undefined;
-                    }
-                }
-            }
-        }
 
         const bindHost = getBindHost();
 
-        if (!container) {
-            const ExposedPorts: AnyMap = {};
-            const addonEnv: StringMap = {};
-            const PortBindings: AnyMap = {};
-            let HealthCheck = undefined;
-            let Mounts: DockerMounts[] = [];
-            const promises = Object.entries(spec.local.ports as { [p: string]: { port: string; type: string } }).map(
-                async ([portType, value]) => {
-                    const dockerPort = `${value.port}/${value.type}`;
-                    ExposedPorts[dockerPort] = {};
-                    addonEnv[`KAPETA_LOCAL_SERVER_PORT_${portType.toUpperCase()}`] = value.port;
-                    const publicPort = await serviceManager.ensureServicePort(
-                        this._systemId,
-                        blockInstance.id,
-                        portType
-                    );
-                    PortBindings[dockerPort] = [
-                        {
-                            HostIp: bindHost,
-                            HostPort: `${publicPort}`,
-                        },
-                    ];
-                }
-            );
-
-            await Promise.all(promises);
-
-            if (spec.local?.env) {
-                Object.entries(spec.local.env).forEach(([key, value]) => {
-                    addonEnv[key] = value as string;
-                });
-            }
-
-            if (spec.local?.mounts) {
-                const mounts = containerManager.createMounts(blockUri.id, spec.local.mounts);
-                Mounts = containerManager.toDockerMounts(mounts);
+        const ExposedPorts: AnyMap = {};
+        const addonEnv: StringMap = {};
+        const PortBindings: AnyMap = {};
+        let HealthCheck = undefined;
+        let Mounts: DockerMounts[] = [];
+        const promises = Object.entries(spec.local.ports as { [p: string]: { port: string; type: string } }).map(
+            async ([portType, value]) => {
+                const dockerPort = `${value.port}/${value.type}`;
+                ExposedPorts[dockerPort] = {};
+                addonEnv[`KAPETA_LOCAL_SERVER_PORT_${portType.toUpperCase()}`] = value.port;
+                const publicPort = await serviceManager.ensureServicePort(this._systemId, blockInstance.id, portType);
+                PortBindings[dockerPort] = [
+                    {
+                        HostIp: bindHost,
+                        HostPort: `${publicPort}`,
+                    },
+                ];
             }
+        );
 
-            if (spec.local?.health) {
-                HealthCheck = containerManager.toDockerHealth(spec.local?.health);
-            }
+        await Promise.all(promises);
 
-            // For windows we need to default to root
-            const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
-
-            logs.addLog(`Creating new container for block: ${containerName}`);
-            container = await containerManager.startContainer({
-                Image: dockerImage,
-                name: containerName,
-                ExposedPorts,
-                HealthCheck,
-                HostConfig: {
-                    Binds: [
-                        `${toLocalBindVolume(kapetaYmlPath)}:/kapeta.yml:ro`,
-                        `${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`,
-                    ],
-                    PortBindings,
-                    Mounts,
-                },
-                Labels: {
-                    instance: blockInstance.id,
-                },
-                Env: [
-                    `KAPETA_INSTANCE_NAME=${blockInstance.ref}`,
-                    `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
-                    ...DOCKER_ENV_VARS,
-                    ...Object.entries({
-                        ...env,
-                        ...addonEnv,
-                    }).map(([key, value]) => `${key}=${value}`),
-                ],
+        if (spec.local?.env) {
+            Object.entries(spec.local.env).forEach(([key, value]) => {
+                addonEnv[key] = value as string;
             });
+        }
 
-            try {
-                if (HealthCheck) {
-                    await containerManager.waitForHealthy(container);
-                } else {
-                    await containerManager.waitForReady(container);
-                }
-            } catch (e: any) {
-                logs.addLog(e.message, 'ERROR');
-            }
+        if (spec.local?.mounts) {
+            const mounts = containerManager.createMounts(blockUri.id, spec.local.mounts);
+            Mounts = containerManager.toDockerMounts(mounts);
         }
 
-        const out = await this._handleContainer(container, logs, true);
+        if (spec.local?.health) {
+            HealthCheck = containerManager.toDockerHealth(spec.local?.health);
+        }
+
+        // For windows we need to default to root
+        const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
+
+        logs.addLog(`Creating new container for block: ${containerName}`);
+        const out = await this.ensureContainer({
+            Image: dockerImage,
+            name: containerName,
+            ExposedPorts,
+            HealthCheck,
+            HostConfig: {
+                Binds: [
+                    `${toLocalBindVolume(kapetaYmlPath)}:/kapeta.yml:ro`,
+                    `${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`,
+                ],
+                PortBindings,
+                Mounts,
+            },
+            Labels: {
+                instance: blockInstance.id,
+            },
+            Env: [
+                `KAPETA_INSTANCE_NAME=${blockInstance.ref}`,
+                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
+                ...DOCKER_ENV_VARS,
+                ...Object.entries({
+                    ...env,
+                    ...addonEnv,
+                }).map(([key, value]) => `${key}=${value}`),
+            ],
+        });
+
         const portTypes = spec.local.ports ? Object.keys(spec.local.ports) : [];
         if (portTypes.length > 0) {
             out.portType = portTypes[0];