@kapeta/local-cluster-service 0.9.1 → 0.10.0

package/dist/esm/src/containerManager.js

@@ -6,9 +6,9 @@ import FSExtra from 'fs-extra';
 import { Docker } from 'node-docker-api';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import ClusterConfiguration from '@kapeta/local-cluster-config';
-import { getBindHost } from './utils/utils';
-import uuid from "node-uuid";
-const LABEL_PORT_PREFIX = 'kapeta_port-';
+import uuid from 'node-uuid';
+import md5 from 'md5';
+export const CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
 const HEALTH_CHECK_MAX = 20;
@@ -141,22 +141,19 @@ class ContainerManager {
         if (!tag) {
             tag = 'latest';
         }
-        if (tag !== 'latest') {
-            if (IMAGE_PULL_CACHE[image]) {
-                const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
-                if (timeSince < cacheForMS) {
-                    return;
-                }
-            }
-            const imageTagList = (await this.docker().image.list())
-                .map((image) => image.data)
-                .filter((imageData) => !!imageData.RepoTags)
-                .map((imageData) => imageData.RepoTags);
-            if (imageTagList.some((imageTags) => imageTags.indexOf(image) > -1)) {
-                console.log('Image found: %s', image);
-                return;
+        if (IMAGE_PULL_CACHE[image]) {
+            const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
+            if (timeSince < cacheForMS) {
+                return false;
             }
-            console.log('Image not found: %s', image);
+        }
+        const imageTagList = (await this.docker().image.list())
+            .map((image) => image.data)
+            .filter((imageData) => !!imageData.RepoTags)
+            .map((imageData) => imageData.RepoTags);
+        if (imageTagList.some((imageTags) => imageTags.indexOf(image) > -1)) {
+            console.log('Image found: %s', image);
+            return false;
         }
         console.log('Pulling image: %s', image);
         await this.docker()
@@ -167,6 +164,7 @@ class ContainerManager {
             .then((stream) => promisifyStream(stream));
         IMAGE_PULL_CACHE[image] = Date.now();
         console.log('Image pulled: %s', image);
+        return true;
     }
     toDockerMounts(mounts) {
         const Mounts = [];
@@ -189,51 +187,58 @@ class ContainerManager {
             Retries: health.retries || 10,
         };
     }
-    async run(image, name, opts) {
-        const PortBindings = {};
-        const Env = [];
-        const Labels = {
-            kapeta: 'true',
-        };
-        await this.pull(image);
-        const bindHost = getBindHost();
-        const ExposedPorts = {};
-        _.forEach(opts.ports, (portInfo, containerPort) => {
-            ExposedPorts['' + containerPort] = {};
-            PortBindings['' + containerPort] = [
-                {
-                    HostPort: '' + portInfo.hostPort,
-                    HostIp: bindHost,
-                },
-            ];
-            Labels[LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
-        });
-        const Mounts = this.toDockerMounts(opts.mounts);
-        _.forEach(opts.env, (value, name) => {
-            Env.push(name + '=' + value);
-        });
-        let HealthCheck = undefined;
-        if (opts.health) {
-            HealthCheck = this.toDockerHealth(opts.health);
-        }
-        const dockerContainer = await this.startContainer({
-            name: name,
-            Image: image,
-            Hostname: name + '.kapeta',
-            Labels,
-            Cmd: opts.cmd,
-            ExposedPorts,
-            Env,
-            HealthCheck,
-            HostConfig: {
-                PortBindings,
-                Mounts,
-            },
-        });
-        if (opts.health) {
-            await this.waitForHealthy(dockerContainer);
+    applyHash(dockerOpts) {
+        if (dockerOpts?.Labels?.HASH) {
+            delete dockerOpts.Labels.HASH;
         }
-        return new ContainerInfo(dockerContainer);
+        const hash = md5(JSON.stringify(dockerOpts));
+        if (!dockerOpts.Labels) {
+            dockerOpts.Labels = {};
+        }
+        dockerOpts.Labels.HASH = hash;
+    }
+    async ensureContainer(opts) {
+        let imagePulled = false;
+        try {
+            imagePulled = await this.pull(opts.Image);
+        }
+        catch (e) {
+            console.warn('Failed to pull image. Continuing...', e);
+        }
+        this.applyHash(opts);
+        if (!opts.name) {
+            console.log('Starting unnamed container: %s', opts.Image);
+            return this.startContainer(opts);
+        }
+        const containerInfo = await this.getContainerByName(opts.name);
+        if (imagePulled) {
+            console.log('New version of image was pulled: %s', opts.Image);
+        }
+        else {
+            // If image was pulled always recreate
+            if (!containerInfo) {
+                console.log('Starting new container: %s', opts.name);
+                return this.startContainer(opts);
+            }
+            const containerData = containerInfo.native.data;
+            if (containerData?.Labels?.HASH === opts.Labels.HASH) {
+                if (!(await containerInfo.isRunning())) {
+                    console.log('Starting previously created container: %s', opts.name);
+                    await containerInfo.start();
+                }
+                else {
+                    console.log('Previously created container already running: %s', opts.name);
+                }
+                return containerInfo.native;
+            }
+        }
+        if (containerInfo) {
+            // Remove the container and start a new one
+            console.log('Replacing previously created container: %s', opts.name);
+            await containerInfo.remove({ force: true });
+        }
+        console.log('Starting new container: %s', opts.name);
+        return this.startContainer(opts);
     }
     async startContainer(opts) {
         const extraHosts = getExtraHosts(this._version);
@@ -411,10 +416,10 @@ export class ContainerInfo {
         const portTypes = {};
         const ports = {};
         _.forEach(inspectResult.Config.Labels, (portType, name) => {
-            if (!name.startsWith(LABEL_PORT_PREFIX)) {
+            if (!name.startsWith(CONTAINER_LABEL_PORT_PREFIX)) {
                 return;
             }
-            const hostPort = name.substr(LABEL_PORT_PREFIX.length);
+            const hostPort = name.substr(CONTAINER_LABEL_PORT_PREFIX.length);
             portTypes[hostPort] = portType;
         });
         _.forEach(inspectResult.HostConfig.PortBindings, (portBindings, containerPortSpec) => {

package/dist/esm/src/instanceManager.js

@@ -447,7 +447,8 @@ export class InstanceManager {
                         changed = true;
                     }
                 }
-                if (instance.desiredStatus === DesiredInstanceStatus.RUN && newStatus === InstanceStatus.STOPPED) {
+                if (instance.desiredStatus === DesiredInstanceStatus.RUN &&
+                    [InstanceStatus.STOPPED, InstanceStatus.FAILED, InstanceStatus.STOPPING].includes(newStatus)) {
                     //If the instance is stopped but we want it to run, start it
                     try {
                         await this.start(instance.systemId, instance.instanceId);
@@ -457,7 +458,8 @@ export class InstanceManager {
                     }
                     return;
                 }
-                if (instance.desiredStatus === DesiredInstanceStatus.STOP && newStatus === InstanceStatus.READY) {
+                if (instance.desiredStatus === DesiredInstanceStatus.STOP &&
+                    [InstanceStatus.READY, InstanceStatus.STARTING, InstanceStatus.UNHEALTHY].includes(newStatus)) {
                     //If the instance is running but we want it to stop, stop it
                     try {
                         await this.stop(instance.systemId, instance.instanceId);

package/dist/esm/src/operatorManager.js

@@ -3,10 +3,11 @@ import md5 from 'md5';
 import { parseKapetaUri } from '@kapeta/nodejs-utils';
 import { serviceManager } from './serviceManager';
 import { storageService } from './storageService';
-import { containerManager } from './containerManager';
+import { CONTAINER_LABEL_PORT_PREFIX, ContainerInfo, containerManager } from './containerManager';
 import FSExtra from 'fs-extra';
 import { definitionsManager } from './definitionsManager';
-import { normalizeKapetaUri } from './utils/utils';
+import { getBindHost, normalizeKapetaUri } from './utils/utils';
+import _ from 'lodash';
 const KIND_OPERATOR = 'core/resource-type-operator';
 class Operator {
     _data;
@@ -132,32 +133,46 @@ class OperatorManager {
         }
         const mounts = containerManager.createMounts(resourceType, operatorData.mounts);
         const containerName = containerBaseName + '-' + md5(nameParts.join('_'));
-        let container = await containerManager.get(containerName);
-        const isRunning = container ? await container.isRunning() : false;
-        if (container && !isRunning) {
-            await container.start();
-        }
-        if (!container) {
-            container = await containerManager.run(operatorData.image, containerName, {
-                mounts,
-                ports,
-                health: operatorData.health,
-                env: operatorData.env,
-                cmd: operatorData.cmd,
-            });
-        }
-        try {
-            if (operatorData.health) {
-                await containerManager.waitForHealthy(container.native);
-            }
-            else {
-                await containerManager.waitForReady(container.native);
-            }
-        }
-        catch (e) {
-            console.error(e.message);
+        const PortBindings = {};
+        const Env = [];
+        const Labels = {
+            kapeta: 'true',
+        };
+        const bindHost = getBindHost();
+        const ExposedPorts = {};
+        _.forEach(ports, (portInfo, containerPort) => {
+            ExposedPorts['' + containerPort] = {};
+            PortBindings['' + containerPort] = [
+                {
+                    HostPort: '' + portInfo.hostPort,
+                    HostIp: bindHost,
+                },
+            ];
+            Labels[CONTAINER_LABEL_PORT_PREFIX + portInfo.hostPort] = portInfo.type;
+        });
+        const Mounts = containerManager.toDockerMounts(mounts);
+        _.forEach(operatorData.env, (value, name) => {
+            Env.push(name + '=' + value);
+        });
+        let HealthCheck = undefined;
+        if (operatorData.health) {
+            HealthCheck = containerManager.toDockerHealth(operatorData.health);
         }
-        return container;
+        const container = await containerManager.ensureContainer({
+            name: containerName,
+            Image: operatorData.image,
+            Hostname: containerName + '.kapeta',
+            Labels,
+            Cmd: operatorData.cmd,
+            ExposedPorts,
+            Env,
+            HealthCheck,
+            HostConfig: {
+                PortBindings,
+                Mounts,
+            },
+        });
+        return new ContainerInfo(container);
     }
 }
 export const operatorManager = new OperatorManager();

package/dist/esm/src/utils/BlockInstanceRunner.d.ts

@@ -14,6 +14,7 @@ export declare class BlockInstanceRunner {
      * Starts local process
      */
     private _startLocalProcess;
+    private ensureContainer;
     private _handleContainer;
     private _startDockerProcess;
     /**

package/dist/esm/src/utils/BlockInstanceRunner.js

@@ -128,27 +128,6 @@ export class BlockInstanceRunner {
             throw new Error(`Missing docker image information: ${JSON.stringify(localContainer)}`);
         }
         const containerName = getBlockInstanceContainerName(blockInstance.id);
-        const logs = new LogData();
-        logs.addLog(`Starting block ${blockInstance.ref}`);
-        let containerInfo = await containerManager.getContainerByName(containerName);
-        let container = containerInfo?.native;
-        console.log('Starting dev container', containerName);
-        if (containerInfo) {
-            console.log(`Dev container already exists. Deleting...`);
-            try {
-                await containerInfo.remove({
-                    force: true,
-                });
-            }
-            catch (e) {
-                throw new Error('Failed to delete existing container: ' + e.message);
-            }
-            container = undefined;
-            containerInfo = undefined;
-        }
-        logs.addLog(`Creating new container for block: ${containerName}`);
-        console.log('Creating new dev container', containerName, dockerImage);
-        await containerManager.pull(dockerImage);
         const startCmd = localContainer.handlers?.onCreate ? localContainer.handlers.onCreate : '';
         const dockerOpts = localContainer.options ?? {};
         const homeDir = localContainer.userHome ? localContainer.userHome : '/root';
@@ -177,8 +156,7 @@ export class BlockInstanceRunner {
         if (localContainer.healthcheck) {
             HealthCheck = containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
-        console.log('Starting dev container', containerName, dockerImage);
-        container = await containerManager.startContainer({
+        return this.ensureContainer({
             Image: dockerImage,
             name: containerName,
             WorkingDir: workingDir,
@@ -205,8 +183,12 @@ export class BlockInstanceRunner {
             },
             ...dockerOpts,
         });
+    }
+    async ensureContainer(opts) {
+        const logs = new LogData();
+        const container = await containerManager.ensureContainer(opts);
         try {
-            if (HealthCheck) {
+            if (opts.HealthCheck) {
                 await containerManager.waitForHealthy(container);
             }
             else {
@@ -284,45 +266,23 @@ export class BlockInstanceRunner {
         }
         const containerName = getBlockInstanceContainerName(blockInstance.id);
         const logs = new LogData();
-        const containerInfo = await containerManager.getContainerByName(containerName);
-        let container = containerInfo?.native;
         // For windows we need to default to root
         const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
-        if (container) {
-            const containerData = container.data;
-            if (containerData.State === 'running') {
-                logs.addLog(`Found existing running container for block: ${containerName}`);
-            }
-            else {
-                logs.addLog(`Found existing container for block: ${containerName}. Starting now`);
-                await container.start();
-            }
-        }
-        else {
-            logs.addLog(`Creating new container for block: ${containerName}`);
-            container = await containerManager.startContainer({
-                Image: dockerImage,
-                name: containerName,
-                Labels: {
-                    instance: blockInstance.id,
-                },
-                Env: [
-                    ...DOCKER_ENV_VARS,
-                    `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
-                    ...Object.entries(env).map(([key, value]) => `${key}=${value}`),
-                ],
-                HostConfig: {
-                    Binds: [`${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`],
-                },
-            });
-            try {
-                await containerManager.waitForReady(container);
-            }
-            catch (e) {
-                logs.addLog(e.message, 'ERROR');
-            }
-        }
-        return this._handleContainer(container, logs);
+        return this.ensureContainer({
+            Image: dockerImage,
+            name: containerName,
+            Labels: {
+                instance: blockInstance.id,
+            },
+            Env: [
+                ...DOCKER_ENV_VARS,
+                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
+                ...Object.entries(env).map(([key, value]) => `${key}=${value}`),
+            ],
+            HostConfig: {
+                Binds: [`${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`],
+            },
+        });
     }
     /**
      *
@@ -345,120 +305,68 @@ export class BlockInstanceRunner {
             throw new Error(`Provider did not have local image: ${providerRef}`);
         }
         const dockerImage = spec?.local?.image;
-        try {
-            await containerManager.pull(dockerImage);
-        }
-        catch (e) {
-            console.warn('Failed to pull image. Continuing...', e);
-        }
         const containerName = getBlockInstanceContainerName(blockInstance.id);
         const logs = new LogData();
-        const containerInfo = await containerManager.getContainerByName(containerName);
-        let container = containerInfo?.native;
-        if (container) {
-            const containerData = container.data;
-            if (containerData.State === 'running') {
-                logs.addLog(`Found existing running container for block: ${containerName}`);
-            }
-            else {
-                if (containerData.State?.ExitCode > 0) {
-                    logs.addLog(`Container exited with code: ${containerData.State.ExitCode}. Deleting...`);
-                    try {
-                        await containerManager.remove(container);
-                    }
-                    catch (e) { }
-                    container = undefined;
-                }
-                else {
-                    logs.addLog(`Found existing container for block: ${containerName}. Starting now`);
-                    try {
-                        await container.start();
-                    }
-                    catch (e) {
-                        console.warn('Failed to start container. Deleting...', e);
-                        try {
-                            await containerManager.remove(container);
-                        }
-                        catch (e) { }
-                        container = undefined;
-                    }
-                }
-            }
-        }
         const bindHost = getBindHost();
-        if (!container) {
-            const ExposedPorts = {};
-            const addonEnv = {};
-            const PortBindings = {};
-            let HealthCheck = undefined;
-            let Mounts = [];
-            const promises = Object.entries(spec.local.ports).map(async ([portType, value]) => {
-                const dockerPort = `${value.port}/${value.type}`;
-                ExposedPorts[dockerPort] = {};
-                addonEnv[`KAPETA_LOCAL_SERVER_PORT_${portType.toUpperCase()}`] = value.port;
-                const publicPort = await serviceManager.ensureServicePort(this._systemId, blockInstance.id, portType);
-                PortBindings[dockerPort] = [
-                    {
-                        HostIp: bindHost,
-                        HostPort: `${publicPort}`,
-                    },
-                ];
-            });
-            await Promise.all(promises);
-            if (spec.local?.env) {
-                Object.entries(spec.local.env).forEach(([key, value]) => {
-                    addonEnv[key] = value;
-                });
-            }
-            if (spec.local?.mounts) {
-                const mounts = containerManager.createMounts(blockUri.id, spec.local.mounts);
-                Mounts = containerManager.toDockerMounts(mounts);
-            }
-            if (spec.local?.health) {
-                HealthCheck = containerManager.toDockerHealth(spec.local?.health);
-            }
-            // For windows we need to default to root
-            const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
-            logs.addLog(`Creating new container for block: ${containerName}`);
-            container = await containerManager.startContainer({
-                Image: dockerImage,
-                name: containerName,
-                ExposedPorts,
-                HealthCheck,
-                HostConfig: {
-                    Binds: [
-                        `${toLocalBindVolume(kapetaYmlPath)}:/kapeta.yml:ro`,
-                        `${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`,
-                    ],
-                    PortBindings,
-                    Mounts,
-                },
-                Labels: {
-                    instance: blockInstance.id,
+        const ExposedPorts = {};
+        const addonEnv = {};
+        const PortBindings = {};
+        let HealthCheck = undefined;
+        let Mounts = [];
+        const promises = Object.entries(spec.local.ports).map(async ([portType, value]) => {
+            const dockerPort = `${value.port}/${value.type}`;
+            ExposedPorts[dockerPort] = {};
+            addonEnv[`KAPETA_LOCAL_SERVER_PORT_${portType.toUpperCase()}`] = value.port;
+            const publicPort = await serviceManager.ensureServicePort(this._systemId, blockInstance.id, portType);
+            PortBindings[dockerPort] = [
+                {
+                    HostIp: bindHost,
+                    HostPort: `${publicPort}`,
                 },
-                Env: [
-                    `KAPETA_INSTANCE_NAME=${blockInstance.ref}`,
-                    `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
-                    ...DOCKER_ENV_VARS,
-                    ...Object.entries({
-                        ...env,
-                        ...addonEnv,
-                    }).map(([key, value]) => `${key}=${value}`),
-                ],
+            ];
+        });
+        await Promise.all(promises);
+        if (spec.local?.env) {
+            Object.entries(spec.local.env).forEach(([key, value]) => {
+                addonEnv[key] = value;
             });
-            try {
-                if (HealthCheck) {
-                    await containerManager.waitForHealthy(container);
-                }
-                else {
-                    await containerManager.waitForReady(container);
-                }
-            }
-            catch (e) {
-                logs.addLog(e.message, 'ERROR');
-            }
         }
-        const out = await this._handleContainer(container, logs, true);
+        if (spec.local?.mounts) {
+            const mounts = containerManager.createMounts(blockUri.id, spec.local.mounts);
+            Mounts = containerManager.toDockerMounts(mounts);
+        }
+        if (spec.local?.health) {
+            HealthCheck = containerManager.toDockerHealth(spec.local?.health);
+        }
+        // For windows we need to default to root
+        const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
+        logs.addLog(`Creating new container for block: ${containerName}`);
+        const out = await this.ensureContainer({
+            Image: dockerImage,
+            name: containerName,
+            ExposedPorts,
+            HealthCheck,
+            HostConfig: {
+                Binds: [
+                    `${toLocalBindVolume(kapetaYmlPath)}:/kapeta.yml:ro`,
+                    `${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`,
+                ],
+                PortBindings,
+                Mounts,
+            },
+            Labels: {
+                instance: blockInstance.id,
+            },
+            Env: [
+                `KAPETA_INSTANCE_NAME=${blockInstance.ref}`,
+                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
+                ...DOCKER_ENV_VARS,
+                ...Object.entries({
+                    ...env,
+                    ...addonEnv,
+                }).map(([key, value]) => `${key}=${value}`),
+            ],
+        });
         const portTypes = spec.local.ports ? Object.keys(spec.local.ports) : [];
         if (portTypes.length > 0) {
             out.portType = portTypes[0];

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.9.1",
+    "version": "0.10.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {