@kapeta/local-cluster-service 0.10.0 → 0.11.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [0.11.0](https://github.com/kapetacom/local-cluster-service/compare/v0.10.1...v0.11.0) (2023-07-31)
+
+
+### Features
+
+* Always get logs from docker ([#53](https://github.com/kapetacom/local-cluster-service/issues/53)) ([5cab8cb](https://github.com/kapetacom/local-cluster-service/commit/5cab8cbf18b38edf99d538e1819e135f0a5bd7e3))
+
+## [0.10.1](https://github.com/kapetacom/local-cluster-service/compare/v0.10.0...v0.10.1) (2023-07-27)
+
+
+### Bug Fixes
+
+* Include port bindings for non-local containers ([#51](https://github.com/kapetacom/local-cluster-service/issues/51)) ([64fd440](https://github.com/kapetacom/local-cluster-service/commit/64fd4409ea9e2dda8e2438d0ec85a8f5a2092b1e))
+
 # [0.10.0](https://github.com/kapetacom/local-cluster-service/compare/v0.9.1...v0.10.0) (2023-07-26)
 
 

package/dist/cjs/src/containerManager.d.ts

@@ -1,5 +1,6 @@
 import { Docker } from 'node-docker-api';
 import { Container } from 'node-docker-api/lib/container';
+import { InstanceInfo, LogEntry } from "./types";
 type StringMap = {
     [key: string]: string;
 };
@@ -52,8 +53,8 @@ declare class ContainerManager {
     initialize(): Promise<void>;
     checkAlive(): Promise<boolean>;
     isAlive(): boolean;
-    getMountPoint(kind: string, mountName: string): string;
-    createMounts(kind: string, mountOpts: StringMap): StringMap;
+    getMountPoint(systemId: string, ref: string, mountName: string): string;
+    createMounts(systemId: string, kind: string, mountOpts: StringMap | null | undefined): Promise<StringMap>;
     ping(): Promise<void>;
     docker(): Docker;
     getContainerByName(containerName: string): Promise<ContainerInfo | undefined>;
@@ -67,11 +68,10 @@ declare class ContainerManager {
     };
     private applyHash;
     ensureContainer(opts: any): Promise<Container>;
+    private createOrUpdateContainer;
     startContainer(opts: any): Promise<Container>;
     waitForReady(container: Container, attempt?: number): Promise<void>;
-    waitForHealthy(container: Container, attempt?: number): Promise<void>;
     _isReady(container: Container): Promise<any>;
-    _isHealthy(container: Container): Promise<boolean>;
     remove(container: Container, opts?: {
         force?: boolean;
     }): Promise<void>;
@@ -81,6 +81,7 @@ declare class ContainerManager {
      * @return {Promise<ContainerInfo>}
      */
     get(name: string): Promise<ContainerInfo | null>;
+    getLogs(instance: InstanceInfo): Promise<LogEntry[]>;
 }
 export declare class ContainerInfo {
     private readonly _container;
@@ -105,6 +106,7 @@ export declare class ContainerInfo {
     inspect(): Promise<any>;
     status(): Promise<DockerState>;
     getPorts(): Promise<PortMap | false>;
+    getLogs(): Promise<LogEntry[]>;
 }
 export declare function getExtraHosts(dockerVersion: string): string[] | undefined;
 /**

package/dist/cjs/src/containerManager.js

@@ -14,6 +14,7 @@ const nodejs_utils_1 = require("@kapeta/nodejs-utils");
 const local_cluster_config_1 = __importDefault(require("@kapeta/local-cluster-config"));
 const node_uuid_1 = __importDefault(require("node-uuid"));
 const md5_1 = __importDefault(require("md5"));
+const utils_1 = require("./utils/utils");
 exports.CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
@@ -21,8 +22,8 @@ const HEALTH_CHECK_MAX = 20;
 const IMAGE_PULL_CACHE_TTL = 30 * 60 * 1000;
 const IMAGE_PULL_CACHE = {};
 exports.HEALTH_CHECK_TIMEOUT = HEALTH_CHECK_INTERVAL * HEALTH_CHECK_MAX * 2;
-const promisifyStream = (stream) => new Promise((resolve, reject) => {
-    stream.on('data', (d) => console.log(d.toString()));
+const promisifyStream = (stream, handler) => new Promise((resolve, reject) => {
+    stream.on('data', handler);
     stream.on('end', resolve);
     stream.on('error', reject);
 });
@@ -101,17 +102,21 @@ class ContainerManager {
     isAlive() {
         return this._alive;
     }
-    getMountPoint(kind, mountName) {
-        const kindUri = (0, nodejs_utils_1.parseKapetaUri)(kind);
-        return path_1.default.join(this._mountDir, kindUri.handle, kindUri.name, mountName);
+    getMountPoint(systemId, ref, mountName) {
+        const kindUri = (0, nodejs_utils_1.parseKapetaUri)(ref);
+        const systemUri = (0, nodejs_utils_1.parseKapetaUri)(systemId);
+        return path_1.default.join(this._mountDir, systemUri.handle, systemUri.name, systemUri.version, kindUri.handle, kindUri.name, kindUri.version, mountName);
     }
-    createMounts(kind, mountOpts) {
+    async createMounts(systemId, kind, mountOpts) {
         const mounts = {};
-        lodash_1.default.forEach(mountOpts, (containerPath, mountName) => {
-            const hostPath = this.getMountPoint(kind, mountName);
-            fs_extra_1.default.mkdirpSync(hostPath);
-            mounts[containerPath] = hostPath;
-        });
+        if (mountOpts) {
+            const mountOptList = Object.entries(mountOpts);
+            for (const [mountName, containerPath] of mountOptList) {
+                const hostPath = this.getMountPoint(systemId, kind, mountName);
+                await fs_extra_1.default.mkdirp(hostPath);
+                mounts[containerPath] = hostPath;
+            }
+        }
         return mounts;
     }
     async ping() {
@@ -162,12 +167,14 @@ class ContainerManager {
             return false;
         }
         console.log('Pulling image: %s', image);
-        await this.docker()
+        const stream = await this.docker()
             .image.create({}, {
             fromImage: imageName,
             tag: tag,
-        })
-            .then((stream) => promisifyStream(stream));
+        });
+        await promisifyStream(stream, (chunk) => {
+            console.log('Data from docker: "%s"', chunk.toString());
+        });
         IMAGE_PULL_CACHE[image] = Date.now();
         console.log('Image pulled: %s', image);
         return true;
@@ -204,6 +211,11 @@ class ContainerManager {
         dockerOpts.Labels.HASH = hash;
     }
     async ensureContainer(opts) {
+        const container = await this.createOrUpdateContainer(opts);
+        await this.waitForReady(container);
+        return container;
+    }
+    async createOrUpdateContainer(opts) {
         let imagePulled = false;
         try {
             imagePulled = await this.pull(opts.Image);
@@ -283,28 +295,6 @@ class ContainerManager {
             }, HEALTH_CHECK_INTERVAL);
         });
     }
-    async waitForHealthy(container, attempt) {
-        if (!attempt) {
-            attempt = 0;
-        }
-        if (attempt >= HEALTH_CHECK_MAX) {
-            throw new Error('Container did not become healthy within the timeout');
-        }
-        if (await this._isHealthy(container)) {
-            return;
-        }
-        return new Promise((resolve, reject) => {
-            setTimeout(async () => {
-                try {
-                    await this.waitForHealthy(container, (attempt ?? 0) + 1);
-                    resolve();
-                }
-                catch (err) {
-                    reject(err);
-                }
-            }, HEALTH_CHECK_INTERVAL);
-        });
-    }
     async _isReady(container) {
         let info;
         try {
@@ -318,16 +308,12 @@ class ContainerManager {
         if (state?.Status === 'exited' || state?.Status === 'removing' || state?.Status === 'dead') {
             throw new Error('Container exited unexpectedly');
         }
-        return infoData?.State?.Running ?? false;
-    }
-    async _isHealthy(container) {
-        try {
-            const info = await container.status();
-            const infoData = info?.data;
-            return infoData?.State?.Health?.Status === 'healthy';
+        if (infoData?.State?.Health) {
+            // If container has health info - wait for it to become healthy
+            return infoData.State.Health.Status === 'healthy';
         }
-        catch (err) {
-            return false;
+        else {
+            return infoData?.State?.Running ?? false;
         }
     }
     async remove(container, opts) {
@@ -357,6 +343,19 @@ class ContainerManager {
         }
         return new ContainerInfo(dockerContainer);
     }
+    async getLogs(instance) {
+        const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
+        const containerInfo = await this.getContainerByName(containerName);
+        if (!containerInfo) {
+            return [{
+                    source: "stdout",
+                    level: "ERROR",
+                    time: Date.now(),
+                    message: "Container not found"
+                }];
+        }
+        return containerInfo.getLogs();
+    }
 }
 class ContainerInfo {
     _container;
@@ -440,6 +439,62 @@ class ContainerInfo {
         });
         return ports;
     }
+    async getLogs() {
+        const logStream = await this.native.logs({
+            stdout: true,
+            stderr: true,
+            follow: false,
+            tail: 100,
+            timestamps: true,
+        });
+        const out = [];
+        await promisifyStream(logStream, (data) => {
+            const buf = data;
+            let offset = 0;
+            while (offset < buf.length) {
+                try {
+                    // Read the docker log format - explained here:
+                    // https://docs.docker.com/engine/api/v1.41/#operation/ContainerAttach
+                    // or here : https://ahmet.im/blog/docker-logs-api-binary-format-explained/
+                    // First byte is stream type
+                    const streamTypeInt = buf.readInt8(offset);
+                    const streamType = streamTypeInt === 1 ? 'stdout' : 'stderr';
+                    // Bytes 4-8 is frame size
+                    const messageLength = buf.readInt32BE(offset + 4);
+                    // After that is the message - with the message length
+                    const dataWithoutStreamType = buf.subarray(offset + 8, offset + 8 + messageLength);
+                    const raw = dataWithoutStreamType.toString();
+                    // Split the message into date and message
+                    const firstSpaceIx = raw.indexOf(' ');
+                    const dateString = raw.substring(0, firstSpaceIx);
+                    const line = raw.substring(firstSpaceIx + 1);
+                    offset = offset + messageLength + 8;
+                    if (!dateString) {
+                        continue;
+                    }
+                    out.push({
+                        time: new Date(dateString).getTime(),
+                        message: line,
+                        level: 'INFO',
+                        source: streamType,
+                    });
+                }
+                catch (err) {
+                    console.error('Error parsing log entry', err);
+                    offset = buf.length;
+                }
+            }
+        });
+        if (out.length === 0) {
+            out.push({
+                time: Date.now(),
+                message: 'No logs found for container',
+                level: 'INFO',
+                source: 'stdout',
+            });
+        }
+        return out;
+    }
 }
 exports.ContainerInfo = ContainerInfo;
 function getExtraHosts(dockerVersion) {

package/dist/cjs/src/definitionsManager.d.ts

@@ -5,6 +5,7 @@ declare class DefinitionsManager {
     clearCache(): void;
     private doCached;
     getDefinitions(kindFilter?: string | string[]): DefinitionInfo[];
+    exists(ref: string): boolean;
     getProviderDefinitions(): DefinitionInfo[];
 }
 export declare const definitionsManager: DefinitionsManager;

package/dist/cjs/src/definitionsManager.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.definitionsManager = void 0;
 const local_cluster_config_1 = __importDefault(require("@kapeta/local-cluster-config"));
+const nodejs_utils_1 = require("@kapeta/nodejs-utils");
 const CACHE_TTL = 60 * 1000; // 1 min
 class DefinitionsManager {
     cache = {};
@@ -37,6 +38,12 @@ class DefinitionsManager {
         const key = this.getKey(kindFilter);
         return this.doCached(key, () => local_cluster_config_1.default.getDefinitions(kindFilter));
     }
+    exists(ref) {
+        const uri = (0, nodejs_utils_1.parseKapetaUri)(ref);
+        return !!this.getDefinitions().find((d) => {
+            return (0, nodejs_utils_1.parseKapetaUri)(`${d.definition.metadata.name}:${d.version}`).id === uri.id;
+        });
+    }
     getProviderDefinitions() {
         return this.doCached('providers', () => local_cluster_config_1.default.getProviderDefinitions());
     }

package/dist/cjs/src/instanceManager.d.ts

@@ -1,4 +1,4 @@
-import { InstanceInfo } from './types';
+import { InstanceInfo, LogEntry } from './types';
 export declare class InstanceManager {
     private _interval;
     private readonly _instances;
@@ -7,6 +7,7 @@ export declare class InstanceManager {
     getInstances(): InstanceInfo[];
     getInstancesForPlan(systemId: string): InstanceInfo[];
     getInstance(systemId: string, instanceId: string): InstanceInfo | undefined;
+    getLogs(systemId: string, instanceId: string): Promise<LogEntry[]>;
     saveInternalInstance(instance: InstanceInfo): Promise<InstanceInfo>;
     /**
      * Method is called when instance is started from the Kapeta SDKs (e.g. NodeJS SDK)

package/dist/cjs/src/instanceManager.js

@@ -56,6 +56,31 @@ class InstanceManager {
         systemId = (0, utils_1.normalizeKapetaUri)(systemId);
         return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
     }
+    async getLogs(systemId, instanceId) {
+        const instance = this.getInstance(systemId, instanceId);
+        if (!instance) {
+            throw new Error(`Instance ${systemId}/${instanceId} not found`);
+        }
+        switch (instance.type) {
+            case types_1.InstanceType.DOCKER:
+                return await containerManager_1.containerManager.getLogs(instance);
+            case types_1.InstanceType.UNKNOWN:
+                return [{
+                        level: 'INFO',
+                        message: 'Instance is starting...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    }];
+            case types_1.InstanceType.LOCAL:
+                return [{
+                        level: 'INFO',
+                        message: 'Instance started outside Kapeta - logs not available...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    }];
+        }
+        return [];
+    }
     async saveInternalInstance(instance) {
         instance.systemId = (0, utils_1.normalizeKapetaUri)(instance.systemId);
         if (instance.ref) {
@@ -106,7 +131,6 @@ class InstanceManager {
                 }
                 instance.desiredStatus = info.desiredStatus;
                 instance.owner = info.owner;
-                instance.internal = undefined;
                 instance.status = types_1.InstanceStatus.STARTING;
                 instance.startedAt = Date.now();
             }
@@ -205,7 +229,7 @@ class InstanceManager {
         this.save();
         try {
             if (instance.type === 'docker') {
-                const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.instanceId);
+                const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
                 const container = await containerManager_1.containerManager.getContainerByName(containerName);
                 if (container) {
                     try {
@@ -281,7 +305,7 @@ class InstanceManager {
             name: blockAsset.data.metadata.name,
             desiredStatus: types_1.DesiredInstanceStatus.RUN,
             owner: types_1.InstanceOwner.INTERNAL,
-            type: types_1.InstanceType.UNKNOWN,
+            type: existingInstance?.type ?? types_1.InstanceType.UNKNOWN,
             status: types_1.InstanceStatus.STARTING,
             startedAt: Date.now(),
         };
@@ -301,41 +325,6 @@ class InstanceManager {
         const startTime = Date.now();
         try {
             const processInfo = await runner.start(blockRef, instanceId, instanceConfig);
-            //emit stdout/stderr via sockets
-            processInfo.output.on('data', (data) => {
-                const payload = {
-                    source: 'stdout',
-                    level: 'INFO',
-                    message: data.toString(),
-                    time: Date.now(),
-                };
-                this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, payload);
-            });
-            processInfo.output.on('exit', (exitCode) => {
-                const timeRunning = Date.now() - startTime;
-                const instance = this.getInstance(systemId, instanceId);
-                if (instance?.status === types_1.InstanceStatus.READY) {
-                    //It's already been running
-                    return;
-                }
-                if (exitCode === 143 || exitCode === 137) {
-                    //Process got SIGTERM (143) or SIGKILL (137)
-                    //TODO: Windows?
-                    return;
-                }
-                if (exitCode !== 0 || timeRunning < MIN_TIME_RUNNING) {
-                    const instance = this.getInstance(systemId, instanceId);
-                    if (instance) {
-                        instance.status = types_1.InstanceStatus.FAILED;
-                        this.save();
-                    }
-                    this.emitSystemEvent(systemId, EVENT_INSTANCE_EXITED, {
-                        error: 'Failed to start instance',
-                        status: EVENT_INSTANCE_EXITED,
-                        instanceId: blockInstance.id,
-                    });
-                }
-            });
             instance.status = types_1.InstanceStatus.READY;
             return this.saveInternalInstance({
                 ...instance,
@@ -344,10 +333,6 @@ class InstanceManager {
                 health: null,
                 portType: processInfo.portType,
                 status: types_1.InstanceStatus.READY,
-                internal: {
-                    logs: processInfo.logs,
-                    output: processInfo.output,
-                },
             });
         }
         catch (e) {
@@ -393,9 +378,7 @@ class InstanceManager {
     save() {
         try {
             storageService_1.storageService.put('instances', this._instances.map((instance) => {
-                const copy = { ...instance };
-                delete copy.internal;
-                return copy;
+                return { ...instance };
             }));
         }
         catch (e) {
@@ -497,7 +480,7 @@ class InstanceManager {
     }
     async getExternalStatus(instance) {
         if (instance.type === types_1.InstanceType.DOCKER) {
-            const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.instanceId);
+            const containerName = (0, utils_1.getBlockInstanceContainerName)(instance.systemId, instance.instanceId);
             const container = await containerManager_1.containerManager.getContainerByName(containerName);
             if (!container) {
                 // If the container doesn't exist, we consider the instance stopped

package/dist/cjs/src/instances/routes.js

@@ -67,15 +67,21 @@ router.post('/:systemId/:instanceId/stop', async (req, res) => {
 /**
  * Get logs for instance in a plan
  */
-router.get('/:systemId/:instanceId/logs', (req, res) => {
+router.get('/:systemId/:instanceId/logs', async (req, res) => {
     const instanceInfo = instanceManager_1.instanceManager.getInstance(req.params.systemId, req.params.instanceId);
     if (!instanceInfo) {
         res.status(404).send({ ok: false });
         return;
     }
-    res.status(202).send({
-        logs: instanceInfo.internal?.logs() ?? [],
-    });
+    try {
+        const logs = await instanceManager_1.instanceManager.getLogs(req.params.systemId, req.params.instanceId);
+        res.status(200).send({
+            logs,
+        });
+    }
+    catch (e) {
+        res.status(500).send({ ok: false, error: e.message });
+    }
 });
 /**
  * Get public address for instance in a plan if available

package/dist/cjs/src/operatorManager.js

@@ -117,13 +117,11 @@ class OperatorManager {
         const operatorData = operator.getData();
         const portTypes = Object.keys(operatorData.ports);
         portTypes.sort();
-        const containerBaseName = 'kapeta-resource';
-        const nameParts = [resourceType.toLowerCase()];
         const ports = {};
         for (let i = 0; i < portTypes.length; i++) {
             const portType = portTypes[i];
             let containerPortInfo = operatorData.ports[portType];
-            const hostPort = await serviceManager_1.serviceManager.ensureServicePort(resourceType, portType);
+            const hostPort = await serviceManager_1.serviceManager.ensureServicePort(systemId, resourceType, portType);
             if (typeof containerPortInfo === 'number' || typeof containerPortInfo === 'string') {
                 containerPortInfo = { port: containerPortInfo, type: 'tcp' };
             }
@@ -131,14 +129,18 @@ class OperatorManager {
                 containerPortInfo.type = 'tcp';
             }
             const portId = containerPortInfo.port + '/' + containerPortInfo.type;
-            nameParts.push(portType + '-' + portId + '-' + hostPort);
             ports[portId] = {
                 type: portType,
                 hostPort,
             };
         }
-        const mounts = containerManager_1.containerManager.createMounts(resourceType, operatorData.mounts);
-        const containerName = containerBaseName + '-' + (0, md5_1.default)(nameParts.join('_'));
+        const mounts = await containerManager_1.containerManager.createMounts(systemId, resourceType, operatorData.mounts);
+        const nameParts = [
+            systemId,
+            resourceType.toLowerCase(),
+            version
+        ];
+        const containerName = `kapeta-resource-${(0, md5_1.default)(nameParts.join('_'))}`;
         const PortBindings = {};
         const Env = [];
         const Labels = {

package/dist/cjs/src/repositoryManager.js

@@ -113,9 +113,11 @@ class RepositoryManager {
             this._installQueue.push(async () => {
                 try {
                     const normalizedRefs = refs.map((ref) => (0, nodejs_utils_1.parseKapetaUri)(ref).id);
-                    const filteredRefs = normalizedRefs.filter((ref) => !INSTALL_ATTEMPTED[ref]);
-                    console.log(filteredRefs);
+                    const filteredRefs = normalizedRefs
+                        .filter((ref) => !INSTALL_ATTEMPTED[ref])
+                        .filter((ref) => !definitionsManager_1.definitionsManager.exists(ref));
                     if (filteredRefs.length > 0) {
+                        console.log(`Auto-installing dependencies: ${filteredRefs.join(', ')}`);
                         filteredRefs.forEach((ref) => (INSTALL_ATTEMPTED[ref] = true));
                         //Auto-install missing asset
                         try {
@@ -206,14 +208,12 @@ class RepositoryManager {
         }
         this._cache[ref] = true;
         if (!installedAsset) {
-            console.log(`Auto-installing missing asset: ${ref}`);
             await this._install([ref]);
         }
         else {
             //Ensure dependencies are installed
             const refs = assetVersion.dependencies.map((dep) => dep.name);
             if (refs.length > 0) {
-                console.log(`Auto-installing dependencies: ${refs.join(', ')}`);
                 await this._install(refs);
             }
         }

package/dist/cjs/src/types.d.ts

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-import EventEmitter from 'events';
 import express from 'express';
 import { Resource } from '@kapeta/schemas';
 import { StringBodyRequest } from './middleware/stringBody';
@@ -50,10 +48,7 @@ export declare enum DesiredInstanceStatus {
 export type ProcessInfo = {
     type: InstanceType;
     pid?: number | string | null;
-    output: EventEmitter;
     portType?: string;
-    logs: () => LogEntry[];
-    stop: () => Promise<void> | void;
 };
 export type InstanceInfo = {
     systemId: string;
@@ -69,10 +64,6 @@ export type InstanceInfo = {
     health?: string | null;
     pid?: number | string | null;
     portType?: string;
-    internal?: {
-        output: EventEmitter;
-        logs: () => LogEntry[];
-    };
 };
 interface ResourceRef {
     blockId: string;

package/dist/cjs/src/utils/BlockInstanceRunner.d.ts

@@ -14,8 +14,6 @@ export declare class BlockInstanceRunner {
      * Starts local process
      */
     private _startLocalProcess;
-    private ensureContainer;
-    private _handleContainer;
     private _startDockerProcess;
     /**
      *
@@ -27,4 +25,7 @@ export declare class BlockInstanceRunner {
      * @private
      */
     _startOperatorProcess(blockInstance: BlockProcessParams, blockUri: KapetaURI, providerDefinition: DefinitionInfo, env: StringMap): Promise<ProcessInfo>;
+    private getDockerPortBindings;
+    private ensureContainer;
+    private _handleContainer;
 }