@kapeta/local-cluster-service 0.0.75 → 0.1.0

package/.github/workflows/main.yml

@@ -0,0 +1,26 @@
+name: Main build
+on:
+    pull_request:
+    push:
+        branches: ['master']
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  token: ${{ secrets.BOT_TOKEN }}
+            - uses: actions/setup-node@v3
+            - run: npm ci
+            - run: npm run build --if-present
+              env:
+                  CI: true
+            - run: npm test -- --passWithNoTests
+            # Probably move this to its own job when it makes sense
+            - name: Semantic Release
+              uses: cycjimmy/semantic-release-action@v3
+              if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+              env:
+                  GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
+                  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

package/CHANGELOG.md

@@ -0,0 +1,14 @@
+# [0.1.0](https://github.com/kapetacom/local-cluster-service/compare/v0.0.76...v0.1.0) (2023-05-06)
+
+
+### Features
+
+* Allow running docker block operators ([0a3992c](https://github.com/kapetacom/local-cluster-service/commit/0a3992c359a119a623ed7d0423e6f7ad814aa8d3))
+
+## [0.0.76](https://github.com/kapetacom/local-cluster-service/compare/v0.0.75...v0.0.76) (2023-05-06)
+
+
+### Bug Fixes
+
+* include docker status in cluster startup response ([0d40253](https://github.com/kapetacom/local-cluster-service/commit/0d402535b7b936fa4f4f480147d8f3103249a6f8))
+* make docker config try more variations before giving up ([f55629e](https://github.com/kapetacom/local-cluster-service/commit/f55629ed3f7167ec7b6810ec16ae6d8068722863))

package/definitions.d.ts

@@ -23,8 +23,8 @@ declare function ProxyRequestHandler(req:Request, res:Response, info:ProxyReques
 
 interface Connection {
     mapping: any
-    from: ResourceRef
-    to: ResourceRef
+    provider: ResourceRef
+    consumer: ResourceRef
 }
 
 interface ResourceInfo {
@@ -36,7 +36,7 @@ interface ResourceInfo {
 interface ProxyRequestInfo {
     address: string
     connection:Connection
-    fromResource:ResourceInfo
-    toResource:ResourceInfo
+    providerResource:ResourceInfo
+    consumerResource:ResourceInfo
     consumerPath:string
 }

package/index.js

@@ -20,7 +20,7 @@ function createServer() {
     app.use('/assets', require('./src/assets/routes'));
     app.use('/providers', require('./src/providers/routes'));
     app.use('/', (err, req, res, next) => {
-        console.error('Request failed: %s %s', req.method, req.originalUrl, err.stack);
+        console.error('Request failed: %s %s', req.method, req.originalUrl, err);
         res.status(500).send({
             ok: false,
             error: err.error ?? err.message
@@ -55,7 +55,7 @@ module.exports = {
 
     /**
      * Starts the local cluster service.
-     * @return {Promise<Integer>} resolves when listening is done with port number. Rejects if listening failed.
+     * @return {Promise<{ host: string, port: nubmer, dockerStatus: boolean}>} resolves when listening is done with port number. Rejects if listening failed.
      */
     start: async function() {
         if (currentServer) {
@@ -63,9 +63,9 @@ module.exports = {
         }
 
         try {
-            await containerManager.ping()
+            await containerManager.initialize()
         } catch (e) {
-            throw new Error('Could not ping docker runtime: ' + e.toString() + '. Make sure docker is running and working.');
+            console.error('Could not ping docker runtime: ' + e.toString() + '. Make sure docker is running and working.');
         }
 
         const clusterPort = storageService.get('cluster','port');
@@ -102,7 +102,7 @@ module.exports = {
                 reject(err);
             });
 
-            currentServer.listen(port, host, () => resolve({host,port}));
+            currentServer.listen(port, host, () => resolve({host,port, dockerStatus: containerManager.isAlive()}));
             currentServer.host = host;
             currentServer.port = port;
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kapeta/local-cluster-service",
-  "version": "0.0.75",
+  "version": "0.1.0",
   "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
   "main": "index.js",
   "repository": {
@@ -16,15 +16,15 @@
     "url": "https://github.com/kapetacom/local-cluster-service/issues"
   },
   "scripts": {
-    "start": "nodemon start.js"
+    "start": "nodemon start.js",
+    "test": "echo its fine"
   },
   "homepage": "https://github.com/kapetacom/local-cluster-service#readme",
   "dependencies": {
     "@kapeta/codegen": "<2",
     "@kapeta/local-cluster-config": "<2",
-    "@kapeta/local-cluster-executor": "<2",
     "@kapeta/nodejs-api-client": "<2",
-    "@kapeta/nodejs-registry-utils": "^0.0.2",
+    "@kapeta/nodejs-registry-utils": "<2",
     "@kapeta/nodejs-utils": "<2",
     "@kapeta/sdk-config": "<2",
     "express": "4.17.1",
@@ -43,5 +43,30 @@
   },
   "devDependencies": {
     "nodemon": "^2.0.2"
+  },
+  "release": {
+    "plugins": [
+      "@semantic-release/commit-analyzer",
+      "@semantic-release/release-notes-generator",
+      [
+        "@semantic-release/changelog",
+        {
+          "changelogFile": "CHANGELOG.md"
+        }
+      ],
+      "@semantic-release/npm",
+      "@semantic-release/github",
+      [
+        "@semantic-release/git",
+        {
+          "assets": [
+            "CHANGELOG.md",
+            "package.json",
+            "package-lock.json"
+          ],
+          "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+        }
+      ]
+    ]
   }
 }

package/src/assetManager.js

@@ -54,7 +54,10 @@ class AssetManager {
      */
     getAssets(assetKinds) {
         if (!assetKinds) {
-            const blockTypeProviders = ClusterConfiguration.getDefinitions('core/block-type');
+            const blockTypeProviders = ClusterConfiguration.getDefinitions([
+                'core/block-type',
+                'core/block-type-operator'
+            ]);
             assetKinds = blockTypeProviders.map(p => {
                 return `${p.definition.metadata.name}:${p.version}`
             });
@@ -82,7 +85,6 @@ class AssetManager {
 
     async getAsset(ref) {
         const uri = parseKapetaUri(ref);
-
         await repositoryManager.ensureAsset(uri.handle, uri.name, uri.version);
 
         let asset = ClusterConfiguration.getDefinitions()

package/src/containerManager.js

@@ -1,40 +1,109 @@
-const {Docker} = require('node-docker-api');
-const _  = require('lodash');
+const {Docker} = require("node-docker-api");
+const path = require("path");
+const _ = require('lodash');
+const FS = require("node:fs");
 
-const LABEL_PORT_PREFIX = 'kapeta_port-';
+const LABEL_PORT_PREFIX = "kapeta_port-";
 
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 1000;
 const HEALTH_CHECK_MAX = 20;
 
-const promisifyStream = (stream) => new Promise((resolve, reject) => {
-    stream.on('data', (d) => console.log(d.toString()))
-    stream.on('end', resolve)
-    stream.on('error', reject)
-});
+const promisifyStream = (stream) =>
+    new Promise((resolve, reject) => {
+        stream.on("data", (d) => console.log(d.toString()));
+        stream.on("end", resolve);
+        stream.on("error", reject);
+    });
 
 class ContainerManager {
     constructor() {
-        this._docker = new Docker();
+        this._docker = null;
         this._alive = false;
     }
 
+    isAlive() {
+        return this._alive;
+    }
+
+    async initialize() {
+        // try
+        const connectOptions = [
+            // use defaults: DOCKER_HOST etc from env, if available
+            undefined,
+            // default linux
+            {socketPath: "/var/run/docker.sock"},
+            // default macOS
+            {socketPath: path.join(os.homedir(), ".docker/run/docker.sock")},
+            // Default http
+            {protocol: "http", host: "localhost", port: 2375},
+            {protocol: "https", host: "localhost", port: 2376},
+        ];
+        for (const opts of connectOptions) {
+            try {
+                const client = new Docker(opts);
+                await client.ping();
+                this._docker = client;
+                return;
+            } catch (err) {
+                // silently ignore bad configs
+            }
+        }
+        throw new Error("Unable to connect to docker");
+    }
+
     async ping() {
         await this._docker.ping();
         this._alive = true;
     }
 
+    async ping() {
+
+        try {
+            const pingResult = await this._docker.ping();
+            if (pingResult !== 'OK') {
+                throw new Error(`Ping failed: ${pingResult}`);
+            }
+        } catch (e) {
+            throw new Error(`Docker not running. Please start the docker daemon before running this command. Error: ${e.message}`);
+        }
+
+        this._alive = true;
+    }
+
+    async ensureAlive() {
+        if (!this._alive) {
+            await this.ping();
+        }
+    }
+
+    async docker() {
+        await this.ensureAlive();
+        return this._docker;
+    }
+
+    async getContainerByName(containerName) {
+        const containers = await this._docker.container.list({all: true});
+        return containers.find(container => {
+            return container.data.Names.indexOf(`/${containerName}`) > -1;
+        });
+    }
+
     async pull(image) {
         let [imageName, tag] = image.split(/:/);
         if (!tag) {
             tag = 'latest';
         }
 
-        await this._docker.image.create({}, {
-            fromImage: imageName,
-            tag: tag
-        }).then(stream => promisifyStream(stream));
-
+        await this._docker.image
+            .create(
+                {},
+                {
+                    fromImage: imageName,
+                    tag: tag,
+                }
+            )
+            .then((stream) => promisifyStream(stream));
     }
 
     /**
@@ -45,24 +114,24 @@ class ContainerManager {
      * @return {Promise<ContainerInfo>}
      */
     async run(image, name, opts) {
-
         const Mounts = [];
         const PortBindings = {};
         const Env = [];
         const Labels = {
-            'kapeta':'true'
+            kapeta: "true",
         };
 
-        console.log('Pulling image: %s', image);
+        console.log("Pulling image: %s", image);
 
         await this.pull(image);
 
-        console.log('Image pulled: %s', image);
+        console.log("Image pulled: %s", image);
 
         _.forEach(opts.ports, (portInfo, containerPort) => {
             PortBindings['' + containerPort] = [
                 {
-                    HostPort: '' + portInfo.hostPort
+                    HostPort: '' + portInfo.hostPort,
+                    HostIp: '127.0.0.1'
                 }
             ];
 
@@ -73,33 +142,35 @@ class ContainerManager {
             Mounts.push({
                 Target,
                 Source,
-                Type: 'bind',
+                Type: "bind",
                 ReadOnly: false,
-                Consistency: 'consistent'
+                Consistency: "consistent",
             });
         });
 
         _.forEach(opts.env, (value, name) => {
-            Env.push(name + '=' + value);
+            Env.push(name + "=" + value);
         });
 
         let HealthCheck = undefined;
 
         if (opts.health) {
             HealthCheck = {
-                Test: [
-                    'CMD-SHELL',
-                    opts.health.cmd
-                ],
-                Interval: opts.health.interval ? opts.health.interval * NANO_SECOND : 5000 * NANO_SECOND,
-                Timeout: opts.health.timeout ? opts.health.timeout * NANO_SECOND : 15000 * NANO_SECOND,
-                Retries: opts.health.retries || 10
+                Test: ["CMD-SHELL", opts.health.cmd],
+                Interval: opts.health.interval
+                    ? opts.health.interval * NANO_SECOND
+                    : 5000 * NANO_SECOND,
+                Timeout: opts.health.timeout
+                    ? opts.health.timeout * NANO_SECOND
+                    : 15000 * NANO_SECOND,
+                Retries: opts.health.retries || 10,
             };
 
-            console.log('Adding health check', HealthCheck);
+            console.log("Adding health check", HealthCheck);
         }
 
-        const dockerContainer = await this._docker.container.create({
+
+        const dockerContainer = await this.startContainer({
             name: name,
             Image: image,
             Labels,
@@ -111,8 +182,6 @@ class ContainerManager {
             }
         });
 
-        await dockerContainer.start();
-
         if (opts.health) {
             await this._waitForHealthy(dockerContainer);
         }
@@ -120,6 +189,14 @@ class ContainerManager {
         return new ContainerInfo(dockerContainer);
     }
 
+    async startContainer(opts) {
+        const dockerContainer = await this._docker.container.create(opts);
+
+        await dockerContainer.start();
+
+        return dockerContainer;
+    }
+
 
     async _waitForHealthy(container, attempt) {
         if (!attempt) {
@@ -127,11 +204,11 @@ class ContainerManager {
         }
 
         if (attempt >= HEALTH_CHECK_MAX) {
-            throw new Error('Operator did not become healthy within the timeout');
+            throw new Error("Operator did not become healthy within the timeout");
         }
 
         if (await this._isHealthy(container)) {
-            console.log('Container became healthy');
+            console.log("Container became healthy");
             return;
         }
 
@@ -139,15 +216,14 @@ class ContainerManager {
             setTimeout(async () => {
                 await this._waitForHealthy(container, attempt + 1);
                 resolve();
-            }, HEALTH_CHECK_INTERVAL)
+            }, HEALTH_CHECK_INTERVAL);
         });
-
     }
 
     async _isHealthy(container) {
         const info = await container.status();
 
-        return info?.data?.State?.Health?.Status === 'healthy';
+        return info?.data?.State?.Health?.Status === "healthy";
     }
 
     /**
@@ -161,9 +237,9 @@ class ContainerManager {
         try {
             dockerContainer = await this._docker.container.get(name);
             await dockerContainer.status();
-        } catch(err) {
+        } catch (err) {
             //Ignore
-            console.log('Container not available - creating it: %s', name);
+            console.log("Container not available - creating it: %s", name);
             dockerContainer = null;
         }
 
@@ -181,7 +257,6 @@ class ContainerInfo {
      * @param {Container} dockerContainer
      */
     constructor(dockerContainer) {
-
         /**
          *
          * @type {Container}
@@ -193,12 +268,10 @@ class ContainerInfo {
     async isRunning() {
         const inspectResult = await this.getStatus();
 
-        if (!inspectResult || 
-            !inspectResult.State) {
+        if (!inspectResult || !inspectResult.State) {
             return false;
         }
 
-
         return inspectResult.State.Running || inspectResult.State.Restarting;
     }
 
@@ -215,7 +288,7 @@ class ContainerInfo {
     }
 
     async remove(opts) {
-        await this._container.delete({ force: !!opts.force });
+        await this._container.delete({force: !!opts.force});
     }
 
     async getPort(type) {
@@ -237,9 +310,11 @@ class ContainerInfo {
     async getPorts() {
         const inspectResult = await this.getStatus();
 
-        if (!inspectResult ||
+        if (
+            !inspectResult ||
             !inspectResult.Config ||
-            !inspectResult.Config.Labels) {
+            !inspectResult.Config.Labels
+        ) {
             return false;
         }
 
@@ -254,26 +329,27 @@ class ContainerInfo {
             const hostPort = name.substr(LABEL_PORT_PREFIX.length);
 
             portTypes[hostPort] = portType;
-
         });
 
-        _.forEach(inspectResult.HostConfig.PortBindings, (portBindings, containerPortSpec) => {
-            let [containerPort, protocol] = containerPortSpec.split(/\//);
+        _.forEach(
+            inspectResult.HostConfig.PortBindings,
+            (portBindings, containerPortSpec) => {
+                let [containerPort, protocol] = containerPortSpec.split(/\//);
 
-            const hostPort = portBindings[0].HostPort;
+                const hostPort = portBindings[0].HostPort;
 
-            const portType = portTypes[hostPort];
+                const portType = portTypes[hostPort];
 
-            ports[portType] = {
-                containerPort,
-                protocol,
-                hostPort
-            };
-        });
+                ports[portType] = {
+                    containerPort,
+                    protocol,
+                    hostPort,
+                };
+            }
+        );
 
         return ports;
     }
 }
 
-
 module.exports = new ContainerManager();

package/src/instanceManager.js

@@ -1,7 +1,7 @@
 const _ = require('lodash');
 const request = require('request');
-
-const {BlockInstanceRunner} = require('@kapeta/local-cluster-executor');
+const EventEmitter = require("events");
+const BlockInstanceRunner = require('./utils/BlockInstanceRunner');
 
 const storageService = require('./storageService');
 const socketManager = require('./socketManager');
@@ -68,6 +68,10 @@ class InstanceManager {
 
             if (instance.status !== newStatus) {
                 instance.status = newStatus;
+                console.log(
+                    'Instance status changed: %s %s -> %s',
+                    instance.systemId, instance.instanceId, instance.status
+                )
                 this._emit(instance.systemId, EVENT_STATUS_CHANGED, instance);
                 changed = true;
             }
@@ -85,7 +89,11 @@ class InstanceManager {
 
         if (instance.type === 'docker') {
             const container = await containerManager.get(instance.pid);
-            return await container.isRunning()
+            if (!container) {
+                console.warn('Container not found: %s', instance.pid);
+                return false;
+            }
+            return await container.isRunning();
         }
 
         //Otherwise its just a normal process.
@@ -185,8 +193,12 @@ class InstanceManager {
         if (instance) {
             instance.status = STATUS_STARTING;
             instance.pid = info.pid;
-            instance.type = info.type;
-            instance.health = healthUrl;
+            if (info.type) {
+                instance.type = info.type;
+            }
+            if (healthUrl) {
+                instance.health = healthUrl;
+            }
             this._emit(systemId, EVENT_STATUS_CHANGED, instance);
         } else {
             instance = {
@@ -268,7 +280,9 @@ class InstanceManager {
         try {
             if (instance.type === 'docker') {
                 const container = await containerManager.get(instance.pid);
-                await container.stop();
+                if (container) {
+                    await container.stop();
+                }
                 return;
             }
             process.kill(instance.pid, 'SIGTERM');
@@ -383,7 +397,8 @@ class InstanceManager {
                     message: e.message,
                     time: Date.now()
                 }
-            ]
+            ];
+
             await this.registerInstance(planRef, instanceId, {
                 type: 'local',
                 pid: null,
@@ -401,7 +416,12 @@ class InstanceManager {
             return this._processes[planRef][instanceId] = {
                 pid: -1,
                 type,
-                logs: () => logs
+                logs: () => logs,
+                stop: () => Promise.resolve(),
+                ref: blockRef,
+                id: instanceId,
+                name: blockInstance.name,
+                output: new EventEmitter()
             };
         }
 
@@ -427,7 +447,11 @@ class InstanceManager {
         }
 
         if (this._processes[planRef][instanceId]) {
-            await this._processes[planRef][instanceId].stop();
+            try {
+                await this._processes[planRef][instanceId].stop();
+            } catch (e) {
+                console.error('Failed to stop process for instance: %s -> %s', planRef, instanceId, e);
+            }
             delete this._processes[planRef][instanceId];
         }
     }

package/src/instances/routes.js

@@ -100,6 +100,16 @@ router.put('/', async (req, res) => {
 
     let instance = JSON.parse(req.stringBody);
 
+    if (req.kapeta.environment === 'docker') {
+        //A bit hacky but we want to avoid overwriting the docker PID with a process PID
+        const oldInstance = instanceManager.getInstance(
+            req.kapeta.systemId,
+            req.kapeta.instanceId
+        );
+        if (oldInstance) {
+            instance.pid = oldInstance.pid;
+        }
+    }
     await instanceManager.registerInstance(
         req.kapeta.systemId,
         req.kapeta.instanceId,

package/src/networkManager.js

@@ -3,10 +3,10 @@ class NetworkManager {
 
     static toConnectionId(connection) {
         return [
-            connection.from.blockId,
-            connection.from.resourceName,
-            connection.to.blockId,
-            connection.to.resourceName
+            connection.provider.blockId,
+            connection.provider.resourceName,
+            connection.consumer.blockId,
+            connection.consumer.resourceName
         ].join('_');
     }
 
@@ -65,8 +65,8 @@ class NetworkManager {
         const traffic = new Traffic(connection, request, consumerMethodId, providerMethodId);
 
         this._ensureConnection(systemId, traffic.connectionId).push(traffic);
-        this._ensureSource(systemId, connection.from.blockId).push(traffic);
-        this._ensureTarget(systemId, connection.to.blockId).push(traffic);
+        this._ensureSource(systemId, connection.provider.blockId).push(traffic);
+        this._ensureTarget(systemId, connection.consumer.blockId).push(traffic);
 
         return traffic;
     }