@kapeta/local-cluster-service 0.11.1 → 0.12.1

package/src/containerManager.ts

@@ -9,10 +9,14 @@ import ClusterConfiguration from '@kapeta/local-cluster-config';
 import { Container } from 'node-docker-api/lib/container';
 import uuid from 'node-uuid';
 import md5 from 'md5';
-import {getBlockInstanceContainerName} from "./utils/utils";
-import {InstanceInfo, LogEntry, LogSource} from "./types";
-import EventEmitter from "events";
-import {LogData} from "./utils/LogData";
+import { getBlockInstanceContainerName } from './utils/utils';
+import { InstanceInfo, LogEntry, LogSource } from './types';
+import { socketManager } from './socketManager';
+import { handlers as ArtifactHandlers } from '@kapeta/nodejs-registry-utils';
+import { progressListener } from './progressListener';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+
+const EVENT_IMAGE_PULL = 'docker-image-pull';
 
 type StringMap = { [key: string]: string };
 
@@ -62,12 +66,11 @@ export const CONTAINER_LABEL_PORT_PREFIX = 'kapeta_port-';
 const NANO_SECOND = 1000000;
 const HEALTH_CHECK_INTERVAL = 3000;
 const HEALTH_CHECK_MAX = 20;
-const IMAGE_PULL_CACHE_TTL = 30 * 60 * 1000;
-const IMAGE_PULL_CACHE: { [key: string]: number } = {};
+
 
 export const HEALTH_CHECK_TIMEOUT = HEALTH_CHECK_INTERVAL * HEALTH_CHECK_MAX * 2;
 
-const promisifyStream = (stream: ReadStream, handler:(d:string|Buffer) => void) =>
+const promisifyStream = (stream: ReadStream, handler: (d: string | Buffer) => void) =>
     new Promise((resolve, reject) => {
         stream.on('data', handler);
         stream.on('end', resolve);
@@ -79,6 +82,7 @@ class ContainerManager {
     private _alive: boolean;
     private _mountDir: string;
     private _version: string;
+    private _lastDockerAccessCheck: number = 0;
 
     constructor() {
         this._docker = null;
@@ -155,24 +159,27 @@ class ContainerManager {
         return this._alive;
     }
 
-    getMountPoint(systemId:string, ref: string, mountName: string) {
+    getMountPoint(systemId: string, ref: string, mountName: string) {
         const kindUri = parseKapetaUri(ref);
-        const systemUri = parseKapetaUri(systemId)
-        return Path.join(this._mountDir,
+        const systemUri = parseKapetaUri(systemId);
+        return Path.join(
+            this._mountDir,
             systemUri.handle,
             systemUri.name,
             systemUri.version,
             kindUri.handle,
             kindUri.name,
-            kindUri.version, mountName);
+            kindUri.version,
+            mountName
+        );
     }
 
-    async createMounts(systemId:string, kind: string, mountOpts: StringMap|null|undefined): Promise<StringMap> {
+    async createMounts(systemId: string, kind: string, mountOpts: StringMap | null | undefined): Promise<StringMap> {
         const mounts: StringMap = {};
 
         if (mountOpts) {
             const mountOptList = Object.entries(mountOpts);
-            for(const [mountName, containerPath] of mountOptList) {
+            for (const [mountName, containerPath] of mountOptList) {
                 const hostPath = this.getMountPoint(systemId, kind, mountName);
                 await FSExtra.mkdirp(hostPath);
                 mounts[containerPath] = hostPath;
@@ -215,19 +222,12 @@ class ContainerManager {
         return undefined;
     }
 
-    async pull(image: string, cacheForMS: number = IMAGE_PULL_CACHE_TTL) {
+    async pull(image: string) {
         let [imageName, tag] = image.split(/:/);
         if (!tag) {
             tag = 'latest';
         }
 
-        if (IMAGE_PULL_CACHE[image]) {
-            const timeSince = Date.now() - IMAGE_PULL_CACHE[image];
-            if (timeSince < cacheForMS) {
-                return false;
-            }
-        }
-
         const imageTagList = (await this.docker().image.list())
             .map((image) => image.data as any)
             .filter((imageData) => !!imageData.RepoTags)
@@ -238,23 +238,151 @@ class ContainerManager {
             return false;
         }
 
-        console.log('Pulling image: %s', image);
-        const stream = await this.docker()
-            .image.create(
-                {},
-                {
-                    fromImage: imageName,
-                    tag: tag,
+        const timeStarted = Date.now();
+        socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: -1 });
+
+        const api = new KapetaAPI();
+        const accessToken = await api.getAccessToken();
+
+        const auth = image.startsWith('docker.kapeta.com/')
+            ? {
+                  username: 'kapeta',
+                  password: accessToken,
+                  serveraddress: 'docker.kapeta.com',
+              }
+            : {};
+
+        const stream = (await this.docker().image.create(auth, {
+            fromImage: imageName,
+            tag: tag,
+        })) as ReadStream;
+
+        const chunks: {
+            [p: string]: {
+                downloading: {
+                    total: number;
+                    current: number;
+                };
+                extracting: {
+                    total: number;
+                    current: number;
+                };
+                done: boolean;
+            };
+        } = {};
+
+        let lastEmitted = Date.now();
+        await promisifyStream(stream, (rawData) => {
+            const lines = rawData.toString().trim().split('\n');
+            lines.forEach((line) => {
+                const data = JSON.parse(line);
+                if (
+                    ![
+                        'Waiting',
+                        'Downloading',
+                        'Extracting',
+                        'Download complete',
+                        'Pull complete',
+                        'Already exists',
+                    ].includes(data.status)
+                ) {
+                    return;
                 }
-            ) as ReadStream;
 
-        await promisifyStream(stream, (chunk) => {
-            console.log('Data from docker: "%s"', chunk.toString());
-        });
+                if (!chunks[data.id]) {
+                    chunks[data.id] = {
+                        downloading: {
+                            total: 0,
+                            current: 0,
+                        },
+                        extracting: {
+                            total: 0,
+                            current: 0,
+                        },
+                        done: false,
+                    };
+                }
+
+                const chunk = chunks[data.id];
+
+                switch (data.status) {
+                    case 'Downloading':
+                        chunk.downloading = data.progressDetail;
+                        break;
+                    case 'Extracting':
+                        chunk.extracting = data.progressDetail;
+                        break;
+                    case 'Download complete':
+                        chunk.downloading.current = chunks[data.id].downloading.total;
+                        break;
+                    case 'Pull complete':
+                        chunk.extracting.current = chunks[data.id].extracting.total;
+                        chunk.done = true;
+                        break;
+                    case 'Already exists':
+                        // Force layer to be done
+                        chunk.downloading.current = 1;
+                        chunk.downloading.total = 1;
+                        chunk.extracting.current = 1;
+                        chunk.extracting.total = 1;
+                        chunk.done = true;
+                        break;
+                }
+            });
+
+            if (Date.now() - lastEmitted < 1000) {
+                return;
+            }
 
-        IMAGE_PULL_CACHE[image] = Date.now();
+            const chunkList = Object.values(chunks);
+            let totals = {
+                downloading: {
+                    total: 0,
+                    current: 0,
+                },
+                extracting: {
+                    total: 0,
+                    current: 0,
+                },
+                total: chunkList.length,
+                done: 0,
+            };
 
-        console.log('Image pulled: %s', image);
+            chunkList.forEach((chunk) => {
+                if (chunk.downloading.current > 0) {
+                    totals.downloading.current += chunk.downloading.current;
+                }
+
+                if (chunk.downloading.total > 0) {
+                    totals.downloading.total += chunk.downloading.total;
+                }
+
+                if (chunk.extracting.current > 0) {
+                    totals.extracting.current += chunk.extracting.current;
+                }
+
+                if (chunk.extracting.total > 0) {
+                    totals.extracting.total += chunk.extracting.total;
+                }
+
+                if (chunk.done) {
+                    totals.done++;
+                }
+            });
+
+            const percent = totals.total > 0 ? (totals.done / totals.total) * 100 : 0;
+            //We emit at most every second to not spam the client
+            socketManager.emitGlobal(EVENT_IMAGE_PULL, {
+                image,
+                percent,
+                status: totals,
+                timeTaken: Date.now() - timeStarted,
+            });
+            lastEmitted = Date.now();
+            //console.log('Pulling image %s: %s % [done: %s, total: %s]', image, Math.round(percent), totals.done, totals.total);
+        });
+
+        socketManager.emitGlobal(EVENT_IMAGE_PULL, { image, percent: 100, timeTaken: Date.now() - timeStarted });
 
         return true;
     }
@@ -297,20 +425,11 @@ class ContainerManager {
     }
 
     public async ensureContainer(opts: any) {
-        const container = await this.createOrUpdateContainer(opts);
-
-        await this.waitForReady(container);
-
-        return container;
+        return await this.createOrUpdateContainer(opts);
     }
 
     private async createOrUpdateContainer(opts: any) {
-        let imagePulled = false;
-        try {
-            imagePulled = await this.pull(opts.Image);
-        } catch (e) {
-            console.warn('Failed to pull image. Continuing...', e);
-        }
+        let imagePulled = await this.pull(opts.Image);
 
         this.applyHash(opts);
         if (!opts.name) {
@@ -448,24 +567,27 @@ class ContainerManager {
         return new ContainerInfo(dockerContainer);
     }
 
-    async getLogs(instance: InstanceInfo):Promise<LogEntry[]> {
+    async getLogs(instance: InstanceInfo): Promise<LogEntry[]> {
         const containerName = getBlockInstanceContainerName(instance.systemId, instance.instanceId);
         const containerInfo = await this.getContainerByName(containerName);
         if (!containerInfo) {
-            return [{
-                source: "stdout",
-                level: "ERROR",
-                time: Date.now(),
-                message: "Container not found"
-            }];
+            return [
+                {
+                    source: 'stdout',
+                    level: 'ERROR',
+                    time: Date.now(),
+                    message: 'Container not found',
+                },
+            ];
         }
 
-        return containerInfo.getLogs()
+        return containerInfo.getLogs();
     }
 }
 
 export class ContainerInfo {
     private readonly _container: Container;
+
     /**
      *
      * @param {Container} dockerContainer
@@ -572,21 +694,20 @@ export class ContainerInfo {
         return ports;
     }
 
-    async getLogs():Promise<LogEntry[]> {
-
-        const logStream = await this.native.logs({
+    async getLogs(): Promise<LogEntry[]> {
+        const logStream = (await this.native.logs({
             stdout: true,
             stderr: true,
             follow: false,
             tail: 100,
             timestamps: true,
-        })  as ReadStream;
+        })) as ReadStream;
 
         const out = [] as LogEntry[];
         await promisifyStream(logStream, (data) => {
             const buf = data as Buffer;
             let offset = 0;
-            while(offset < buf.length) {
+            while (offset < buf.length) {
                 try {
                     // Read the docker log format - explained here:
                     // https://docs.docker.com/engine/api/v1.41/#operation/ContainerAttach
@@ -594,7 +715,7 @@ export class ContainerInfo {
 
                     // First byte is stream type
                     const streamTypeInt = buf.readInt8(offset);
-                    const streamType:LogSource = streamTypeInt === 1 ? 'stdout' : 'stderr';
+                    const streamType: LogSource = streamTypeInt === 1 ? 'stdout' : 'stderr';
 
                     // Bytes 4-8 is frame size
                     const messageLength = buf.readInt32BE(offset + 4);
@@ -619,7 +740,7 @@ export class ContainerInfo {
                     });
                 } catch (err) {
                     console.error('Error parsing log entry', err);
-                    offset = buf.length
+                    offset = buf.length;
                 }
             }
         });

package/src/definitionsManager.ts

@@ -1,5 +1,5 @@
 import ClusterConfiguration, { DefinitionInfo } from '@kapeta/local-cluster-config';
-import {parseKapetaUri} from "@kapeta/nodejs-utils";
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
 
 const CACHE_TTL = 60 * 1000; // 1 min
 
@@ -48,15 +48,19 @@ class DefinitionsManager {
     }
 
     public exists(ref: string) {
-        const uri = parseKapetaUri(ref);
-        return !!this.getDefinitions().find((d) => {
-            return parseKapetaUri(`${d.definition.metadata.name}:${d.version}`).id === uri.id;
-        });
+        return !!this.getDefinition(ref);
     }
 
     public getProviderDefinitions() {
         return this.doCached('providers', () => ClusterConfiguration.getProviderDefinitions());
     }
+
+    public getDefinition(ref: string) {
+        const uri = parseKapetaUri(ref);
+        return this.getDefinitions().find((d) => {
+            return parseKapetaUri(`${d.definition.metadata.name}:${d.version}`).id === uri.id;
+        });
+    }
 }
 
 export const definitionsManager = new DefinitionsManager();

package/src/instanceManager.ts

@@ -9,8 +9,11 @@ import { assetManager } from './assetManager';
 import { containerManager, HEALTH_CHECK_TIMEOUT } from './containerManager';
 import { configManager } from './configManager';
 import { DesiredInstanceStatus, InstanceInfo, InstanceOwner, InstanceStatus, InstanceType, LogEntry } from './types';
-import { BlockInstance } from '@kapeta/schemas';
+import {BlockDefinitionSpec, BlockInstance, Plan} from '@kapeta/schemas';
 import { getBlockInstanceContainerName, normalizeKapetaUri } from './utils/utils';
+import { KIND_OPERATOR, operatorManager } from './operatorManager';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import { definitionsManager } from './definitionsManager';
 
 const CHECK_INTERVAL = 5000;
 const DEFAULT_HEALTH_PORT_TYPE = 'rest';
@@ -26,7 +29,7 @@ export class InstanceManager {
 
     private readonly _instances: InstanceInfo[] = [];
 
-    private readonly instanceLocks:AsyncLock = new AsyncLock();
+    private readonly instanceLocks: AsyncLock = new AsyncLock();
 
     constructor() {
         this._instances = storageService.section('instances', []);
@@ -61,7 +64,17 @@ export class InstanceManager {
 
         systemId = normalizeKapetaUri(systemId);
 
-        return this._instances.filter((instance) => instance.systemId === systemId);
+        const planInfo = definitionsManager.getDefinition(systemId);
+
+        if (!planInfo) {
+            return [];
+        }
+
+        const plan = planInfo.definition as Plan;
+
+        const instanceIds = plan.spec.blocks.map((block) => block.id);
+
+        return this._instances.filter((instance) => instance.systemId === systemId && instanceIds.includes(instance.instanceId));
     }
 
     public getInstance(systemId: string, instanceId: string) {
@@ -70,15 +83,16 @@ export class InstanceManager {
         return this._instances.find((i) => i.systemId === systemId && i.instanceId === instanceId);
     }
 
-
     private async exclusive<T = any>(systemId: string, instanceId: string, fn: () => Promise<T>) {
         systemId = normalizeKapetaUri(systemId);
         const key = `${systemId}/${instanceId}`;
-        return this.instanceLocks.acquire(key, fn);
+        //console.log(`Acquiring lock for ${key}`, this.instanceLocks.isBusy(key));
+        const result = await this.instanceLocks.acquire(key, fn);
+        //console.log(`Releasing lock for ${key}`, this.instanceLocks.isBusy(key));
+        return result;
     }
 
-
-    public async getLogs(systemId: string, instanceId: string):Promise<LogEntry[]> {
+    public async getLogs(systemId: string, instanceId: string): Promise<LogEntry[]> {
         const instance = this.getInstance(systemId, instanceId);
         if (!instance) {
             throw new Error(`Instance ${systemId}/${instanceId} not found`);
@@ -86,23 +100,27 @@ export class InstanceManager {
 
         switch (instance.type) {
             case InstanceType.DOCKER:
-                 return await containerManager.getLogs(instance);
+                return await containerManager.getLogs(instance);
 
             case InstanceType.UNKNOWN:
-                return [{
-                    level: 'INFO',
-                    message: 'Instance is starting...',
-                    time: Date.now(),
-                    source: 'stdout',
-                }];
+                return [
+                    {
+                        level: 'INFO',
+                        message: 'Instance is starting...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    },
+                ];
 
             case InstanceType.LOCAL:
-                return [{
-                    level: 'INFO',
-                    message: 'Instance started outside Kapeta - logs not available...',
-                    time: Date.now(),
-                    source: 'stdout',
-                }];
+                return [
+                    {
+                        level: 'INFO',
+                        message: 'Instance started outside Kapeta - logs not available...',
+                        time: Date.now(),
+                        source: 'stdout',
+                    },
+                ];
         }
 
         return [];
@@ -167,7 +185,10 @@ export class InstanceManager {
             const healthUrl = this.getHealthUrl(info, address);
 
             if (instance) {
-                if (instance.status === InstanceStatus.STOPPING && instance.desiredStatus === DesiredInstanceStatus.STOP) {
+                if (
+                    instance.status === InstanceStatus.STOPPING &&
+                    instance.desiredStatus === DesiredInstanceStatus.STOP
+                ) {
                     //If instance is stopping do not interfere
                     return;
                 }
@@ -239,7 +260,7 @@ export class InstanceManager {
     public markAsStopped(systemId: string, instanceId: string) {
         return this.exclusive(systemId, instanceId, async () => {
             systemId = normalizeKapetaUri(systemId);
-            const instance = _.find(this._instances, {systemId, instanceId});
+            const instance = _.find(this._instances, { systemId, instanceId });
             if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
                 instance.status = InstanceStatus.STOPPED;
                 instance.pid = null;
@@ -247,7 +268,7 @@ export class InstanceManager {
                 this.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
                 this.save();
             }
-        })
+        });
     }
 
     public async startAllForPlan(systemId: string): Promise<InstanceInfo[]> {
@@ -281,12 +302,11 @@ export class InstanceManager {
         return settled.map((p) => (p.status === 'fulfilled' ? p.value : null)).filter((p) => !!p) as InstanceInfo[];
     }
 
-
     public async stop(systemId: string, instanceId: string) {
         return this.stopInner(systemId, instanceId, true);
     }
 
-    private async stopInner(systemId: string, instanceId: string, changeDesired:boolean = false) {
+    private async stopInner(systemId: string, instanceId: string, changeDesired: boolean = false) {
         return this.exclusive(systemId, instanceId, async () => {
             systemId = normalizeKapetaUri(systemId);
             const instance = this.getInstance(systemId, instanceId);
@@ -298,8 +318,7 @@ export class InstanceManager {
                 return;
             }
 
-            if (changeDesired &&
-                instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
+            if (changeDesired && instance.desiredStatus !== DesiredInstanceStatus.EXTERNAL) {
                 instance.desiredStatus = DesiredInstanceStatus.STOP;
             }
 
@@ -359,7 +378,7 @@ export class InstanceManager {
                 throw new Error('Plan not found: ' + systemId);
             }
 
-            const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, {id: instanceId}) : null;
+            const blockInstance = plan.spec && plan.spec.blocks ? _.find(plan.spec.blocks, { id: instanceId }) : null;
             if (!blockInstance) {
                 throw new Error('Block instance not found: ' + instanceId);
             }
@@ -412,6 +431,28 @@ export class InstanceManager {
             // Save the instance before starting it, so that we can track the status
             await this.saveInternalInstance(instance);
 
+            const blockSpec = blockAsset.data.spec as BlockDefinitionSpec;
+            if (blockSpec.consumers) {
+                const promises = blockSpec.consumers.map((consumer) => {
+                    const consumerUri = parseKapetaUri(consumer.kind);
+                    const asset = definitionsManager.getDefinition(consumer.kind);
+                    if (!asset) {
+                        // Definition not found
+                        return Promise.resolve();
+                    }
+
+                    if (KIND_OPERATOR.toLowerCase() !== asset.definition.kind.toLowerCase()) {
+                        // Not an operator
+                        return Promise.resolve();
+                    }
+
+                    console.log('Ensuring resource: %s in %s', consumerUri.id, systemId);
+                    return operatorManager.ensureResource(systemId, consumerUri.fullName, consumerUri.version);
+                });
+
+                await Promise.all(promises);
+            }
+
             if (existingInstance) {
                 // Check if the instance is already running - but after we've commmuicated the desired status
                 const currentStatus = await this.requestInstanceStatus(existingInstance);
@@ -439,7 +480,7 @@ export class InstanceManager {
                     status: InstanceStatus.READY,
                 });
             } catch (e: any) {
-                console.warn('Failed to start instance', e);
+                console.warn('Failed to start instance: ', systemId, instanceId, blockRef, e.message);
                 const logs: LogEntry[] = [
                     {
                         source: 'stdout',
@@ -451,11 +492,12 @@ export class InstanceManager {
 
                 const out = await this.saveInternalInstance({
                     ...instance,
-                    type: InstanceType.LOCAL,
+                    type: InstanceType.UNKNOWN,
                     pid: null,
                     health: null,
                     portType: DEFAULT_HEALTH_PORT_TYPE,
                     status: InstanceStatus.FAILED,
+                    errorMessage: e.message ?? 'Failed to start - Check logs for details.',
                 });
 
                 this.emitInstanceEvent(systemId, instanceId, EVENT_INSTANCE_LOG, logs[0]);
@@ -471,17 +513,16 @@ export class InstanceManager {
         });
     }
 
-    public async restart(systemId: string, instanceId: string) {
+    /**
+     * Stops an instance but does not remove it from the list of active instances
+     *
+     * It will be started again next time the system checks the status of the instance
+     *
+     * We do it this way to not cause the user to wait for the instance to start again
+     */
+    public async prepareForRestart(systemId: string, instanceId: string) {
         systemId = normalizeKapetaUri(systemId);
         await this.stopInner(systemId, instanceId);
-
-        const existingInstance = this.getInstance(systemId, instanceId);
-        if (existingInstance?.desiredStatus === DesiredInstanceStatus.STOP) {
-            // Internal instance was marked as stopped - abort restart
-            return existingInstance;
-        }
-
-        return this.start(systemId, instanceId);
     }
 
     public async stopAll() {
@@ -553,8 +594,7 @@ export class InstanceManager {
                     const skipUpdate =
                         (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
                         ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
-                            instance.status === InstanceStatus.STOPPING &&
-                            instance.desiredStatus === DesiredInstanceStatus.STOP) ||
+                            instance.status === InstanceStatus.STOPPING) ||
                         (newStatus === InstanceStatus.STOPPED &&
                             instance.status === InstanceStatus.STARTING &&
                             instance.desiredStatus === DesiredInstanceStatus.RUN);
@@ -608,7 +648,7 @@ export class InstanceManager {
                     //If the instance is unhealthy, try to restart it
                     console.log('Restarting unhealthy instance', instance);
                     try {
-                        await this.restart(instance.systemId, instance.instanceId);
+                        await this.prepareForRestart(instance.systemId, instance.instanceId);
                     } catch (e) {
                         console.warn('Failed to restart instance', instance.systemId, instance.instanceId, e);
                     }

package/src/instances/routes.ts

@@ -86,7 +86,7 @@ router.get('/:systemId/:instanceId/logs', async (req: Request, res: Response) =>
         res.status(200).send({
             logs,
         });
-    } catch (e:any) {
+    } catch (e: any) {
         res.status(500).send({ ok: false, error: e.message });
     }
 });
@@ -146,8 +146,10 @@ router.put('/', async (req: KapetaBodyRequest, res: Response) => {
         const oldInstance = instanceManager.getInstance(req.kapeta!.systemId, req.kapeta!.instanceId);
         if (oldInstance) {
             instance.pid = oldInstance.pid;
+            instance.desiredStatus = oldInstance.desiredStatus;
         }
         instance.type = InstanceType.DOCKER;
+        instance.owner = InstanceOwner.INTERNAL;
     } else {
         // Coming from user starting the instance outside of kapeta
         instance.type = InstanceType.LOCAL;