@kapeta/local-cluster-service 0.37.0 → 0.38.0

package/src/containerManager.ts

@@ -14,12 +14,12 @@ import ClusterConfiguration from '@kapeta/local-cluster-config';
 import uuid from 'node-uuid';
 import md5 from 'md5';
 import { getBlockInstanceContainerName } from './utils/utils';
-import { Health, InstanceInfo, LogEntry, LogSource } from './types';
+import { InstanceInfo, LogEntry, LogSource } from './types';
 import { KapetaAPI } from '@kapeta/nodejs-api-client';
 import { taskManager, Task } from './taskManager';
 import { EventEmitter } from 'node:events';
 import StreamValues from 'stream-json/streamers/StreamValues';
-import { Stream } from 'stream';
+import { LocalInstanceHealth } from '@kapeta/schemas';
 
 type StringMap = { [key: string]: string };
 
@@ -100,7 +100,7 @@ enum DockerPullEventTypes {
 
 type DockerPullEventType = DockerPullEventTypes | string;
 
-const processJsonStream = <T>(purpose: string, stream: Stream, handler: (d: JSONMessage<T>) => void) =>
+const processJsonStream = <T>(purpose: string, stream: NodeJS.ReadableStream, handler: (d: JSONMessage<T>) => void) =>
     new Promise<void>((resolve, reject) => {
         const jsonStream = StreamValues.withParser();
         jsonStream.on('data', (data: any) => {
@@ -296,15 +296,8 @@ class ContainerManager {
     }
 
     async getContainerByName(containerName: string): Promise<ContainerInfo | undefined> {
-        const containers = await this.docker().listContainers({ all: true });
-        const out = containers.find((container) => {
-            return container.Names.indexOf(`/${containerName}`) > -1;
-        });
-
-        if (out) {
-            return this.get(out.Id);
-        }
-        return undefined;
+        // The container can be fetched by name or by id using the same API call
+        return this.get(containerName);
     }
 
     async pull(image: string) {
@@ -398,6 +391,11 @@ class ContainerManager {
 
                 const chunk = chunks[data.id];
 
+                if (data.stream) {
+                    // Emit raw output to the task log
+                    task.addLog(data.stream);
+                }
+
                 switch (data.status) {
                     case DockerPullEventTypes.PreparingPhase:
                     case DockerPullEventTypes.WaitingPhase:
@@ -529,7 +527,7 @@ class ContainerManager {
         return Mounts;
     }
 
-    toDockerHealth(health: Health) {
+    toDockerHealth(health: LocalInstanceHealth) {
         return {
             Test: ['CMD-SHELL', health.cmd],
             Interval: health.interval ? health.interval * NANO_SECOND : 5000 * NANO_SECOND,
@@ -679,7 +677,7 @@ class ContainerManager {
         let dockerContainer = null;
 
         try {
-            dockerContainer = await this.docker().getContainer(name);
+            dockerContainer = this.docker().getContainer(name);
             await dockerContainer.stats();
         } catch (err) {
             //Ignore
@@ -707,7 +705,7 @@ class ContainerManager {
             ];
         }
 
-        return containerInfo.getLogs();
+        return await containerInfo.getLogs();
     }
 
     async stopLogListening(systemId: string, instanceId: string) {
@@ -782,6 +780,34 @@ class ContainerManager {
             // Ignore
         }
     }
+
+    buildDockerImage(dockerFile: string, imageName: string) {
+        const taskName = `Building docker image: ${imageName}`;
+        const processor = async (task: Task) => {
+            const timeStarted = Date.now();
+            const stream = await this.docker().buildImage(
+                {
+                    context: Path.dirname(dockerFile),
+                    src: [Path.basename(dockerFile)],
+                },
+                {
+                    t: imageName,
+                    dockerfile: Path.basename(dockerFile),
+                }
+            );
+
+            await processJsonStream<string>(`image:build:${imageName}`, stream, (data) => {
+                if (data.stream) {
+                    // Emit raw output to the task log
+                    task.addLog(data.stream);
+                }
+            });
+        };
+
+        return taskManager.add(`docker:image:build:${imageName}`, processor, {
+            name: taskName,
+        });
+    }
 }
 
 function readLogBuffer(logBuffer: Buffer) {
@@ -1038,17 +1064,78 @@ export class ContainerInfo {
         });
 
         const out = readLogBuffer(logs);
+        if (out.length > 0) {
+            return out;
+        }
 
-        if (out.length === 0) {
-            out.push({
-                time: Date.now(),
-                message: 'No logs found for container',
-                level: 'INFO',
-                source: 'stdout',
-            });
+        const status = await this.status();
+        const healthLogs: LogEntry[] = status?.Health?.Log
+            ? status?.Health?.Log.map((log) => {
+                  return {
+                      source: 'stdout',
+                      level: log.ExitCode === 0 ? 'INFO' : 'ERROR',
+                      time: Date.now(),
+                      message: 'Health check: ' + log.Output,
+                  };
+              })
+            : [];
+
+        if (status?.Running) {
+            return [
+                {
+                    source: 'stdout',
+                    level: 'INFO',
+                    time: Date.now(),
+                    message: 'Container is starting...',
+                },
+                ...healthLogs,
+            ];
         }
 
-        return out;
+        if (status?.Restarting) {
+            return [
+                {
+                    source: 'stdout',
+                    level: 'INFO',
+                    time: Date.now(),
+                    message: 'Container is restarting...',
+                },
+                ...healthLogs,
+            ];
+        }
+        if (status?.Paused) {
+            return [
+                {
+                    source: 'stdout',
+                    level: 'INFO',
+                    time: Date.now(),
+                    message: 'Container is paused...',
+                },
+                ...healthLogs,
+            ];
+        }
+
+        if (status?.Error) {
+            return [
+                {
+                    source: 'stderr',
+                    level: 'ERROR',
+                    time: Date.now(),
+                    message: 'Container failed to start:\n' + status.Error,
+                },
+                ...healthLogs,
+            ];
+        }
+
+        return [
+            {
+                source: 'stdout',
+                level: 'INFO',
+                time: Date.now(),
+                message: 'Container not running',
+                ...healthLogs,
+            },
+        ];
     }
 }
 

package/src/instanceManager.ts

@@ -28,12 +28,11 @@ import {
     KIND_BLOCK_TYPE_EXECUTABLE,
     KIND_BLOCK_TYPE_OPERATOR,
     KIND_RESOURCE_OPERATOR,
-    LocalImageOptions,
     LogEntry,
     OperatorInstanceInfo,
     OperatorInstancePort,
 } from './types';
-import { BlockDefinitionSpec, BlockInstance, Plan } from '@kapeta/schemas';
+import { BlockDefinitionSpec, LocalInstance, Plan } from '@kapeta/schemas';
 import { getBlockInstanceContainerName, getResolvedConfiguration } from './utils/utils';
 import { operatorManager } from './operatorManager';
 import { normalizeKapetaUri, parseKapetaUri } from '@kapeta/nodejs-utils';
@@ -293,7 +292,9 @@ export class InstanceManager {
             systemId = normalizeKapetaUri(systemId);
             const instance = _.find(this._instances, { systemId, instanceId });
             if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
-                instance.status = InstanceStatus.STOPPED;
+                if (instance.status != InstanceStatus.FAILED) {
+                    instance.status = InstanceStatus.STOPPED;
+                }
                 instance.pid = null;
                 instance.health = null;
                 socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
@@ -385,7 +386,7 @@ export class InstanceManager {
             throw new Error(`Operator block has no local definition: ${blockRef}`);
         }
 
-        const localConfig = operatorDefinition.definition.spec.local as LocalImageOptions;
+        const localConfig = operatorDefinition.definition.spec.local as LocalInstance;
 
         let instance = await this.start(systemId, instanceId);
         if (instance instanceof Task) {
@@ -666,8 +667,7 @@ export class InstanceManager {
 
                         const out = await this.saveInternalInstance({
                             ...instance,
-                            type: InstanceType.UNKNOWN,
-                            pid: null,
+                            type: InstanceType.DOCKER,
                             health: null,
                             portType: DEFAULT_HEALTH_PORT_TYPE,
                             status: InstanceStatus.FAILED,
@@ -811,7 +811,6 @@ export class InstanceManager {
                 if (instance.status !== newStatus) {
                     const oldStatus = instance.status;
                     const skipUpdate =
-                        (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
                         ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
                             instance.status === InstanceStatus.STOPPING) ||
                         (newStatus === InstanceStatus.STOPPED &&
@@ -835,7 +834,7 @@ export class InstanceManager {
 
                 if (
                     instance.desiredStatus === DesiredInstanceStatus.RUN &&
-                    [InstanceStatus.STOPPED, InstanceStatus.FAILED, InstanceStatus.STOPPING].includes(newStatus)
+                    [InstanceStatus.STOPPED, InstanceStatus.STOPPING].includes(newStatus)
                 ) {
                     //If the instance is stopped but we want it to run, start it
                     try {
@@ -928,11 +927,18 @@ export class InstanceManager {
             }
 
             if (statusType === 'created') {
+                if (state.ExitCode !== 0) {
+                    // Failed during creation
+                    return InstanceStatus.FAILED;
+                }
                 return InstanceStatus.STARTING;
             }
 
             if (statusType === 'exited' || statusType === 'dead') {
-                return InstanceStatus.STOPPED;
+                if (state.ExitCode === 0) {
+                    return InstanceStatus.STOPPED;
+                }
+                return InstanceStatus.FAILED;
             }
 
             if (statusType === 'removing') {
@@ -1013,7 +1019,7 @@ export class InstanceManager {
         }
 
         if (parseKapetaUri(provider.kind).fullName === KIND_BLOCK_TYPE_OPERATOR) {
-            const localConfig = provider.data.spec.local as LocalImageOptions;
+            const localConfig = provider.data.spec.local as LocalInstance;
             return localConfig.singleton ?? false;
         }
 

package/src/operatorManager.ts

@@ -21,11 +21,10 @@ import {
     EnvironmentType,
     KIND_BLOCK_TYPE_OPERATOR,
     KIND_RESOURCE_OPERATOR,
-    LocalImageOptions,
     OperatorInfo,
     StringMap,
 } from './types';
-import { BlockInstance, Resource } from '@kapeta/schemas';
+import { BlockInstance, LocalInstance, Resource } from '@kapeta/schemas';
 import { definitionsManager } from './definitionsManager';
 import { getBindHost, toPortInfo } from './utils/utils';
 import { parseKapetaUri, normalizeKapetaUri } from '@kapeta/nodejs-utils';
@@ -42,7 +41,7 @@ class Operator {
         this._data = data;
     }
 
-    getLocalData(): LocalImageOptions {
+    getLocalData(): LocalInstance {
         return this._data.definition.spec.local;
     }
 

package/src/progressListener.ts

@@ -7,6 +7,7 @@ import { spawn } from '@kapeta/nodejs-process';
 import { socketManager } from './socketManager';
 import { LogEntry } from './types';
 import { format } from 'node:util';
+import { Task } from './taskManager';
 
 export class ProgressListener {
     private readonly systemId: string | undefined;
@@ -17,7 +18,7 @@ export class ProgressListener {
         this.instanceId = instanceId;
     }
 
-    private emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
+    protected emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
         const logEntry: LogEntry = {
             ...payload,
             source: 'stdout',
@@ -135,3 +136,16 @@ export class ProgressListener {
         });
     }
 }
+
+export class TaskProgressListener extends ProgressListener {
+    private readonly task: Task;
+
+    constructor(task: Task) {
+        super();
+        this.task = task;
+    }
+
+    protected emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
+        this.task.addLog(payload.message, payload.level);
+    }
+}

package/src/repositoryManager.ts

@@ -10,7 +10,7 @@ import { Actions, AssetVersion, Config, RegistryService } from '@kapeta/nodejs-r
 import { definitionsManager } from './definitionsManager';
 import { Task, taskManager } from './taskManager';
 import { normalizeKapetaUri, parseKapetaUri, parseVersion } from '@kapeta/nodejs-utils';
-import { ProgressListener } from './progressListener';
+import { TaskProgressListener } from './progressListener';
 import { RepositoryWatcher } from './RepositoryWatcher';
 import { SourceOfChange } from './types';
 import { cacheManager } from './cacheManager';
@@ -191,16 +191,18 @@ class RepositoryManager extends EventEmitter {
     private async scheduleInstallation(refs: string[]): Promise<Task[]> {
         //We make sure to only install one asset at a time - otherwise unexpected things might happen
         const createInstaller = (ref: string) => {
-            return async () => {
+            return async (task: Task) => {
                 if (await definitionsManager.exists(ref)) {
                     return;
                 }
+
+                const progressListener = new TaskProgressListener(task);
                 //console.log(`Installing asset: ${ref}`);
                 //Auto-install missing asset
                 try {
                     //We change to a temp dir to avoid issues with the current working directory
                     process.chdir(os.tmpdir());
-                    await Actions.install(new ProgressListener(), [ref], {});
+                    await Actions.install(progressListener, [ref], {});
                 } catch (e) {
                     console.error(`Failed to install asset: ${ref}`, e);
                     throw e;

package/src/taskManager.ts

@@ -7,10 +7,12 @@
  * Class that handles processing background tasks.
  */
 import { socketManager } from './socketManager';
+import { LogLevel } from './types';
 
 const EVENT_TASK_UPDATED = 'task-updated';
 const EVENT_TASK_ADDED = 'task-added';
 const EVENT_TASK_REMOVED = 'task-removed';
+const EVENT_TASK_LOG = 'task-log';
 
 export type TaskRunner<T> = (task: Task<T>) => Promise<T>;
 
@@ -94,6 +96,15 @@ export class Task<T = void> implements TaskData<T> {
         socketManager.emitGlobal(EVENT_TASK_UPDATED, this.toData());
     }
 
+    public addLog(log: string, level: LogLevel = 'INFO') {
+        socketManager.emitGlobal(EVENT_TASK_LOG, {
+            id: this.id,
+            message: log,
+            level,
+            time: Date.now(),
+        });
+    }
+
     async wait(): Promise<T> {
         return this.future.promise;
     }

package/src/types.ts

@@ -73,20 +73,6 @@ export interface Health {
     retries?: number;
 }
 
-export type PortInfo = { port: number; type: 'tcp' | 'udp' } | number | string;
-
-export type LocalImageOptions<Credentials = AnyMap, Options = AnyMap> = {
-    image: string;
-    ports: { [key: string]: PortInfo };
-    credentials?: Credentials;
-    options?: Options;
-    cmd?: string;
-    env?: AnyMap;
-    health?: Health;
-    singleton?: boolean;
-    mounts?: { [key: string]: string };
-};
-
 export type InstanceInfo = {
     systemId: string;
     instanceId: string;
@@ -105,11 +91,6 @@ export type InstanceInfo = {
     portType?: string;
 };
 
-interface ResourceRef {
-    blockId: string;
-    resourceName: string;
-}
-
 export type ProxyRequestHandler = (req: StringBodyRequest, res: express.Response, info: ProxyRequestInfo) => void;
 
 export interface OperatorInstancePort {

package/src/utils/BlockInstanceRunner.ts

@@ -18,19 +18,13 @@ import {
 } from '../containerManager';
 import { LogData } from './LogData';
 import { clusterService } from '../clusterService';
-import {
-    AnyMap,
-    BlockProcessParams,
-    InstanceType,
-    KIND_BLOCK_TYPE_OPERATOR,
-    LocalImageOptions,
-    ProcessInfo,
-    StringMap,
-} from '../types';
+import { AnyMap, BlockProcessParams, InstanceType, KIND_BLOCK_TYPE_OPERATOR, ProcessInfo, StringMap } from '../types';
 import { definitionsManager } from '../definitionsManager';
 import Docker from 'dockerode';
 import OS from 'node:os';
+import Path from 'node:path';
 import { taskManager } from '../taskManager';
+import { LocalDevContainer, LocalInstance } from '@kapeta/schemas';
 
 const KAPETA_SYSTEM_ID = 'KAPETA_SYSTEM_ID';
 const KAPETA_BLOCK_REF = 'KAPETA_BLOCK_REF';
@@ -189,6 +183,8 @@ export class BlockInstanceRunner {
             throw new Error('Missing target kind in block definition');
         }
 
+        const realLocalPath = await FSExtra.realpath(baseDir);
+
         const kindUri = parseKapetaUri(assetVersion.definition.kind);
 
         const providerVersion = await getProvider(kindUri);
@@ -205,17 +201,29 @@ export class BlockInstanceRunner {
             throw new Error(`Target not found: ${targetKindUri.id}`);
         }
 
-        const localContainer = targetVersion.definition.spec.local;
+        const localContainer = targetVersion.definition.spec.local as LocalDevContainer;
 
         if (!localContainer) {
             throw new Error(`Missing local container information from target: ${targetKindUri.id}`);
         }
 
-        const dockerImage = localContainer.image;
-        if (!dockerImage) {
+        let dockerImage = localContainer.image;
+        const isDockerImage = !localContainer.type || localContainer.type.toLowerCase() === 'docker';
+        const isDockerFile = Boolean(localContainer.type && localContainer.type.toLowerCase() === 'dockerfile');
+        if (isDockerImage && !dockerImage) {
             throw new Error(`Missing docker image information: ${JSON.stringify(localContainer)}`);
         }
 
+        if (isDockerFile) {
+            dockerImage = blockInfo.fullName + ':local';
+            const dockerFile = Path.join(realLocalPath, localContainer.file ?? 'Dockerfile');
+            if (!FSExtra.existsSync(dockerFile)) {
+                throw new Error(`Dockerfile not found at: ${dockerFile}`);
+            }
+            const task = containerManager.buildDockerImage(dockerFile, blockInfo.fullName + ':local');
+            await task.wait();
+        }
+
         const containerName = await getBlockInstanceContainerName(this._systemId, blockInstance.id, targetKindUri.id);
         const startCmd = localContainer.handlers?.onCreate ? localContainer.handlers.onCreate : '';
         const dockerOpts = localContainer.options ?? {};
@@ -242,8 +250,6 @@ export class BlockInstanceRunner {
             HealthCheck = containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
 
-        const realLocalPath = await FSExtra.realpath(baseDir);
-
         const Mounts = containerManager.toDockerMounts({
             [workingDir]: toLocalBindVolume(realLocalPath),
         });
@@ -391,7 +397,7 @@ export class BlockInstanceRunner {
             throw new Error(`Provider did not have local image: ${providerRef}`);
         }
 
-        const local = spec.local as LocalImageOptions;
+        const local = spec.local as LocalInstance;
 
         const dockerImage = local.image;
         const operatorUri = local.singleton ? parseKapetaUri(providerRef) : blockUri;

package/src/utils/commandLineUtils.ts

@@ -4,7 +4,7 @@
  */
 
 import { spawn, hasApp } from '@kapeta/nodejs-process';
-import { taskManager } from '../taskManager';
+import { Task, taskManager } from '../taskManager';
 
 export async function hasCLI() {
     return hasApp('kap');
@@ -17,11 +17,19 @@ export async function ensureCLI() {
 
     return taskManager.add(
         `cli:install`,
-        () => {
+        (task: Task) => {
             const process = spawn('npm', ['install', '-g', '@kapeta/kap'], {
                 shell: true,
             });
 
+            process.process.stdout?.on('data', (data: any) => {
+                task.addLog(data.toString(), 'INFO');
+            });
+
+            process.process.stderr?.on('data', (data: any) => {
+                task.addLog(data.toString(), 'ERROR');
+            });
+
             return process.wait();
         },
         {

package/src/utils/utils.ts

@@ -6,9 +6,9 @@
 import FS from 'node:fs';
 import YAML from 'yaml';
 import md5 from 'md5';
-import { EntityList } from '@kapeta/schemas';
+import { EntityList, LocalInstancePort, LocalInstancePortType } from '@kapeta/schemas';
 import _ from 'lodash';
-import { AnyMap, KIND_BLOCK_TYPE_OPERATOR, PortInfo } from '../types';
+import { AnyMap, KIND_BLOCK_TYPE_OPERATOR } from '../types';
 import ClusterConfiguration from '@kapeta/local-cluster-config';
 import { definitionsManager } from '../definitionsManager';
 import { normalizeKapetaUri, parseKapetaUri } from '@kapeta/nodejs-utils';
@@ -40,13 +40,13 @@ export async function getBlockInstanceContainerName(systemId: string, instanceId
     return `kapeta-block-instance-${md5(normalizeKapetaUri(systemId) + instanceId)}`;
 }
 
-export function toPortInfo(port: PortInfo) {
+export function toPortInfo(port: LocalInstancePort) {
     if (typeof port === 'number' || typeof port === 'string') {
         return { port: parseInt(`${port}`), type: 'tcp' };
     }
 
     if (!port.type) {
-        port.type = 'tcp';
+        port.type = LocalInstancePortType.TCP;
     }
 
     return port;