@kapeta/local-cluster-service 0.36.1 → 0.38.0

package/src/instanceManager.ts

@@ -25,14 +25,16 @@ import {
     InstanceOwner,
     InstanceStatus,
     InstanceType,
-    LocalImageOptions,
+    KIND_BLOCK_TYPE_EXECUTABLE,
+    KIND_BLOCK_TYPE_OPERATOR,
+    KIND_RESOURCE_OPERATOR,
     LogEntry,
     OperatorInstanceInfo,
     OperatorInstancePort,
 } from './types';
-import { BlockDefinitionSpec, BlockInstance, Plan } from '@kapeta/schemas';
+import { BlockDefinitionSpec, LocalInstance, Plan } from '@kapeta/schemas';
 import { getBlockInstanceContainerName, getResolvedConfiguration } from './utils/utils';
-import { KIND_BLOCK_OPERATOR, KIND_RESOURCE_OPERATOR, operatorManager } from './operatorManager';
+import { operatorManager } from './operatorManager';
 import { normalizeKapetaUri, parseKapetaUri } from '@kapeta/nodejs-utils';
 import { definitionsManager } from './definitionsManager';
 import { Task, taskManager } from './taskManager';
@@ -290,7 +292,9 @@ export class InstanceManager {
             systemId = normalizeKapetaUri(systemId);
             const instance = _.find(this._instances, { systemId, instanceId });
             if (instance && instance.owner === InstanceOwner.EXTERNAL && instance.status !== InstanceStatus.STOPPED) {
-                instance.status = InstanceStatus.STOPPED;
+                if (instance.status != InstanceStatus.FAILED) {
+                    instance.status = InstanceStatus.STOPPED;
+                }
                 instance.pid = null;
                 instance.health = null;
                 socketManager.emitSystemEvent(systemId, EVENT_STATUS_CHANGED, instance);
@@ -313,12 +317,13 @@ export class InstanceManager {
         return taskManager.add(
             `plan:start:${systemId}`,
             async () => {
-                let promises: Promise<InstanceInfo>[] = [];
-                let errors = [];
-                for (let blockInstance of Object.values(plan.spec.blocks as BlockInstance[])) {
+                const promises: Promise<InstanceInfo>[] = [];
+                const errors = [];
+                const instanceIds = await this.getAllInstancesExceptKind(systemId, KIND_BLOCK_TYPE_EXECUTABLE);
+                for (const instanceId of instanceIds) {
                     try {
                         promises.push(
-                            this.start(systemId, blockInstance.id).then((taskOrInstance) => {
+                            this.start(systemId, instanceId).then((taskOrInstance) => {
                                 if (taskOrInstance instanceof Task) {
                                     return taskOrInstance.wait();
                                 }
@@ -381,7 +386,7 @@ export class InstanceManager {
             throw new Error(`Operator block has no local definition: ${blockRef}`);
         }
 
-        const localConfig = operatorDefinition.definition.spec.local as LocalImageOptions;
+        const localConfig = operatorDefinition.definition.spec.local as LocalInstance;
 
         let instance = await this.start(systemId, instanceId);
         if (instance instanceof Task) {
@@ -662,8 +667,7 @@ export class InstanceManager {
 
                         const out = await this.saveInternalInstance({
                             ...instance,
-                            type: InstanceType.UNKNOWN,
-                            pid: null,
+                            type: InstanceType.DOCKER,
                             health: null,
                             portType: DEFAULT_HEALTH_PORT_TYPE,
                             status: InstanceStatus.FAILED,
@@ -807,7 +811,6 @@ export class InstanceManager {
                 if (instance.status !== newStatus) {
                     const oldStatus = instance.status;
                     const skipUpdate =
-                        (newStatus === InstanceStatus.STOPPED && instance.status === InstanceStatus.FAILED) ||
                         ([InstanceStatus.READY, InstanceStatus.UNHEALTHY].includes(newStatus) &&
                             instance.status === InstanceStatus.STOPPING) ||
                         (newStatus === InstanceStatus.STOPPED &&
@@ -831,7 +834,7 @@ export class InstanceManager {
 
                 if (
                     instance.desiredStatus === DesiredInstanceStatus.RUN &&
-                    [InstanceStatus.STOPPED, InstanceStatus.FAILED, InstanceStatus.STOPPING].includes(newStatus)
+                    [InstanceStatus.STOPPED, InstanceStatus.STOPPING].includes(newStatus)
                 ) {
                     //If the instance is stopped but we want it to run, start it
                     try {
@@ -924,11 +927,18 @@ export class InstanceManager {
             }
 
             if (statusType === 'created') {
+                if (state.ExitCode !== 0) {
+                    // Failed during creation
+                    return InstanceStatus.FAILED;
+                }
                 return InstanceStatus.STARTING;
             }
 
             if (statusType === 'exited' || statusType === 'dead') {
-                return InstanceStatus.STOPPED;
+                if (state.ExitCode === 0) {
+                    return InstanceStatus.STOPPED;
+                }
+                return InstanceStatus.FAILED;
             }
 
             if (statusType === 'removing') {
@@ -1008,8 +1018,8 @@ export class InstanceManager {
             return false;
         }
 
-        if (parseKapetaUri(provider.kind).fullName === KIND_BLOCK_OPERATOR) {
-            const localConfig = provider.data.spec.local as LocalImageOptions;
+        if (parseKapetaUri(provider.kind).fullName === KIND_BLOCK_TYPE_OPERATOR) {
+            const localConfig = provider.data.spec.local as LocalInstance;
             return localConfig.singleton ?? false;
         }
 
@@ -1025,6 +1035,37 @@ export class InstanceManager {
         return block.data.kind;
     }
 
+    /**
+     * Get the kind of an asset. Use the maxDepth parameter to specify how deep to look for the
+     * kind. For example, if maxDepth is 2, the method will look for the kind of the asset and then
+     * the kind of the kind.
+     * @param assetRef The asset reference
+     * @param maxDepth The maximum depth to look for the kind
+     * @returns The kind of the asset or null if not found
+     */
+    private async getDeepKindForAssetRef(assetRef: string, maxDepth: number): Promise<string | null> {
+        if (maxDepth <= 0) {
+            return null;
+        }
+
+        try {
+            const asset = await assetManager.getAsset(assetRef);
+            if (!asset || !asset.data.kind) {
+                return null;
+            }
+
+            if (maxDepth === 1) {
+                return asset.data.kind;
+            } else {
+                // Recurse with the kind of the current block and one less depth
+                return await this.getDeepKindForAssetRef(asset.data.kind, maxDepth - 1);
+            }
+        } catch (error) {
+            console.error('Error fetching kind for assetRef:', assetRef, error);
+            return null;
+        }
+    }
+
     private async isUsingKind(ref: string, kind: string): Promise<boolean> {
         const assetKind = await this.getKindForAssetRef(ref);
         if (!assetKind) {
@@ -1048,6 +1089,36 @@ export class InstanceManager {
 
         return out;
     }
+
+    /**
+     * Get the ids for all block instances except the ones of the specified kind
+     * @param systemId The plan reference id
+     * @param kind The kind to exclude. Can be a string or an array of strings
+     * @returns An array of block instance ids
+     */
+    private async getAllInstancesExceptKind(systemId: string, kind: string | string[]): Promise<string[]> {
+        const plan = await assetManager.getPlan(systemId);
+        if (!plan?.spec?.blocks) {
+            return [];
+        }
+        const out: string[] = [];
+        const excludedKinds = kind instanceof Array ? kind : [kind];
+        for (const block of plan.spec.blocks) {
+            const blockKindOfKind = await this.getDeepKindForAssetRef(block.block.ref, 2);
+            if (!blockKindOfKind) {
+                continue;
+            }
+
+            const shouldIncludeBlock =
+                excludedKinds.some((excludedKind) => excludedKind === parseKapetaUri(blockKindOfKind).fullName) ===
+                false;
+            if (shouldIncludeBlock) {
+                out.push(block.id);
+            }
+        }
+
+        return out;
+    }
 }
 
 export const instanceManager = new InstanceManager();

package/src/operatorManager.ts

@@ -16,8 +16,15 @@ import {
     containerManager,
 } from './containerManager';
 import FSExtra from 'fs-extra';
-import { AnyMap, EnvironmentType, LocalImageOptions, OperatorInfo, StringMap } from './types';
-import { BlockInstance, Resource } from '@kapeta/schemas';
+import {
+    AnyMap,
+    EnvironmentType,
+    KIND_BLOCK_TYPE_OPERATOR,
+    KIND_RESOURCE_OPERATOR,
+    OperatorInfo,
+    StringMap,
+} from './types';
+import { BlockInstance, LocalInstance, Resource } from '@kapeta/schemas';
 import { definitionsManager } from './definitionsManager';
 import { getBindHost, toPortInfo } from './utils/utils';
 import { parseKapetaUri, normalizeKapetaUri } from '@kapeta/nodejs-utils';
@@ -25,8 +32,6 @@ import _ from 'lodash';
 import AsyncLock from 'async-lock';
 import { taskManager } from './taskManager';
 
-export const KIND_RESOURCE_OPERATOR = 'core/resource-type-operator';
-export const KIND_BLOCK_OPERATOR = 'core/block-type-operator';
 const KIND_PLAN = 'core/plan';
 
 class Operator {
@@ -36,7 +41,7 @@ class Operator {
         this._data = data;
     }
 
-    getLocalData(): LocalImageOptions {
+    getLocalData(): LocalInstance {
         return this._data.definition.spec.local;
     }
 
@@ -68,7 +73,7 @@ class OperatorManager {
      * Get operator definition for resource type
      */
     async getOperator(fullName: string, version: string) {
-        const operators = await definitionsManager.getDefinitions([KIND_RESOURCE_OPERATOR, KIND_BLOCK_OPERATOR]);
+        const operators = await definitionsManager.getDefinitions([KIND_RESOURCE_OPERATOR, KIND_BLOCK_TYPE_OPERATOR]);
 
         const operator: DefinitionInfo | undefined = operators.find(
             (operator) =>

package/src/progressListener.ts

@@ -7,6 +7,7 @@ import { spawn } from '@kapeta/nodejs-process';
 import { socketManager } from './socketManager';
 import { LogEntry } from './types';
 import { format } from 'node:util';
+import { Task } from './taskManager';
 
 export class ProgressListener {
     private readonly systemId: string | undefined;
@@ -17,7 +18,7 @@ export class ProgressListener {
         this.instanceId = instanceId;
     }
 
-    private emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
+    protected emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
         const logEntry: LogEntry = {
             ...payload,
             source: 'stdout',
@@ -135,3 +136,16 @@ export class ProgressListener {
         });
     }
 }
+
+export class TaskProgressListener extends ProgressListener {
+    private readonly task: Task;
+
+    constructor(task: Task) {
+        super();
+        this.task = task;
+    }
+
+    protected emitLog(payload: Omit<LogEntry, 'time' | 'source'>) {
+        this.task.addLog(payload.message, payload.level);
+    }
+}

package/src/repositoryManager.ts

@@ -10,7 +10,7 @@ import { Actions, AssetVersion, Config, RegistryService } from '@kapeta/nodejs-r
 import { definitionsManager } from './definitionsManager';
 import { Task, taskManager } from './taskManager';
 import { normalizeKapetaUri, parseKapetaUri, parseVersion } from '@kapeta/nodejs-utils';
-import { ProgressListener } from './progressListener';
+import { TaskProgressListener } from './progressListener';
 import { RepositoryWatcher } from './RepositoryWatcher';
 import { SourceOfChange } from './types';
 import { cacheManager } from './cacheManager';
@@ -191,16 +191,18 @@ class RepositoryManager extends EventEmitter {
     private async scheduleInstallation(refs: string[]): Promise<Task[]> {
         //We make sure to only install one asset at a time - otherwise unexpected things might happen
         const createInstaller = (ref: string) => {
-            return async () => {
+            return async (task: Task) => {
                 if (await definitionsManager.exists(ref)) {
                     return;
                 }
+
+                const progressListener = new TaskProgressListener(task);
                 //console.log(`Installing asset: ${ref}`);
                 //Auto-install missing asset
                 try {
                     //We change to a temp dir to avoid issues with the current working directory
                     process.chdir(os.tmpdir());
-                    await Actions.install(new ProgressListener(), [ref], {});
+                    await Actions.install(progressListener, [ref], {});
                 } catch (e) {
                     console.error(`Failed to install asset: ${ref}`, e);
                     throw e;

package/src/taskManager.ts

@@ -7,10 +7,12 @@
  * Class that handles processing background tasks.
  */
 import { socketManager } from './socketManager';
+import { LogLevel } from './types';
 
 const EVENT_TASK_UPDATED = 'task-updated';
 const EVENT_TASK_ADDED = 'task-added';
 const EVENT_TASK_REMOVED = 'task-removed';
+const EVENT_TASK_LOG = 'task-log';
 
 export type TaskRunner<T> = (task: Task<T>) => Promise<T>;
 
@@ -94,6 +96,15 @@ export class Task<T = void> implements TaskData<T> {
         socketManager.emitGlobal(EVENT_TASK_UPDATED, this.toData());
     }
 
+    public addLog(log: string, level: LogLevel = 'INFO') {
+        socketManager.emitGlobal(EVENT_TASK_LOG, {
+            id: this.id,
+            message: log,
+            level,
+            time: Date.now(),
+        });
+    }
+
     async wait(): Promise<T> {
         return this.future.promise;
     }

package/src/types.ts

@@ -8,6 +8,11 @@ import { Connection, Resource } from '@kapeta/schemas';
 import { StringBodyRequest } from './middleware/stringBody';
 import { KapetaRequest } from './middleware/kapeta';
 
+export const KIND_RESOURCE_OPERATOR = 'core/resource-type-operator';
+export const KIND_BLOCK_TYPE = 'core/block-type';
+export const KIND_BLOCK_TYPE_OPERATOR = 'core/block-type-operator';
+export const KIND_BLOCK_TYPE_EXECUTABLE = 'core/block-type-executable';
+
 export type StringMap = { [key: string]: string };
 export type AnyMap = { [key: string]: any };
 export type SourceOfChange = 'user' | 'filesystem';
@@ -68,20 +73,6 @@ export interface Health {
     retries?: number;
 }
 
-export type PortInfo = { port: number; type: 'tcp' | 'udp' } | number | string;
-
-export type LocalImageOptions<Credentials = AnyMap, Options = AnyMap> = {
-    image: string;
-    ports: { [key: string]: PortInfo };
-    credentials?: Credentials;
-    options?: Options;
-    cmd?: string;
-    env?: AnyMap;
-    health?: Health;
-    singleton?: boolean;
-    mounts?: { [key: string]: string };
-};
-
 export type InstanceInfo = {
     systemId: string;
     instanceId: string;
@@ -100,11 +91,6 @@ export type InstanceInfo = {
     portType?: string;
 };
 
-interface ResourceRef {
-    blockId: string;
-    resourceName: string;
-}
-
 export type ProxyRequestHandler = (req: StringBodyRequest, res: express.Response, info: ProxyRequestInfo) => void;
 
 export interface OperatorInstancePort {

package/src/utils/BlockInstanceRunner.ts

@@ -18,13 +18,14 @@ import {
 } from '../containerManager';
 import { LogData } from './LogData';
 import { clusterService } from '../clusterService';
-import { AnyMap, BlockProcessParams, InstanceType, LocalImageOptions, ProcessInfo, StringMap } from '../types';
+import { AnyMap, BlockProcessParams, InstanceType, KIND_BLOCK_TYPE_OPERATOR, ProcessInfo, StringMap } from '../types';
 import { definitionsManager } from '../definitionsManager';
 import Docker from 'dockerode';
 import OS from 'node:os';
+import Path from 'node:path';
 import { taskManager } from '../taskManager';
+import { LocalDevContainer, LocalInstance } from '@kapeta/schemas';
 
-const KIND_BLOCK_TYPE_OPERATOR = 'core/block-type-operator';
 const KAPETA_SYSTEM_ID = 'KAPETA_SYSTEM_ID';
 const KAPETA_BLOCK_REF = 'KAPETA_BLOCK_REF';
 const KAPETA_INSTANCE_ID = 'KAPETA_INSTANCE_ID';
@@ -182,6 +183,8 @@ export class BlockInstanceRunner {
             throw new Error('Missing target kind in block definition');
         }
 
+        const realLocalPath = await FSExtra.realpath(baseDir);
+
         const kindUri = parseKapetaUri(assetVersion.definition.kind);
 
         const providerVersion = await getProvider(kindUri);
@@ -198,17 +201,29 @@ export class BlockInstanceRunner {
             throw new Error(`Target not found: ${targetKindUri.id}`);
         }
 
-        const localContainer = targetVersion.definition.spec.local;
+        const localContainer = targetVersion.definition.spec.local as LocalDevContainer;
 
         if (!localContainer) {
             throw new Error(`Missing local container information from target: ${targetKindUri.id}`);
         }
 
-        const dockerImage = localContainer.image;
-        if (!dockerImage) {
+        let dockerImage = localContainer.image;
+        const isDockerImage = !localContainer.type || localContainer.type.toLowerCase() === 'docker';
+        const isDockerFile = Boolean(localContainer.type && localContainer.type.toLowerCase() === 'dockerfile');
+        if (isDockerImage && !dockerImage) {
             throw new Error(`Missing docker image information: ${JSON.stringify(localContainer)}`);
         }
 
+        if (isDockerFile) {
+            dockerImage = blockInfo.fullName + ':local';
+            const dockerFile = Path.join(realLocalPath, localContainer.file ?? 'Dockerfile');
+            if (!FSExtra.existsSync(dockerFile)) {
+                throw new Error(`Dockerfile not found at: ${dockerFile}`);
+            }
+            const task = containerManager.buildDockerImage(dockerFile, blockInfo.fullName + ':local');
+            await task.wait();
+        }
+
         const containerName = await getBlockInstanceContainerName(this._systemId, blockInstance.id, targetKindUri.id);
         const startCmd = localContainer.handlers?.onCreate ? localContainer.handlers.onCreate : '';
         const dockerOpts = localContainer.options ?? {};
@@ -235,8 +250,6 @@ export class BlockInstanceRunner {
             HealthCheck = containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
 
-        const realLocalPath = await FSExtra.realpath(baseDir);
-
         const Mounts = containerManager.toDockerMounts({
             [workingDir]: toLocalBindVolume(realLocalPath),
         });
@@ -384,7 +397,7 @@ export class BlockInstanceRunner {
             throw new Error(`Provider did not have local image: ${providerRef}`);
         }
 
-        const local = spec.local as LocalImageOptions;
+        const local = spec.local as LocalInstance;
 
         const dockerImage = local.image;
         const operatorUri = local.singleton ? parseKapetaUri(providerRef) : blockUri;

package/src/utils/commandLineUtils.ts

@@ -4,7 +4,7 @@
  */
 
 import { spawn, hasApp } from '@kapeta/nodejs-process';
-import { taskManager } from '../taskManager';
+import { Task, taskManager } from '../taskManager';
 
 export async function hasCLI() {
     return hasApp('kap');
@@ -17,11 +17,19 @@ export async function ensureCLI() {
 
     return taskManager.add(
         `cli:install`,
-        () => {
+        (task: Task) => {
             const process = spawn('npm', ['install', '-g', '@kapeta/kap'], {
                 shell: true,
             });
 
+            process.process.stdout?.on('data', (data: any) => {
+                task.addLog(data.toString(), 'INFO');
+            });
+
+            process.process.stderr?.on('data', (data: any) => {
+                task.addLog(data.toString(), 'ERROR');
+            });
+
             return process.wait();
         },
         {

package/src/utils/utils.ts

@@ -6,13 +6,12 @@
 import FS from 'node:fs';
 import YAML from 'yaml';
 import md5 from 'md5';
-import { EntityList } from '@kapeta/schemas';
+import { EntityList, LocalInstancePort, LocalInstancePortType } from '@kapeta/schemas';
 import _ from 'lodash';
-import { AnyMap, PortInfo } from '../types';
+import { AnyMap, KIND_BLOCK_TYPE_OPERATOR } from '../types';
 import ClusterConfiguration from '@kapeta/local-cluster-config';
 import { definitionsManager } from '../definitionsManager';
 import { normalizeKapetaUri, parseKapetaUri } from '@kapeta/nodejs-utils';
-import { KIND_BLOCK_OPERATOR } from '../operatorManager';
 import { assetManager } from '../assetManager';
 
 export async function getBlockInstanceContainerName(systemId: string, instanceId: string, blockType?: string) {
@@ -32,7 +31,7 @@ export async function getBlockInstanceContainerName(systemId: string, instanceId
         throw new Error(`Block type ${blockType} not found`);
     }
     if (
-        parseKapetaUri(typeDefinition.definition.kind).fullName === KIND_BLOCK_OPERATOR &&
+        parseKapetaUri(typeDefinition.definition.kind).fullName === KIND_BLOCK_TYPE_OPERATOR &&
         typeDefinition.definition.spec?.local?.singleton
     ) {
         return `kapeta-instance-operator-${md5(normalizeKapetaUri(systemId) + normalizeKapetaUri(blockType))}`;
@@ -41,13 +40,13 @@ export async function getBlockInstanceContainerName(systemId: string, instanceId
     return `kapeta-block-instance-${md5(normalizeKapetaUri(systemId) + instanceId)}`;
 }
 
-export function toPortInfo(port: PortInfo) {
+export function toPortInfo(port: LocalInstancePort) {
     if (typeof port === 'number' || typeof port === 'string') {
         return { port: parseInt(`${port}`), type: 'tcp' };
     }
 
     if (!port.type) {
-        port.type = 'tcp';
+        port.type = LocalInstancePortType.TCP;
     }
 
     return port;