@kapeta/local-cluster-service 0.40.4 → 0.40.5

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [0.40.5](https://github.com/kapetacom/local-cluster-service/compare/v0.40.4...v0.40.5) (2024-04-09)
+
+
+### Bug Fixes
+
+* Move most / big env vars to config file instead ([#137](https://github.com/kapetacom/local-cluster-service/issues/137)) ([21c5c50](https://github.com/kapetacom/local-cluster-service/commit/21c5c50341665c2919b9c1ed718b933a10a459fb))
+
 ## [0.40.4](https://github.com/kapetacom/local-cluster-service/compare/v0.40.3...v0.40.4) (2024-04-04)
 
 

package/dist/cjs/src/operatorManager.d.ts

@@ -22,7 +22,7 @@ declare class OperatorManager {
     /**
      * Get operator definition for resource type
      */
-    getOperator(fullName: string, version: string): Promise<Operator>;
+    getOperator(fullName: string, version: string): Promise<Operator | null>;
     /**
      * Get information about a specific consumed resource
      */
@@ -35,7 +35,7 @@ declare class OperatorManager {
      * @param kind the full name - e.g. myhandle/rabbitmq
      * @param version the version of the operator
      */
-    ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo>;
+    ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo | null>;
 }
 export declare const operatorManager: OperatorManager;
 export {};

package/dist/cjs/src/operatorManager.js

@@ -61,7 +61,8 @@ class OperatorManager {
             throw new Error(`Unknown operator type: ${fullName}:${version}`);
         }
         if (!operator.definition.spec || !operator.definition.spec.local) {
-            throw new Error(`Operator missing local definition: ${fullName}:${version}`);
+            console.warn(`Operator missing local definition: ${fullName}:${version}`);
+            return null;
         }
         return new Operator(operator);
     }
@@ -95,6 +96,16 @@ class OperatorManager {
         }
         const kindUri = (0, nodejs_utils_1.parseKapetaUri)(blockResource.kind);
         const operator = await this.getOperator(resourceType, kindUri.version);
+        if (!operator) {
+            return {
+                host: '',
+                port: 0,
+                type: '',
+                protocol: '',
+                options: {},
+                credentials: {},
+            };
+        }
         const credentials = operator.getCredentials();
         if (ensureContainer) {
             await this.ensureOperator(systemId, resourceType, kindUri.version);
@@ -120,6 +131,9 @@ class OperatorManager {
     }
     async getOperatorPorts(systemId, kind, version) {
         const operator = await this.getOperator(kind, version);
+        if (!operator) {
+            return {};
+        }
         const operatorData = operator.getLocalData();
         const portTypes = Object.keys(operatorData.ports);
         portTypes.sort();
@@ -150,6 +164,9 @@ class OperatorManager {
         const key = `${systemId}#${kind}:${version}`;
         return await this.operatorLock.acquire(key, async () => {
             const operator = await this.getOperator(kind, version);
+            if (!operator) {
+                return null;
+            }
             const operatorData = operator.getLocalData();
             const ports = await this.getOperatorPorts(systemId, kind, version);
             const nameParts = [systemId, kind.toLowerCase(), version];

package/dist/cjs/src/utils/BlockInstanceRunner.d.ts

@@ -25,4 +25,5 @@ export declare class BlockInstanceRunner {
     private getServiceBlockPortBindings;
     private ensureContainer;
     private _handleContainer;
+    private getFileHash;
 }

package/dist/cjs/src/utils/BlockInstanceRunner.js

@@ -23,9 +23,12 @@ const node_path_1 = __importDefault(require("node:path"));
 const taskManager_1 = require("../taskManager");
 const InternalConfigProvider_1 = require("./InternalConfigProvider");
 const config_mapper_1 = require("@kapeta/config-mapper");
+const crypto_1 = __importDefault(require("crypto"));
 const KAPETA_SYSTEM_ID = 'KAPETA_SYSTEM_ID';
 const KAPETA_BLOCK_REF = 'KAPETA_BLOCK_REF';
 const KAPETA_INSTANCE_ID = 'KAPETA_INSTANCE_ID';
+// The maximum length of an environment variable - this is to avoid hitting the command line length limits
+const MAX_ENV_LENGTH = 256;
 /**
  * Needed when running local docker containers as part of plan
  * @type {string[]}
@@ -111,21 +114,28 @@ class BlockInstanceRunner {
         const realBaseDir = await fs_extra_1.default.realpath(baseDir);
         const internalConfigProvider = await (0, InternalConfigProvider_1.createInternalConfigProvider)(this._systemId, blockInstance.id, assetVersion);
         // Resolve the environment variables
-        const envVars = await (0, config_mapper_1.resolveKapetaVariables)(realBaseDir, internalConfigProvider);
+        const variables = await (0, config_mapper_1.resolveKapetaVariables)(realBaseDir, internalConfigProvider);
         // Write out the config templates if they exist
-        await (0, config_mapper_1.writeConfigTemplates)(envVars, realBaseDir);
+        await (0, config_mapper_1.writeConfigTemplates)(variables, realBaseDir);
+        const env = await (0, config_mapper_1.getEnvironmentVariables)(variables);
+        // Gets the tmp path to write the config file
+        const configFilePath = (0, config_mapper_1.getConfigFilePath)(blockInstance.id);
+        // Write the config to a file - will be read by the SDKs
+        await (0, config_mapper_1.writeEnvConfigFile)(variables, configFilePath);
+        // Add the config file path to the environment variables
+        env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = configFilePath;
         let processInfo;
         if (providerVersion.definition.kind === types_1.KIND_BLOCK_TYPE_OPERATOR) {
-            processInfo = await this._startOperatorProcess(blockInstance, blockUri, providerVersion, envVars);
+            processInfo = await this._startOperatorProcess(blockInstance, blockUri, providerVersion, env);
         }
         else {
             //We need a port type to know how to connect to the block consistently
             const portTypes = getServiceProviderPorts(assetVersion, providerVersion);
             if (blockUri.version === 'local') {
-                processInfo = await this._startLocalProcess(blockInstance, blockUri, envVars, assetVersion);
+                processInfo = await this._startLocalProcess(blockInstance, blockUri, env, assetVersion);
             }
             else {
-                processInfo = await this._startDockerProcess(blockInstance, blockUri, envVars, assetVersion);
+                processInfo = await this._startDockerProcess(blockInstance, blockUri, env, assetVersion);
             }
             if (portTypes.length > 0) {
                 processInfo.portType = portTypes[0];
@@ -200,6 +210,14 @@ class BlockInstanceRunner {
         if (localContainer.healthcheck) {
             HealthCheck = containerManager_1.containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
+        if (env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]) {
+            // If we have a config file, we need to bind it to the container and adjust the env var
+            const localConfig = `/${config_mapper_1.KAPETA_ENV_CONFIG_FILE}`;
+            Binds.push(`${(0, containerManager_1.toLocalBindVolume)(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+            // We also provide the hash to detect changes to the config file
+            env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]);
+            env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = localConfig;
+        }
         const Mounts = isDockerImage
             ? // For docker images we mount the local directory to the working directory
                 containerManager_1.containerManager.toDockerMounts({
@@ -208,6 +226,15 @@ class BlockInstanceRunner {
             : // For dockerfiles we don't mount anything
                 [];
         const systemUri = (0, nodejs_utils_1.parseKapetaUri)(this._systemId);
+        const Env = [
+            ...customEnvs,
+            ...DOCKER_ENV_VARS,
+            `KAPETA_LOCAL_CLUSTER_PORT=${clusterService_1.clusterService.getClusterServicePort()}`,
+            ...Object.entries({
+                ...env,
+                ...addonEnv,
+            }).map(([key, value]) => `${key}=${value}`),
+        ];
         return this.ensureContainer({
             ...dockerOpts,
             Image: dockerImage,
@@ -222,15 +249,7 @@ class BlockInstanceRunner {
             HealthCheck,
             ExposedPorts,
             Cmd: startCmd ? startCmd.split(/\s+/g) : [],
-            Env: [
-                ...customEnvs,
-                ...DOCKER_ENV_VARS,
-                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService_1.clusterService.getClusterServicePort()}`,
-                ...Object.entries({
-                    ...env,
-                    ...addonEnv,
-                }).map(([key, value]) => `${key}=${value}`),
-            ],
+            Env,
             HostConfig: {
                 ...customHostConfigs,
                 Binds: [
@@ -275,6 +294,15 @@ class BlockInstanceRunner {
         // For windows we need to default to root
         const innerHome = process.platform === 'win32' ? '/root/.kapeta' : local_cluster_config_1.default.getKapetaBasedir();
         const systemUri = (0, nodejs_utils_1.parseKapetaUri)(this._systemId);
+        const Binds = [`${(0, containerManager_1.toLocalBindVolume)(local_cluster_config_1.default.getKapetaBasedir())}:${innerHome}`];
+        if (env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]) {
+            // If we have a config file, we need to bind it to the container and adjust the env var
+            const localConfig = `/${config_mapper_1.KAPETA_ENV_CONFIG_FILE}`;
+            Binds.push(`${(0, containerManager_1.toLocalBindVolume)(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+            // We also provide the hash to detect changes to the config file
+            env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]);
+            env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = localConfig;
+        }
         return this.ensureContainer({
             Image: dockerImage,
             name: containerName,
@@ -293,7 +321,7 @@ class BlockInstanceRunner {
                 }).map(([key, value]) => `${key}=${value}`),
             ],
             HostConfig: {
-                Binds: [`${(0, containerManager_1.toLocalBindVolume)(local_cluster_config_1.default.getKapetaBasedir())}:${innerHome}`],
+                Binds,
                 PortBindings,
             },
         });
@@ -360,6 +388,14 @@ class BlockInstanceRunner {
                     `${(0, containerManager_1.toLocalBindVolume)(kapetaYmlPath)}:/kapeta.yml:ro`,
                     `${(0, containerManager_1.toLocalBindVolume)(local_cluster_config_1.default.getKapetaBasedir())}:${innerHome}`,
                 ];
+            if (!local.singleton && env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]) {
+                // If we have a config file, we need to bind it to the container and adjust the env var
+                const localConfig = `/${config_mapper_1.KAPETA_ENV_CONFIG_FILE}`;
+                Binds.push(`${(0, containerManager_1.toLocalBindVolume)(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+                // We also provide the hash to detect changes to the config file
+                env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]);
+                env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = localConfig;
+            }
             const systemUri = (0, nodejs_utils_1.parseKapetaUri)(this._systemId);
             console.log(`Ensuring container for operator block: ${containerName} [singleton: ${!!local.singleton}]`);
             logs.addLog(`Ensuring container for operator block: ${containerName}`);
@@ -436,5 +472,13 @@ class BlockInstanceRunner {
             pid: container.id,
         };
     }
+    async getFileHash(filePath) {
+        const content = await fs_extra_1.default.readFile(filePath);
+        const hash = crypto_1.default.createHash('md5');
+        //passing the data to be hashed
+        const data = hash.update(content);
+        //Creating the hash in the required format
+        return data.digest('hex');
+    }
 }
 exports.BlockInstanceRunner = BlockInstanceRunner;

package/dist/esm/src/operatorManager.d.ts

@@ -22,7 +22,7 @@ declare class OperatorManager {
     /**
      * Get operator definition for resource type
      */
-    getOperator(fullName: string, version: string): Promise<Operator>;
+    getOperator(fullName: string, version: string): Promise<Operator | null>;
     /**
      * Get information about a specific consumed resource
      */
@@ -35,7 +35,7 @@ declare class OperatorManager {
      * @param kind the full name - e.g. myhandle/rabbitmq
      * @param version the version of the operator
      */
-    ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo>;
+    ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo | null>;
 }
 export declare const operatorManager: OperatorManager;
 export {};

package/dist/esm/src/operatorManager.js

@@ -61,7 +61,8 @@ class OperatorManager {
             throw new Error(`Unknown operator type: ${fullName}:${version}`);
         }
         if (!operator.definition.spec || !operator.definition.spec.local) {
-            throw new Error(`Operator missing local definition: ${fullName}:${version}`);
+            console.warn(`Operator missing local definition: ${fullName}:${version}`);
+            return null;
         }
         return new Operator(operator);
     }
@@ -95,6 +96,16 @@ class OperatorManager {
         }
         const kindUri = (0, nodejs_utils_1.parseKapetaUri)(blockResource.kind);
         const operator = await this.getOperator(resourceType, kindUri.version);
+        if (!operator) {
+            return {
+                host: '',
+                port: 0,
+                type: '',
+                protocol: '',
+                options: {},
+                credentials: {},
+            };
+        }
         const credentials = operator.getCredentials();
         if (ensureContainer) {
             await this.ensureOperator(systemId, resourceType, kindUri.version);
@@ -120,6 +131,9 @@ class OperatorManager {
     }
     async getOperatorPorts(systemId, kind, version) {
         const operator = await this.getOperator(kind, version);
+        if (!operator) {
+            return {};
+        }
         const operatorData = operator.getLocalData();
         const portTypes = Object.keys(operatorData.ports);
         portTypes.sort();
@@ -150,6 +164,9 @@ class OperatorManager {
         const key = `${systemId}#${kind}:${version}`;
         return await this.operatorLock.acquire(key, async () => {
             const operator = await this.getOperator(kind, version);
+            if (!operator) {
+                return null;
+            }
             const operatorData = operator.getLocalData();
             const ports = await this.getOperatorPorts(systemId, kind, version);
             const nameParts = [systemId, kind.toLowerCase(), version];

package/dist/esm/src/utils/BlockInstanceRunner.d.ts

@@ -25,4 +25,5 @@ export declare class BlockInstanceRunner {
     private getServiceBlockPortBindings;
     private ensureContainer;
     private _handleContainer;
+    private getFileHash;
 }

package/dist/esm/src/utils/BlockInstanceRunner.js

@@ -23,9 +23,12 @@ const node_path_1 = __importDefault(require("node:path"));
 const taskManager_1 = require("../taskManager");
 const InternalConfigProvider_1 = require("./InternalConfigProvider");
 const config_mapper_1 = require("@kapeta/config-mapper");
+const crypto_1 = __importDefault(require("crypto"));
 const KAPETA_SYSTEM_ID = 'KAPETA_SYSTEM_ID';
 const KAPETA_BLOCK_REF = 'KAPETA_BLOCK_REF';
 const KAPETA_INSTANCE_ID = 'KAPETA_INSTANCE_ID';
+// The maximum length of an environment variable - this is to avoid hitting the command line length limits
+const MAX_ENV_LENGTH = 256;
 /**
  * Needed when running local docker containers as part of plan
  * @type {string[]}
@@ -111,21 +114,28 @@ class BlockInstanceRunner {
         const realBaseDir = await fs_extra_1.default.realpath(baseDir);
         const internalConfigProvider = await (0, InternalConfigProvider_1.createInternalConfigProvider)(this._systemId, blockInstance.id, assetVersion);
         // Resolve the environment variables
-        const envVars = await (0, config_mapper_1.resolveKapetaVariables)(realBaseDir, internalConfigProvider);
+        const variables = await (0, config_mapper_1.resolveKapetaVariables)(realBaseDir, internalConfigProvider);
         // Write out the config templates if they exist
-        await (0, config_mapper_1.writeConfigTemplates)(envVars, realBaseDir);
+        await (0, config_mapper_1.writeConfigTemplates)(variables, realBaseDir);
+        const env = await (0, config_mapper_1.getEnvironmentVariables)(variables);
+        // Gets the tmp path to write the config file
+        const configFilePath = (0, config_mapper_1.getConfigFilePath)(blockInstance.id);
+        // Write the config to a file - will be read by the SDKs
+        await (0, config_mapper_1.writeEnvConfigFile)(variables, configFilePath);
+        // Add the config file path to the environment variables
+        env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = configFilePath;
         let processInfo;
         if (providerVersion.definition.kind === types_1.KIND_BLOCK_TYPE_OPERATOR) {
-            processInfo = await this._startOperatorProcess(blockInstance, blockUri, providerVersion, envVars);
+            processInfo = await this._startOperatorProcess(blockInstance, blockUri, providerVersion, env);
         }
         else {
             //We need a port type to know how to connect to the block consistently
             const portTypes = getServiceProviderPorts(assetVersion, providerVersion);
             if (blockUri.version === 'local') {
-                processInfo = await this._startLocalProcess(blockInstance, blockUri, envVars, assetVersion);
+                processInfo = await this._startLocalProcess(blockInstance, blockUri, env, assetVersion);
             }
             else {
-                processInfo = await this._startDockerProcess(blockInstance, blockUri, envVars, assetVersion);
+                processInfo = await this._startDockerProcess(blockInstance, blockUri, env, assetVersion);
             }
             if (portTypes.length > 0) {
                 processInfo.portType = portTypes[0];
@@ -200,6 +210,14 @@ class BlockInstanceRunner {
         if (localContainer.healthcheck) {
             HealthCheck = containerManager_1.containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
+        if (env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]) {
+            // If we have a config file, we need to bind it to the container and adjust the env var
+            const localConfig = `/${config_mapper_1.KAPETA_ENV_CONFIG_FILE}`;
+            Binds.push(`${(0, containerManager_1.toLocalBindVolume)(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+            // We also provide the hash to detect changes to the config file
+            env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]);
+            env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = localConfig;
+        }
         const Mounts = isDockerImage
             ? // For docker images we mount the local directory to the working directory
                 containerManager_1.containerManager.toDockerMounts({
@@ -208,6 +226,15 @@ class BlockInstanceRunner {
             : // For dockerfiles we don't mount anything
                 [];
         const systemUri = (0, nodejs_utils_1.parseKapetaUri)(this._systemId);
+        const Env = [
+            ...customEnvs,
+            ...DOCKER_ENV_VARS,
+            `KAPETA_LOCAL_CLUSTER_PORT=${clusterService_1.clusterService.getClusterServicePort()}`,
+            ...Object.entries({
+                ...env,
+                ...addonEnv,
+            }).map(([key, value]) => `${key}=${value}`),
+        ];
         return this.ensureContainer({
             ...dockerOpts,
             Image: dockerImage,
@@ -222,15 +249,7 @@ class BlockInstanceRunner {
             HealthCheck,
             ExposedPorts,
             Cmd: startCmd ? startCmd.split(/\s+/g) : [],
-            Env: [
-                ...customEnvs,
-                ...DOCKER_ENV_VARS,
-                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService_1.clusterService.getClusterServicePort()}`,
-                ...Object.entries({
-                    ...env,
-                    ...addonEnv,
-                }).map(([key, value]) => `${key}=${value}`),
-            ],
+            Env,
             HostConfig: {
                 ...customHostConfigs,
                 Binds: [
@@ -275,6 +294,15 @@ class BlockInstanceRunner {
         // For windows we need to default to root
         const innerHome = process.platform === 'win32' ? '/root/.kapeta' : local_cluster_config_1.default.getKapetaBasedir();
         const systemUri = (0, nodejs_utils_1.parseKapetaUri)(this._systemId);
+        const Binds = [`${(0, containerManager_1.toLocalBindVolume)(local_cluster_config_1.default.getKapetaBasedir())}:${innerHome}`];
+        if (env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]) {
+            // If we have a config file, we need to bind it to the container and adjust the env var
+            const localConfig = `/${config_mapper_1.KAPETA_ENV_CONFIG_FILE}`;
+            Binds.push(`${(0, containerManager_1.toLocalBindVolume)(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+            // We also provide the hash to detect changes to the config file
+            env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]);
+            env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = localConfig;
+        }
         return this.ensureContainer({
             Image: dockerImage,
             name: containerName,
@@ -293,7 +321,7 @@ class BlockInstanceRunner {
                 }).map(([key, value]) => `${key}=${value}`),
             ],
             HostConfig: {
-                Binds: [`${(0, containerManager_1.toLocalBindVolume)(local_cluster_config_1.default.getKapetaBasedir())}:${innerHome}`],
+                Binds,
                 PortBindings,
             },
         });
@@ -360,6 +388,14 @@ class BlockInstanceRunner {
                     `${(0, containerManager_1.toLocalBindVolume)(kapetaYmlPath)}:/kapeta.yml:ro`,
                     `${(0, containerManager_1.toLocalBindVolume)(local_cluster_config_1.default.getKapetaBasedir())}:${innerHome}`,
                 ];
+            if (!local.singleton && env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]) {
+                // If we have a config file, we need to bind it to the container and adjust the env var
+                const localConfig = `/${config_mapper_1.KAPETA_ENV_CONFIG_FILE}`;
+                Binds.push(`${(0, containerManager_1.toLocalBindVolume)(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+                // We also provide the hash to detect changes to the config file
+                env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[config_mapper_1.KAPETA_CONFIG_ENV_VAR]);
+                env[config_mapper_1.KAPETA_CONFIG_ENV_VAR] = localConfig;
+            }
             const systemUri = (0, nodejs_utils_1.parseKapetaUri)(this._systemId);
             console.log(`Ensuring container for operator block: ${containerName} [singleton: ${!!local.singleton}]`);
             logs.addLog(`Ensuring container for operator block: ${containerName}`);
@@ -436,5 +472,13 @@ class BlockInstanceRunner {
             pid: container.id,
         };
     }
+    async getFileHash(filePath) {
+        const content = await fs_extra_1.default.readFile(filePath);
+        const hash = crypto_1.default.createHash('md5');
+        //passing the data to be hashed
+        const data = hash.update(content);
+        //Creating the hash in the required format
+        return data.digest('hex');
+    }
 }
 exports.BlockInstanceRunner = BlockInstanceRunner;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.40.4",
+    "version": "0.40.5",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -49,7 +49,7 @@
     "homepage": "https://github.com/kapetacom/local-cluster-service#readme",
     "dependencies": {
         "@kapeta/codegen": "^1.3.0",
-        "@kapeta/config-mapper": "^1.1.1",
+        "@kapeta/config-mapper": "^1.2.0",
         "@kapeta/local-cluster-config": "^0.4.1",
         "@kapeta/nodejs-api-client": ">=0.2.0 <2",
         "@kapeta/nodejs-process": "^1.2.0",

package/src/operatorManager.ts

@@ -83,7 +83,8 @@ class OperatorManager {
         }
 
         if (!operator.definition.spec || !operator.definition.spec.local) {
-            throw new Error(`Operator missing local definition: ${fullName}:${version}`);
+            console.warn(`Operator missing local definition: ${fullName}:${version}`);
+            return null;
         }
 
         return new Operator(operator);
@@ -138,6 +139,16 @@ class OperatorManager {
 
         const kindUri = parseKapetaUri(blockResource.kind);
         const operator = await this.getOperator(resourceType, kindUri.version);
+        if (!operator) {
+            return {
+                host: '',
+                port: 0,
+                type: '',
+                protocol: '',
+                options: {},
+                credentials: {},
+            };
+        }
         const credentials = operator.getCredentials();
         if (ensureContainer) {
             await this.ensureOperator(systemId, resourceType, kindUri.version);
@@ -168,6 +179,9 @@ class OperatorManager {
 
     async getOperatorPorts(systemId: string, kind: string, version: string) {
         const operator = await this.getOperator(kind, version);
+        if (!operator) {
+            return {};
+        }
 
         const operatorData = operator.getLocalData();
 
@@ -201,13 +215,16 @@ class OperatorManager {
      * @param kind the full name - e.g. myhandle/rabbitmq
      * @param version the version of the operator
      */
-    async ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo> {
+    async ensureOperator(systemId: string, kind: string, version: string): Promise<ContainerInfo | null> {
         systemId = normalizeKapetaUri(systemId);
 
         const key = `${systemId}#${kind}:${version}`;
 
         return await this.operatorLock.acquire(key, async () => {
             const operator = await this.getOperator(kind, version);
+            if (!operator) {
+                return null;
+            }
 
             const operatorData = operator.getLocalData();
 

package/src/utils/BlockInstanceRunner.ts

@@ -34,12 +34,24 @@ import Path from 'node:path';
 import { taskManager } from '../taskManager';
 import { LocalDevContainer, LocalInstance } from '@kapeta/schemas';
 import { createInternalConfigProvider } from './InternalConfigProvider';
-import { resolveKapetaVariables, writeConfigTemplates } from '@kapeta/config-mapper';
+import {
+    getConfigFilePath,
+    getEnvironmentVariables,
+    KAPETA_CONFIG_ENV_VAR,
+    KAPETA_ENV_CONFIG_FILE,
+    resolveKapetaVariables,
+    writeConfigTemplates,
+    writeEnvConfigFile,
+} from '@kapeta/config-mapper';
+import crypto from 'crypto';
 
 const KAPETA_SYSTEM_ID = 'KAPETA_SYSTEM_ID';
 const KAPETA_BLOCK_REF = 'KAPETA_BLOCK_REF';
 const KAPETA_INSTANCE_ID = 'KAPETA_INSTANCE_ID';
 
+// The maximum length of an environment variable - this is to avoid hitting the command line length limits
+const MAX_ENV_LENGTH = 256;
+
 /**
  * Needed when running local docker containers as part of plan
  * @type {string[]}
@@ -145,23 +157,34 @@ export class BlockInstanceRunner {
         );
 
         // Resolve the environment variables
-        const envVars = await resolveKapetaVariables(realBaseDir, internalConfigProvider);
+        const variables = await resolveKapetaVariables(realBaseDir, internalConfigProvider);
 
         // Write out the config templates if they exist
-        await writeConfigTemplates(envVars, realBaseDir);
+        await writeConfigTemplates(variables, realBaseDir);
+
+        const env = await getEnvironmentVariables(variables);
+
+        // Gets the tmp path to write the config file
+        const configFilePath = getConfigFilePath(blockInstance.id);
+
+        // Write the config to a file - will be read by the SDKs
+        await writeEnvConfigFile(variables, configFilePath);
+
+        // Add the config file path to the environment variables
+        env[KAPETA_CONFIG_ENV_VAR] = configFilePath;
 
         let processInfo: ProcessInfo;
 
         if (providerVersion.definition.kind === KIND_BLOCK_TYPE_OPERATOR) {
-            processInfo = await this._startOperatorProcess(blockInstance, blockUri, providerVersion, envVars);
+            processInfo = await this._startOperatorProcess(blockInstance, blockUri, providerVersion, env);
         } else {
             //We need a port type to know how to connect to the block consistently
             const portTypes = getServiceProviderPorts(assetVersion, providerVersion);
 
             if (blockUri.version === 'local') {
-                processInfo = await this._startLocalProcess(blockInstance, blockUri, envVars, assetVersion);
+                processInfo = await this._startLocalProcess(blockInstance, blockUri, env, assetVersion);
             } else {
-                processInfo = await this._startDockerProcess(blockInstance, blockUri, envVars, assetVersion);
+                processInfo = await this._startDockerProcess(blockInstance, blockUri, env, assetVersion);
             }
 
             if (portTypes.length > 0) {
@@ -269,6 +292,15 @@ export class BlockInstanceRunner {
             HealthCheck = containerManager.toDockerHealth({ cmd: localContainer.healthcheck });
         }
 
+        if (env[KAPETA_CONFIG_ENV_VAR]) {
+            // If we have a config file, we need to bind it to the container and adjust the env var
+            const localConfig = `/${KAPETA_ENV_CONFIG_FILE}`;
+            Binds.push(`${toLocalBindVolume(env[KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+            // We also provide the hash to detect changes to the config file
+            env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[KAPETA_CONFIG_ENV_VAR]);
+            env[KAPETA_CONFIG_ENV_VAR] = localConfig;
+        }
+
         const Mounts = isDockerImage
             ? // For docker images we mount the local directory to the working directory
               containerManager.toDockerMounts({
@@ -279,6 +311,16 @@ export class BlockInstanceRunner {
 
         const systemUri = parseKapetaUri(this._systemId);
 
+        const Env = [
+            ...customEnvs,
+            ...DOCKER_ENV_VARS,
+            `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
+            ...Object.entries({
+                ...env,
+                ...addonEnv,
+            }).map(([key, value]) => `${key}=${value}`),
+        ];
+
         return this.ensureContainer({
             ...dockerOpts,
             Image: dockerImage,
@@ -293,15 +335,7 @@ export class BlockInstanceRunner {
             HealthCheck,
             ExposedPorts,
             Cmd: startCmd ? startCmd.split(/\s+/g) : [],
-            Env: [
-                ...customEnvs,
-                ...DOCKER_ENV_VARS,
-                `KAPETA_LOCAL_CLUSTER_PORT=${clusterService.getClusterServicePort()}`,
-                ...Object.entries({
-                    ...env,
-                    ...addonEnv,
-                }).map(([key, value]) => `${key}=${value}`),
-            ],
+            Env,
             HostConfig: {
                 ...customHostConfigs,
                 Binds: [
@@ -372,6 +406,16 @@ export class BlockInstanceRunner {
         const innerHome = process.platform === 'win32' ? '/root/.kapeta' : ClusterConfig.getKapetaBasedir();
         const systemUri = parseKapetaUri(this._systemId);
 
+        const Binds = [`${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`];
+        if (env[KAPETA_CONFIG_ENV_VAR]) {
+            // If we have a config file, we need to bind it to the container and adjust the env var
+            const localConfig = `/${KAPETA_ENV_CONFIG_FILE}`;
+            Binds.push(`${toLocalBindVolume(env[KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+            // We also provide the hash to detect changes to the config file
+            env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[KAPETA_CONFIG_ENV_VAR]);
+            env[KAPETA_CONFIG_ENV_VAR] = localConfig;
+        }
+
         return this.ensureContainer({
             Image: dockerImage,
             name: containerName,
@@ -390,7 +434,7 @@ export class BlockInstanceRunner {
                 }).map(([key, value]) => `${key}=${value}`),
             ],
             HostConfig: {
-                Binds: [`${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`],
+                Binds,
                 PortBindings,
             },
         });
@@ -486,6 +530,15 @@ export class BlockInstanceRunner {
                           `${toLocalBindVolume(ClusterConfig.getKapetaBasedir())}:${innerHome}`,
                       ];
 
+                if (!local.singleton && env[KAPETA_CONFIG_ENV_VAR]) {
+                    // If we have a config file, we need to bind it to the container and adjust the env var
+                    const localConfig = `/${KAPETA_ENV_CONFIG_FILE}`;
+                    Binds.push(`${toLocalBindVolume(env[KAPETA_CONFIG_ENV_VAR])}:${localConfig}:ro`);
+                    // We also provide the hash to detect changes to the config file
+                    env['KAPETA_CONFIG_HASH'] = await this.getFileHash(env[KAPETA_CONFIG_ENV_VAR]);
+                    env[KAPETA_CONFIG_ENV_VAR] = localConfig;
+                }
+
                 const systemUri = parseKapetaUri(this._systemId);
 
                 console.log(
@@ -583,4 +636,13 @@ export class BlockInstanceRunner {
             pid: container.id,
         };
     }
+
+    private async getFileHash(filePath: string) {
+        const content = await FSExtra.readFile(filePath);
+        const hash = crypto.createHash('md5');
+        //passing the data to be hashed
+        const data = hash.update(content);
+        //Creating the hash in the required format
+        return data.digest('hex');
+    }
 }