@flui-cloud/cli 0.0.1

package/lib/cli/src/services/cli-cluster-creator.service.js

@@ -0,0 +1,655 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var CliClusterCreatorService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CliClusterCreatorService = void 0;
+const common_1 = require("@nestjs/common");
+const cluster_entity_1 = require("../../../src/modules/infrastructure/clusters/entities/cluster.entity");
+const cluster_node_entity_1 = require("../../../src/modules/infrastructure/clusters/entities/cluster-node.entity");
+const infrastructure_operations_entity_1 = require("../../../src/modules/infrastructure/servers/entities/infrastructure-operations.entity");
+const cli_cluster_repository_1 = require("../lib/repositories/cli-cluster.repository");
+const cli_node_repository_1 = require("../lib/repositories/cli-node.repository");
+const cli_operation_repository_1 = require("../lib/repositories/cli-operation.repository");
+const provider_factory_1 = require("../../../src/modules/providers/services/provider.factory");
+const encryption_service_1 = require("../../../src/modules/shared/encryption/services/encryption.service");
+const cli_k3s_script_service_1 = require("./cli-k3s-script.service");
+const nip_base_domain_util_1 = require("../lib/nip-base-domain.util");
+const label_service_1 = require("../../../src/modules/infrastructure/shared/services/label.service");
+const cli_ca_service_1 = require("./cli-ca.service");
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const os = __importStar(require("node:os"));
+const https = __importStar(require("node:https"));
+const hetzner_firewall_service_1 = require("../../../src/modules/providers/services/hetzner-firewall.service");
+const cli_firewall_repository_1 = require("../lib/repositories/cli-firewall.repository");
+const cli_ssh_service_1 = require("./cli-ssh.service");
+const cli_logger_service_1 = require("./cli-logger.service");
+const cli_vnet_repository_1 = require("../lib/repositories/cli-vnet.repository");
+/**
+ * CLI Cluster Creator Service
+ *
+ * Creates K3s clusters synchronously without Bull queue.
+ * This service bypasses the queue processor and creates infrastructure directly.
+ */
+let CliClusterCreatorService = CliClusterCreatorService_1 = class CliClusterCreatorService {
+    constructor(clusterRepository, nodeRepository, operationRepository, providerFactory, encryptionService, k3sScriptService, labelService, caService, firewallService, firewallRepository, sshService, cliLoggerService, vnetRepository) {
+        this.clusterRepository = clusterRepository;
+        this.nodeRepository = nodeRepository;
+        this.operationRepository = operationRepository;
+        this.providerFactory = providerFactory;
+        this.encryptionService = encryptionService;
+        this.k3sScriptService = k3sScriptService;
+        this.labelService = labelService;
+        this.caService = caService;
+        this.firewallService = firewallService;
+        this.firewallRepository = firewallRepository;
+        this.sshService = sshService;
+        this.cliLoggerService = cliLoggerService;
+        this.vnetRepository = vnetRepository;
+        this.logger = new common_1.Logger(CliClusterCreatorService_1.name);
+    }
+    /** Log to both NestJS stdout and operation log file */
+    log(operationId, message, level = 'INFO') {
+        if (level === 'ERROR')
+            this.logger.error(message);
+        else if (level === 'WARN')
+            this.logger.warn(message);
+        else
+            this.logger.log(message);
+        this.cliLoggerService.writeLog(operationId, message, level);
+    }
+    /**
+     * Create cluster synchronously (bypasses Bull queue)
+     */
+    async createClusterSync(cluster, operation) {
+        const opId = operation.id;
+        this.log(opId, `[CLI Mode] Creating cluster ${cluster.name} synchronously...`);
+        try {
+            // Update operation status
+            operation.status = infrastructure_operations_entity_1.OperationStatus.IN_PROGRESS;
+            operation.currentStepIndex = 0;
+            await this.operationRepository.save(operation);
+            // Get cloud provider
+            const provider = this.providerFactory.getProvider(cluster.provider);
+            // For CLI mode, we'll use provider-generated SSH keys or user-specified ones
+            // This avoids the AccessService database dependency
+            // Decrypt K3s token
+            const k3sToken = this.encryptionService.decrypt(cluster.k3sTokenEncrypted);
+            // Get firewall ID from metadata if pre-created
+            const metadata = operation.metadata;
+            const firewallId = metadata?.firewallId;
+            // Step 1: Create master node
+            this.log(opId, 'Creating master node...');
+            operation.currentStepIndex = 1;
+            await this.operationRepository.save(operation);
+            const masterServerName = `${cluster.name}-master`;
+            const masterLabels = [
+                { key: 'managed-by', value: 'flui-cloud' },
+                { key: 'flui-cluster-id', value: cluster.id },
+                { key: 'flui-resource-type', value: 'cluster-node' },
+                { key: 'flui-node-type', value: 'master' },
+            ];
+            // Generate bootstrap SSH key for master node
+            const masterBootstrapKey = await this.generateBootstrapKey(provider, cluster.id, masterServerName);
+            // Get CA public key for enrollment
+            const caPublicKey = await this.caService.getCaPublicKey();
+            const decrypted = this.decryptClusterSecrets(cluster);
+            const { postgresPassword, redisPassword, grafanaPassword, encryptionKey, fluiApiKey, providerToken, providerScalewayAccessKey, providerScalewaySecretKey, providerRegions, zitadelMasterkey, zitadelDbAdminPassword, zitadelDbUserPassword, zitadelAdminTempPassword, jwtSecret, adminEmail, adminPassword, } = decrypted;
+            const clusterMeta = cluster.metadata;
+            const envVnet = clusterMeta?.envVnet;
+            // Create master node entity FIRST to get the database ID
+            const masterNode = this.nodeRepository.create({
+                cluster,
+                clusterId: cluster.id,
+                providerResourceId: '', // Will be updated after server creation
+                serverName: masterServerName,
+                nodeType: cluster_node_entity_1.NodeType.MASTER,
+                status: cluster_node_entity_1.NodeStatus.CREATING,
+                ipAddress: '', // Will be updated after server creation
+                metadata: {},
+            });
+            await this.nodeRepository.save(masterNode);
+            this.log(opId, `Master node database record created: ${masterNode.id}`);
+            // Resolve Flui shared storage config (NFS+fscache, see scaling doc §14).
+            // Default: enabled. Master gets a Volume hosting the NFS export; workers
+            // mount it via NFSv4 + fscache. Disabled at creation via --no-shared-storage.
+            const sharedStorageEnabled = cluster.sharedStorageEnabled !== false;
+            const sharedStorageVolumeSizeGb = cluster.sharedStorageVolumeSizeGb ?? 20;
+            const masterUserData = await this.k3sScriptService.generateMasterScript({
+                serverId: masterNode.id, // Use database node ID
+                clusterId: cluster.id,
+                clusterName: cluster.name,
+                k3sToken,
+                instanceId: `${cluster.name}-master`,
+                instanceName: masterServerName,
+                provider: cluster.provider,
+                caPublicKey,
+                operationId: operation.id,
+                deployObservabilityStack: cluster.clusterType === cluster_entity_1.ClusterType.OBSERVABILITY,
+                postgresPassword,
+                redisPassword,
+                grafanaPassword,
+                authMode: clusterMeta?.authMode || 'local',
+                jwtSecret,
+                adminEmail,
+                adminPassword,
+                encryptionKey,
+                zitadelMasterkey,
+                zitadelDbAdminPassword,
+                zitadelDbUserPassword,
+                zitadelAdminTempPassword,
+                fluiApiKey,
+                providerApiKey: providerToken,
+                providerScalewayAccessKey,
+                providerScalewaySecretKey,
+                providerRegions,
+                clusterRegion: cluster.region,
+                instanceType: cluster.nodeSize,
+                clusterFirewallId: firewallId || '',
+                nipIoCertEnabled: !clusterMeta?.zitadelDomain,
+                acmeStaging: !!clusterMeta?.acmeStaging,
+                nipHostnameToken: cluster.nipHostnameToken || null,
+                envVnet: envVnet
+                    ? {
+                        vnetProviderResourceId: envVnet.vnetProviderResourceId,
+                        vnetProvider: cluster.provider,
+                        vnetName: 'flui-env-vnet',
+                        vnetIpRange: envVnet.vnetIpRange,
+                        subnetProviderResourceId: envVnet.subnetProviderResourceId,
+                        subnetIpRange: envVnet.subnetIpRange,
+                        subnetType: envVnet.subnetType,
+                        networkZone: envVnet.networkZone,
+                    }
+                    : undefined,
+                sharedStorage: sharedStorageEnabled
+                    ? {
+                        enabled: true,
+                        volumeSizeGb: sharedStorageVolumeSizeGb,
+                    }
+                    : undefined,
+            });
+            const masterServer = await provider.createServer({
+                name: masterServerName,
+                image: cluster.image,
+                server_type: cluster.nodeSize,
+                location: cluster.region,
+                ssh_keys: [masterBootstrapKey.keyId],
+                labels: masterLabels,
+                user_data: masterUserData,
+                firewalls: firewallId ? [firewallId] : undefined,
+                networks: envVnet ? [envVnet.vnetProviderResourceId] : undefined,
+                attachedVolumes: sharedStorageEnabled
+                    ? [
+                        {
+                            name: `${cluster.name}-flui-shared`,
+                            sizeGb: sharedStorageVolumeSizeGb,
+                            labels: masterLabels,
+                        },
+                    ]
+                    : undefined,
+            });
+            // Persist Flui shared storage Volume id for cleanup at destroy time.
+            if (sharedStorageEnabled && masterServer.attachedVolumes?.[0]) {
+                cluster.sharedStorageVolumeId =
+                    masterServer.attachedVolumes[0].volumeId;
+                cluster.sharedStorageVolumeSizeGb =
+                    masterServer.attachedVolumes[0].sizeGb ?? sharedStorageVolumeSizeGb;
+            }
+            masterNode.providerResourceId = masterServer.serverId;
+            masterNode.ipAddress = masterServer.ipAddress;
+            masterNode.privateIp = masterServer.privateIp;
+            await this.nodeRepository.save(masterNode);
+            if (envVnet) {
+                await this.vnetRepository.attachServerToVNet(envVnet.vnetProviderResourceId, masterServer.serverId);
+            }
+            cluster.masterIpAddress = masterServer.ipAddress;
+            cluster.nodeCount = 1;
+            await this.clusterRepository.save(cluster);
+            this.log(opId, `Master node created: ${masterServer.ipAddress}`);
+            // Step 2: Wait for observability stack to be ready
+            this.log(opId, 'Waiting for observability stack deployment...');
+            operation.currentStepIndex = 2;
+            await this.operationRepository.save(operation);
+            await this.waitForObservabilityStackReady(masterServer.ipAddress, 1800000, // 30 min timeout (deployment can take time)
+            cluster.nipHostnameToken);
+            this.log(opId, '✅ Observability stack is fully deployed and ready');
+            // Step 3a: Fetch real kubeconfig from K3s master via SSH
+            this.log(opId, 'Fetching kubeconfig from K3s master...');
+            const kubeconfig = await this.fetchKubeconfig(cluster.masterIpAddress);
+            cluster.kubeconfigEncrypted = this.encryptionService.encrypt(kubeconfig);
+            await this.clusterRepository.save(cluster);
+            this.log(opId, '✅ Kubeconfig generated and encrypted');
+            // Step 3b: Patch Kubernetes secret with SSH CA keys + bootstrap seeder vars
+            await this.createApiCredentialsSecret(opId, cluster.masterIpAddress, {
+                fluiApiKey,
+                providerToken,
+                providerScalewayAccessKey,
+                providerScalewaySecretKey,
+                providerRegions,
+                clusterRegion: cluster.region,
+                instanceType: cluster.nodeSize,
+                envVnet,
+                bootstrapNodePrivateIp: masterServer.privateIp,
+                provider: cluster.provider,
+                sharedStorageVolumeId: cluster.sharedStorageVolumeId ?? undefined,
+                sharedStorageVolumeSizeGb: cluster.sharedStorageVolumeSizeGb ?? undefined,
+            });
+            // Step 3c: Informational — Zitadel PAT injected on demand via sync-auth-domain
+            if (cluster.clusterType === cluster_entity_1.ClusterType.OBSERVABILITY) {
+                this.log(opId, 'ℹ️  Zitadel service account PAT will be injected when sync-auth-domain is called.');
+                this.log(opId, '   After DNS is configured, call: POST /api/v1/clusters/:id/dns-zone/sync-auth-domain');
+            }
+            masterNode.status = cluster_node_entity_1.NodeStatus.READY;
+            await this.nodeRepository.save(masterNode);
+            // Step 4: Create worker nodes if needed
+            const workerCount = metadata?.workerCount || 0;
+            if (workerCount > 0) {
+                this.log(opId, `Creating ${workerCount} worker nodes...`);
+                operation.currentStepIndex = 4;
+                await this.operationRepository.save(operation);
+                for (let i = 0; i < workerCount; i++) {
+                    const workerServerName = `${cluster.name}-worker-${i + 1}`;
+                    const workerLabels = [
+                        { key: 'managed-by', value: 'flui-cloud' },
+                        { key: 'flui-cluster-id', value: cluster.id },
+                        { key: 'flui-resource-type', value: 'cluster-node' },
+                        { key: 'flui-node-type', value: 'worker' },
+                    ];
+                    // Generate bootstrap SSH key for worker node
+                    const workerBootstrapKey = await this.generateBootstrapKey(provider, cluster.id, workerServerName);
+                    // Create worker node entity FIRST to get the database ID
+                    const workerNode = this.nodeRepository.create({
+                        cluster,
+                        clusterId: cluster.id,
+                        providerResourceId: '', // Will be updated after server creation
+                        serverName: workerServerName,
+                        nodeType: cluster_node_entity_1.NodeType.WORKER,
+                        status: cluster_node_entity_1.NodeStatus.CREATING,
+                        ipAddress: '', // Will be updated after server creation
+                        metadata: {},
+                    });
+                    await this.nodeRepository.save(workerNode);
+                    this.log(opId, `Worker node ${i + 1} database record created: ${workerNode.id}`);
+                    // Worker shared storage config: mount NFS from master via fscache.
+                    // Worker only needs masterPrivateIp; the master is already up at
+                    // this point so we read its private IP from the saved entity.
+                    const workerSharedStorage = sharedStorageEnabled && masterNode.privateIp
+                        ? {
+                            enabled: true,
+                            masterPrivateIp: masterNode.privateIp,
+                        }
+                        : undefined;
+                    const workerUserData = await this.k3sScriptService.generateWorkerScript({
+                        serverId: workerNode.id, // Use database node ID
+                        clusterId: cluster.id,
+                        clusterName: cluster.name,
+                        k3sToken,
+                        masterIp: cluster.masterIpAddress,
+                        instanceId: `${cluster.name}-worker-${i + 1}`,
+                        instanceName: workerServerName,
+                        provider: cluster.provider,
+                        caPublicKey,
+                        operationId: operation.id,
+                        sharedStorage: workerSharedStorage,
+                    });
+                    const workerServer = await provider.createServer({
+                        name: workerServerName,
+                        image: cluster.image,
+                        server_type: cluster.nodeSize,
+                        location: cluster.region,
+                        ssh_keys: [workerBootstrapKey.keyId],
+                        labels: workerLabels,
+                        user_data: workerUserData,
+                        firewalls: firewallId ? [firewallId] : undefined,
+                        networks: envVnet ? [envVnet.vnetProviderResourceId] : undefined,
+                    });
+                    workerNode.providerResourceId = workerServer.serverId;
+                    workerNode.ipAddress = workerServer.ipAddress;
+                    workerNode.privateIp = workerServer.privateIp;
+                    await this.nodeRepository.save(workerNode);
+                    if (envVnet) {
+                        await this.vnetRepository.attachServerToVNet(envVnet.vnetProviderResourceId, workerServer.serverId);
+                    }
+                    this.log(opId, `Worker node ${i + 1}/${workerCount} created: ${workerServer.ipAddress}`);
+                }
+                cluster.nodeCount = 1 + workerCount;
+                await this.clusterRepository.save(cluster);
+            }
+            // Update firewall repository with cluster ID if firewall was used
+            if (firewallId) {
+                try {
+                    const firewallRecord = await this.firewallRepository.findById(firewallId);
+                    if (firewallRecord) {
+                        firewallRecord.clusterId = cluster.id;
+                        // Get server IDs from cluster nodes
+                        const serverIds = cluster.nodes.map((node) => node.providerResourceId);
+                        firewallRecord.appliedToServerIds = serverIds;
+                        await this.firewallRepository.save(firewallRecord);
+                        // Update Hetzner firewall labels with cluster ID
+                        try {
+                            const existingLabels = Object.fromEntries(firewallRecord.labels.map((l) => [l.key, l.value]));
+                            const updatedLabels = {
+                                ...existingLabels,
+                                'flui-cluster-id': cluster.id,
+                                'flui-cluster-name': cluster.name,
+                            };
+                            await this.firewallService.updateFirewallLabels(firewallId, updatedLabels);
+                            this.log(opId, `✅ Firewall ${firewallId} labels updated on Hetzner with cluster ID`);
+                        }
+                        catch (labelError) {
+                            this.log(opId, `Failed to update Hetzner firewall labels: ${labelError.message}`, 'WARN');
+                        }
+                        this.log(opId, `✅ Firewall ${firewallId} configured for cluster ${cluster.name}`);
+                    }
+                }
+                catch (error) {
+                    this.log(opId, `Failed to update firewall record: ${error.message}`, 'WARN');
+                }
+            }
+            // Mark cluster as READY
+            cluster.status = cluster_entity_1.ClusterStatus.READY;
+            await this.clusterRepository.save(cluster);
+            // Mark operation as COMPLETED
+            operation.status = infrastructure_operations_entity_1.OperationStatus.COMPLETED;
+            operation.currentStepIndex = operation.totalSteps;
+            await this.operationRepository.save(operation);
+            this.log(opId, `Cluster ${cluster.name} created successfully!`);
+        }
+        catch (error) {
+            this.log(opId, `Failed to create cluster: ${error.message}`, 'ERROR');
+            // Mark cluster as FAILED
+            cluster.status = cluster_entity_1.ClusterStatus.ERROR;
+            await this.clusterRepository.save(cluster);
+            // Mark operation as FAILED
+            operation.status = infrastructure_operations_entity_1.OperationStatus.FAILED;
+            operation.metadata = {
+                ...operation.metadata,
+                error: error.message,
+                errorStack: error.stack,
+            };
+            await this.operationRepository.save(operation);
+            throw error;
+        }
+    }
+    /**
+     * Generate bootstrap SSH key for a node
+     * Returns the SSH key ID from the cloud provider and local key paths
+     */
+    decryptClusterSecrets(cluster) {
+        const meta = (cluster.metadata ?? {});
+        const tryDecrypt = (v) => v ? this.encryptionService.decrypt(v) : '';
+        const encryptionKeyPath = path.join(os.homedir(), '.flui', 'encryption.key');
+        return {
+            postgresPassword: this.encryptionService.decrypt(meta.postgresPasswordEncrypted),
+            redisPassword: this.encryptionService.decrypt(meta.redisPasswordEncrypted),
+            grafanaPassword: this.encryptionService.decrypt(meta.grafanaPasswordEncrypted),
+            encryptionKey: fs.existsSync(encryptionKeyPath)
+                ? fs.readFileSync(encryptionKeyPath, 'utf-8').trim()
+                : '',
+            fluiApiKey: meta.fluiApiKey || '',
+            providerToken: tryDecrypt(meta.providerTokenEncrypted),
+            providerScalewayAccessKey: tryDecrypt(meta.providerScalewayAccessKeyEncrypted),
+            providerScalewaySecretKey: tryDecrypt(meta.providerScalewaySecretKeyEncrypted),
+            providerRegions: meta.providerRegions || '',
+            zitadelMasterkey: tryDecrypt(meta.zitadelMasterkeyEncrypted),
+            zitadelDbAdminPassword: tryDecrypt(meta.zitadelDbAdminPasswordEncrypted),
+            zitadelDbUserPassword: tryDecrypt(meta.zitadelDbUserPasswordEncrypted),
+            zitadelAdminTempPassword: tryDecrypt(meta.zitadelAdminTempPasswordEncrypted),
+            jwtSecret: tryDecrypt(meta.jwtSecretEncrypted),
+            adminEmail: meta.adminEmail || '',
+            adminPassword: tryDecrypt(meta.adminPasswordEncrypted),
+        };
+    }
+    async generateBootstrapKey(provider, clusterId, nodeName) {
+        const bootstrapDir = path.join(os.homedir(), '.flui', 'bootstrap-keys', clusterId);
+        // Ensure bootstrap directory exists
+        if (!fs.existsSync(bootstrapDir)) {
+            fs.mkdirSync(bootstrapDir, { recursive: true, mode: 0o700 });
+        }
+        const privateKeyPath = path.join(bootstrapDir, `${nodeName}-bootstrap`);
+        const publicKeyPath = `${privateKeyPath}.pub`;
+        this.logger.debug(`Generating bootstrap SSH key for ${nodeName}...`);
+        // Generate ED25519 SSH keypair
+        (0, node_child_process_1.execFileSync)('ssh-keygen', [
+            '-t',
+            'ed25519',
+            '-f',
+            privateKeyPath,
+            '-N',
+            '',
+            '-C',
+            `flui-bootstrap-${nodeName}`,
+        ], { stdio: 'pipe' });
+        // Set correct permissions
+        fs.chmodSync(privateKeyPath, 0o600);
+        fs.chmodSync(publicKeyPath, 0o644);
+        // Read public key content
+        const publicKeyContent = fs.readFileSync(publicKeyPath, 'utf-8').trim();
+        // Upload to cloud provider with labels (Record<string, string> format)
+        const bootstrapLabels = {
+            'managed-by': 'flui-cloud',
+            'flui-cluster-id': clusterId,
+            'flui-resource-type': 'ssh-key',
+            'flui-ssh-key-name': nodeName,
+        };
+        this.logger.debug(`Uploading bootstrap key to provider for ${nodeName}...`);
+        const sshKey = await provider.createSSHKey(`flui-bootstrap-${clusterId}-${nodeName}`, publicKeyContent, bootstrapLabels);
+        this.logger.log(`Bootstrap key created for ${nodeName} (ID: ${sshKey.id})`);
+        return {
+            keyId: sshKey.id.toString(),
+            publicKeyPath,
+            privateKeyPath,
+        };
+    }
+    /**
+     * Get CA enrollment script for cloud-init
+     */
+    async getCaEnrollmentScript() {
+        return await this.caService.getEnrollmentScript();
+    }
+    /**
+     * Wait for observability stack to be ready by polling health endpoint
+     */
+    async waitForObservabilityStackReady(masterIp, timeout, nipHostnameToken) {
+        const startTime = Date.now();
+        const baseDomain = (0, nip_base_domain_util_1.buildNipBaseDomain)(masterIp, nipHostnameToken);
+        const healthUrl = `https://app.${baseDomain}/`;
+        let lastStatus = '';
+        this.logger.log(`Polling health endpoint: ${healthUrl}`);
+        this.logger.log(`Timeout: ${timeout / 1000}s (${timeout / 60000} minutes)`);
+        const agent = new https.Agent({ rejectUnauthorized: false });
+        const checkUrl = (url) => new Promise((resolve) => {
+            const req = https.get(url, { agent }, (res) => {
+                resolve(res.statusCode ?? null);
+                res.resume();
+            });
+            req.setTimeout(8000, () => {
+                req.destroy();
+                resolve(null);
+            });
+            req.on('error', () => resolve(null));
+        });
+        while (Date.now() - startTime < timeout) {
+            const status = await checkUrl(healthUrl);
+            if (status !== null && status >= 200 && status < 400) {
+                this.logger.log(`✅ Observability stack ready (HTTP ${status} from ${healthUrl})`);
+                return;
+            }
+            if (status !== null && lastStatus !== `pending-${status}`) {
+                this.logger.log(`Bootstrap in progress (HTTP ${status} from Traefik)`);
+                lastStatus = `pending-${status}`;
+            }
+            else if (status === null && lastStatus !== 'unreachable') {
+                if (Date.now() - startTime > 120000) {
+                    this.logger.warn(`Health check unreachable (will retry...)`);
+                }
+                lastStatus = 'unreachable';
+            }
+            await new Promise((resolve) => setTimeout(resolve, 10000));
+        }
+        throw new Error(`Timeout waiting for observability stack to be ready after ${timeout / 1000}s`);
+    }
+    /**
+     * Wait for server to be ready
+     */
+    async waitForServerReady(provider, serverId, timeout) {
+        const startTime = Date.now();
+        while (Date.now() - startTime < timeout) {
+            const server = await provider.getServerStatus(serverId);
+            if (server.status === 'running' || server.status === 'ready') {
+                return;
+            }
+            if (server.status === 'error' || server.status === 'failed') {
+                throw new Error(`Server ${serverId} failed to start: ${server.status}`);
+            }
+            // Wait 5 seconds before checking again
+            await new Promise((resolve) => setTimeout(resolve, 5000));
+        }
+        throw new Error(`Timeout waiting for server ${serverId} to be ready`);
+    }
+    /**
+     * Fetch the real kubeconfig from the K3s master via SSH.
+     * K3s generates a valid kubeconfig with client certificates at
+     * /etc/rancher/k3s/k3s.yaml. We read it and replace 127.0.0.1
+     * with the actual master IP so it's accessible remotely.
+     */
+    async fetchKubeconfig(masterIp) {
+        const raw = await this.sshService.sshExec(masterIp, 'sudo cat /etc/rancher/k3s/k3s.yaml');
+        // K3s writes server: https://127.0.0.1:6443 — replace with real IP
+        return raw.replaceAll('127.0.0.1', masterIp);
+    }
+    /**
+     * Patch the existing flui-secrets Kubernetes secret with SSH CA keys.
+     *
+     * The bootstrap script (00-secrets.yaml) creates flui-secrets with
+     * DB_PASSWORD, REDIS_PASSWORD, ENCRYPTION_KEY, GRAFANA_ADMIN_PASSWORD.
+     * This method adds SSH CA keys so the API can SSH to servers using
+     * the same CA as the CLI (unified SSH access).
+     */
+    async createApiCredentialsSecret(operationId, masterIp, bootstrap) {
+        this.log(operationId, 'Patching Kubernetes secret with SSH CA keys + bootstrap vars...');
+        try {
+            // Load CLI CA keys to share with API for unified SSH access
+            const caKeyDir = path.join(os.homedir(), '.flui', 'ca');
+            const caPrivateKey = fs
+                .readFileSync(path.join(caKeyDir, 'ca_key'), 'utf8')
+                .trim();
+            const caPublicKey = fs
+                .readFileSync(path.join(caKeyDir, 'ca_key.pub'), 'utf8')
+                .trim();
+            // Base64-encode for Kubernetes Opaque secret data field
+            const b64 = (s) => Buffer.from(s).toString('base64');
+            // Encode the entire patch JSON as base64, decode on the server, then apply.
+            // This avoids all shell escaping issues since sshExec wraps commands
+            // in double quotes and JSON contains double quotes that would conflict.
+            const data = {
+                SSH_CA_PRIVATE_KEY: b64(caPrivateKey),
+                SSH_CA_PUBLIC_KEY: b64(caPublicKey),
+                FLUI_CLI_API_KEY: b64(bootstrap.fluiApiKey),
+                PROVIDER_HETZNER_API_KEY: bootstrap.provider === 'hetzner'
+                    ? b64(bootstrap.providerToken)
+                    : b64(''),
+                PROVIDER_SCALEWAY_ACCESS_KEY: b64(bootstrap.providerScalewayAccessKey || ''),
+                PROVIDER_SCALEWAY_SECRET_KEY: b64(bootstrap.providerScalewaySecretKey || ''),
+                PROVIDER_REGIONS: b64(bootstrap.providerRegions),
+                CLUSTER_REGION: b64(bootstrap.clusterRegion),
+                INSTANCE_TYPE: b64(bootstrap.instanceType),
+            };
+            if (bootstrap.envVnet) {
+                data.FLUI_VNET_PROVIDER_RESOURCE_ID = b64(bootstrap.envVnet.vnetProviderResourceId);
+                data.FLUI_VNET_PROVIDER = b64(bootstrap.provider);
+                data.FLUI_VNET_NAME = b64('flui-env-vnet');
+                data.FLUI_VNET_IP_RANGE = b64(bootstrap.envVnet.vnetIpRange);
+                data.FLUI_SUBNET_PROVIDER_RESOURCE_ID = b64(bootstrap.envVnet.subnetProviderResourceId);
+                data.FLUI_SUBNET_IP_RANGE = b64(bootstrap.envVnet.subnetIpRange);
+                data.FLUI_SUBNET_TYPE = b64(bootstrap.envVnet.subnetType);
+                data.FLUI_SUBNET_NETWORK_ZONE = b64(bootstrap.envVnet.networkZone);
+            }
+            if (bootstrap.bootstrapNodePrivateIp) {
+                data.FLUI_BOOTSTRAP_NODE_PRIVATE_IP = b64(bootstrap.bootstrapNodePrivateIp);
+            }
+            if (bootstrap.sharedStorageVolumeId) {
+                data.FLUI_SHARED_STORAGE_VOLUME_ID = b64(bootstrap.sharedStorageVolumeId);
+            }
+            if (bootstrap.sharedStorageVolumeSizeGb) {
+                data.FLUI_SHARED_STORAGE_VOLUME_GB = b64(String(bootstrap.sharedStorageVolumeSizeGb));
+            }
+            const patchJson = JSON.stringify({ data });
+            const patchBase64 = Buffer.from(patchJson).toString('base64');
+            const writeAndPatchCmd = `echo ${patchBase64} | base64 -d > /tmp/flui-ssh-ca-patch.json` +
+                ` && (kubectl patch secret flui-secrets -n flui-system --type merge` +
+                `      --patch-file /tmp/flui-ssh-ca-patch.json` +
+                `   || kubectl patch secret flui-secrets -n default --type merge` +
+                `      --patch-file /tmp/flui-ssh-ca-patch.json)` +
+                ` && rm -f /tmp/flui-ssh-ca-patch.json` +
+                ` && (kubectl rollout restart deployment/flui-api -n flui-system 2>/dev/null` +
+                ` || kubectl rollout restart deployment/flui-api -n default 2>/dev/null` +
+                ` || true)`;
+            await this.sshService.sshExec(masterIp, writeAndPatchCmd);
+            this.log(operationId, '✅ Kubernetes secret patched with SSH CA keys and bootstrap vars');
+        }
+        catch (error) {
+            this.log(operationId, `Failed to patch Kubernetes secret: ${error.message}`, 'WARN');
+            this.log(operationId, 'You can add SSH CA manually later.', 'WARN');
+        }
+    }
+};
+exports.CliClusterCreatorService = CliClusterCreatorService;
+exports.CliClusterCreatorService = CliClusterCreatorService = CliClusterCreatorService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [cli_cluster_repository_1.CliClusterRepository,
+        cli_node_repository_1.CliNodeRepository,
+        cli_operation_repository_1.CliOperationRepository,
+        provider_factory_1.ProviderFactory,
+        encryption_service_1.EncryptionService,
+        cli_k3s_script_service_1.CliK3sScriptService,
+        label_service_1.LabelService,
+        cli_ca_service_1.CliCaService,
+        hetzner_firewall_service_1.HetznerFirewallService,
+        cli_firewall_repository_1.CliFirewallRepository,
+        cli_ssh_service_1.CliSshService,
+        cli_logger_service_1.CliLoggerService,
+        cli_vnet_repository_1.CliVnetRepository])
+], CliClusterCreatorService);