@flui-cloud/cli 0.0.1

package/lib/cli/src/services/cli-clusters.service.js

@@ -0,0 +1,762 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var CliClustersService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CliClustersService = void 0;
+const common_1 = require("@nestjs/common");
+const node_crypto_1 = require("node:crypto");
+const uuid_1 = require("uuid");
+const config_storage_1 = require("../lib/config-storage");
+const cluster_entity_1 = require("../../../src/modules/infrastructure/clusters/entities/cluster.entity");
+const hostname_mode_enum_1 = require("../../../src/modules/dns/enums/hostname-mode.enum");
+const nip_token_util_1 = require("../../../src/modules/dns/utils/nip-token.util");
+const infrastructure_operations_entity_1 = require("../../../src/modules/infrastructure/servers/entities/infrastructure-operations.entity");
+const cli_cluster_repository_1 = require("../lib/repositories/cli-cluster.repository");
+const cli_operation_repository_1 = require("../lib/repositories/cli-operation.repository");
+const cli_node_repository_1 = require("../lib/repositories/cli-node.repository");
+const cli_firewall_repository_1 = require("../lib/repositories/cli-firewall.repository");
+const encryption_service_1 = require("../../../src/modules/shared/encryption/services/encryption.service");
+const cli_ssh_service_1 = require("./cli-ssh.service");
+const cli_ca_service_1 = require("./cli-ca.service");
+const provider_factory_1 = require("../../../src/modules/providers/services/provider.factory");
+const hetzner_firewall_service_1 = require("../../../src/modules/providers/services/hetzner-firewall.service");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const os = __importStar(require("node:os"));
+/**
+ * CLI Clusters Service
+ *
+ * Simplified version of ClustersService for CLI usage.
+ * Uses file-based repositories and Bull queue mock.
+ * Integrates SSH and CA management for secure server access.
+ */
+let CliClustersService = CliClustersService_1 = class CliClustersService {
+    constructor(clusterRepository, operationRepository, nodeRepository, firewallRepository, encryptionService, sshService, caService, providerFactory, firewallService, infrastructureQueue) {
+        this.clusterRepository = clusterRepository;
+        this.operationRepository = operationRepository;
+        this.nodeRepository = nodeRepository;
+        this.firewallRepository = firewallRepository;
+        this.encryptionService = encryptionService;
+        this.sshService = sshService;
+        this.caService = caService;
+        this.providerFactory = providerFactory;
+        this.firewallService = firewallService;
+        this.infrastructureQueue = infrastructureQueue;
+        this.logger = new common_1.Logger(CliClustersService_1.name);
+    }
+    /**
+     * Create a new cluster
+     */
+    async create(createClusterDto) {
+        this.logger.log(`Creating cluster: ${createClusterDto.name}`);
+        // Get or create SSH key pair
+        const { publicKey: sshPublicKey } = await this.sshService.getOrCreateSshKey();
+        this.logger.log('SSH key ready for cluster provisioning');
+        // Get or create CA certificate
+        const caPublicKey = await this.caService.getCaPublicKey();
+        this.logger.log('CA certificate ready for cluster provisioning');
+        // Generate and encrypt K3s token
+        const k3sToken = this.generateK3sToken();
+        const k3sTokenEncrypted = this.encryptionService.encrypt(k3sToken);
+        // Generate and encrypt observability stack passwords
+        const postgresPassword = this.generateSecurePassword(64);
+        const redisPassword = this.generateSecurePassword(64);
+        const grafanaPassword = this.generateSecurePassword(64);
+        // Generate CLI API key (pre-seeded into flui-secrets for BootstrapSeeder)
+        const fluiApiKey = `flui_${(0, uuid_1.v4)().replaceAll('-', '')}`;
+        // Read provider credentials for bootstrap injection.
+        // Hetzner uses a single API token; Scaleway uses an Access Key ID + Secret Key pair.
+        // Both shapes are propagated to the cluster pod so BootstrapSeeder can re-create
+        // the right credential rows server-side without an extra HTTP roundtrip.
+        const configStorage = new config_storage_1.ConfigStorage();
+        let providerToken = '';
+        let providerScalewayAccessKey = '';
+        let providerScalewaySecretKey = '';
+        let providerRegions = '';
+        if (createClusterDto.provider === 'scaleway') {
+            const creds = configStorage.getCredentials('scaleway');
+            providerScalewayAccessKey = creds?.accessKey || '';
+            providerScalewaySecretKey = creds?.secretKey || '';
+            // Scaleway authenticates HTTP calls with the Secret Key (X-Auth-Token);
+            // keep providerToken populated so legacy code paths still work.
+            providerToken = providerScalewaySecretKey;
+            providerRegions = 'fr-par,nl-ams,pl-waw';
+        }
+        else {
+            providerToken = configStorage.getToken(createClusterDto.provider) || '';
+            providerRegions = 'nbg1,fsn1,hel1,ash,hil';
+        }
+        // Generate JWT secret and admin credentials for local auth mode
+        const jwtSecret = this.generateSecurePassword(64);
+        const adminEmail = createClusterDto.metadata?.adminEmail || '';
+        const adminPassword = this.generateSecurePassword(24);
+        // Generate Zitadel secrets (only used by OBSERVABILITY clusters)
+        const zitadelMasterkey = this.generateSecurePassword(32);
+        const zitadelDbAdminPassword = this.generateSecurePassword(32);
+        const zitadelDbUserPassword = this.generateSecurePassword(32);
+        const zitadelAdminTempPassword = this.generateSecurePassword(24);
+        // Get worker count from DTO
+        const workerCount = createClusterDto.workerCount;
+        // Determine cluster type from metadata (same logic as API)
+        const metadata = createClusterDto.metadata || {};
+        const clusterType = metadata.isObservabilityCluster
+            ? cluster_entity_1.ClusterType.OBSERVABILITY
+            : cluster_entity_1.ClusterType.WORKLOAD;
+        const hostnameMode = createClusterDto.endpointHostnameMode ?? hostname_mode_enum_1.HostnameMode.IP;
+        let nipHostnameToken = null;
+        if (hostnameMode === hostname_mode_enum_1.HostnameMode.IP) {
+            if (createClusterDto.nipHostnameToken) {
+                if (!(0, nip_token_util_1.isValidNipHostnameToken)(createClusterDto.nipHostnameToken)) {
+                    throw new Error('nipHostnameToken must match [a-z0-9-], 1-30 chars, no leading/trailing dash.');
+                }
+                nipHostnameToken = createClusterDto.nipHostnameToken;
+            }
+            else {
+                nipHostnameToken = (0, nip_token_util_1.generateNipHostnameToken)();
+            }
+            this.logger.log(`Cluster ${createClusterDto.name} nip.io hostname token: ${nipHostnameToken}`);
+        }
+        // Create cluster entity with SSH and CA info
+        const cluster = this.clusterRepository.create({
+            name: createClusterDto.name,
+            provider: createClusterDto.provider,
+            region: createClusterDto.region,
+            nodeSize: createClusterDto.nodeSize,
+            image: createClusterDto.image || 'ubuntu-24.04',
+            nodeCount: 0, // Will be updated during creation
+            status: cluster_entity_1.ClusterStatus.CREATING,
+            clusterType,
+            k3sTokenEncrypted,
+            k3sVersion: createClusterDto.k3sVersion || 'v1.35.4+k3s1',
+            endpointHostnameMode: hostnameMode,
+            nipHostnameToken,
+            sharedStorageEnabled: createClusterDto.sharedStorageEnabled,
+            sharedStorageVolumeSizeGb: createClusterDto.sharedStorageVolumeSizeGb,
+            metadata: {
+                ...createClusterDto.metadata,
+                sshPublicKey,
+                caPublicKey,
+                // Store encrypted passwords for observability stack
+                postgresPasswordEncrypted: this.encryptionService.encrypt(postgresPassword),
+                redisPasswordEncrypted: this.encryptionService.encrypt(redisPassword),
+                grafanaPasswordEncrypted: this.encryptionService.encrypt(grafanaPassword),
+                // Auth mode — used by env:credentials to know which sections to show
+                authMode: metadata.authMode || 'local',
+                // Store encrypted JWT secret and admin credentials for local auth
+                jwtSecretEncrypted: this.encryptionService.encrypt(jwtSecret),
+                adminEmail,
+                adminPasswordEncrypted: this.encryptionService.encrypt(adminPassword),
+                // Store encrypted Zitadel secrets
+                zitadelMasterkeyEncrypted: this.encryptionService.encrypt(zitadelMasterkey),
+                zitadelDbAdminPasswordEncrypted: this.encryptionService.encrypt(zitadelDbAdminPassword),
+                zitadelDbUserPasswordEncrypted: this.encryptionService.encrypt(zitadelDbUserPassword),
+                zitadelAdminTempPasswordEncrypted: this.encryptionService.encrypt(zitadelAdminTempPassword),
+                // Bootstrap seeder vars — injected into flui-secrets, read by BootstrapSeeder at API startup
+                fluiApiKey, // plain text — written into K8s secret, not stored encrypted here
+                providerTokenEncrypted: this.encryptionService.encrypt(providerToken),
+                providerScalewayAccessKeyEncrypted: providerScalewayAccessKey
+                    ? this.encryptionService.encrypt(providerScalewayAccessKey)
+                    : '',
+                providerScalewaySecretKeyEncrypted: providerScalewaySecretKey
+                    ? this.encryptionService.encrypt(providerScalewaySecretKey)
+                    : '',
+                providerRegions,
+            },
+            sshKeyIds: createClusterDto.sshKeys || [],
+        });
+        await this.clusterRepository.save(cluster);
+        // Create operation
+        const operation = this.operationRepository.create({
+            operationType: infrastructure_operations_entity_1.OperationType.CREATE_CLUSTER,
+            status: infrastructure_operations_entity_1.OperationStatus.PENDING,
+            resourceId: cluster.id,
+            resourceType: 'cluster',
+            resourceName: cluster.name,
+            provider: cluster.provider,
+            totalSteps: 5 + workerCount,
+            currentStepIndex: 0,
+            metadata: {
+                clusterId: cluster.id,
+                clusterConfig: createClusterDto,
+                estimatedDurationInSeconds: 80,
+                targetNodeCount: workerCount + 1,
+                workerCount,
+                sshPublicKey,
+                caPublicKey,
+                firewallId: createClusterDto.metadata?.firewallId,
+                sourceCidrs: createClusterDto.metadata?.sourceCidrs,
+            },
+        });
+        await this.operationRepository.save(operation);
+        // Add job to queue (our mock queue will execute synchronously)
+        await this.infrastructureQueue.add('create-cluster', {
+            clusterId: cluster.id,
+            operationId: operation.id,
+        });
+        return { cluster, operation };
+    }
+    /**
+     * Get cluster by ID
+     */
+    async findOne(id) {
+        return this.clusterRepository.findOne({
+            where: { id },
+            relations: ['nodes'],
+        });
+    }
+    /**
+     * Get kubeconfig for cluster
+     */
+    async getKubeconfig(clusterId) {
+        const cluster = await this.findOne(clusterId);
+        if (!cluster?.kubeconfigEncrypted) {
+            return null;
+        }
+        return this.encryptionService.decrypt(cluster.kubeconfigEncrypted);
+    }
+    /**
+     * Delete cluster with orphaned VPS cleanup
+     */
+    async remove(id) {
+        const cluster = await this.findOne(id);
+        if (!cluster) {
+            throw new Error(`Cluster ${id} not found`);
+        }
+        this.logger.log(`Deleting observability cluster ${id} (${cluster.name})`);
+        this.logger.log(`Cluster details: ${JSON.stringify({
+            id: cluster.id,
+            name: cluster.name,
+            provider: cluster.provider,
+            region: cluster.region,
+            status: cluster.status,
+            nodeCount: cluster.nodeCount,
+        })}`);
+        // Update status to DELETING
+        cluster.status = cluster_entity_1.ClusterStatus.DELETING;
+        await this.clusterRepository.save(cluster);
+        try {
+            // Get provider instance
+            const provider = this.providerFactory.getProvider(cluster.provider);
+            this.logger.log(`Using provider: ${cluster.provider}`);
+            // PHASE 1: Delete nodes tracked in local storage
+            const localNodes = await this.nodeRepository.find({
+                where: { clusterId: cluster.id },
+            });
+            this.logger.log(`Phase 1: Found ${localNodes.length} nodes in local storage`);
+            if (localNodes.length > 0) {
+                this.logger.log(`Local nodes details: ${JSON.stringify(localNodes.map((n) => ({
+                    id: n.id,
+                    serverName: n.serverName,
+                    providerResourceId: n.providerResourceId,
+                    nodeType: n.nodeType,
+                    ipAddress: n.ipAddress,
+                })))}`);
+            }
+            const deletedServerIds = new Set();
+            const deletionActions = [];
+            // STEP 1: Initiate deletion for all tracked servers
+            for (const node of localNodes) {
+                if (node.providerResourceId) {
+                    this.logger.log(`Initiating deletion for tracked server ${node.serverName} (${node.providerResourceId})`);
+                    try {
+                        const result = await provider.deleteServer({
+                            server_id: node.providerResourceId,
+                            provider: cluster.provider,
+                            force: true, // Force delete even if server is running
+                        });
+                        if (result.actionId) {
+                            deletionActions.push({
+                                serverId: node.providerResourceId,
+                                actionId: result.actionId,
+                                serverName: node.serverName,
+                            });
+                            this.logger.log(`✓ Deletion initiated for ${node.serverName} (Action ID: ${result.actionId})`);
+                        }
+                        else {
+                            // If no actionId, mark as deleted immediately (might be already deleted)
+                            deletedServerIds.add(node.providerResourceId);
+                            this.logger.log(`✓ Server ${node.serverName} deleted (no action tracking needed)`);
+                        }
+                    }
+                    catch (error) {
+                        // Log error but continue with other nodes
+                        this.logger.error(`✗ Failed to delete tracked server ${node.serverName}:`, error.message);
+                    }
+                    // Remove node from file storage
+                    await this.nodeRepository.remove(node);
+                }
+            }
+            // STEP 2: Wait for all deletion actions to complete
+            if (deletionActions.length > 0) {
+                this.logger.log(`Waiting for ${deletionActions.length} server deletion(s) to complete...`);
+                // Check if provider supports waitForActionCompletion (Hetzner does)
+                if (typeof provider.waitForActionCompletion === 'function') {
+                    for (const action of deletionActions) {
+                        try {
+                            this.logger.log(`Waiting for server ${action.serverName} deletion to complete (Action ID: ${action.actionId})...`);
+                            await provider.waitForActionCompletion(action.actionId);
+                            deletedServerIds.add(action.serverId);
+                            this.logger.log(`✓ Server ${action.serverName} fully deleted`);
+                        }
+                        catch (error) {
+                            this.logger.error(`✗ Failed to wait for server ${action.serverName} deletion:`, error.message);
+                            // Even if waiting fails, mark as deleted to continue cleanup
+                            deletedServerIds.add(action.serverId);
+                        }
+                    }
+                }
+                else {
+                    this.logger.warn('Provider does not support action tracking. Proceeding without waiting.');
+                    // Mark all as deleted if provider doesn't support action tracking
+                    deletionActions.forEach((action) => deletedServerIds.add(action.serverId));
+                }
+                this.logger.log(`✓ All tracked server deletions completed (${deletedServerIds.size} servers deleted)`);
+            }
+            // PHASE 2: Scan provider for orphaned VPS with matching observability-id
+            this.logger.log(`Phase 2: Scanning ${cluster.provider} for orphaned VPS...`);
+            const allServers = await provider.listServersAsDto();
+            this.logger.log(`Found ${allServers.length} total servers on provider`);
+            if (allServers.length > 0) {
+                this.logger.log(`All servers summary: ${JSON.stringify(allServers.map((s) => ({
+                    id: s.provider_resource_id,
+                    name: s.name,
+                    labels: s.labels?.map((l) => `${l.key}=${l.value}`),
+                })))}`);
+            }
+            // Filter servers that belong to this observability cluster
+            const orphanedServers = allServers.filter((server) => {
+                // Check if already deleted in Phase 1
+                if (deletedServerIds.has(server.provider_resource_id)) {
+                    this.logger.log(`Skipping server ${server.name} - already deleted in Phase 1`);
+                    return false;
+                }
+                // Check if server belongs to this observability cluster
+                const hasObservabilityId = server.labels?.some((label) => label.key === 'flui-cluster-id' && label.value === cluster.id);
+                // Check if server is managed by flui
+                const isFluiManaged = server.labels?.some((label) => label.key === 'managed-by' && label.value === 'flui-cloud');
+                const isMatch = hasObservabilityId && isFluiManaged;
+                this.logger.log(`Server ${server.name}: isFluiManaged=${isFluiManaged}, hasObservabilityId=${hasObservabilityId}, isMatch=${isMatch}`);
+                return isMatch;
+            });
+            this.logger.log(`Found ${orphanedServers.length} orphaned VPS to clean up`);
+            // Delete orphaned VPS with safety checks
+            const orphanedDeletionActions = [];
+            for (const server of orphanedServers) {
+                if (this.verifySafeDeletion(server, cluster.id)) {
+                    try {
+                        this.logger.log(`Initiating deletion for orphaned VPS ${server.name} (${server.provider_resource_id})`);
+                        const result = await provider.deleteServer({
+                            server_id: server.provider_resource_id,
+                            provider: cluster.provider,
+                            force: true, // Force delete even if server is running
+                        });
+                        if (result.actionId) {
+                            orphanedDeletionActions.push({
+                                serverId: server.provider_resource_id,
+                                actionId: result.actionId,
+                                serverName: server.name,
+                            });
+                            this.logger.log(`✓ Deletion initiated for ${server.name} (Action ID: ${result.actionId})`);
+                        }
+                        else {
+                            this.logger.log(`✓ Orphaned VPS ${server.name} deleted (no action tracking needed)`);
+                        }
+                    }
+                    catch (error) {
+                        this.logger.error(`✗ Failed to delete orphaned VPS ${server.name}:`, error.message);
+                    }
+                }
+                else {
+                    this.logger.warn(`⚠ Skipped VPS ${server.name} - safety check failed`);
+                }
+            }
+            // Wait for all orphaned server deletions to complete
+            if (orphanedDeletionActions.length > 0) {
+                this.logger.log(`Waiting for ${orphanedDeletionActions.length} orphaned VPS deletion(s) to complete...`);
+                if (typeof provider.waitForActionCompletion === 'function') {
+                    for (const action of orphanedDeletionActions) {
+                        try {
+                            this.logger.log(`Waiting for orphaned VPS ${action.serverName} deletion to complete (Action ID: ${action.actionId})...`);
+                            await provider.waitForActionCompletion(action.actionId);
+                            this.logger.log(`✓ Orphaned VPS ${action.serverName} fully deleted`);
+                        }
+                        catch (error) {
+                            this.logger.error(`✗ Failed to wait for orphaned VPS ${action.serverName} deletion:`, error.message);
+                        }
+                    }
+                }
+                this.logger.log(`✓ All orphaned VPS deletions completed (${orphanedDeletionActions.length} servers deleted)`);
+            }
+            // PHASE 2.5: Delete cluster firewalls (GUARANTEED CLEANUP)
+            this.logger.log(`Phase 2.5: Deleting cluster firewalls from ${cluster.provider}...`);
+            try {
+                // STEP 1: Find ALL firewalls for this cluster
+                this.logger.log(`Searching for firewalls with label flui-cluster-id=${cluster.id}`);
+                const clusterFirewalls = await this.firewallService.listFirewalls({
+                    clusterId: cluster.id,
+                });
+                this.logger.log(`Found ${clusterFirewalls.length} firewall(s) for cluster ${cluster.id}`);
+                // STEP 1b: FALLBACK - Search by name pattern if no firewalls found by label
+                let allFirewalls = [...clusterFirewalls];
+                if (clusterFirewalls.length === 0) {
+                    this.logger.log(`No firewalls found by label. Searching by name pattern...`);
+                    // Search for firewalls with observability pattern:
+                    // - Old pattern: flui-observability-{clusterId}
+                    // - New pattern: flui-observability-firewall-{timestamp}
+                    const allProviderFirewalls = await this.firewallService.listFirewalls({});
+                    const nameMatchedFirewalls = allProviderFirewalls.filter((fw) => fw.name === `flui-observability-${cluster.id}` ||
+                        fw.name.startsWith('flui-observability-firewall-') ||
+                        fw.name.startsWith('flui-firewall-temp-'));
+                    if (nameMatchedFirewalls.length > 0) {
+                        this.logger.log(`Found ${nameMatchedFirewalls.length} firewall(s) by name pattern:`);
+                        nameMatchedFirewalls.forEach((fw) => {
+                            this.logger.log(`  - ${fw.name} (${fw.id})`);
+                        });
+                        allFirewalls.push(...nameMatchedFirewalls);
+                    }
+                    else {
+                        this.logger.log(`No firewalls found with observability name patterns`);
+                    }
+                }
+                if (allFirewalls.length === 0) {
+                    this.logger.log('✓ No firewalls to delete for this cluster');
+                }
+                else {
+                    // STEP 2: Delete each firewall with retry logic for "resource_in_use" errors
+                    const deletePromises = allFirewalls.map(async (fw) => {
+                        const maxRetries = 5;
+                        const retryDelay = 5000; // 5 seconds
+                        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+                            try {
+                                this.logger.log(`Deleting firewall ${fw.name} (${fw.id}) - Attempt ${attempt}/${maxRetries}`);
+                                await this.firewallService.deleteFirewall(fw.id);
+                                this.logger.log(`✓ Deleted firewall ${fw.name}`);
+                                return { success: true, id: fw.id, name: fw.name };
+                            }
+                            catch (error) {
+                                const isResourceInUse = error.message?.includes('resource_in_use') ||
+                                    error.message?.includes('still in use');
+                                if (isResourceInUse && attempt < maxRetries) {
+                                    this.logger.warn(`Firewall ${fw.name} is still in use. Waiting ${retryDelay}ms before retry ${attempt + 1}/${maxRetries}...`);
+                                    await new Promise((resolve) => setTimeout(resolve, retryDelay));
+                                    continue; // Retry
+                                }
+                                this.logger.error(`✗ Failed to delete firewall ${fw.name} after ${attempt} attempts: ${error.message}`);
+                                return { success: false, id: fw.id, name: fw.name, error };
+                            }
+                        }
+                        return {
+                            success: false,
+                            id: fw.id,
+                            name: fw.name,
+                            error: new Error('Max retries exceeded'),
+                        };
+                    });
+                    const results = await Promise.all(deletePromises);
+                    const failedCount = results.filter((r) => !r.success).length;
+                    // STEP 3: VERIFICATION - Re-scan to ensure all deleted
+                    this.logger.log('Verifying all firewalls deleted...');
+                    const remainingFirewalls = await this.firewallService.listFirewalls({
+                        clusterId: cluster.id,
+                    });
+                    if (remainingFirewalls.length > 0) {
+                        this.logger.warn(`⚠️  WARNING: ${remainingFirewalls.length} firewall(s) still exist for cluster after deletion!`);
+                        // RETRY deletion for remaining firewalls
+                        for (const fw of remainingFirewalls) {
+                            try {
+                                this.logger.log(`RETRY: Deleting firewall ${fw.name} (${fw.id})`);
+                                await this.firewallService.deleteFirewall(fw.id);
+                                this.logger.log(`✓ RETRY SUCCESS: Deleted firewall ${fw.name}`);
+                            }
+                            catch (error) {
+                                this.logger.error(`✗ RETRY FAILED: Could not delete firewall ${fw.name}: ${error.message}`);
+                            }
+                        }
+                        // Final verification
+                        const finalCheck = await this.firewallService.listFirewalls({
+                            clusterId: cluster.id,
+                        });
+                        if (finalCheck.length > 0) {
+                            this.logger.error(`❌ CRITICAL: ${finalCheck.length} firewall(s) could not be deleted. Manual cleanup required:`);
+                            finalCheck.forEach((fw) => {
+                                this.logger.error(`   - Firewall ID: ${fw.id}, Name: ${fw.name}`);
+                            });
+                        }
+                        else {
+                            this.logger.log('✓ All firewalls successfully deleted after retry');
+                        }
+                    }
+                    else {
+                        this.logger.log(`✓ Successfully deleted all ${allFirewalls.length} firewall(s)`);
+                    }
+                    if (failedCount > 0) {
+                        this.logger.warn(`${failedCount} firewall deletion(s) failed on first attempt`);
+                    }
+                    // STEP 4: Clean up firewall from local storage
+                    this.logger.log('Cleaning up firewall from local storage...');
+                    for (const fw of allFirewalls) {
+                        try {
+                            await this.firewallRepository.delete(fw.id);
+                            this.logger.log(`✓ Removed firewall ${fw.id} from local storage`);
+                        }
+                        catch (error) {
+                            this.logger.warn(`Failed to remove firewall ${fw.id} from local storage: ${error.message}`);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                this.logger.error(`Phase 2.5 failed: ${error.message}. Continuing with cluster deletion...`);
+                this.logger.error(`Error stack: ${error.stack}`);
+            }
+            this.logger.log('Phase 2.5 completed: Firewall cleanup finished');
+            // PHASE 3: Clean up bootstrap SSH keys
+            this.logger.log(`Phase 3: Scanning ${cluster.provider} for bootstrap SSH keys...`);
+            if (provider.listSSHKeys && provider.deleteSSHKey) {
+                const allKeys = await provider.listSSHKeys();
+                this.logger.log(`Found ${allKeys.length} total SSH keys on provider`);
+                // Filter bootstrap keys for this observability cluster
+                const bootstrapKeys = allKeys.filter((key) => {
+                    const tags = key.tags || {};
+                    const isFluiManaged = tags['managed-by'] === 'flui-cloud';
+                    const hasMatchingObsId = tags['flui-cluster-id'] === cluster.id;
+                    const isBootstrapKey = tags['flui-resource-type'] === 'ssh-key';
+                    return isFluiManaged && hasMatchingObsId && isBootstrapKey;
+                });
+                this.logger.log(`Found ${bootstrapKeys.length} bootstrap SSH keys to clean up`);
+                // Delete bootstrap keys
+                for (const key of bootstrapKeys) {
+                    try {
+                        this.logger.log(`Deleting bootstrap SSH key ${key.name} (${key.id})`);
+                        await provider.deleteSSHKey(key.id.toString());
+                        this.logger.log(`✓ Deleted bootstrap SSH key ${key.name}`);
+                    }
+                    catch (error) {
+                        this.logger.error(`✗ Failed to delete bootstrap SSH key ${key.name}:`, error.message);
+                    }
+                }
+                // Clean up local bootstrap key files
+                const bootstrapDir = path.join(os.homedir(), '.flui', 'bootstrap-keys', cluster.id);
+                if (fs.existsSync(bootstrapDir)) {
+                    try {
+                        fs.rmSync(bootstrapDir, { recursive: true, force: true });
+                        this.logger.log(`✓ Cleaned up local bootstrap keys: ${bootstrapDir}`);
+                    }
+                    catch (error) {
+                        this.logger.warn(`Failed to cleanup local bootstrap keys: ${error.message}`);
+                    }
+                }
+            }
+            else {
+                this.logger.warn(`Provider ${cluster.provider} does not support SSH key management, skipping Phase 3`);
+            }
+            // Detach + delete the Flui-managed shared storage Volume (if any).
+            // We do this AFTER servers are deleted so the volume is already detached
+            // at the provider; the explicit detachVolume call is a belt-and-braces
+            // safety net for cases where the provider doesn't auto-detach.
+            if (cluster.sharedStorageVolumeId) {
+                const volumeRef = this.formatVolumeRef(cluster.sharedStorageVolumeId, cluster.provider, cluster.region);
+                this.logger.log(`[volume-cleanup] Starting detach+delete for ${volumeRef}`);
+                if (provider.detachVolume) {
+                    try {
+                        await provider.detachVolume(volumeRef);
+                    }
+                    catch (detachErr) {
+                        this.logger.warn(`[volume-cleanup] Detach failed for ${volumeRef}: ${detachErr.message} — will retry delete anyway`);
+                    }
+                }
+                if (provider.deleteVolume) {
+                    // Scaleway can take a few seconds to release the volume reference
+                    // after the server is gone. Retry with backoff before giving up.
+                    const maxAttempts = 6;
+                    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+                        try {
+                            await provider.deleteVolume(volumeRef);
+                            this.logger.log(`[volume-cleanup] ✓ Deleted shared storage volume ${volumeRef}`);
+                            break;
+                        }
+                        catch (volErr) {
+                            const msg = volErr.message;
+                            const lastAttempt = attempt === maxAttempts;
+                            if (lastAttempt) {
+                                this.logger.warn(`[volume-cleanup] FAILED after ${attempt} attempts: ${msg}. ` +
+                                    `Run "flui env orphan-volumes --cleanup" to recover.`);
+                            }
+                            else {
+                                this.logger.warn(`[volume-cleanup] attempt ${attempt}/${maxAttempts} failed (${msg}); retrying in 5s...`);
+                                await new Promise((r) => setTimeout(r, 5000));
+                            }
+                        }
+                    }
+                }
+                else {
+                    this.logger.warn(`[volume-cleanup] Provider ${cluster.provider} has no deleteVolume — leaving ${volumeRef} at provider`);
+                }
+            }
+            // Remove cluster from file storage
+            await this.clusterRepository.remove(cluster);
+            this.logger.log(`✓ Successfully deleted cluster ${cluster.name}`);
+            this.logger.log(`Total: ${localNodes.length} tracked + ${orphanedServers.length} orphaned VPS cleaned up`);
+        }
+        catch (error) {
+            // If deletion fails, revert status to ERROR
+            cluster.status = cluster_entity_1.ClusterStatus.ERROR;
+            await this.clusterRepository.save(cluster);
+            this.logger.error(`Failed to delete cluster ${id}:`, error);
+            throw error;
+        }
+    }
+    /**
+     * Verify that a server is safe to delete
+     * Checks for all required flui labels
+     */
+    formatVolumeRef(volumeId, provider, region) {
+        if (provider !== 'scaleway')
+            return volumeId;
+        if (volumeId.includes(':'))
+            return volumeId;
+        const zone = /^[a-z]{2}-[a-z]{3}$/.test(region) ? `${region}-1` : region;
+        return `${zone}:${volumeId}`;
+    }
+    verifySafeDeletion(server, expectedObservabilityId) {
+        const labels = server.labels || [];
+        // MUST have managed-by = flui-cloud
+        const isFluiManaged = labels.some((label) => label.key === 'managed-by' && label.value === 'flui-cloud');
+        // MUST have matching cluster-id
+        const hasMatchingObsId = labels.some((label) => label.key === 'flui-cluster-id' &&
+            label.value === expectedObservabilityId);
+        // MUST have node-type (master or worker)
+        const hasNodeType = labels.some((label) => label.key === 'flui-node-type' &&
+            (label.value === 'master' || label.value === 'worker'));
+        // MUST have resource-type = cluster-node
+        const isClusterNode = labels.some((label) => label.key === 'flui-resource-type' && label.value === 'cluster-node');
+        const isSafe = isFluiManaged && hasMatchingObsId && hasNodeType && isClusterNode;
+        if (!isSafe) {
+            this.logger.warn(`Safety check FAILED for server ${server.name}:`, {
+                isFluiManaged,
+                hasMatchingObsId,
+                hasNodeType,
+                isClusterNode,
+                labels: labels.map((l) => `${l.key}=${l.value}`),
+            });
+        }
+        return isSafe;
+    }
+    /**
+     * Get or create encryption key from ~/.flui/encryption.key
+     * Same pattern as CA: generate once, reuse forever.
+     * This key is shared between CLI and API (local dev) and deployed
+     * to K8s clusters as ENCRYPTION_KEY in the flui-secrets Secret.
+     */
+    getOrCreateEncryptionKey() {
+        const keyDir = path.join(os.homedir(), '.flui');
+        const keyFilePath = path.join(keyDir, 'encryption.key');
+        // Reuse existing key
+        if (fs.existsSync(keyFilePath)) {
+            const existingKey = fs.readFileSync(keyFilePath, 'utf-8').trim();
+            if (existingKey.length === 64) {
+                this.logger.log('Reusing encryption key from ~/.flui/encryption.key');
+                return existingKey;
+            }
+            this.logger.warn('Invalid encryption key in ~/.flui/encryption.key, regenerating');
+        }
+        // Generate new key
+        const newKey = (0, node_crypto_1.randomBytes)(32).toString('hex');
+        if (!fs.existsSync(keyDir)) {
+            fs.mkdirSync(keyDir, { recursive: true });
+        }
+        fs.writeFileSync(keyFilePath, newKey, { encoding: 'utf-8', mode: 0o600 });
+        this.logger.log('Generated new encryption key at ~/.flui/encryption.key');
+        return newKey;
+    }
+    /**
+     * Generate random K3s token
+     */
+    generateK3sToken() {
+        const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+        let token = '';
+        for (let i = 0; i < 64; i++) {
+            token += chars.charAt((0, node_crypto_1.randomInt)(chars.length));
+        }
+        return token;
+    }
+    /**
+     * Generate secure password for observability stack components
+     * Uses safe characters for bash variables: a-zA-Z0-9-_@.+:=
+     */
+    generateSecurePassword(length = 64) {
+        const lower = 'abcdefghijklmnopqrstuvwxyz';
+        const upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+        const digits = '0123456789';
+        const symbols = '-_@.+:=';
+        const charset = lower + upper + digits + symbols;
+        const chars = [];
+        chars.push(lower.charAt((0, node_crypto_1.randomInt)(lower.length)), upper.charAt((0, node_crypto_1.randomInt)(upper.length)), digits.charAt((0, node_crypto_1.randomInt)(digits.length)), symbols.charAt((0, node_crypto_1.randomInt)(symbols.length)));
+        for (let i = 4; i < length; i++) {
+            chars.push(charset.charAt((0, node_crypto_1.randomInt)(charset.length)));
+        }
+        for (let i = chars.length - 1; i > 0; i--) {
+            const j = (0, node_crypto_1.randomInt)(i + 1);
+            [chars[i], chars[j]] = [chars[j], chars[i]];
+        }
+        return chars.join('');
+    }
+};
+exports.CliClustersService = CliClustersService;
+exports.CliClustersService = CliClustersService = CliClustersService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __param(9, (0, common_1.Inject)('BullQueue_infrastructure')),
+    __metadata("design:paramtypes", [cli_cluster_repository_1.CliClusterRepository,
+        cli_operation_repository_1.CliOperationRepository,
+        cli_node_repository_1.CliNodeRepository,
+        cli_firewall_repository_1.CliFirewallRepository,
+        encryption_service_1.EncryptionService,
+        cli_ssh_service_1.CliSshService,
+        cli_ca_service_1.CliCaService,
+        provider_factory_1.ProviderFactory,
+        hetzner_firewall_service_1.HetznerFirewallService, Object])
+], CliClustersService);