@flui-cloud/cli 0.0.1

package/lib/cli/src/services/cli-observability-cluster.service.js

@@ -0,0 +1,525 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var CliObservabilityClusterService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CliObservabilityClusterService = void 0;
+const common_1 = require("@nestjs/common");
+const cluster_entity_1 = require("../../../src/modules/infrastructure/clusters/entities/cluster.entity");
+const cli_cluster_repository_1 = require("../lib/repositories/cli-cluster.repository");
+const nip_base_domain_util_1 = require("../lib/nip-base-domain.util");
+const cli_node_repository_1 = require("../lib/repositories/cli-node.repository");
+const cli_clusters_service_1 = require("./cli-clusters.service");
+const cli_operation_repository_1 = require("../lib/repositories/cli-operation.repository");
+const cli_ssh_service_1 = require("./cli-ssh.service");
+const infrastructure_operations_entity_1 = require("../../../src/modules/infrastructure/servers/entities/infrastructure-operations.entity");
+const net = __importStar(require("node:net"));
+const https = __importStar(require("node:https"));
+/**
+ * CLI Observability Cluster Service
+ *
+ * Simplified version of ObservabilityClusterService for CLI usage.
+ * Uses file-based repositories instead of TypeORM.
+ */
+let CliObservabilityClusterService = CliObservabilityClusterService_1 = class CliObservabilityClusterService {
+    constructor(clusterRepository, nodeRepository, clustersService, operationRepository, sshService) {
+        this.clusterRepository = clusterRepository;
+        this.nodeRepository = nodeRepository;
+        this.clustersService = clustersService;
+        this.operationRepository = operationRepository;
+        this.sshService = sshService;
+        this.logger = new common_1.Logger(CliObservabilityClusterService_1.name);
+    }
+    /**
+     * Get the observability cluster
+     */
+    async getObservabilityCluster() {
+        const cluster = await this.clusterRepository.findOne({
+            where: {
+                metadata: { isObservabilityCluster: true },
+            },
+        });
+        if (!cluster) {
+            return null;
+        }
+        // Manually load nodes from separate repository
+        const nodes = await this.nodeRepository.find({
+            where: { clusterId: cluster.id },
+        });
+        cluster.nodes = nodes;
+        return cluster;
+    }
+    /**
+     * Check if observability cluster exists
+     */
+    async hasObservabilityCluster() {
+        const cluster = await this.getObservabilityCluster();
+        return cluster !== null;
+    }
+    /**
+     * Get observability service endpoints
+     */
+    async getObservabilityEndpoints(clusterId) {
+        const cluster = await this.clusterRepository.findOne({
+            where: { id: clusterId },
+        });
+        if (!cluster?.masterIpAddress) {
+            return {};
+        }
+        const ip = cluster.masterIpAddress;
+        const baseDomain = (0, nip_base_domain_util_1.buildNipBaseDomain)(ip, cluster.nipHostnameToken);
+        return {
+            prometheus: `cluster-internal (kubectl port-forward -n flui-system svc/prometheus 9090)`,
+            grafana: `cluster-internal (kubectl port-forward -n flui-system svc/grafana 3000)`,
+            loki: `cluster-internal (kubectl port-forward -n flui-system svc/loki 3100)`,
+            postgres: `cluster-internal (kubectl port-forward -n flui-system svc/postgres 5432)`,
+            redis: `cluster-internal (kubectl port-forward -n flui-system svc/redis 6379)`,
+            fluiApi: `https://api.${baseDomain}`,
+            fluiWeb: `https://app.${baseDomain}`,
+        };
+    }
+    /**
+     * Create observability cluster
+     */
+    async createObservabilityCluster(provider, region, nodeSize, workerCount = 0, firewallId, sourceCidrs, authMode = 'local', envVnet, adminEmail, acmeStaging, diskSizeGb, options) {
+        const createDto = {
+            name: `observability-cluster-${Date.now()}`,
+            provider: provider,
+            region,
+            nodeSize,
+            workerCount,
+            image: 'ubuntu-24.04',
+            diskSizeGb,
+            sharedStorageEnabled: options?.sharedStorageEnabled,
+            sharedStorageVolumeSizeGb: options?.sharedStorageVolumeSizeGb,
+            metadata: {
+                isObservabilityCluster: true,
+                firewallId,
+                sourceCidrs,
+                authMode,
+                envVnet,
+                adminEmail,
+                acmeStaging,
+            },
+        };
+        const { cluster } = await this.clustersService.create(createDto);
+        return cluster.id;
+    }
+    /**
+     * Deploy observability stack (Prometheus, Grafana, Loki)
+     */
+    async deployObservabilityStack(clusterId) {
+        // TODO: Implement observability stack deployment
+        this.logger.warn('Observability stack deployment not implemented in CLI mode');
+    }
+    /**
+     * Delete observability cluster
+     */
+    async deleteObservabilityCluster() {
+        // Local clusters.json can accumulate stale observability entries (e.g. a
+        // previous destroy died mid-way, or a create crashed after persisting).
+        // findOne would only catch one — so we iterate and purge all matches.
+        const clusters = await this.clusterRepository.find({
+            where: { metadata: { isObservabilityCluster: true } },
+        });
+        if (clusters.length === 0) {
+            throw new Error('No observability cluster found');
+        }
+        if (clusters.length > 1) {
+            this.logger.warn(`Found ${clusters.length} observability cluster records — removing all to clear stale state`);
+        }
+        let lastError = null;
+        for (const cluster of clusters) {
+            try {
+                await this.clustersService.remove(cluster.id);
+            }
+            catch (error) {
+                lastError = error;
+                this.logger.error(`Failed to remove cluster ${cluster.id} (${cluster.name}): ${error.message}`);
+            }
+        }
+        if (lastError)
+            throw lastError;
+    }
+    /**
+     * Get cluster operation by cluster ID
+     */
+    async getClusterOperation(clusterId) {
+        return this.operationRepository.findOne({
+            where: {
+                resourceId: clusterId,
+                resourceType: 'cluster',
+            },
+            order: {
+                createdAt: 'DESC',
+            },
+        });
+    }
+    /**
+     * Wait for cluster to be ready by polling operation status
+     * @param clusterId Cluster ID to wait for
+     * @param timeoutMs Timeout in milliseconds (default: 10 minutes)
+     * @param pollIntervalMs Polling interval in milliseconds (default: 10 seconds)
+     * @returns Promise that resolves when cluster is ready
+     * @throws Error if timeout is reached or operation fails
+     */
+    async waitForClusterReady(clusterId, timeoutMs = 600000, pollIntervalMs = 10000) {
+        const startTime = Date.now();
+        const maxAttempts = Math.ceil(timeoutMs / pollIntervalMs);
+        this.logger.log(`Waiting for cluster ${clusterId} to be ready (timeout: ${timeoutMs}ms, poll interval: ${pollIntervalMs}ms)`);
+        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+            const operation = await this.getClusterOperation(clusterId);
+            if (!operation) {
+                throw new Error(`No operation found for cluster ${clusterId}. Cluster may not exist.`);
+            }
+            this.logger.debug(`Poll attempt ${attempt}/${maxAttempts}: Operation status = ${operation.status}, progress = ${operation.progress}%`);
+            // Check if operation completed successfully
+            if (operation.status === infrastructure_operations_entity_1.OperationStatus.COMPLETED) {
+                this.logger.log(`Cluster ${clusterId} is ready (took ${Date.now() - startTime}ms)`);
+                return;
+            }
+            // Check if operation failed
+            if (operation.status === infrastructure_operations_entity_1.OperationStatus.FAILED) {
+                const errorMsg = operation.metadata?.error || 'Unknown error during cluster creation';
+                throw new Error(`Cluster creation failed: ${errorMsg}`);
+            }
+            // Check timeout
+            if (Date.now() - startTime >= timeoutMs) {
+                throw new Error(`Timeout waiting for cluster ${clusterId} to be ready after ${timeoutMs}ms. Current status: ${operation.status}, progress: ${operation.progress}%`);
+            }
+            // Wait before next poll (unless it's the last attempt)
+            if (attempt < maxAttempts) {
+                await this.sleep(pollIntervalMs);
+            }
+        }
+        throw new Error(`Failed to confirm cluster readiness after ${maxAttempts} attempts`);
+    }
+    /**
+     * Wait for master node IP address to become available
+     * Polls the cluster repository until masterIpAddress is populated
+     */
+    async waitForMasterIp(clusterId, timeoutMs = 600000, pollIntervalMs = 5000) {
+        const startTime = Date.now();
+        this.logger.log(`Waiting for master IP of cluster ${clusterId} (timeout: ${timeoutMs}ms)`);
+        while (Date.now() - startTime < timeoutMs) {
+            const cluster = await this.clusterRepository.findOne({
+                where: { id: clusterId },
+            });
+            if (!cluster) {
+                throw new Error(`Cluster ${clusterId} not found. It may have been deleted.`);
+            }
+            if (cluster.status === cluster_entity_1.ClusterStatus.ERROR) {
+                throw new Error('Cluster creation failed before master IP was assigned.');
+            }
+            if (cluster.masterIpAddress) {
+                this.logger.log(`Master IP available: ${cluster.masterIpAddress}`);
+                return cluster.masterIpAddress;
+            }
+            await this.sleep(pollIntervalMs);
+        }
+        throw new Error(`Timeout waiting for master IP address after ${timeoutMs}ms`);
+    }
+    /**
+     * Wait for TCP port to become reachable on a host
+     */
+    async waitForPortReady(host, port = 22, timeoutMs = 600000, pollIntervalMs = 5000) {
+        const startTime = Date.now();
+        this.logger.log(`Waiting for port ${port} on ${host} (timeout: ${timeoutMs}ms)`);
+        while (Date.now() - startTime < timeoutMs) {
+            const isReachable = await this.checkTcpPort(host, port);
+            if (isReachable) {
+                this.logger.log(`Port ${port} is reachable on ${host}`);
+                return;
+            }
+            await this.sleep(pollIntervalMs);
+        }
+        throw new Error(`Timeout waiting for port ${port} on ${host} after ${timeoutMs}ms`);
+    }
+    /**
+     * Wait for SSH to be fully ready (CA enrolled + cert auth working)
+     * Attempts an actual SSH command with retry until it succeeds.
+     * This is needed because TCP port 22 can be open before cloud-init
+     * has finished configuring the CA for certificate authentication.
+     *
+     * @param sshTestFn Function that attempts an SSH command and throws on failure
+     */
+    async waitForSshAuth(sshTestFn, timeoutMs = 600000, pollIntervalMs = 10000) {
+        const startTime = Date.now();
+        this.logger.log(`Waiting for SSH authentication to be ready (timeout: ${timeoutMs}ms)`);
+        let lastError;
+        while (Date.now() - startTime < timeoutMs) {
+            try {
+                await sshTestFn();
+                this.logger.log('SSH authentication is ready');
+                return;
+            }
+            catch (error) {
+                lastError = error;
+            }
+            await this.sleep(pollIntervalMs);
+        }
+        if (lastError) {
+            this.logger.error(`SSH exec failed: ${lastError.message}`);
+        }
+        throw new Error(`Timeout waiting for SSH authentication after ${timeoutMs}ms`);
+    }
+    /**
+     * Poll operation status in background and call onReady when COMPLETED.
+     * Returns a stop function and a done promise that resolves when the
+     * poller finishes (after onReady/onFailed callback completes or stop() is called).
+     */
+    pollOperationUntilReady(clusterId, onReady, onFailed, pollIntervalMs = 10000) {
+        let stopped = false;
+        const poll = async () => {
+            while (!stopped) {
+                try {
+                    const operation = await this.getClusterOperation(clusterId);
+                    if (!operation) {
+                        this.logger.warn(`No operation found for cluster ${clusterId} during background poll`);
+                        await this.sleep(pollIntervalMs);
+                        continue;
+                    }
+                    if (operation.status === infrastructure_operations_entity_1.OperationStatus.COMPLETED) {
+                        this.logger.log(`Background poll: cluster ${clusterId} is READY`);
+                        if (!stopped) {
+                            await onReady();
+                        }
+                        return;
+                    }
+                    if (operation.status === infrastructure_operations_entity_1.OperationStatus.FAILED) {
+                        const errorMsg = operation.metadata?.error ||
+                            'Unknown error during cluster creation';
+                        this.logger.error(`Background poll: cluster creation failed: ${errorMsg}`);
+                        if (onFailed && !stopped) {
+                            onFailed(errorMsg);
+                        }
+                        return;
+                    }
+                }
+                catch (error) {
+                    this.logger.debug(`Background poll error: ${error.message}`);
+                }
+                await this.sleep(pollIntervalMs);
+            }
+        };
+        const done = poll().catch((error) => {
+            this.logger.error(`Background poller crashed: ${error.message}`);
+        });
+        return {
+            stop: () => {
+                stopped = true;
+            },
+            done,
+        };
+    }
+    /**
+     * Check observability services health via HTTP endpoints
+     * @param masterIp Master node IP address
+     * @returns Object with service health status
+     */
+    async checkObservabilityServices(masterIp, _nipHostnameToken) {
+        const lookups = [
+            { ns: 'flui-observability', name: 'vmsingle', key: 'prometheus' },
+            { ns: 'flui-observability', name: 'grafana', key: 'grafana' },
+            { ns: 'flui-observability', name: 'loki', key: 'loki' },
+            { ns: 'flui-system', name: 'postgres', key: 'postgres' },
+            { ns: 'flui-system', name: 'redis', key: 'redis' },
+            { ns: 'flui-system', name: 'flui-api', key: 'fluiApi' },
+            { ns: 'flui-system', name: 'flui-web', key: 'fluiWeb' },
+        ];
+        const result = {
+            prometheus: 'unreachable',
+            grafana: 'unreachable',
+            loki: 'unreachable',
+            postgres: 'unreachable',
+            redis: 'unreachable',
+            fluiApi: 'unreachable',
+            fluiWeb: 'unreachable',
+        };
+        try {
+            const combined = `(kubectl -n flui-observability get deploy,statefulset -o json 2>/dev/null || echo '{"items":[]}') ; ` +
+                `echo '---FLUI-SEP---' ; ` +
+                `(kubectl -n flui-system get deploy,statefulset -o json 2>/dev/null || echo '{"items":[]}')`;
+            const raw = await this.sshService.sshExec(masterIp, combined);
+            const [obsRaw, sysRaw] = raw.split('---FLUI-SEP---');
+            const collect = (json) => {
+                try {
+                    const parsed = JSON.parse(json);
+                    return parsed.items ?? [];
+                }
+                catch {
+                    return [];
+                }
+            };
+            const items = [...collect(obsRaw || ''), ...collect(sysRaw || '')];
+            for (const lookup of lookups) {
+                const item = items.find((i) => i.metadata?.name === lookup.name &&
+                    i.metadata?.namespace === lookup.ns);
+                const replicas = item?.status?.replicas ?? 0;
+                const ready = item?.status?.readyReplicas ?? 0;
+                result[lookup.key] =
+                    replicas > 0 && ready === replicas ? 'healthy' : 'unreachable';
+            }
+        }
+        catch {
+            // On SSH failure, leave defaults (all unreachable).
+        }
+        return result;
+    }
+    /**
+     * Check if a TCP port is reachable on a host
+     */
+    checkTcpPort(host, port, timeout = 5000) {
+        return new Promise((resolve) => {
+            const socket = new net.Socket();
+            const timer = setTimeout(() => {
+                socket.destroy();
+                resolve(false);
+            }, timeout);
+            socket.on('connect', () => {
+                clearTimeout(timer);
+                socket.destroy();
+                resolve(true);
+            });
+            socket.on('error', () => {
+                clearTimeout(timer);
+                resolve(false);
+            });
+            socket.connect(port, host);
+        });
+    }
+    /**
+     * Sleep helper
+     */
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    async waitForValidTls(url, timeoutMs = 300_000, intervalMs = 15_000, acmeStaging = false) {
+        // Production: rely on Node's default chain validation (LE prod root is in the OS trust store).
+        // Staging: chain is signed by LE staging fake CA (not in OS trust store), so we connect with
+        // rejectUnauthorized=false but then inspect the served cert issuer to ensure it's a real LE
+        // cert and not Traefik's self-signed default.
+        const agent = acmeStaging
+            ? new https.Agent({ rejectUnauthorized: false })
+            : undefined;
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            const valid = await new Promise((resolve) => {
+                const req = https.request(url, { method: 'HEAD', agent }, (res) => {
+                    if (!acmeStaging) {
+                        resolve(true);
+                        return;
+                    }
+                    const cert = res.socket.getPeerCertificate?.();
+                    const issuerO = cert?.issuer?.O ?? '';
+                    resolve(issuerO.includes("Let's Encrypt"));
+                });
+                req.on('error', () => resolve(false));
+                req.setTimeout(10_000, () => {
+                    req.destroy();
+                    resolve(false);
+                });
+                req.end();
+            });
+            if (valid)
+                return true;
+            await this.sleep(intervalMs);
+        }
+        return false;
+    }
+    async waitForOidcReady(apiBaseUrl, timeoutMs = 300_000, intervalMs = 10_000, acmeStaging = false) {
+        const url = `${apiBaseUrl.replace(/\/$/, '')}/api/v1/health/oidc`;
+        const agent = acmeStaging
+            ? new https.Agent({ rejectUnauthorized: false })
+            : undefined;
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            const ready = await new Promise((resolve) => {
+                const req = https.get(url, { timeout: 10_000, agent }, (res) => {
+                    if (acmeStaging) {
+                        const cert = res.socket.getPeerCertificate?.();
+                        const issuerO = cert?.issuer?.O ?? '';
+                        if (!issuerO.includes("Let's Encrypt")) {
+                            res.resume();
+                            resolve(false);
+                            return;
+                        }
+                    }
+                    if (res.statusCode !== 200) {
+                        res.resume();
+                        resolve(false);
+                        return;
+                    }
+                    let body = '';
+                    res.on('data', (chunk) => (body += chunk));
+                    res.on('end', () => {
+                        try {
+                            resolve(!!JSON.parse(body).ready);
+                        }
+                        catch {
+                            resolve(false);
+                        }
+                    });
+                });
+                req.on('error', () => resolve(false));
+                req.on('timeout', () => {
+                    req.destroy();
+                    resolve(false);
+                });
+            });
+            if (ready)
+                return true;
+            await this.sleep(intervalMs);
+        }
+        return false;
+    }
+};
+exports.CliObservabilityClusterService = CliObservabilityClusterService;
+exports.CliObservabilityClusterService = CliObservabilityClusterService = CliObservabilityClusterService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [cli_cluster_repository_1.CliClusterRepository,
+        cli_node_repository_1.CliNodeRepository,
+        cli_clusters_service_1.CliClustersService,
+        cli_operation_repository_1.CliOperationRepository,
+        cli_ssh_service_1.CliSshService])
+], CliObservabilityClusterService);

package/lib/cli/src/services/cli-ssh.service.d.ts

@@ -0,0 +1,90 @@
+import { CliCaService } from './cli-ca.service';
+/**
+ * CLI SSH Management Service
+ *
+ * Manages SSH access with ephemeral certificates:
+ * - Generates ephemeral ED25519 keypair for each connection
+ * - Gets certificate signed by CA (5-minute validity)
+ * - Uses certificate for SSH authentication
+ * - Cleans up temporary files after connection
+ */
+export declare class CliSshService {
+    private readonly caService;
+    private readonly logger;
+    private readonly fluiDir;
+    private readonly sshDir;
+    private readonly privateKeyPath;
+    private readonly publicKeyPath;
+    constructor(caService: CliCaService);
+    /**
+     * Ensure ~/.flui/ssh directory exists
+     */
+    private ensureSshDir;
+    /**
+     * Get or generate SSH key pair
+     * Returns the public key content for server provisioning
+     */
+    getOrCreateSshKey(): Promise<{
+        publicKey: string;
+        privateKeyPath: string;
+        publicKeyPath: string;
+    }>;
+    /**
+     * Get SSH public key content
+     */
+    getPublicKey(): Promise<string>;
+    /**
+     * Get SSH private key path for SSH connections
+     */
+    getPrivateKeyPath(): string;
+    /**
+     * Generate ephemeral keypair and get signed certificate
+     * Returns paths to private key and certificate
+     */
+    private generateEphemeralKeypair;
+    /**
+     * SSH into a server using ephemeral certificate
+     */
+    sshConnect(host: string, username?: string): Promise<void>;
+    /**
+     * Execute command on remote server via SSH with ephemeral certificate
+     */
+    sshExec(host: string, command: string, username?: string): Promise<string>;
+    /**
+     * Open an SSH session to `host` that requests one or more local port
+     * forwards (`-L localPort:remoteHost:remotePort`) and runs `remoteCommand`
+     * on the remote side. Stays in foreground until the user kills it or the
+     * remote command exits. Returns the child's exit info.
+     */
+    sshForward(opts: {
+        host: string;
+        username?: string;
+        forwards: Array<{
+            localPort: number;
+            remotePort: number;
+            remoteHost?: string;
+        }>;
+        remoteCommand?: string;
+        onReady?: () => void;
+        /** Number of `Forwarding from …` lines expected on stderr before declaring readiness. */
+        expectedForwardLines?: number;
+    }): Promise<{
+        status: number | null;
+        signal: NodeJS.Signals | null;
+    }>;
+    /**
+     * Get remote log file via SSH
+     */
+    getRemoteLog(host: string, logPath: string, username?: string): Promise<string>;
+    /**
+     * Tail remote log file via SSH
+     */
+    tailRemoteLog(host: string, logPath: string, lines?: number, username?: string): Promise<string>;
+    /**
+     * Stream remote log file via SSH with tail -f
+     * Returns a cleanup function that should be called to stop streaming and cleanup resources
+     */
+    streamRemoteLog(host: string, logPath: string, username?: string, onData?: (data: string) => void): Promise<{
+        cleanup: () => void;
+    }>;
+}