manageiq-appliance_console 5.3.2 → 6.1.0

data/lib/manageiq/appliance_console/message_configuration.rb

@@ -0,0 +1,199 @@
+require 'active_support/core_ext/module/delegation'
+require 'pathname'
+
+module ManageIQ
+  module ApplianceConsole
+    class MessageConfiguration
+      attr_reader :message_keystore_username, :message_keystore_password,
+                  :message_server_host, :message_server_port,
+                  :miq_config_dir_path, :config_dir_path, :sample_config_dir_path,
+                  :client_properties_path,
+                  :keystore_dir_path, :truststore_path, :keystore_path,
+                  :messaging_yaml_sample_path, :messaging_yaml_path,
+                  :ca_cert_path
+
+      BASE_DIR                          = "/opt/kafka".freeze
+      LOGS_DIR                          = "#{BASE_DIR}/logs".freeze
+      CONFIG_DIR                        = "#{BASE_DIR}/config".freeze
+      SAMPLE_CONFIG_DIR                 = "#{BASE_DIR}/config-sample".freeze
+      MIQ_CONFIG_DIR                    = ManageIQ::ApplianceConsole::RAILS_ROOT.join("config").freeze
+
+      def initialize(options = {})
+        @message_server_port        = options[:message_server_port] || 9093
+        @message_keystore_username  = options[:message_keystore_username] || "admin"
+        @message_keystore_password  = options[:message_keystore_password]
+
+        @miq_config_dir_path        = Pathname.new(MIQ_CONFIG_DIR)
+        @config_dir_path            = Pathname.new(CONFIG_DIR)
+        @sample_config_dir_path     = Pathname.new(SAMPLE_CONFIG_DIR)
+
+        @client_properties_path     = config_dir_path.join("client.properties")
+        @keystore_dir_path          = config_dir_path.join("keystore")
+        @truststore_path            = keystore_dir_path.join("truststore.jks")
+        @keystore_path              = keystore_dir_path.join("keystore.jks")
+
+        @messaging_yaml_sample_path = miq_config_dir_path.join("messaging.kafka.yml")
+        @messaging_yaml_path        = miq_config_dir_path.join("messaging.yml")
+        @ca_cert_path               = keystore_dir_path.join("ca-cert")
+      end
+
+      def already_configured?
+        installed_file_found = false
+        installed_files.each do |f|
+          if File.exist?(f)
+            installed_file_found = true
+            say("Installed file #{f} found.")
+          end
+        end
+        installed_file_found
+      end
+
+      def ask_questions
+        return false unless valid_environment?
+
+        ask_for_parameters
+        show_parameters
+        return false unless agree("\nProceed? (Y/N): ")
+
+        return false unless host_reachable?(message_server_host, "Message Server Host:")
+
+        true
+      end
+
+      def create_client_properties
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        return if file_found?(client_properties_path)
+
+        algorithm = message_server_host.ipaddress? ? "" : "HTTPS"
+        protocol = secure? ? "SASL_SSL" : "PLAINTEXT"
+        content = secure? ? secure_client_properties_content(algorithm, protocol) : unsecure_client_properties_content(algorithm, protocol)
+
+        File.write(client_properties_path, content)
+      end
+
+      def secure_client_properties_content(algorithm, protocol)
+        secure_content = <<~CLIENT_PROPERTIES
+          ssl.truststore.location=#{truststore_path}
+          ssl.truststore.password=#{message_keystore_password}
+        CLIENT_PROPERTIES
+
+        unsecure_client_properties_content(algorithm, protocol) + secure_content
+      end
+
+      def unsecure_client_properties_content(algorithm, protocol)
+        <<~CLIENT_PROPERTIES
+          ssl.endpoint.identification.algorithm=#{algorithm}
+
+          sasl.mechanism=PLAIN
+          security.protocol=#{protocol}
+          sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\
+            username=#{message_keystore_username} \\
+            password=#{message_keystore_password} ;
+        CLIENT_PROPERTIES
+      end
+
+      def configure_messaging_yaml
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        return if file_found?(messaging_yaml_path)
+
+        messaging_yaml = YAML.load_file(messaging_yaml_sample_path)
+
+        messaging_yaml["production"].delete("username")
+        messaging_yaml["production"].delete("password")
+
+        messaging_yaml["production"]["hostname"]          = message_server_host
+        messaging_yaml["production"]["port"]              = message_server_port
+        messaging_yaml["production"]["sasl.mechanism"]    = "PLAIN"
+        messaging_yaml["production"]["sasl.username"]     = message_keystore_username
+        messaging_yaml["production"]["sasl.password"]     = ManageIQ::Password.try_encrypt(message_keystore_password)
+
+        if secure?
+          messaging_yaml["production"]["security.protocol"] = "SASL_SSL"
+          messaging_yaml["production"]["ssl.ca.location"]   = ca_cert_path.to_path
+        else
+          messaging_yaml["production"]["security.protocol"] = "PLAINTEXT"
+        end
+
+        File.write(messaging_yaml_path, messaging_yaml.to_yaml)
+      end
+
+      def remove_installed_files
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        installed_files.each { |f| FileUtils.rm_rf(f) }
+      end
+
+      def valid_environment?
+        if already_configured?
+          unconfigure if agree("\nAlready configured on this Appliance, Un-Configure first? (Y/N): ")
+          return false unless agree("\nProceed with Configuration? (Y/N): ")
+        end
+        true
+      end
+
+      def file_found?(path)
+        return false unless File.exist?(path)
+
+        say("\tWARNING: #{path} already exists. Taking no action.")
+        true
+      end
+
+      def files_found?(path_list)
+        return false unless path_list.all? { |path| File.exist?(path) }
+
+        path_list.each { |path| file_found?(path) }
+        true
+      end
+
+      def file_contains?(path, content)
+        return false unless File.exist?(path)
+
+        content.split("\n").each do |l|
+          l.gsub!("/", "\\/")
+          l.gsub!(/password=.*$/, "password=") # Remove the password as it can have special characters that grep can not match.
+          return false unless File.foreach(path).grep(/#{l}/).any?
+        end
+
+        say("Content already exists in #{path}. Taking no action.")
+        true
+      end
+
+      def host_reachable?(host, what)
+        require 'net/ping'
+        say("Checking connectivity to #{host} ... ")
+        unless Net::Ping::External.new(host).ping
+          say("Failed.\nCould not connect to #{host},")
+          say("the #{what} must be reachable by name.")
+          return false
+        end
+        say("Succeeded.")
+        true
+      end
+
+      def configure_messaging_type(value)
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        result = ManageIQ::ApplianceConsole::Utilities.rake_run("evm:settings:set", ["/prototype/messaging_type=#{value}"])
+        raise parse_errors(result).join(', ') if result.failure?
+      end
+
+      def restart_evmserverd
+        say("Restart evmserverd if it is running...")
+        evmserverd_service = LinuxAdmin::Service.new("evmserverd")
+        evmserverd_service.restart if evmserverd_service.running?
+      end
+
+      def unconfigure
+        configure_messaging_type("miq_queue") # Settings.prototype.messaging_type = 'miq_queue'
+        restart_evmserverd
+        remove_installed_files
+      end
+
+      def secure?
+        message_server_port == 9_093
+      end
+    end
+  end
+end

data/lib/manageiq/appliance_console/message_configuration_client.rb

@@ -0,0 +1,96 @@
+require "awesome_spawn"
+require "fileutils"
+require "linux_admin"
+require 'net/scp'
+require "manageiq/appliance_console/message_configuration"
+
+module ManageIQ
+  module ApplianceConsole
+    class MessageClientConfiguration < MessageConfiguration
+      attr_reader :message_server_password, :message_server_username, :installed_files,
+                  :message_truststore_path_src, :message_ca_cert_path_src
+
+      def initialize(options = {})
+        super(options)
+
+        @message_server_host         = options[:message_server_host]
+        @message_server_username     = options[:message_server_usernamed] || "root"
+        @message_server_password     = options[:message_server_password]
+
+        @message_truststore_path_src = options[:message_truststore_path_src] || truststore_path
+        @message_ca_cert_path_src    = options[:message_ca_cert_path_src] || ca_cert_path
+
+        @installed_files = [client_properties_path, messaging_yaml_path, truststore_path]
+      end
+
+      def configure
+        begin
+          MessageServerConfiguration.new.unconfigure if MessageServerConfiguration.configured?
+          configure_messaging_yaml          # Set up the local message client in case EVM is actually running on this, Message Server
+          create_client_properties          # Create the client.properties configuration fle
+          fetch_truststore_from_server      # Fetch the Java Keystore from the Kafka Server
+          configure_messaging_type("kafka") # Settings.prototype.messaging_type = 'kafka'
+          restart_evmserverd
+        rescue AwesomeSpawn::CommandResultError => e
+          say(e.result.output)
+          say(e.result.error)
+          say("")
+          say("Failed to Configure the Message Client- #{e}")
+          return false
+        rescue => e
+          say("Failed to Configure the Message Client- #{e}")
+          return false
+        end
+        true
+      end
+
+      def ask_for_parameters
+        say("\nMessage Client Parameters:\n\n")
+
+        @message_server_host         = ask_for_string("Message Server Hostname or IP address")
+        @message_server_port         = ask_for_integer("Message Server Port number", (1..65_535), 9_093).to_i
+        @message_server_username     = ask_for_string("Message Server Username", message_server_username)
+        @message_server_password     = ask_for_password("Message Server Password")
+        @message_truststore_path_src = ask_for_string("Message Server Truststore Path", truststore_path)
+        @message_ca_cert_path_src    = ask_for_string("Message Server CA Cert Path", ca_cert_path)
+        @message_keystore_username   = ask_for_string("Message Keystore Username", message_keystore_username) if secure?
+        @message_keystore_password   = ask_for_password("Message Keystore Password") if secure?
+      end
+
+      def show_parameters
+        say("\nMessage Client Configuration:\n")
+        say("Message Client Details:\n")
+        say("  Message Server Hostname:   #{message_server_host}\n")
+        say("  Message Server Username:   #{message_server_username}\n")
+        say("  Message Keystore Username: #{message_keystore_username}\n")
+      end
+
+      def fetch_truststore_from_server
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        fetch_from_server(message_truststore_path_src, truststore_path)
+      end
+
+      def fetch_ca_cert_from_server
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        fetch_from_server(message_ca_cert_path_src, ca_cert_path)
+      end
+
+      private
+
+      def fetch_from_server(src_file, dst_file)
+        return if file_found?(dst_file)
+
+        Net::SCP.start(message_server_host, message_server_username, :password => message_server_password) do |scp|
+          scp.download!(src_file, dst_file)
+        end
+
+        File.exist?(dst_file)
+      rescue => e
+        say("Failed to fetch #{src_file} from server: #{e.message}")
+        false
+      end
+    end
+  end
+end

data/lib/manageiq/appliance_console/message_configuration_server.rb

@@ -0,0 +1,319 @@
+require "awesome_spawn"
+require "fileutils"
+require "linux_admin"
+require "manageiq/appliance_console/message_configuration"
+
+module ManageIQ
+  module ApplianceConsole
+    class MessageServerConfiguration < MessageConfiguration
+      attr_reader :jaas_config_path,
+                  :server_properties_path, :server_properties_sample_path,
+                  :ca_cert_srl_path, :ca_key_path, :cert_file_path, :cert_signed_path,
+                  :keystore_files, :installed_files, :message_persistent_disk
+
+      PERSISTENT_DIRECTORY = Pathname.new("/var/lib/kafka/persistent_data").freeze
+      PERSISTENT_NAME = "kafka_messages".freeze
+
+      def initialize(options = {})
+        super(options)
+
+        @message_server_host           = options[:message_server_use_ipaddr] == true ? my_ipaddr : options[:message_server_host] || my_hostname
+        @message_persistent_disk       = LinuxAdmin::Disk.new(:path => options[:message_persistent_disk]) unless options[:message_persistent_disk].nil?
+
+        @jaas_config_path              = config_dir_path.join("kafka_server_jaas.conf")
+        @server_properties_path        = config_dir_path.join("server.properties")
+        @server_properties_sample_path = sample_config_dir_path.join("server.properties")
+
+        @ca_cert_srl_path              = keystore_dir_path.join("ca-cert.srl")
+        @ca_key_path                   = keystore_dir_path.join("ca-key")
+        @cert_file_path                = keystore_dir_path.join("cert-file")
+        @cert_signed_path              = keystore_dir_path.join("cert-signed")
+
+        @keystore_files  = [ca_cert_path, ca_cert_srl_path, ca_key_path, cert_file_path, cert_signed_path, truststore_path, keystore_path]
+        @installed_files = [jaas_config_path, client_properties_path, server_properties_path, messaging_yaml_path, LOGS_DIR] + keystore_files
+      end
+
+      def configure
+        begin
+          configure_persistent_disk         # Configure the persistent message store on a different disk
+          create_jaas_config                # Create the message server jaas config file
+          create_client_properties          # Create the client.properties config
+          create_logs_directory             # Create the logs directory:
+          configure_firewall                # Open the firewall for message port 9093
+          configure_keystore                # Populate the Java Keystore
+          create_server_properties          # Update the /opt/message/config/server.properties
+          configure_messaging_yaml          # Set up the local message client in case EVM is actually running on this, Message Server
+          configure_messaging_type("kafka") # Settings.prototype.messaging_type = 'kafka'
+          restart_services
+        rescue AwesomeSpawn::CommandResultError => e
+          say(e.result.output)
+          say(e.result.error)
+          say("")
+          say("Failed to Configure the Message Server- #{e}")
+          return false
+        rescue => e
+          say("Failed to Configure the Message Server- #{e}")
+          return false
+        end
+        true
+      end
+
+      def restart_services
+        say("Starting zookeeper and configure it to start on reboots ...")
+        LinuxAdmin::Service.new("zookeeper").start.enable
+
+        say("Starting kafka and configure it to start on reboots ...")
+        LinuxAdmin::Service.new("kafka").start.enable
+
+        restart_evmserverd
+      end
+
+      def ask_for_parameters
+        say("\nMessage Server Parameters:\n\n")
+
+        @message_server_host       = ask_for_string("Message Server Hostname or IP address", message_server_host)
+        @message_keystore_username = ask_for_string("Message Keystore Username", message_keystore_username)
+        @message_keystore_password = ask_for_password("Message Keystore Password")
+        @message_persistent_disk   = ask_for_persistent_disk
+      end
+
+      def ask_for_persistent_disk
+        choose_disk if use_new_disk
+      end
+
+      def use_new_disk
+        agree("Configure a new persistent disk volume? (Y/N): ")
+      end
+
+      def choose_disk
+        ask_for_disk("Persistent disk")
+      end
+
+      def show_parameters
+        say("\nMessage Server Configuration:\n")
+        say("Message Server Details:\n")
+        say("    Message Server Hostname: #{message_server_host}\n")
+        say("  Message Keystore Username: #{message_keystore_username}\n")
+        say("    Persistent message disk: #{message_persistent_disk.path}\n") if message_persistent_disk
+      end
+
+      def unconfigure
+        super
+
+        unconfigure_firewall
+        deactivate_services
+      end
+
+      def self.configured?
+        LinuxAdmin::Service.new("kafka").running? ||
+          LinuxAdmin::Service.new("zookeeper").running?
+      end
+
+      private
+
+      def my_ipaddr
+        LinuxAdmin::IpAddress.new.address
+      end
+
+      def my_hostname
+        LinuxAdmin::Hosts.new.hostname
+      end
+
+      def configure_persistent_disk
+        return true unless message_persistent_disk
+
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        deactivate_services # Just in case they are running.
+
+        FileUtils.mkdir_p(PERSISTENT_DIRECTORY)
+        LogicalVolumeManagement.new(:disk => message_persistent_disk, :mount_point => PERSISTENT_DIRECTORY, :name => PERSISTENT_NAME).setup
+        FileUtils.chmod(0o755, PERSISTENT_DIRECTORY)
+        FileUtils.chown("kafka", "kafka", PERSISTENT_DIRECTORY)
+
+        true
+      end
+
+      def activate_new_persistent_disk
+        return true unless message_persistent_disk
+
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        data = File.read(server_properties_path)
+        data.gsub!(/^log.dirs=.*$/, "log.dirs=#{PERSISTENT_DIRECTORY}")
+        File.write(server_properties_path, data)
+
+        true
+      end
+
+      def create_jaas_config
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        content = <<~JAAS
+          KafkaServer {
+            org.apache.kafka.common.security.plain.PlainLoginModule required
+            username=#{message_keystore_username}
+            password=#{message_keystore_password}
+            user_admin=#{message_keystore_password} ;
+          };
+        JAAS
+
+        File.write(jaas_config_path, content) unless file_found?(jaas_config_path)
+      end
+
+      def create_logs_directory
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        return if file_found?(LOGS_DIR)
+
+        FileUtils.mkdir_p(LOGS_DIR)
+        FileUtils.chmod(0o755, LOGS_DIR)
+        FileUtils.chown("kafka", "kafka", LOGS_DIR)
+      end
+
+      def configure_firewall
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        modify_firewall(:add_port)
+      end
+
+      def configure_keystore
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        return if files_found?(keystore_files)
+
+        keystore_params = assemble_keystore_params
+
+        # Generte a Java keystore and key pair, creating keystore.jks
+        # :stdin_data provides the -storepass twice to confirm and an extra CR to accept the same password for -keypass
+        AwesomeSpawn.run!("keytool", :params => keystore_params, :stdin_data => "#{message_keystore_password}\n#{message_keystore_password}\n\n")
+
+        # Use openssl to create a new CA cert, creating ca-cert and ca-key
+        AwesomeSpawn.run!("openssl", :env    => {"PASSWORD" => message_keystore_password},
+                                     :params => ["req", "-new", "-x509", {"-keyout"  => ca_key_path,
+                                                                          "-out"     => ca_cert_path,
+                                                                          "-days"    => 10_000,
+                                                                          "-passout" => "env:PASSWORD",
+                                                                          "-subj"    => '/CN=something'}])
+
+        # Import the CA cert into the trust store, creating truststore.jks
+        # :stdin_data provides the -storepass argument and yes to confirm
+        AwesomeSpawn.run!("keytool", :params     => {"-keystore" => truststore_path,
+                                                     "-alias"    => "CARoot",
+                                                     "-import"   => nil,
+                                                     "-file"     => ca_cert_path},
+                                     :stdin_data => "#{message_keystore_password}\n#{message_keystore_password}\nyes\n")
+
+        # Generate a certificate signing request (CSR) for an existing Java keystore, creating cert-file
+        # :stdin_data provides the -storepass argument
+        AwesomeSpawn.run!("keytool", :params     => {"-keystore" => keystore_path,
+                                                     "-alias"    => keystore_params["-alias"],
+                                                     "-certreq"  => nil,
+                                                     "-file"     => cert_file_path},
+                                     :stdin_data => "#{message_keystore_password}\n")
+
+        # Use openssl to sign the certificate with the "CA" certificate, creating ca-cert.srl and cert-signed
+        AwesomeSpawn.run!("openssl", :env    => {"PASSWORD" => message_keystore_password},
+                                     :params => ["x509", "-req", {"-CA"             => ca_cert_path,
+                                                                  "-CAkey"          => ca_key_path,
+                                                                  "-in"             => cert_file_path,
+                                                                  "-out"            => cert_signed_path,
+                                                                  "-days"           => 10_000,
+                                                                  "-CAcreateserial" => nil,
+                                                                  "-passin"         => "env:PASSWORD"}])
+
+        # Import a root or intermediate CA certificate to an existing Java keystore, updating keystore.jks
+        # :stdin_data provides the -storepass argument and yes to confirm
+        AwesomeSpawn.run!("keytool", :params     => {"-keystore" => keystore_path,
+                                                     "-alias"    => "CARoot",
+                                                     "-import"   => nil,
+                                                     "-file"     => ca_cert_path},
+                                     :stdin_data => "#{message_keystore_password}\nyes\n")
+
+        # Import a signed primary certificate to an existing Java keystore, updating keystore.jks
+        # :stdin_data provides the -storepass argument
+        AwesomeSpawn.run!("keytool", :params     => {"-keystore" => keystore_path,
+                                                     "-alias"    => keystore_params["-alias"],
+                                                     "-import"   => nil,
+                                                     "-file"     => cert_signed_path},
+                                     :stdin_data => "#{message_keystore_password}\n")
+      end
+
+      def create_server_properties
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        if message_server_host.ipaddress?
+          ident_algorithm = ""
+          client_auth = "none"
+        else
+          ident_algorithm = "HTTPS"
+          client_auth = "required"
+        end
+
+        content = <<~SERVER_PROPERTIES
+
+          listeners=SASL_SSL://:#{message_server_port}
+
+          ssl.endpoint.identification.algorithm=#{ident_algorithm}
+          ssl.keystore.location=#{keystore_path}
+          ssl.keystore.password=#{message_keystore_password}
+          ssl.key.password=#{message_keystore_password}
+
+          ssl.truststore.location=#{truststore_path}
+          ssl.truststore.password=#{message_keystore_password}
+
+          ssl.client.auth=#{client_auth}
+
+          sasl.enabled.mechanisms=PLAIN
+          sasl.mechanism.inter.broker.protocol=PLAIN
+
+          security.inter.broker.protocol=SASL_SSL
+        SERVER_PROPERTIES
+
+        return if file_contains?(server_properties_path, content)
+
+        FileUtils.cp(server_properties_sample_path, server_properties_path)
+        File.write(server_properties_path, content, :mode => "a")
+
+        activate_new_persistent_disk
+      end
+
+      def unconfigure_firewall
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        modify_firewall(:remove_port)
+      end
+
+      def deactivate_services
+        say(__method__.to_s.tr("_", " ").titleize)
+
+        LinuxAdmin::Service.new("zookeeper").stop
+        LinuxAdmin::Service.new("kafka").stop
+      end
+
+      def assemble_keystore_params
+        keystore_params = {"-keystore" => keystore_path,
+                           "-validity" => 10_000,
+                           "-genkey"   => nil,
+                           "-keyalg"   => "RSA"}
+
+        if message_server_host.ipaddress?
+          keystore_params["-alias"] = "localhost"
+          keystore_params["-ext"] = "san=ip:#{message_server_host}"
+        else
+          keystore_params["-alias"] = message_server_host
+          keystore_params["-ext"] = "san=dns:#{message_server_host}"
+        end
+
+        keystore_params["-dname"] = "cn=#{keystore_params["-alias"]}"
+
+        keystore_params
+      end
+
+      def modify_firewall(action)
+        AwesomeSpawn.run!("firewall-cmd", :params => {action => "#{message_server_port}/tcp", :permanent => nil})
+        AwesomeSpawn.run!("firewall-cmd --reload")
+      end
+    end
+  end
+end