cem_acpt 0.1.0

data/lib/cem_acpt/platform/base.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative File.join(__dir__, '..', 'image_name_builder')
+require_relative File.join(__dir__, '..', 'logging')
+
+module CemAcpt::Platform
+  # Base class for all platform classes. This class provides an API for
+  # interacting with the underlying platform.
+  class Base
+    include CemAcpt::Logging
+
+    attr_reader :config, :test_data, :local_port, :node_name, :image_name
+
+    # @param conf [CemAcpt::Config] the config object.
+    # @param single_test_data [Hash] the test data for the current test.
+    # @param local_port [Integer] the local port to use for the test.
+    def initialize(conf, single_test_data, local_port)
+      raise ArgumentError, 'single_test_data must be a Hash' unless single_test_data.is_a?(Hash)
+
+      @config = conf.get('node_data')
+      @test_data = single_test_data
+      @local_port = local_port
+      @node_name = @test_data[:node_name] || random_node_name
+      @image_name = conf.has?('image_name_builder') ? image_name_builder(conf, single_test_data) : @test_data[:image_name]
+    end
+
+    # Node should return a hash of all data about a created node.
+    def node
+      raise NotImplementedError, '#node must be implemented by subclass'
+    end
+
+    # Provision a node. Will be called asynchronously.
+    def provision
+      raise NotImplementedError, '#provision must be implemented by subclass'
+    end
+
+    # Destroy a node. Will be called asynchronously.
+    def destroy
+      raise NotImplementedError, '#destroy must be implemented by subclass'
+    end
+
+    # Tests to see if a node is ready to accept connections from the test suite.
+    def ready?
+      raise NotImplementedError, '#ready? must be implemented by subclass'
+    end
+
+    # Upload and install a Puppet module package on the node. Blocking call.
+    def install_puppet_module_package(_module_pkg_path, _remote_path)
+      raise NotImplementedError, '#install_puppet_module_package must be implemented by subclass'
+    end
+
+    # Generates a random node name.
+    def random_node_name
+      "acpt-test-#{SecureRandom.hex(10)}"
+    end
+
+    # Builds an image name if the config specifies to use the image name builder.
+    def image_name_builder(conf, tdata)
+      @image_name_builder ||= CemAcpt::ImageNameBuilder.new(conf).build(tdata)
+    end
+
+    # Returns a command provider specified by the Platform module of the specific platform.
+    def self.command_provider
+      raise NotImplementedError, '#command_provider must be implemented by subclass'
+    end
+
+    # Applies a Puppet manifest on the given node.
+    def self.apply_manifest(_instance_name, _manifest, _opts = {})
+      raise NotImplementedError, '#apply_manifest must be implemented by subclass'
+    end
+
+    # Runs a shell command on the given node.
+    def self.run_shell(_instance_name, _cmd, _opts = {})
+      raise NotImplementedError, '#run_shell must be implemented by subclass'
+    end
+  end
+end

data/lib/cem_acpt/platform/gcp/cmd.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require_relative File.join(__dir__, '..', 'base', 'cmd.rb')
+
+module CemAcpt::Platform::Gcp
+  # This class provides methods to run gcloud commands. It allows for default values to be
+  # set for the project, zone, and user and can also find these values from the local config.
+  # Additionally, this class provides a way to run SSH commands against GCP VMs using IAP.
+  class Cmd < CemAcpt::Platform::CmdBase
+    def initialize(project: nil, zone: nil, out_format: nil, filter: nil, user_name: nil, local_port: nil)
+      super
+      require 'net/ssh'
+      require 'net/ssh/proxy/command'
+      require 'net/scp'
+
+      @project = project unless project.nil?
+      @zone = zone unless zone.nil?
+      @default_out_format = out_format
+      @default_filter = filter
+      @user_name = user_name
+      @local_port = local_port
+      raise 'gcloud command not available' unless gcloud?
+      raise 'gcloud is not authenticated' unless authenticated?
+    end
+
+    # Returns either the project name passed in during object initialization
+    # or the project name from the local config.
+    def project
+      @project ||= project_from_config&.chomp
+    end
+
+    # Returns either the zone name passed in during object initialization
+    # or the zone name from the local config.
+    def zone
+      @zone ||= zone_from_config&.chomp
+    end
+
+    # Returns either the user name passed in during object initialization
+    # or queries the current active, authenticated user from gcloud and returns the name.
+    def user_name
+      @user_name ||= authenticated_user_name&.chomp
+    end
+
+    # Returns the format string passed in during object initialization or
+    # the default format string.
+    def format(out_format = nil)
+      out_format&.chomp || @default_out_format&.chomp
+    end
+
+    # Returns the filter string passed in during object initialization or
+    # the default filter string.
+    def filter(out_filter = nil)
+      out_filter&.chomp || @default_filter&.chomp
+    end
+
+    def local_port
+      @local_port ||= rand(49_512..65_535)
+    end
+
+    # Returns a formatted hash of ssh options to be used with Net::SSH.start.
+    # If you pass in a GCP VM instance name, this method will configure the
+    # IAP tunnel ProxyCommand to use. If you pass in an opts hash, it will
+    # merge the options with the default options.
+    def ssh_opts(instance_name: nil, use_proxy_command: true, opts: {})
+      base_opts = default_ssh_opts
+      if use_proxy_command
+        base_opts[:proxy] = proxy_command(instance_name, port: base_opts[:port])
+        base_opts[:host_key_alias] = vm_alias(instance_name)
+      end
+      base_opts.merge(opts).reject { |_, v| v.nil? }
+    end
+
+    # Runs `gcloud` commands locally.
+    def local_exec(command, out_format: 'json', out_filter: nil, project_flag: true)
+      cmd_parts = ['gcloud', command]
+      cmd_parts << "--project=#{project.chomp}" unless !project_flag || project.nil?
+      cmd_parts << "--format=#{format(out_format)}" if format(out_format)
+      cmd_parts << "--filter=\"#{filter(out_filter)}\"" if filter(out_filter)
+      final_command = cmd_parts.join(' ')
+      stdout, stderr, status = Open3.capture3(final_command)
+      raise "gcloud command '#{final_command}' failed: #{stderr}" unless status.success?
+
+      begin
+        return ::JSON.parse(stdout) if format(out_format) == 'json'
+      rescue ::JSON::ParserError
+        return stdout
+      end
+
+      stdout
+    end
+
+    # Stops a GCP VM instance.
+    def stop_instance(instance_name)
+      local_exec("compute instances stop #{instance_name}")
+    end
+
+    # Deletes a GCP VM instance.
+    def delete_instance(instance_name)
+      local_exec("compute instances delete #{instance_name} --quiet")
+    rescue StandardError
+      # Ignore errors when deleting instances.
+    end
+
+    # Returns a Hash describing a GCP VM instance.
+    def vm_describe(instance_name, out_format: 'json', out_filter: nil)
+      local_exec("compute instances describe #{instance_name}", out_format: out_format, out_filter: out_filter)
+    end
+
+    # This function runs the specified command as the currently authenticated user
+    # on the given CGP VM via SSH. Using `ssh` does not invoke the gcloud command
+    # and is dependent on the system's SSH configuration.
+    def ssh(instance_name, cmd, ignore_command_errors: false, use_proxy_command: true, opts: {})
+      Net::SSH.start(instance_name, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |ssh|
+        result = ssh.exec!(cmd)
+        if result.exitstatus != 0 && !ignore_command_errors
+          abridged_cmd = cmd.length > 100 ? "#{cmd[0..100]}..." : cmd
+          raise "Failed to run SSH command \"#{abridged_cmd}\" on #{instance_name}: #{result}"
+        end
+        result
+      end
+    end
+
+    # Uploads a file to the specified VM.
+    def scp_upload(instance_name, local, remote, use_proxy_command: true, scp_opts: {}, opts: {})
+      raise "Local file #{local} does not exist" unless File.exist?(local)
+
+      hostname = use_proxy_command ? vm_alias(instance_name) : instance_name
+      Net::SCP.start(hostname, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |scp|
+        scp.upload(local, remote, scp_opts).wait
+      end
+    end
+
+    # Downloads a file from the specified VM.
+    def scp_download(instance_name, remote, local, use_proxy_command: true, scp_opts: {}, opts: {})
+      raise "Local file #{local} does not exist" unless File.exist?(local)
+
+      hostname = use_proxy_command ? vm_alias(instance_name) : instance_name
+      Net::SCP.start(hostname, user_name, ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |scp|
+        scp.download(remote, local, scp_opts).wait
+      end
+    end
+
+    def ssh_ready?(instance_name, timeout = 300, opts: {})
+      with_timed_retry(timeout) do
+        logger.debug("Testing SSH connection to #{instance_name}")
+        Net::SSH.start(instance_name, user_name, ssh_opts(instance_name: instance_name, opts: opts)) do |_|
+          true
+        end
+      end
+    end
+
+    # This function spawns a background thread to run a GCP IAP tunnel, run the given
+    # code block, then kill the thread. The code block will be yielded ssh_opts that
+    # are used to configure SSH connections over the IAP tunnel. The IAP tunnel is
+    # run in it's own thread and will not block the main thread. The IAP tunnel is
+    # killed when the code block exits. All SSH connections made in the code block
+    # must be made to the host '127.0.0.1' using the yielded ssh_opts and not the VM name.
+    # @param instance_name [String] The name of the GCP VM instance to connect to.
+    # @param instance_port [Integer] The port to connect to on the GCP VM instance.
+    # @param disable_connection_check [Boolean] If true, the connection check will be disabled.
+    def with_iap_tunnel(instance_name, instance_port = 22, disable_connection_check: false)
+      return unless block_given?
+
+      cmd = [
+        'compute start-iap-tunnel',
+        "#{instance_name} #{instance_port}",
+        "--local-host-port=localhost:#{local_port}",
+      ]
+      cmd << '--disable-connection-check' if disable_connection_check
+      tunnel_ssh_opts = {
+        proxy: nil,
+        port: local_port,
+        forward_agent: false,
+      }
+      begin
+        thread = Thread.new do
+          local_exec(cmd.join(' '))
+        end
+        yield ssh_opts(use_proxy_command: false, opts: tunnel_ssh_opts)
+        thread.exit
+      rescue IOError
+        # Ignore errors when killing the thread.
+      ensure
+        thread.exit
+      end
+    end
+
+    def project_from_config
+      local_exec('config get-value project', out_format: nil, project_flag: false)
+    end
+
+    def zone_from_config
+      local_exec('config get-value compute/zone', out_format: nil, project_flag: false)
+    end
+
+    def run_shell(instance_name, cmd, opts = {})
+      ssh_opts = opts.key?(:ssh_opts) ? opts[:ssh_opts] : {}
+      command = [
+        'sudo -n -u root -i',
+        cmd,
+      ]
+      ssh(
+        instance_name,
+        command.join(' '),
+        ignore_command_errors: true,
+        use_proxy_command: false,
+        opts: ssh_opts,
+      )
+    end
+
+    def apply_manifest(instance_name, manifest, opts = {})
+      require 'tempfile'
+
+      temp_manifest = Tempfile.new('acpt_manifest')
+      temp_manifest.write(manifest)
+      temp_manifest.close
+      begin
+        scp_upload(
+          instance_name,
+          temp_manifest.path,
+          '/tmp/acpt_manifest.pp',
+          opts: opts[:ssh_opts],
+        )
+      ensure
+        temp_manifest.unlink
+      end
+      apply_cmd = [
+        'sudo -n -u root -i',
+        "#{opts[:puppet_path]} apply /tmp/acpt_manifest.pp"
+      ]
+      apply_cmd << '--trace' if opts[:apply][:trace]
+      apply_cmd << "--hiera_config=#{opts[:apply][:hiera_config]}" if opts[:apply][:hiera_config]
+      apply_cmd << '--debug' if opts[:apply][:debug]
+      apply_cmd << '--noop' if opts[:apply][:noop]
+      apply_cmd << '--detailed-exitcodes' if opts[:apply][:detailed_exitcodes]
+
+      ssh(
+        instance_name,
+        apply_cmd.join(' '),
+        ignore_command_errors: true,
+        opts: opts[:ssh_opts]
+      )
+    end
+
+    private
+
+    def vm_alias(vm)
+      "compute.#{vm_describe(vm)['id']}"
+    end
+
+    # Default options for Net::SSH
+    def default_ssh_opts
+      {
+        verify_host_key: :accept_new_or_local_tunnel,
+        keys: [File.join(ENV['HOME'], '.ssh', 'google_compute_engine')],
+        keys_only: true,
+        config: false,
+        check_host_ip: false,
+        non_interactive: true,
+        port: 22,
+        user: user_name,
+        user_known_hosts_file: File.join(ENV['HOME'], '.ssh', 'google_compute_known_hosts'),
+      }
+    end
+
+    # Returns a Proxy::Command object to use for the SSH option ProxyCommand.
+    # This works in the same way that `gcloud compute ssh --tunnel-through-iap` does.
+    def proxy_command(instance_name, port: 22, quiet: true, verbosity: nil)
+      proxy_command = [
+        'gcloud compute start-iap-tunnel',
+        "#{instance_name} #{port}",
+        '--listen-on-stdin',
+        "--project=#{project}",
+        "--zone=#{zone}",
+      ]
+      proxy_command << '--no-user-output-enabled --quiet' if quiet
+      proxy_command << "--verbosity=#{verbosity}" unless quiet || verbosity.nil? || verbosity.empty?
+      Net::SSH::Proxy::Command.new(proxy_command.join(' '))
+    end
+
+    # This function checks to ensure that the gcloud tool is available on the system.
+    def gcloud?
+      ver = local_exec('--version', out_format: nil, project_flag: false)
+      !ver.nil? && !ver.empty?
+    rescue RuntimeError, Errno::ENOENT
+      false
+    end
+
+    # This function checks to ensure that a user is authenticated with gcloud.
+    def authenticated?
+      local_exec('auth list', project_flag: false).any? { |a| a['status'] == 'ACTIVE' }
+    rescue RuntimeError, Errno::ENOENT
+      false
+    end
+
+    # This function returns the currently authenticated user's name sanitized for use in SSH
+    def authenticated_user_name
+      uname = nil
+      begin
+        uname = local_exec('config get-value account', out_format: nil, project_flag: false)
+      rescue RuntimeError
+        auth_json = local_exec('auth list')
+        uname = auth_json.find { |x| x['status'] == 'ACTIVE' }['account']
+      end
+
+      raise 'failed to find authenticated user name' unless uname
+
+      uname.split(%r{[^a-zA-Z0-9_]}).join('_')
+    end
+  end
+end