cem_acpt 0.2.5 → 0.6.1

data/lib/cem_acpt/context.rb

@@ -1,153 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'platform'
-require_relative 'utils'
-require_relative 'shared_objects'
-require_relative 'test_data'
-require_relative 'test_runner/run_handler'
-require_relative 'logging'
-
-module CemAcpt
-  # Context provides the context in which the RunHandler creates and starts Runners.
-  module Context
-    class ContextError < StandardError; end
-
-    class << self
-      include CemAcpt::Logging
-
-      KEY_PATH = File.join([ENV['HOME'], '.ssh', 'acpt_test_key']).freeze
-      KH_PATH = File.join([ENV['HOME'], '.ssh', 'acpt_test_known_hosts']).freeze
-
-      def log(msg, level = :info)
-        real_msg = "CONTEXT: #{msg}"
-        logger.send(level, real_msg)
-      end
-
-      # Builds the Puppet module package
-      # @param opts [Hash] config opts
-      # @return [String] The path to the Puppet module package
-      def build_module_package(opts = {})
-        module_dir = opts[:module_dir] || __dir__
-        pkg_path = CemAcpt::Utils::Puppet.build_module_package(module_dir)
-        log("Module package built at #{pkg_path}")
-        pkg_path
-      end
-
-      # Creates a SSH key and a SSH known hosts file for the acceptance test suite
-      def new_test_ssh_key
-        log('Creating ephemeral SSH key and known hosts file for acceptance test suites...')
-        @ssh_priv_key, @ssh_pub_key = CemAcpt::Utils::SSH.ephemeral_ssh_key
-        @ssh_known_hosts = CemAcpt::Utils::SSH.acpt_known_hosts
-        CemAcpt::Utils::SSH.set_ssh_file_permissions(@ssh_priv_key, @ssh_pub_key, @ssh_known_hosts)
-        log('Successfully created SSH files...')
-        log("SSH private key: #{@ssh_priv_key}", :debug)
-        log("SSH public key: #{@ssh_pub_key}", :debug)
-        log("SSH known hosts: #{@ssh_known_hosts}", :debug)
-      end
-
-      # Deletes acceptance test suite SSH files
-      def clean_test_ssh_key
-        log('Deleting ephemeral ssh keys and acpt_known_hosts if they exist...')
-        [@ssh_priv_key, @ssh_pub_key, @ssh_known_hosts].map { |f| File.delete(f) if File.exist?(f) }
-      end
-
-      # Prints a period to the terminal every 5 seconds in a single line to keep the terminal
-      # alive. This is used when running in CI mode. Does nothing unless the option `:CI` is
-      # `true`, or the environment variables CI or GITHUB_ACTION are set to a truthy value.
-      # @param opts [Hash] config opts
-      def keep_terminal_alive(opts = {})
-        @keep_terminal_alive = if opts[:CI] || ENV['CI'] || ENV['GITHUB_ACTION']
-                                 CemAcpt::Utils::Terminal.keep_terminal_alive
-                               end
-      end
-
-      def clean_up_test_suite(opts)
-        @ctx.node_inventory.clear!
-        @ctx.node_inventory.clean_local_files
-        clean_test_ssh_key unless opts[:no_ephemeral_ssh_key]
-        @run_handler&.destroy_test_nodes
-        @keep_terminal_alive&.kill
-      end
-    end
-
-    # Creates a context (CemAcpt::Context::Ctx) object for the RunHandler to create and start Runners.
-    # Provides the following objects for the Runners: a config object,
-    # the test data hash, the node inventory, and the local port allocator.
-    # Additionally, it creates the platform-specific node objects for each
-    # test suite in the test data. It then calls the provided block with
-    # the context objects nodes, config, test_data, and node_inventory.
-    # @param config_opts [Hash] the config options
-    def self.with(**opts)
-      @opts = opts
-      @start_time = Time.now
-      raise CemAcpt::Context::ContextError, 'CemAcpt::Context.with requires a block' unless block_given?
-
-      config_file = @opts[:config_file] || File.expand_path('./cem_acpt_config.yaml')
-      logger.info("Running acceptance test suite at #{@start_time}")
-      logger.debug("Config opts: #{@opts}")
-      logger.debug("Config file: #{config_file}")
-      logger.info("Using module directory: #{@opts[:module_dir]}")
-      keep_terminal_alive(@opts)
-      Dir.chdir(opts[:module_dir]) do
-        new_test_ssh_key unless @opts[:no_ephemeral_ssh_key]
-        pkg_path = build_module_package(@opts)
-        @ctx = CemAcpt::Context::Ctx.new(opts: @opts, config_file: config_file, module_package_path: pkg_path)
-        logger.debug("Created Ctx object #{@ctx}")
-        @run_handler = CemAcpt::TestRunner::RunHandler.new(@ctx)
-        logger.debug("Created RunHandler object #{@run_handler}")
-        yield @run_handler
-      end
-      @exit_code = @run_handler.exit_code
-    rescue StandardError => e
-      logger.fatal("Acceptance test suite encountered an error: #{e.message}")
-      logger.fatal(e.backtrace.join("\n"))
-      @exit_code = 1
-    ensure
-      clean_up_test_suite(@opts)
-      total_minutes = ((Time.now - @start_time) / 60).round
-      logger.info("Test suite finished in ~#{total_minutes} minutes")
-      @exit_code || 1
-    end
-
-    # Ctx holds the context objects for the RunHandler to create and start Runners.
-    class Ctx
-      attr_reader :config, :test_data, :module_package_path, :node_inventory, :local_port_allocator
-
-      def initialize(opts: nil, config_file: nil, module_package_path: nil)
-        @config = CemAcpt::Config.new.load(opts: opts, config_file: config_file)
-        @test_data = CemAcpt::TestData.acceptance_test_data(@config)
-        @module_package_path = module_package_path
-        @node_inventory = CemAcpt::NodeInventory.new
-        @local_port_allocator = CemAcpt::LocalPortAllocator.new
-        prep_environment
-      end
-
-      def nodes
-        return @nodes if defined?(@nodes)
-
-        raise CemAcpt::Error, 'No platform(s) specified' unless @config.has?('platform') || @config.has?('platforms')
-
-        @nodes = nodes_from_platforms
-        @nodes
-      end
-
-      private
-
-      def prep_environment
-        @node_inventory.clean_local_files
-      end
-
-      def nodes_from_platforms
-        nodes = {}
-        if @config.has?('platform')
-          nodes[@config.get('platform')] = CemAcpt::Platform.use(@config.get('platform'), @config, @test_data, @local_port_allocator)
-        elsif @config.has?('platforms')
-          config.get('platforms').each do |platform|
-            nodes[platform] = CemAcpt::Platform.use(platform, @config, @test_data, @local_port_allocator)
-          end
-        end
-        nodes
-      end
-    end
-  end
-end

data/lib/cem_acpt/platform/base/cmd.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module CemAcpt::Platform
-  require_relative File.join(__dir__, '..', '..', 'logging.rb')
-
-  class CmdError < StandardError; end
-
-  # Base class for command providers. Provides an API for subclasses to implement.
-  class CmdBase
-    include CemAcpt::Logging
-
-    attr_reader :env
-
-    def initialize(*_args, env: {}, **_kwargs)
-      @env = env
-    end
-
-    def local_exec(*_args, **_kwargs)
-      raise NotImplementedError, '#local_exec must be implemented by a subclass'
-    end
-
-    def ssh(_instance_name, _command, _ignore_command_errors: false, _opts: {})
-      raise NotImplementedError, '#ssh must be implemented by a subclass'
-    end
-
-    def scp_upload(_instance_name, _local, _remote, _scp_opts: {}, _opts: {})
-      raise NotImplementedError, '#scp_upload must be implemented by a subclass'
-    end
-
-    def scp_download(_instance_name, _local, _remote, _scp_opts: {}, _opts: {})
-      raise NotImplementedError, '#scp_download must be implemented by a subclass'
-    end
-
-    def ssh_ready?(_instance_name, _timeout = 300, _opts: {})
-      raise NotImplementedError, '#ssh_ready? must be implemented by a subclass'
-    end
-
-    def apply_manifest(_instance_name, _manifest, _opts: {})
-      raise NotImplementedError, '#create_manifest_on_node must be implemented by a subclass'
-    end
-
-    def run_shell(_instance_name, _command, _opts: {})
-      raise NotImplementedError, '#run_shell must be implemented by a subclass'
-    end
-
-    def trim_output(output)
-      if output.include?("\n")
-        output.split("\n").map(&:strip).reject(&:empty?).join("\n")
-      else
-        output.strip
-      end
-    end
-
-    def with_timed_retry(timeout = 300)
-      return unless block_given?
-
-      last_error = nil
-      start_time = Time.now
-      while Time.now - start_time < timeout
-        begin
-          output = yield
-          return output
-        rescue StandardError => e
-          last_error = e
-          sleep(10)
-        end
-      end
-      raise last_error
-    end
-  end
-end

data/lib/cem_acpt/platform/gcp/cmd.rb

@@ -1,345 +0,0 @@
-# frozen_string_literal: true
-
-module CemAcpt::Platform::Gcp
-  require 'json'
-  require 'open3'
-  require_relative File.join(__dir__, '..', 'base', 'cmd.rb')
-
-  # This class provides methods to run gcloud commands. It allows for default values to be
-  # set for the project, zone, and user and can also find these values from the local config.
-  # Additionally, this class provides a way to run SSH commands against GCP VMs using IAP.
-  class Cmd < CemAcpt::Platform::CmdBase
-    def initialize(project: nil, zone: nil, out_format: nil, filter: nil, user_name: nil, local_port: nil, ssh_key: nil)
-      super(env: { 'CLOUDSDK_PYTHON_SITEPACKAGES' => '1' })
-      require 'net/ssh'
-      require 'net/ssh/proxy/command'
-
-      @project = project unless project.nil?
-      @zone = zone unless zone.nil?
-      @default_out_format = out_format
-      @default_filter = filter
-      @user_name = user_name
-      @local_port = local_port
-      @ssh_key = ssh_key
-      raise CemAcpt::Platform::CmdError, 'gcloud command not available' unless gcloud?
-      raise CemAcpt::Platform::CmdError, 'gcloud is not authenticated' unless authenticated?
-    end
-
-    # Returns either the project name passed in during object initialization
-    # or the project name from the local config.
-    def project
-      @project ||= project_from_config&.chomp
-    end
-
-    # Returns either the zone name passed in during object initialization
-    # or the zone name from the local config.
-    def zone
-      @zone ||= zone_from_config&.chomp
-    end
-
-    # Returns either the user name passed in during object initialization
-    # or queries the current active, authenticated user from gcloud and returns the name.
-    def user_name
-      @user_name ||= authenticated_user_name
-    end
-
-    # Returns the format string passed in during object initialization or
-    # the default format string.
-    def format(out_format = nil)
-      out_format&.chomp || @default_out_format
-    end
-
-    # Returns the filter string passed in during object initialization or
-    # the default filter string.
-    def filter(out_filter = nil)
-      out_filter&.chomp || @default_filter
-    end
-
-    def local_port
-      @local_port ||= rand(49_512..65_535)
-    end
-
-    def ssh_key
-      return @ssh_key unless @ssh_key.nil?
-
-      if File.exist?(File.join([ENV['HOME'], '.ssh', 'acpt_test_key']))
-        @ssh_key = File.join([ENV['HOME'], '.ssh', 'acpt_test_key'])
-      else
-        logger.debug("Test SSH key not found at #{File.join([ENV['HOME'], '.ssh', 'acpt_test_key'])}, using default")
-        @ssh_key = File.join([ENV['HOME'], '.ssh', 'google_compute_engine'])
-      end
-      @ssh_key
-    end
-
-    # Returns a formatted hash of ssh options to be used with Net::SSH.start.
-    # If you pass in a GCP VM instance name, this method will configure the
-    # IAP tunnel ProxyCommand to use. If you pass in an opts hash, it will
-    # merge the options with the default options.
-    def ssh_opts(instance_name: nil, use_proxy_command: true, opts: {})
-      base_opts = default_ssh_opts
-      if use_proxy_command
-        base_opts[:proxy] = proxy_command(instance_name, port: base_opts[:port])
-        base_opts[:host_key_alias] = vm_alias(instance_name)
-      end
-      base_opts.merge(opts).reject { |_, v| v.nil? }
-    end
-
-    # Runs `gcloud` commands locally.
-    def local_exec(command, out_format: 'json', out_filter: nil, project_flag: true)
-      final_command = format_gcloud_command(command, out_format: out_format, out_filter: out_filter, project_flag: project_flag)
-      stdout, stderr, status = Open3.capture3(env, final_command)
-      raise "gcloud command '#{final_command}' failed: #{stderr}" unless status.success?
-
-      if format(out_format) == 'json'
-        begin
-          ::JSON.parse(stdout)
-        rescue ::JSON::ParserError
-          stdout.chomp
-        end
-      else
-        stdout.chomp
-      end
-    end
-
-    # Stops a GCP VM instance.
-    def stop_instance(instance_name)
-      local_exec("compute instances stop #{instance_name}")
-    end
-
-    # Deletes a GCP VM instance.
-    def delete_instance(instance_name)
-      local_exec("compute instances delete #{instance_name} --quiet")
-    rescue StandardError
-      # Ignore errors when deleting instances.
-    end
-
-    # Returns a Hash describing a GCP VM instance.
-    def vm_describe(instance_name, out_format: 'json', out_filter: nil)
-      local_exec("compute instances describe #{instance_name}", out_format: out_format, out_filter: out_filter)
-    end
-
-    # This function runs the specified command as the currently authenticated user
-    # on the given CGP VM via SSH. Using `ssh` does not invoke the gcloud command
-    # and is dependent on the system's SSH configuration.
-    def ssh(instance_name, cmd, ignore_command_errors: false, use_proxy_command: true, opts: {})
-      Net::SSH.start(instance_name, user_name,
-                     ssh_opts(instance_name: instance_name, use_proxy_command: use_proxy_command, opts: opts)) do |ssh|
-        logger.debug("Running SSH command '#{cmd}' on instance '#{instance_name}'")
-        result = ssh.exec!(cmd)
-        if result.exitstatus != 0 && !ignore_command_errors
-          abridged_cmd = cmd.length > 100 ? "#{cmd[0..100]}..." : cmd
-          raise "Failed to run SSH command \"#{abridged_cmd}\" on #{instance_name}: #{result}"
-        end
-        result
-      end
-    end
-
-    # Uploads a file to the specified VM.
-    def scp_upload(instance_name, local, remote, use_proxy_command: true, scp_opts: {}, opts: {})
-      raise "Local file #{local} does not exist" unless File.exist?(local)
-
-      cmd = [
-        'compute',
-        'scp',
-      ]
-      cmd << '--strict-host-key-checking=no'
-      cmd << "--tunnel-through-iap" if use_proxy_command
-      cmd << "--recurse" if scp_opts[:recurse]
-      cmd << "#{local} #{instance_name}:#{remote}"
-      local_exec(cmd.join(' '), out_format: 'json')
-    end
-
-    # Downloads a file from the specified VM.
-    def scp_download(instance_name, remote, local, use_proxy_command: true, scp_opts: {}, opts: {})
-      raise "Local file #{local} does not exist" unless File.exist?(local)
-
-      cmd = [
-        'compute',
-        'scp',
-      ]
-      cmd << '--strict-host-key-checking=no'
-      cmd << '--tunnel-through-iap' if use_proxy_command
-      cmd << '--recurse' if scp_opts[:recurse]
-      cmd << "#{instance_name}:#{remote} #{local}"
-      local_exec(cmd.join(' '), out_format: 'json')
-    end
-
-    def ssh_ready?(instance_name, opts: {})
-      ssh_options = ssh_opts(instance_name: instance_name, opts: opts)
-      logger.debug("Testing SSH connection to #{instance_name} with options #{ssh_options}")
-      gcloud_ssh(instance_name, 'echo "SSH is ready"')
-      logger.debug('Removing ecdsa & ed25519 host keys from config due to bug in jruby-openssl')
-      gcloud_ssh(instance_name, 'sudo sed -E -i "s;HostKey /etc/ssh/ssh_host_(ecdsa|ed25519)_key;;g" /etc/ssh/sshd_config')
-      logger.debug('Restarting SSH service')
-      gcloud_ssh(instance_name, 'sudo systemctl restart sshd', ignore_command_errors: true)
-      logger.info("SSH connection to #{instance_name} is ready")
-      true
-    rescue StandardError => e
-      logger.debug("SSH connection to #{instance_name} failed: #{e}")
-      false
-    end
-
-    # This function spawns a background thread to run a GCP IAP tunnel, run the given
-    # code block, then kill the thread. The code block will be yielded ssh_opts that
-    # are used to configure SSH connections over the IAP tunnel. The IAP tunnel is
-    # run in it's own thread and will not block the main thread. The IAP tunnel is
-    # killed when the code block exits. All SSH connections made in the code block
-    # must be made to the host '127.0.0.1' using the yielded ssh_opts and not the VM name.
-    # @param instance_name [String] The name of the GCP VM instance to connect to.
-    # @param instance_port [Integer] The port to connect to on the GCP VM instance.
-    # @param disable_connection_check [Boolean] If true, the connection check will be disabled.
-    def with_iap_tunnel(instance_name, instance_port = 22, disable_connection_check: false, &block)
-      cmd = [
-        'compute start-iap-tunnel',
-        "#{instance_name} #{instance_port}",
-        "--local-host-port=localhost:#{local_port}",
-      ]
-      cmd << '--disable-connection-check' if disable_connection_check
-      tunnel_ssh_opts = {
-        proxy: nil,
-        port: local_port,
-        forward_agent: false,
-      }
-      final_cmd = format_gcloud_command(cmd.join(' '), out_format: nil, project_flag: false)
-      tunnel_pid = Process.spawn(env, final_cmd)
-      begin
-        block.call(ssh_opts(use_proxy_command: false, opts: tunnel_ssh_opts))
-      ensure
-        Process.kill('TERM', tunnel_pid)
-      end
-    end
-
-    def project_from_config
-      local_exec('config get-value project', out_format: nil, project_flag: false)
-    end
-
-    def zone_from_config
-      local_exec('config get-value compute/zone', out_format: nil, project_flag: false)
-    end
-
-    def run_shell(instance_name, cmd, opts = {})
-      ssh_opts = opts.key?(:ssh_opts) ? opts[:ssh_opts] : {}
-      command = [
-        'sudo -n -u root -i',
-        cmd,
-      ]
-      ssh(
-        instance_name,
-        command.join(' '),
-        ignore_command_errors: true,
-        opts: ssh_opts,
-      )
-    end
-
-    def apply_manifest(instance_name, manifest, opts = {})
-      unless opts[:apply][:no_upload]
-        with_temp_manifest_file(manifest) do |tf|
-          upload_temp_manifest(instance_name, tf.path, remote_path: '/tmp/acpt_manifest.pp', opts: opts)
-        end
-      end
-      apply_cmd = [opts[:puppet_path], 'apply', '/tmp/acpt_manifest.pp']
-      apply_cmd << '--trace' if opts[:apply][:trace]
-      apply_cmd << "--hiera_config=#{opts[:apply][:hiera_config]}" if opts[:apply][:hiera_config]
-      apply_cmd << '--debug' if opts[:apply][:debug]
-      apply_cmd << '--noop' if opts[:apply][:noop]
-      apply_cmd << '--detailed-exitcodes' if opts[:apply][:detailed_exitcodes]
-
-      run_shell(
-        instance_name,
-        apply_cmd.join(' '),
-        opts
-      )
-    end
-
-    private
-
-    def format_gcloud_command(command, out_format: 'json', out_filter: nil, project_flag: true)
-      cmd_parts = ['gcloud', command]
-      cmd_parts << "--project=#{project.chomp}" unless !project_flag || project.nil?
-      cmd_parts << "--format=#{format(out_format)}" if format(out_format)
-      cmd_parts << "--filter=\"#{filter(out_filter)}\"" if filter(out_filter)
-      cmd_parts.join(' ')
-    end
-
-    def with_temp_manifest_file(manifest, file_name: 'acpt_manifest')
-      require 'tempfile'
-      tf = Tempfile.new(file_name)
-      tf.write(manifest)
-      tf.close
-      begin
-        yield tf
-      ensure
-        tf.unlink
-      end
-    end
-
-    def upload_temp_manifest(instance_name, local_path, remote_path: '/tmp/acpt_manifest.pp', opts: {})
-      scp_upload(instance_name, local_path, remote_path, opts: opts[:ssh_opts]) unless opts[:apply][:no_upload]
-    end
-
-    def gcloud_ssh(instance_name, command, ignore_command_errors: false)
-      local_exec("compute ssh --ssh-key-file #{ssh_key} --tunnel-through-iap #{instance_name} --command='#{command}'")
-    rescue StandardError => e
-      raise e unless ignore_command_errors
-    end
-
-    def vm_alias(vm)
-      "compute.#{vm_describe(vm)['id']}"
-    end
-
-    # Default options for Net::SSH
-    def default_ssh_opts
-      {
-        auth_methods: ['publickey'],
-        check_host_ip: false,
-        compression: true,
-        config: false,
-        keys: [ssh_key],
-        keys_only: true,
-        kex: ['diffie-hellman-group-exchange-sha256'], # ecdh algos cause jruby to shit the bed
-        non_interactive: true,
-        port: 22,
-        user: user_name,
-        user_known_hosts_file: File.join(ENV['HOME'], '.ssh', 'acpt_test_known_hosts'),
-        verify_host_key: :never,
-      }
-    end
-
-    # Returns a Proxy::Command object to use for the SSH option ProxyCommand.
-    # This works in the same way that `gcloud compute ssh --tunnel-through-iap` does.
-    def proxy_command(instance_name, port: 22, quiet: true, verbosity: 'debug')
-      proxy_command = [
-        'gcloud compute start-iap-tunnel',
-        "#{instance_name} #{port}",
-        '--listen-on-stdin',
-        "--project=#{project}",
-        "--zone=#{zone}",
-      ]
-      #proxy_command << '--no-user-output-enabled --quiet' if quiet
-      proxy_command << "--verbosity=#{verbosity}" unless quiet || verbosity.nil? || verbosity.empty?
-      Net::SSH::Proxy::Command.new(proxy_command.join(' '))
-    end
-
-    # This function checks to ensure that the gcloud tool is available on the system.
-    def gcloud?
-      ver = local_exec('--version', out_format: nil, project_flag: false)
-      !ver.nil? && !ver.empty?
-    rescue RuntimeError, Errno::ENOENT
-      false
-    end
-
-    # This function checks to ensure that a user is authenticated with gcloud.
-    def authenticated?
-      local_exec('auth list', project_flag: false).any? { |a| a['status'] == 'ACTIVE' }
-    rescue RuntimeError, Errno::ENOENT
-      false
-    end
-
-    # This function returns the currently authenticated user's name sanitized for use in SSH
-    def authenticated_user_name
-      local_exec('compute os-login describe-profile', project_flag: false)['posixAccounts'][0]['username']
-    rescue StandardError => e
-      raise "failed to find authenticated user name from os-login profile: #{e.message}"
-    end
-  end
-end