cem_acpt 0.7.3 → 0.8.1

data/lib/cem_acpt/test_runner.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'fileutils'
+require 'securerandom'
+require_relative 'action_result'
 require_relative 'goss'
 require_relative 'logging'
 require_relative 'platform'
@@ -17,14 +20,17 @@ module CemAcpt
       include CemAcpt::Logging
 
       attr_reader :duration, :exit_code
+      attr_accessor :run_data # This is opened up mainly for windows use.
 
       def initialize(config)
         @config = config
         @run_data = {}
         @duration = 0
         @exit_code = 0
-        @results = nil
+        @results = []
         @http_statuses = []
+        @provisioned = false
+        @destroyed = false
       end
 
       def inspect
@@ -38,149 +44,222 @@ module CemAcpt
       def run
         @run_data = {}
         @start_time = Time.now
-        logger.with_ci_group("CemAcpt v#{CemAcpt::VERSION} run started at #{@start_time}") do
-          logger.info('CemAcpt') { "Using module directory: #{config.get('module_dir')}..." }
-          Dir.chdir(config.get('module_dir')) do
-            keep_terminal_alive
-            @run_data[:private_key], @run_data[:public_key], @run_data[:known_hosts] = new_ephemeral_ssh_keys
-            logger.info('CemAcpt') { 'Created ephemeral SSH key pair...' }
-            @run_data[:module_package_path] = build_module_package
-            logger.info('CemAcpt') { "Created module package: #{@run_data[:module_package_path]}..." }
-            @run_data[:test_data] = new_test_data
-            logger.info('CemAcpt') { 'Created test data...' }
-            logger.verbose('CemAcpt') { "Test data: #{@run_data[:test_data]}" }
-            @run_data[:nodes] = new_node_data
-            logger.info('CemAcpt') { 'Created node data...' }
-            logger.verbose('CemAcpt') { "Node data: #{@run_data[:nodes]}" }
-            @instance_names_ips = provision_test_nodes
-            logger.info('CemAcpt') { 'Provisioned test nodes...' }
-            logger.debug('CemAcpt') { "Instance names and IPs: #{@instance_names_ips}" }
-            @results = run_tests(@instance_names_ips.map { |_, v| v['ip'] },
-                                 config.get('actions.only'),
-                                 config.get('actions.except'))
+        module_dir = config.get('module_dir')
+        @old_dir = Dir.pwd
+        Dir.chdir(module_dir)
+        logger.start_ci_group("CemAcpt v#{CemAcpt::VERSION} run started at #{@start_time}")
+        logger.info('CemAcpt::TestRunner') { "Using module directory: #{module_dir}..." }
+        @run_data[:private_key], @run_data[:public_key], @run_data[:known_hosts] = new_ephemeral_ssh_keys
+        logger.info('CemAcpt::TestRunner') { 'Created ephemeral SSH key pair...' }
+        @run_data[:module_package_path] = build_module_package
+        logger.info('CemAcpt::TestRunner') { "Created module package: #{@run_data[:module_package_path]}..." }
+        @run_data[:test_data] = new_test_data
+        logger.info('CemAcpt::TestRunner') { 'Created test data...' }
+        logger.verbose('CemAcpt::TestRunner') { "Test data: #{@run_data[:test_data]}" }
+        @run_data[:nodes] = new_node_data
+        logger.info('CemAcpt::TestRunner') { 'Created node data...' }
+        logger.verbose('CemAcpt::TestRunner') { "Node data: #{@run_data[:nodes]}" }
+        @instance_names_ips = provision_test_nodes
+        logger.info('CemAcpt::TestRunner') { "Instance names and IPs class: #{@instance_names_ips.class}" }
+        @provisioned = true
+        logger.info('CemAcpt::TestRunner') { 'Provisioned test nodes...' }
+        logger.debug('CemAcpt::TestRunner') { "Instance names and IPs: #{@instance_names_ips}" }
+        # Verifying that we're running on windows nodes or not
+        if config.get('tests').first.include? 'windows'
+          logger.info('CemAcpt') { 'Running on windows nodes...' }
+          upload_module_to_bucket
+
+          @instance_names_ips.each do |k, v|
+            # Login_info here is basically a super charged version of a hash from
+            # instance_names_ips. It contains the username, password, and ip of the
+            # windows node, as well as the test name that will be run on that node.
+            login_info = CemAcpt::Utils.get_windows_login_info(k, v)
+            win_node = CemAcpt::Utils::WinRMRunner::WinNode.new(login_info, @run_data[:win_remote_module_name])
+            win_node.run
           end
         end
+        @results = run_tests(@instance_names_ips.map { |_, v| v['ip'] },
+                              config.get('actions.only'),
+                              config.get('actions.except'))
+      rescue StandardError => e
+        logger.error('CemAcpt::TestRunner') { 'Run failed due to error...' }
+        @results << ActionResult.new(e)
       ensure
+        logger.end_ci_group
         clean_up
         process_test_results
+        Dir.chdir(@old_dir) if @old_dir
+        @results
       end
 
-      def clean_up(trap_context = false)
-        kill_keep_terminal_alive unless trap_context
-
-        return no_destroy if config.get('no_destroy_nodes')
+      def clean_up(_trap_context = false)
+        logger.start_ci_group("CemAcpt v#{CemAcpt::VERSION} run finished at #{Time.now}")
+        logger.debug('CemAcpt::TestRunner') { "Starting clean up, provisioned: #{@provisioned}, destroyed: #{@destroyed}" }
 
-        clean_ephemeral_ssh_keys
-        destroy_test_nodes
+        if config.get('no_destroy_nodes')
+          logger.warn('CemAcpt::TestRunner') { 'Not destroying test nodes because no-destroy-nodes is set...' }
+          @provisioner&.show
+          logger.info('CemAcpt') { "Test SSH Keys:\n  Private Key: #{@run_data[:private_key]}\n  Public Key:#{@run_data[:public_key]}" }
+        else
+          cleanup_bucket # Clean up bucket if we're testing the cem_windows module
+          clean_ephemeral_ssh_keys
+          destroy_test_nodes
+        end
+      rescue StandardError => e
+        logger.verbose('CemAcpt::TestRunner') { "Error cleaning up: #{e}" }
+        logger.verbose('CemAcpt::TestRunner') { e.backtrace.join("\n") }
+      ensure
+        logger.end_ci_group
       end
 
       private
 
       attr_reader :config
 
-      # @return [Thread] The thread that keeps the terminal alive
-      def keep_terminal_alive
-        return unless config.ci?
-
-        @keep_terminal_alive ||= CemAcpt::Utils::Terminal.keep_terminal_alive
-      end
-
-      def kill_keep_terminal_alive
-        return if @trap_context
-
-        keep_terminal_alive&.kill
-      end
-
       # @return [String] The path to the module package
       def build_module_package
-        CemAcpt::Utils::Puppet.build_module_package(config.get('module_dir'))
+        if config.get('tests').first.include? 'windows'
+          CemAcpt::Utils.package_win_module(config.get('module_dir'))
+        else
+          CemAcpt::Utils::Puppet.build_module_package(config.get('module_dir'))
+        end
       end
 
       # @return [Array<String>] The paths to the ssh private key, public key, and known hosts file
       def new_ephemeral_ssh_keys
         return [nil, nil, nil] if config.get('no_ephemeral_ssh_key')
 
+        logger.info('CemAcpt::TestRunner') { 'Creating ephemeral SSH keys...' }
         CemAcpt::Utils::SSH::Ephemeral.create
       end
 
       def clean_ephemeral_ssh_keys
         return if config.get('no_ephemeral_ssh_key') || config.get('no_destroy_nodes')
 
+        logger.info('CemAcpt::TestRunner') { 'Cleaning ephemeral SSH keys...' }
         CemAcpt::Utils::SSH::Ephemeral.clean
       end
 
       def new_test_data
+        logger.debug('CemAcpt::TestRunner') { 'Creating new test data...' }
         CemAcpt::TestData.acceptance_test_data(config)
       end
 
       def new_node_data
+        logger.debug('CemAcpt::TestRunner') { 'Creating new node data...' }
         CemAcpt::Platform.use(config.get('platform.name'), config, @run_data)
       end
 
       def provision_test_nodes
+        logger.info('CemAcpt::TestRunner') { 'Provisioning test nodes...' }
         @provisioner = CemAcpt::Provision.new_provisioner(config, @run_data)
         @provisioner.provision
       end
 
       def destroy_test_nodes
-        return no_destroy if config.get('no_destroy_nodes')
-
-        logger.with_ci_group("CemAcpt v#{CemAcpt::VERSION} run finished at #{Time.now}") { @provisioner&.destroy }
+        logger.info('CemAcpt::TestRunner') { 'Destroying test nodes if necessary...' }
+        if !@provisioned
+          logger.warn('CemAcpt::TestRunner') { 'Test nodes not provisioned, nothing to destroy...' }
+        elsif @destroyed
+          logger.warn('CemAcpt::TestRunner') { 'Test nodes already destroyed, not destroying...' }
+        else
+          logger.info('CemAcpt::TestRunner') { 'Test nodes are provisioned and not destroyed, destroying...' }
+          @provisioner&.destroy
+          @destroyed = true
+        end
       end
 
       def no_destroy
-        logger.warn('CemAcpt') { 'Not destroying test nodes...' }
-        logger.with_ci_group("CemAcpt v#{CemAcpt::VERSION} run finished at #{Time.now}") do
-          @provisioner&.show
-          logger.info('CemAcpt') { "Test SSH Keys:\n  Private Key: #{@run_data[:private_key]}\n  Public Key:#{@run_data[:public_key]}" }
-        end
+        logger.warn('CemAcpt::TestRunner') { 'Not destroying test nodes because no-destroy-nodes is set...' }
+        @provisioner&.show
+        logger.info('CemAcpt') { "Test SSH Keys:\n  Private Key: #{@run_data[:private_key]}\n  Public Key:#{@run_data[:public_key]}" }
       end
 
       def run_tests(hosts, only_actions, except_actions)
-        only_actions = [] if only_actions.nil?
-        except_actions = [] if except_actions.nil?
-        CemAcpt::Goss::Api.run_actions_async(hosts, only: only_actions, except: except_actions)
-      end
-
-      def result_log_formatter
-        @result_log_formatter ||= LogFormatter::GossActionResponse.new(config, @instance_names_ips)
+        logger.info('CemAcpt::TestRunner') { 'Running tests...' }
+        logger.verbose('CemAcpt::TestRunner') { "Hosts: #{hosts}" }
+        logger.verbose('CemAcpt::TestRunner') { "Only actions: #{only_actions}" }
+        logger.verbose('CemAcpt::TestRunner') { "Except actions: #{except_actions}" }
+        api_results = CemAcpt::Goss::Api.run_actions_async(hosts,
+                                                           only: only_actions || [],
+                                                           except: except_actions || [])
+        res = []
+        api_results.close unless api_results.closed?
+        while (r = api_results.pop)
+          res << ActionResult.new(r)
+        end
+        res
       end
 
       def process_test_results
-        if @results.nil?
-          logger.error('CemAcpt') { 'No test results to process' }
+        if @results.nil? || @results.empty?
+          logger.error('CemAcpt::TestRunner') { 'No test results to process' }
           @exit_code = 1
         else
+          logger.info('CemAcpt::TestRunner') { "Processing #{@results.size} test result(s)..." }
           until @results.empty?
             result = @results.pop
             @http_statuses << result.http_status
             log_test_result(result)
           end
           if @http_statuses.empty?
-            logger.error('CemAcpt') { 'No test results to process' }
+            logger.error('CemAcpt::TestRunner') { 'No test results to process' }
             @exit_code = 1
           else
-            @exit_code = @http_statuses.any? { |s| s.to_i != 200 } ? 1 : 0
+            @exit_code = (@http_statuses.any? { |s| s.to_i != 200 }) ? 1 : 0
           end
         end
         @duration = Time.now - @start_time
-        logger.info('CemAcpt') { "Test suite finished after ~#{duration.round} seconds." }
+        logger.info('CemAcpt::TestRunner') { "Test suite finished after ~#{duration.round} seconds." }
       end
 
       def log_test_result(result)
-        logger.with_ci_group("Test results for #{result_log_formatter.test_name(result)}") do
-          logger.info(result_log_formatter.summary(result))
-          formatted_results = result_log_formatter.results(result)
-          formatted_results.each do |r|
-            if r.start_with?('Passed:')
-              logger.verbose { r }
-            elsif r.start_with?('Skipped:')
-              logger.info { r }
-            else
-              logger.error { r }
-            end
+        result_log_formatter = LogFormatter.new_formatter(result, config, @instance_names_ips)
+        logger.start_ci_group("Test results for #{result_log_formatter.test_name(result)}")
+        return log_error_test_result(result_log_formatter, result) if result.error?
+
+        logger.info(result_log_formatter.summary(result))
+        formatted_results = result_log_formatter.results(result)
+        formatted_results.each do |r|
+          if r.start_with?('Passed:')
+            logger.verbose { r }
+          elsif r.start_with?('Skipped:')
+            logger.info { r }
+          else
+            logger.error { r }
           end
         end
+      ensure
+        logger.end_ci_group
+      end
+
+      def log_error_test_result(formatter, result)
+        logger.fatal { formatter.results(result).join("\n") }
+      end
+
+      # Upload the cem_windows module to the bucket if we're testing the cem_windows module
+      # This should only be done once per cem_acpt run. It's important to update the module_package_path
+      # in the run_data to reflect the new module path if we do end up changing the module name
+      def upload_module_to_bucket
+        @run_data[:win_remote_module_name] = SecureRandom.uuid << File.split(@run_data[:module_package_path]).last
+        @run_data[:win_remote_module_path] = File.join('gs://win_cem_acpt', @run_data[:win_remote_module_name])
+        # Upload the module from the local host to the bucket
+        logger.info('CemAcpt') { "Uploading #{@run_data[:module_pakage_path]} to #{@run_data[:win_remote_module_path]}..." }
+        CemAcpt::Utils::Shell.run_cmd("gcloud storage cp #{@run_data[:module_package_path]} #{@run_data[:win_remote_module_path]}")
+        logger.debug('CemAcpt') { 'Successfully uploaded module' }
+      end
+
+      # We have to clean up our gcp bucket after we're done testing. This will limit the duplicated
+      # modules existing in the bucket.
+      def cleanup_bucket
+        if @run_data[:win_remote_module_path]
+          logger.info('CemAcpt::TestRunner') { "Cleaning up bucket #{@run_data[:win_remote_module_path]}..." }
+          # Cleanup the module from the bucket
+          cleanup_cmd = CemAcpt::Utils::Shell.run_cmd("gcloud storage rm #{@run_data[:win_remote_module_path]}", raise_on_error: false)
+          logger.debug('CemAcpt::TestRunner') { "Removed module from bucket: #{cleanup_cmd}" } unless cleanup_cmd.nil? || cleanup_cmd.empty?
+        else
+          logger.info('CemAcpt::TestRunner') { 'No module to clean up in bucket...' }
+        end
       end
     end
   end

data/lib/cem_acpt/utils/shell.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'stringio'
+
+module CemAcpt
+  # Error class for shell commands
+  class ShellCommandError < StandardError; end
+
+  module Utils
+    # Generic utilities for running local shell commands
+    module Shell
+      # Runs a command in a subshell and returns the Process::Status
+      # and the string output of the command.
+      # @param cmd [String] The command to run
+      # @param env [Hash] A hash of environment variables to set
+      # @param output [IO] An IO object that implements #:<< to write the output of the
+      #   command to in real time. Typically this is a Logger object. Defaults to $stdout.
+      #   If the object responds to #:debug, the command will be logged at the debug level.
+      # @param raise_on_fail [Boolean] Whether to raise an error if the command fails
+      # @return [String] The string output of the command
+      def self.run_cmd(cmd, env = {}, output: $stdout, raise_on_fail: true)
+        io_outerr = StringIO.new
+        if output.respond_to?(:debug)
+          output.debug('CemAcpt::Utils::Shell') { "Running command:\n\t#{cmd}\nWith environment:\n\t#{env}" }
+        else
+          output << "Running command:\n\t#{cmd}\nWith environment:\n\t#{env}\n"
+        end
+        val = Open3.popen2e(env, cmd) do |stdin, outerr, wait_thr|
+          stdin.close
+          outerr.sync = true
+          output_thread = Thread.new do
+            while (line = outerr.readline_nonblock)
+              output << line if output
+              io_outerr.write(line) unless line.chomp.empty?
+            end
+          rescue IO::WaitReadable
+            begin
+              IO.select([outerr])
+              retry
+            rescue IOError
+              # outerr closed, won't retry
+            end
+          rescue EOFError
+            # outerr closed, won't retry
+          end
+          wait_thr.join
+          output_thread.exit
+          wait_thr.value
+        end
+        io_string = io_outerr.string
+        raise CemAcpt::ShellCommandError, "Error running command: #{cmd}\n#{io_string}" if raise_on_fail && !val.success?
+
+        io_string
+      end
+
+      # Mimics the behavior of the `which` command.
+      # @param cmd [String] The command to find
+      # @return [String] The path to the command
+      # @return [nil] If the command is not found
+      def self.which(cmd)
+        return cmd if File.executable?(cmd) && !File.directory?(cmd)
+
+        exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+        ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+          exts.each do |ext|
+            exe = File.join(path, "#{cmd}#{ext}")
+            return exe if File.executable?(exe) && !File.directory?(exe)
+          end
+        end
+        nil
+      end
+
+      # IO monkey patch for non-blocking readline
+      class ::IO
+        def readline_nonblock
+          rlnb = []
+          rnlb << read_nonblock(1) while rlnb[-1] != "\n"
+          rlnb.join
+        rescue IO::WaitReadable => blocking
+          raise blocking if rlnb.empty?
+
+          rlnb.join
+        rescue EOFError
+          rlnb.join
+        end
+      end
+    end
+  end
+end

data/lib/cem_acpt/utils/ssh.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'open3'
+require_relative 'shell'
 require_relative '../logging'
 
 module CemAcpt
@@ -31,10 +31,8 @@ module CemAcpt
         def create(key_name, **options)
           delete(key_name) # Delete existing keys with same name
           cmd = new_keygen_cmd(key_name, **options)
-          logger.debug("Creating SSH key with command: #{cmd}")
-          _stdout, stderr, status = Open3.capture3(cmd)
-          raise "Failed to create SSH key! #{stderr}" unless status.success?
-
+          logger.debug('CemAcpt::Utils::SSH::Keygen') { "Creating SSH key with command: #{cmd}" }
+          CemAcpt::Utils::Shell.run_cmd(cmd, output: logger)
           key_paths(key_name)
         end
 
@@ -42,7 +40,7 @@ module CemAcpt
           priv_key = key_path(key_name)
           pub_key = key_path(key_name, public_key: true)
           if ::File.file?(priv_key)
-            logger.debug("Deleting private key: #{priv_key}")
+            logger.debug('CemAcpt::Utils::SSH::Keygen') { "Deleting private key: #{priv_key}" }
             ::File.delete(priv_key)
           end
           if ::File.file?(pub_key)
@@ -53,20 +51,11 @@ module CemAcpt
 
         private
 
-        FIND_BIN_PATH_COMMANDS = [
-          "#{ENV['SHELL']} -c 'command -v ssh-keygen'",
-          "#{ENV['SHELL']} -c 'which ssh-keygen'",
-        ].freeze
-
-        def find_bin_path(find_cmd = FIND_BIN_PATH_COMMANDS.first)
-          bin_path, stderr, status = Open3.capture3(find_cmd)
-          raise "Cannot find ssh-keygen with command #{find_cmd}: #{stderr}" unless status.success?
-
-          bin_path.chomp
-        rescue StandardError => e
-          return find_bin_path(FIND_BIN_PATH_COMMANDS.last) unless FIND_BIN_PATH_COMMANDS.last == find_cmd
+        def find_bin_path
+          bin_path = CemAcpt::Utils::Shell.which('ssh-keygen')
+          raise 'Cannot find command ssh-keygen in PATH, make sure it is installed' if bin_path.nil?
 
-          raise e
+          bin_path
         end
 
         def new_keygen_cmd(key_name, **options)
@@ -100,12 +89,10 @@ module CemAcpt
       end
 
       def self.ssh_keygen
-        bin_path = `#{ENV['SHELL']} -c 'command -v ssh-keygen'`.chomp
-        raise 'Cannot find ssh-keygen! Install it and verify PATH' unless bin_path
+        bin_path = CemAcpt::Utils::Shell.which('ssh-keygen')
+        raise 'Cannot find command ssh-keygen in PATH, make sure it is installed' if bin_path.nil?
 
         bin_path
-      rescue StandardError => e
-        raise "Cannot find ssh-keygen! Install it and verify PATH. Orignal error: #{e}"
       end
 
       def self.default_keydir

data/lib/cem_acpt/utils/terminal.rb

@@ -1,14 +1,11 @@
 # frozen_string_literal: true
 
-require 'concurrent-ruby'
-
 module CemAcpt
   module Utils
     # Terminal-related utilities
     module Terminal
       def self.keep_terminal_alive
-        executor = Concurrent::SingleThreadExecutor.new
-        executor.post do
+        Thread.new do
           loop do
             $stdout.print(".\r")
             sleep(1)
@@ -20,7 +17,6 @@ module CemAcpt
             sleep(1)
           end
         end
-        executor
       end
     end
   end