beaker 2.29.1 → 2.30.0

data/lib/beaker/test_case.rb

@@ -1,11 +1,4 @@
-[ 'host', 'dsl' ].each do |lib|
-  require "beaker/#{lib}"
-end
-
-require 'tempfile'
-require 'benchmark'
-require 'stringio'
-require 'rbconfig'
+require 'beaker/runner/native/test_case'
 
 module Beaker
   # This class represents a single test case. A test case is necessarily
@@ -18,173 +11,6 @@ module Beaker
   #
   # See {Beaker::DSL} for more information about writing tests
   # using the DSL.
-  class TestCase
-    include Beaker::DSL
-
-    # The Exception raised by Ruby's STDLIB's test framework (Ruby 1.9)
-    TEST_EXCEPTION_CLASS = ::MiniTest::Assertion
-
-    # Necessary for implementing {Beaker::DSL::Helpers#confine}.
-    # Assumed to be an array of valid {Beaker::Host} objects for
-    # this test case.
-    attr_accessor :hosts
-
-    # Necessary for many methods in {Beaker::DSL}. Assumed to be
-    # an instance of {Beaker::Logger}.
-    attr_accessor :logger
-
-    # Necessary for many methods in {Beaker::DSL::Helpers}.  Assumed to be
-    # a hash.
-    attr_accessor :metadata
-
-    #The full log for this test
-    attr_accessor :sublog
-
-    #The result for the last command run
-    attr_accessor :last_result
-
-    # A Hash of 'product name' => 'version installed', only set when
-    # products are installed via git or PE install steps. See the 'git' or
-    # 'pe' directories within 'ROOT/setup' for examples.
-    attr_reader :version
-
-    # Parsed command line options.
-    attr_reader :options
-
-    # The path to the file which contains this test case.
-    attr_reader :path
-
-    # I don't know why this is here
-    attr_reader :fail_flag
-
-    # The user that is running this tests home directory, needed by 'net/ssh'.
-    attr_reader :usr_home
-
-    # A Symbol denoting the status of this test (:fail, :pending,
-    # :skipped, :pass).
-    attr_reader :test_status
-
-    # The exception that may have stopped this test's execution.
-    attr_reader :exception
-
-    # @deprecated
-    # The amount of time taken to execute the test. Unused, probably soon
-    # to be removed or refactored.
-    attr_reader :runtime
-
-    # An Array of Procs to be called after test execution has stopped
-    # (whether by exception or not).
-    attr_reader :teardown_procs
-
-    # @deprecated
-    # Legacy accessor from when test files would only contain one remote
-    # action.  Contains the Result of the last call to utilize
-    # {Beaker::DSL::Helpers#on}.  Do not use as it is not safe
-    # in test files that use multiple calls to
-    # {Beaker::DSL::Helpers#on}.
-    attr_accessor :result
-
-    # @param [Hosts,Array<Host>] these_hosts The hosts to execute this test
-    #                                        against/on.
-    # @param [Logger] logger A logger that implements
-    #                        {Beaker::Logger}'s interface.
-    # @param [Hash{Symbol=>String}] options Parsed command line options.
-    # @param [String] path The local path to a test file to be executed.
-    def initialize(these_hosts, logger, options={}, path=nil)
-      @hosts   = these_hosts
-      @logger = logger
-      @sublog = ""
-      @options = options
-      @path    = path
-      @usr_home = options[:home]
-      @test_status = :pass
-      @exception = nil
-      @runtime = nil
-      @teardown_procs = []
-      @metadata = {}
-      set_current_test_filename(@path ? File.basename(@path, '.rb') : nil)
-
-
-      #
-      # We put this on each wrapper (rather than the class) so that methods
-      # defined in the tests don't leak out to other tests.
-      class << self
-        def run_test
-          @logger.start_sublog
-          @logger.last_result = nil
-
-          set_current_step_name(nil)
-
-          #add arbitrary role methods
-          roles = []
-          @hosts.each do |host|
-            roles << host[:roles]
-          end
-          add_role_def( roles.flatten.uniq )
-
-          @runtime = Benchmark.realtime do
-            begin
-              test = File.read(path)
-              eval test,nil,path,1
-            rescue FailTest, TEST_EXCEPTION_CLASS => e
-              @test_status = :fail
-              @exception   = e
-            rescue PendingTest
-              @test_status = :pending
-            rescue SkipTest
-              @test_status = :skip
-            rescue StandardError, ScriptError, SignalException => e
-              log_and_fail_test(e)
-            ensure
-              @teardown_procs.each do |teardown|
-                begin
-                  teardown.call
-                rescue StandardError, SignalException, TEST_EXCEPTION_CLASS => e
-                  log_and_fail_test(e)
-                end
-              end
-            end
-          end
-          @sublog = @logger.get_sublog
-          @last_result = @logger.last_result
-          return self
-        end
-
-        private
-
-        # Log an error and mark the test as failed, passing through an
-        # exception so it can be displayed at the end of the total run.
-        #
-        # We break out the complete exception backtrace and log each line
-        # individually as well.
-        #
-        # @param exception [Exception] exception to fail with
-        def log_and_fail_test(exception)
-          logger.error("#{exception.class}: #{exception.message}")
-          bt = exception.backtrace
-          logger.pretty_backtrace(bt).each_line do |line|
-            logger.error(line)
-          end
-          @test_status = :error
-          @exception   = exception
-        end
-      end
-    end
-
-    # The TestCase as a hash
-    # @api public
-    # @note The visibility and semantics of this method are valid, but the
-    #   structure of the Hash it returns may change without notice
-    #
-    # @return [Hash] A Hash representation of this test.
-    def to_hash
-      hash = {}
-      hash['HOSTS'] = {}
-      @hosts.each do |host|
-        hash['HOSTS'][host.name] = host.overrides
-      end
-      hash
-    end
-
+  class TestCase < Beaker::Runner::Native::TestCase
   end
 end

data/lib/beaker/test_suite.rb

@@ -1,407 +1,17 @@
-# -*- coding: utf-8 -*-
-require 'nokogiri'
-require 'fileutils'
-[ 'test_case', 'logger' ].each do |lib|
-  require "beaker/#{lib}"
-end
+require 'beaker/runner/native/test_suite'
+require 'beaker/runner/mini_test/test_suite'
 
 module Beaker
-  #A collection of {TestCase} objects are considered a {TestSuite}.
-  #Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
   class TestSuite
-
-    #Holds the output of a test suite, formats in plain text or xml
-    class TestSuiteResult
-      attr_accessor :start_time, :stop_time, :total_tests
-
-      #Create a {TestSuiteResult} instance.
-      #@param [Hash{Symbol=>String}] options Options for this object
-      #@option options [Logger] :logger The Logger object to report information to
-      #@param [String] name The name of the {TestSuite} that the results are for
-      def initialize( options, name )
-        @options = options
-        @logger = options[:logger]
-        @name = name
-        @test_cases = []
-        #Set some defaults, just in case you attempt to print without including them
-        start_time = Time.at(0)
-        stop_time = Time.at(1)
-      end
-
-      #Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
-      #@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
-      def add_test_case( test_case )
-        @test_cases << test_case
-      end
-
-      #How many {TestCase} instances are in this {TestSuiteResult}
-      def test_count
-        @test_cases.length
-      end
-
-      #How many passed {TestCase} instances are in this {TestSuiteResult}
-      def passed_tests
-        @test_cases.select { |c| c.test_status == :pass }.length
-      end
-
-      #How many errored {TestCase} instances are in this {TestSuiteResult}
-      def errored_tests
-        @test_cases.select { |c| c.test_status == :error }.length
-      end
-
-      #How many failed {TestCase} instances are in this {TestSuiteResult}
-      def failed_tests
-        @test_cases.select { |c| c.test_status == :fail }.length
-      end
-
-      #How many skipped {TestCase} instances are in this {TestSuiteResult}
-      def skipped_tests
-        @test_cases.select { |c| c.test_status == :skip }.length
-      end
-
-      #How many pending {TestCase} instances are in this {TestSuiteResult}
-      def pending_tests
-        @test_cases.select {|c| c.test_status == :pending}.length
-      end
-
-      #How many {TestCase} instances failed in this {TestSuiteResult}
-      def sum_failed
-        failed_tests + errored_tests
-      end
-
-      #Did all the {TestCase} instances in this {TestSuiteResult} pass?
-      def success?
-        sum_failed == 0
-      end
-
-      #Did one or more {TestCase} instances in this {TestSuiteResult} fail?
-      def failed?
-        !success?
-      end
-
-      #The sum of all {TestCase} runtimes in this {TestSuiteResult}
-      def elapsed_time
-        @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
-      end
-
-      #Plain text summay of test suite
-      #@param [Logger] summary_logger The logger we will print the summary to
-      def summarize(summary_logger)
-
-        summary_logger.notify <<-HEREDOC
-      Test Suite: #{@name} @ #{start_time}
-
-      - Host Configuration Summary -
-        HEREDOC
-
-        average_test_time = elapsed_time / test_count
-
-        summary_logger.notify %Q[
-
-              - Test Case Summary for suite '#{@name}' -
-       Total Suite Time: %.2f seconds
-      Average Test Time: %.2f seconds
-              Attempted: #{test_count}
-                 Passed: #{passed_tests}
-                 Failed: #{failed_tests}
-                Errored: #{errored_tests}
-                Skipped: #{skipped_tests}
-                Pending: #{pending_tests}
-                  Total: #{@total_tests}
-
-      - Specific Test Case Status -
-        ] % [elapsed_time, average_test_time]
-
-        grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
-
-        summary_logger.notify "Failed Tests Cases:"
-        (grouped_summary[:fail] || []).each do |test_case|
-          print_test_result(test_case)
-        end
-
-        summary_logger.notify "Errored Tests Cases:"
-        (grouped_summary[:error] || []).each do |test_case|
-          print_test_result(test_case)
-        end
-
-        summary_logger.notify "Skipped Tests Cases:"
-        (grouped_summary[:skip] || []).each do |test_case|
-          print_test_result(test_case)
-        end
-
-        summary_logger.notify "Pending Tests Cases:"
-        (grouped_summary[:pending] || []).each do |test_case|
-          print_test_result(test_case)
-        end
-
-        summary_logger.notify("\n\n")
-      end
-
-      #A convenience method for printing the results of a {TestCase}
-      #@param [TestCase] test_case The {TestCase} to examine and print results for
-      def print_test_result(test_case)
-        test_reported = if test_case.exception
-                          "reported: #{test_case.exception.inspect}"
-                        else
-                          test_case.test_status
-                        end
-        @logger.notify "  Test Case #{test_case.path} #{test_reported}"
-      end
-
-      # Writes Junit XML of this {TestSuiteResult}
-      #
-      # @param [String] xml_file      Path to the XML file (from Beaker's running directory)
-      # @param [String] file_to_link  Path to the paired file that should be linked
-      #                               from this one (this is relative to the XML
-      #                               file itself, so it would just be the different
-      #                               file name if they're in the same directory)
-      # @param [Boolean] time_sort    Whether the test results should be output in
-      #                               order of time spent in the test, or in the
-      #                               order of test execution (default)
-      #
-      # @return nil
-      # @api private
-      def write_junit_xml(xml_file, file_to_link = nil, time_sort = false)
-        stylesheet = File.join(@options[:project_root], @options[:xml_stylesheet])
-
-        begin
-          LoggerJunit.write_xml(xml_file, stylesheet) do |doc, suites|
-
-            meta_info = Nokogiri::XML::Node.new('meta_test_info', doc)
-            unless file_to_link.nil?
-              meta_info['page_active'] = time_sort ? 'performance' : 'execution'
-              meta_info['link_url'] = file_to_link
-            else
-              meta_info['page_active'] = 'no-links'
-              meta_info['link_url'] = ''
-            end
-            suites.add_child(meta_info)
-
-            suite = Nokogiri::XML::Node.new('testsuite', doc)
-            suite['name']     = @name
-            suite['tests']    = test_count
-            suite['errors']   = errored_tests
-            suite['failures'] = failed_tests
-            suite['skip']     = skipped_tests
-            suite['pending']  = pending_tests
-            suite['total']    = @total_tests
-            suite['time']     = "%f" % (stop_time - start_time)
-            properties = Nokogiri::XML::Node.new('properties', doc)
-            @options.each_pair do | name, value |
-              property = Nokogiri::XML::Node.new('property', doc)
-              property['name']  = name
-              property['value'] = value
-              properties.add_child(property)
-            end
-            suite.add_child(properties)
-
-            test_cases_to_report = @test_cases
-            test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
-            test_cases_to_report.each do |test|
-              item = Nokogiri::XML::Node.new('testcase', doc)
-              item['classname'] = File.dirname(test.path)
-              item['name']      = File.basename(test.path)
-              item['time']      = "%f" % test.runtime
-
-              # Did we fail?  If so, report that.
-              # We need to remove the escape character from colorized text, the
-              # substitution of other entities is handled well by Rexml
-              if test.test_status == :fail || test.test_status == :error then
-                status = Nokogiri::XML::Node.new('failure', doc)
-                status['type'] =  test.test_status.to_s
-                if test.exception then
-                  status['message'] = test.exception.to_s.gsub(/\e/, '')
-                  data = LoggerJunit.format_cdata(test.exception.backtrace.join('\n'))
-                  status.add_child(status.document.create_cdata(data))
-                end
-                item.add_child(status)
-              end
-
-              if test.test_status == :skip
-                status = Nokogiri::XML::Node.new('skip', doc)
-                status['type'] =  test.test_status.to_s
-                item.add_child(status)
-              end
-
-              if test.test_status == :pending
-                status = Nokogiri::XML::Node.new('pending', doc)
-                status['type'] =  test.test_status.to_s
-                item.add_child(status)
-              end
-
-              if test.sublog then
-                stdout = Nokogiri::XML::Node.new('system-out', doc)
-                data = LoggerJunit.format_cdata(test.sublog)
-                stdout.add_child(stdout.document.create_cdata(data))
-                item.add_child(stdout)
-              end
-
-              if test.last_result and test.last_result.stderr and not test.last_result.stderr.empty? then
-                stderr = Nokogiri::XML::Node.new('system-err', doc)
-                data = LoggerJunit.format_cdata(test.last_result.stderr)
-                stderr.add_child(stderr.document.create_cdata(data))
-                item.add_child(stderr)
-              end
-
-              suite.add_child(item)
-            end
-            suites.add_child(suite)
-          end
-        rescue Exception => e
-          @logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
-        end
-
-      end
-    end
-
-    attr_reader :name, :options, :fail_mode
-
-    #Create {TestSuite} instance
-    #@param [String] name The name of the {TestSuite}
-    #@param [Array<Host>] hosts An Array of Hosts to act upon.
-    #@param [Hash{Symbol=>String}] options Options for this object
-    #@option options [Logger] :logger The Logger object to report information to
-    #@option options [String] :log_dir The directory where text run logs will be written
-    #@option options [String] :xml_dir The directory where JUnit XML file will be written
-    #@option options [String] :xml_file The name of the JUnit XML file to be written to
-    #@option options [String] :project_root The full path to the Beaker lib directory
-    #@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
-    #@param [Symbol] fail_mode One of :slow, :fast
-    #@param [Time] timestamp Beaker execution start time
-    def initialize(name, hosts, options, timestamp, fail_mode=nil)
-      @logger     = options[:logger]
-      @test_cases = []
-      @test_files = options[name]
-      @name       = name.to_s.gsub(/\s+/, '-')
-      @hosts      = hosts
-      @run        = false
-      @options    = options
-      @fail_mode  = fail_mode || @options[:fail_mode] || :slow
-      @test_suite_results = TestSuiteResult.new(@options, name)
-      @timestamp = timestamp
-
-      report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
-
-    rescue => e
-      report_and_raise(@logger, e, "TestSuite: initialize")
-    end
-
-    #Execute all the {TestCase} instances and then report the results as both plain text and xml.  The text result
-    #is reported to a newly created run log.
-    #Execution is dependent upon the fail_mode.  If mode is :fast then stop running any additional {TestCase} instances
-    #after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
-    def run
-      @run = true
-      start_time = Time.now
-
-      #Create a run log for this TestSuite.
-      run_log = log_path("#{@name}-run.log", @options[:log_dated_dir])
-      @logger.add_destination(run_log)
-
-      # This is an awful hack to maintain backward compatibility until tests
-      # are ported to use logger.  Still in use in PuppetDB tests
-      Beaker.const_set(:Log, @logger) unless defined?( Log )
-
-      @test_suite_results.start_time = start_time
-      @test_suite_results.total_tests = @test_files.length
-
-      @test_files.each do |test_file|
-        @logger.info "Begin #{test_file}"
-        start = Time.now
-        test_case = TestCase.new(@hosts, @logger, options, test_file).run_test
-        duration = Time.now - start
-        @test_suite_results.add_test_case(test_case)
-        @test_cases << test_case
-
-        state = test_case.test_status == :skip ? 'skipp' : test_case.test_status
-        msg = "#{test_file} #{state}ed in %.2f seconds" % duration.to_f
-        case test_case.test_status
-        when :pass
-          @logger.success msg
-        when :skip
-          @logger.warn msg
-        when :fail
-          @logger.error msg
-          break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
-        when :error
-          @logger.warn msg
-          break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
-        end
-      end
-      @test_suite_results.stop_time = Time.now
-
-      # REVISIT: This changes global state, breaking logging in any future runs
-      # of the suite – or, at least, making them highly confusing for anyone who
-      # has not studied the implementation in detail. --daniel 2011-03-14
-      @test_suite_results.summarize( Logger.new(log_path("#{name}-summary.txt", @options[:log_dated_dir]), STDOUT) )
-
-      junit_file_log  = log_path(@options[:xml_file], @options[:xml_dated_dir])
-      if @options[:xml_time_enabled]
-        junit_file_time = log_path(@options[:xml_time], @options[:xml_dated_dir])
-        @test_suite_results.write_junit_xml( junit_file_log, @options[:xml_time] )
-        @test_suite_results.write_junit_xml( junit_file_time, @options[:xml_file], true )
-      else
-        @test_suite_results.write_junit_xml( junit_file_log )
-      end
-      #All done with this run, remove run log
-      @logger.remove_destination(run_log)
-
-      # Allow chaining operations...
-      return self
-    end
-
-    #Execute all the TestCases in this suite.
-    #This is a wrapper that catches any failures generated during TestSuite::run.
-    def run_and_raise_on_failure
-      begin
-        run
-        return self if @test_suite_results.success?
-      rescue => e
-        #failed during run
-        report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
+    def self.runner(runner)
+      case runner
+      when "native"
+        ::Beaker::Runner::Native::TestSuite
+      when "minitest"
+        ::Beaker::Runner::MiniTest::TestSuite
       else
-        #failed during test
-        report_and_raise(@logger, RuntimeError.new("Failed while running the #{name} suite"), "TestSuite: report_and_raise_on_failure")
-      end
-    end
-
-    # Gives a full file path for output to be written to, maintaining the latest symlink
-    # @param [String] name The file name that we want to write to.
-    # @param [String] log_dir The desired output directory.
-    #                         A symlink will be made from ./basedir/latest to that.
-    # @example
-    #   log_path('output.txt', 'log/2014-06-02_16_31_22')
-    #
-    #     This will create the structure:
-    #
-    #   ./log/2014-06-02_16_31_22/output.txt
-    #   ./log/latest -> 2014-06-02_16_31_22
-    #
-    # @example
-    #   log_path('foo.log', 'log/man/date')
-    #
-    #     This will create the structure:
-    #
-    #   ./log/man/date/foo.log
-    #   ./log/latest -> man/date
-    def log_path(name, log_dir)
-      FileUtils.mkdir_p(log_dir) unless File.directory?(log_dir)
-
-      base_dir = log_dir
-      link_dir = ''
-      while File.dirname(base_dir) != '.' do
-        link_dir = link_dir == '' ? File.basename(base_dir) : File.join(File.basename(base_dir), link_dir)
-        base_dir = File.dirname(base_dir)
-      end
-
-      latest = File.join(base_dir, "latest")
-      if !File.exist?(latest) or File.symlink?(latest) then
-        File.delete(latest) if File.exist?(latest) || File.symlink?(latest)
-        File.symlink(link_dir, latest)
+        nil
       end
-
-      File.join(log_dir, name)
     end
-
   end
 end