beaker 3.22.0 → 3.23.0

data/lib/beaker/test_suite_result.rb

@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+require 'fileutils'
+[ 'test_case', 'logger' , 'test_suite', 'logger_junit'].each do |lib|
+  require "beaker/#{lib}"
+end
+
+module Beaker
+  #Holds the output of a test suite, formats in plain text or xml
+  class TestSuiteResult
+    attr_accessor :start_time, :stop_time, :total_tests
+
+    #Create a {TestSuiteResult} instance.
+    #@param [Hash{Symbol=>String}] options Options for this object
+    #@option options [Logger] :logger The Logger object to report information to
+    #@param [String] name The name of the {TestSuite} that the results are for
+    def initialize( options, name )
+      @options = options
+      @logger = options[:logger]
+      @name = name
+      @test_cases = []
+      #Set some defaults, just in case you attempt to print without including them
+      start_time = Time.at(0)
+      stop_time = Time.at(1)
+    end
+
+    #Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
+    #@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
+    def add_test_case( test_case )
+      @test_cases << test_case
+    end
+
+    #How many {TestCase} instances are in this {TestSuiteResult}
+    def test_count
+      @test_cases.length
+    end
+
+    #How many passed {TestCase} instances are in this {TestSuiteResult}
+    def passed_tests
+      @test_cases.select { |c| c.test_status == :pass }.length
+    end
+
+    #How many errored {TestCase} instances are in this {TestSuiteResult}
+    def errored_tests
+      @test_cases.select { |c| c.test_status == :error }.length
+    end
+
+    #How many failed {TestCase} instances are in this {TestSuiteResult}
+    def failed_tests
+      @test_cases.select { |c| c.test_status == :fail }.length
+    end
+
+    #How many skipped {TestCase} instances are in this {TestSuiteResult}
+    def skipped_tests
+      @test_cases.select { |c| c.test_status == :skip }.length
+    end
+
+    #How many pending {TestCase} instances are in this {TestSuiteResult}
+    def pending_tests
+      @test_cases.select {|c| c.test_status == :pending}.length
+    end
+
+    #How many {TestCase} instances failed in this {TestSuiteResult}
+    def sum_failed
+      failed_tests + errored_tests
+    end
+
+    #Did all the {TestCase} instances in this {TestSuiteResult} pass?
+    def success?
+      sum_failed == 0
+    end
+
+    #Did one or more {TestCase} instances in this {TestSuiteResult} fail?
+    def failed?
+      !success?
+    end
+
+    #The sum of all {TestCase} runtimes in this {TestSuiteResult}
+    def elapsed_time
+      @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
+    end
+
+    #Plain text summay of test suite
+    #@param [Logger] summary_logger The logger we will print the summary to
+    def summarize(summary_logger)
+
+      summary_logger.notify <<-HEREDOC
+      Test Suite: #{@name} @ #{start_time}
+
+      - Host Configuration Summary -
+        HEREDOC
+
+      average_test_time = elapsed_time / test_count
+
+      summary_logger.notify %Q[
+
+              - Test Case Summary for suite '#{@name}' -
+       Total Suite Time: %.2f seconds
+      Average Test Time: %.2f seconds
+              Attempted: #{test_count}
+                 Passed: #{passed_tests}
+                 Failed: #{failed_tests}
+                Errored: #{errored_tests}
+                Skipped: #{skipped_tests}
+                Pending: #{pending_tests}
+                  Total: #{@total_tests}
+
+      - Specific Test Case Status -
+        ] % [elapsed_time, average_test_time]
+
+      grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
+
+      summary_logger.notify "Failed Tests Cases:"
+      (grouped_summary[:fail] || []).each do |test_case|
+        summary_logger.notify print_test_result(test_case)
+      end
+
+      summary_logger.notify "Errored Tests Cases:"
+      (grouped_summary[:error] || []).each do |test_case|
+        summary_logger.notify print_test_result(test_case)
+      end
+
+      summary_logger.notify "Skipped Tests Cases:"
+      (grouped_summary[:skip] || []).each do |test_case|
+        summary_logger.notify print_test_result(test_case)
+      end
+
+      summary_logger.notify "Pending Tests Cases:"
+      (grouped_summary[:pending] || []).each do |test_case|
+        summary_logger.notify print_test_result(test_case)
+      end
+
+      summary_logger.notify("\n\n")
+    end
+
+    #A convenience method for printing the results of a {TestCase}
+    #@param [TestCase] test_case The {TestCase} to examine and print results for
+    def print_test_result(test_case)
+      if test_case.exception
+        test_file_trace = ""
+        test_case.exception.backtrace.each do |line|
+          if line.include?(test_case.path)
+            test_file_trace = "\r\n    Test line: #{line}"
+            break
+          end
+        end if test_case.exception.backtrace && test_case.path
+        test_reported = "reported: #{test_case.exception.inspect}#{test_file_trace}"
+      else
+        test_case.test_status
+      end
+      "  Test Case #{test_case.path} #{test_reported}"
+    end
+
+    # Writes Junit XML of this {TestSuiteResult}
+    #
+    # @param [String] xml_file      Path to the XML file (from Beaker's running directory)
+    # @param [String] file_to_link  Path to the paired file that should be linked
+    #                               from this one (this is relative to the XML
+    #                               file itself, so it would just be the different
+    #                               file name if they're in the same directory)
+    # @param [Boolean] time_sort    Whether the test results should be output in
+    #                               order of time spent in the test, or in the
+    #                               order of test execution (default)
+    #
+    # @return nil
+    # @api private
+    def write_junit_xml(xml_file, file_to_link = nil, time_sort = false)
+      stylesheet = File.join(@options[:project_root], @options[:xml_stylesheet])
+
+      begin
+        LoggerJunit.write_xml(xml_file, stylesheet) do |doc, suites|
+
+          meta_info = suites.add_element(REXML::Element.new('meta_test_info'))
+          unless file_to_link.nil?
+            time_sort ? meta_info.add_attribute('page_active', 'performance') : meta_info.add_attribute('page_active', 'execution')
+            meta_info.add_attribute('link_url', file_to_link)
+          else
+            meta_info.add_attribute('page_active', 'no-links')
+            meta_info.add_attribute('link_url', '')
+          end
+
+          suite = suites.add_element(REXML::Element.new('testsuite'))
+          suite.add_attributes(
+            [
+              ['name' , @name],
+              ['tests', test_count],
+              ['errors', errored_tests],
+              ['failures', failed_tests],
+              ['skipped', skipped_tests],
+              ['pending', pending_tests],
+              ['total', @total_tests],
+              ['time', "%f" % (stop_time - start_time)]
+          ])
+          properties = suite.add_element(REXML::Element.new('properties'))
+          @options.each_pair do |name,value|
+            property = properties.add_element(REXML::Element.new('property'))
+            property.add_attributes([['name', name], ['value', value.to_s || '']])
+          end
+
+          test_cases_to_report = @test_cases
+          test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
+          test_cases_to_report.each do |test|
+            item = suite.add_element(REXML::Element.new('testcase'))
+            item.add_attributes(
+              [
+                ['classname', File.dirname(test.path)],
+                ['name', File.basename(test.path)],
+                ['time', "%f" % test.runtime]
+              ])
+
+            test.exports.each do |export|
+              export.keys.each do |key|
+                item.add_attribute(key.to_s.tr(" ", "_"), export[key])
+              end
+            end
+
+            #Report failures
+            if test.test_status == :fail || test.test_status == :error
+              status = item.add_element(REXML::Element.new('failure'))
+              status.add_attribute('type', test.test_status.to_s)
+              if test.exception
+                status.add_attribute('message', test.exception.to_s.gsub(/\e/,''))
+                data = LoggerJunit.format_cdata(test.exception.backtrace.join('\n'))
+                REXML::CData.new(data, true, status)
+              end
+            end
+
+            if test.test_status == :skip
+              status = item.add_element(REXML::Element.new('skipped'))
+              status.add_attribute('type', test.test_status.to_s)
+            end
+
+            if test.test_status == :pending
+              status = item.add_element(REXML::Element.new('pending'))
+              status.add_attribute('type', test.test_status.to_s)
+            end
+
+            if test.sublog
+              stdout = item.add_element(REXML::Element.new('system-out'))
+              data = LoggerJunit.format_cdata(test.sublog)
+              REXML::CData.new(data, true, stdout)
+            end
+
+            if test.last_result and test.last_result.stderr and not test.last_result.stderr.empty?
+              stderr = item.add_element('system-err')
+              data = LoggerJunit.format_cdata(test.last_result.stderr)
+              REXML::CData.new(data, true, stderr)
+            end
+          end
+        end
+      rescue Exception => e
+        @logger.error "failure in XML output: \n#{e.to_s}" + e.backtrace.join("\n")
+      end
+    end
+
+  end
+end

data/lib/beaker/version.rb

@@ -1,5 +1,5 @@
 module Beaker
   module Version
-    STRING = '3.22.0'
+    STRING = '3.23.0'
   end
 end

data/spec/beaker/logger_junit_spec.rb

@@ -61,10 +61,8 @@ module Beaker
       it 'opens the given file for writing, and writes the doc to it' do
         mock_doc = Object.new
         doc_xml = 'flibbity-floo'
-        allow( mock_doc ).to receive( :to_xml ) { doc_xml }
-        mock_file_handle = Object.new
-        expect( mock_file_handle ).to receive( :write ).with( doc_xml )
-        expect( File ).to receive( :open ).with( xml_file, 'w' ).and_yield( mock_file_handle )
+        allow( mock_doc ).to receive( :write ).with(File, 2)
+        expect( File ).to receive( :open ).with( xml_file, 'w' )
         LoggerJunit.finish(mock_doc, xml_file)
       end
 
@@ -90,4 +88,4 @@ module Beaker
 
     end
   end
-end
+end

data/spec/beaker/test_suite_spec.rb

@@ -117,18 +117,16 @@ module Beaker
         expect( tsr.passed_tests).to be === 1
 
       end
-
-
     end
 
-    describe TestSuite::TestSuiteResult do
+    describe TestSuiteResult do
 
       let( :options )           { make_opts.merge({ :logger => double().as_null_object }) }
       let( :hosts )             { make_hosts() }
       let( :testcase1 )         { Beaker::TestCase.new( hosts, options[:logger], options) }
       let( :testcase2 )         { Beaker::TestCase.new( hosts, options[:logger], options) }
       let( :testcase3 )         { Beaker::TestCase.new( hosts, options[:logger], options) }
-      let( :test_suite_result ) { TestSuite::TestSuiteResult.new( options, "my_suite") }
+      let( :test_suite_result ) { TestSuiteResult.new( options, "my_suite") }
 
       it 'supports adding test cases' do
         expect( test_suite_result.test_count ).to be === 0
@@ -254,11 +252,8 @@ module Beaker
                                          :log_dated_dir => '.',
                                          :xml_dated_dir => '.'}) }
         let(:rb_test) { 'my_ruby_file.rb' }
+
         before(:each) do
-          @nokogiri_mock = Hash.new
-          allow( @nokogiri_mock ).to receive( :add_child )
-          allow( Nokogiri::XML::Node ).to receive( :new ) { @nokogiri_mock }
-          allow( LoggerJunit ).to receive( :write_xml ).and_yield( Object.new, @nokogiri_mock )
           @files = [ rb_test, rb_test, rb_test]
           @ts    = Beaker::TestSuite.new( 'name', hosts, options, Time.now, :fast )
           @tsr   = @ts.instance_variable_get( :@test_suite_results )
@@ -270,6 +265,9 @@ module Beaker
             allow( tc ).to receive( :sublog ).and_return( false )
             @test_cases << tc
           end
+          @rexml_mock = REXML::Element.new("testsuites")
+          allow(REXML::Element).to receive( :add_element ).and_call_original
+          allow( LoggerJunit ).to receive( :write_xml ).and_yield( Object.new, @rexml_mock )
         end
 
         it 'doesn\'t re-order test cases themselves on time_sort' do
@@ -292,12 +290,14 @@ module Beaker
           inner_value = {'second' => '2nd'}
           @test_cases.each do |tc|
             tc.instance_variable_set(:@runtime, 0)
-            tc.instance_variable_set(:@exports, [{'oh' => 'hai', 'first' => inner_value}])
+            tc.instance_variable_set(:@exports, [{'oh hey' => 'hai', 'first' => inner_value}])
             @tsr.add_test_case( tc )
           end
-          @tsr.write_junit_xml( 'fakeFilePath08' )
-          expect( @nokogiri_mock['oh'] ).to    eq('hai')
-          expect( @nokogiri_mock['first'] ).to eq(inner_value)
+          @tsr.write_junit_xml( 'fakeFilePath08')
+          @rexml_mock.elements.each("//testcase") do |e|
+            expect(e.attributes["oh_hey"].to_s).to eq('hai')
+            expect(e.attributes["first"]).to eq(inner_value.to_s)
+          end
         end
 
         it 'writes @export array of hashes properly' do
@@ -308,14 +308,12 @@ module Beaker
             @tsr.add_test_case( tc )
           end
           @tsr.write_junit_xml( 'fakeFilePath08' )
-          expect( @nokogiri_mock[:yes] ).to eq('hello')
-          expect( @nokogiri_mock[:uh] ).to  eq('sher')
+          @rexml_mock.elements.each("//testcase") do |e|
+            expect(e.attributes["yes"].to_s).to eq('hello')
+            expect(e.attributes["uh"].to_s).to eq('sher')
+          end
         end
 
-        # this isn't the best test as the @nokogiri_mock is a single hash.
-        #   it really should be an array of hashes or nested, to ensure each case
-        #   gets the correct key/value.  But i could not get it to work properly with
-        #   the various calls to ::Node and #add_child
         it 'writes @export hashes per test case properly' do
           expect( @tsr.instance_variable_get( :@logger ) ).to receive( :error ).never
           @test_cases.each_with_index do |tc,index|
@@ -324,14 +322,14 @@ module Beaker
             @tsr.add_test_case( tc )
           end
           @tsr.write_junit_xml( 'fakeFilePath08' )
-          @test_cases.each_with_index do |tc,index|
-            expect( @nokogiri_mock["yes_#{index}"] ).to eq("hello#{index}")
+          index = 0
+          @rexml_mock.elements.each("//testcase") do |e|
+            expect(e.attributes["yes_#{index}"]).to eq("hello#{index}")
+            index += 1
           end
         end
-
       end
 
-
     end
 
     describe '#log_path' do
@@ -359,7 +357,6 @@ module Beaker
           testsuite.log_path('foo.txt', 'a/b/c/d/e/f')
           expect( File.symlink?('a/latest') ).to be_truthy
         end
-
       end
 
       describe 'builds the symlink directory correctly' do
@@ -376,7 +373,6 @@ module Beaker
           testsuite.log_path('foo.txt', 'f/g/h/i/j/k')
           expect( File.readlink('f/latest') ).to be === 'g/h/i/j/k'
         end
-
       end
 
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beaker
 version: !ruby/object:Gem::Version
-  version: 3.22.0
+  version: 3.23.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-09 00:00:00.000000000 Z
+date: 2017-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -66,20 +66,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.10'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.4.2
+- !ruby/object:Gem::Dependency
+  name: rb-readline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: hocon
   requirement: !ruby/object:Gem::Requirement
@@ -305,21 +319,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: beaker-docker
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: beaker-docker
+  name: beaker-aws
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -333,21 +347,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: beaker-aws
+  name: beaker-vmpooler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: beaker-vmpooler
+  name: beaker-google
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -361,7 +375,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: beaker-google
+  name: beaker-vagrant
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -375,7 +389,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: beaker-vagrant
+  name: beaker-vmware
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -389,7 +403,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: beaker-vmware
+  name: beaker-openstack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -403,7 +417,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: beaker-openstack
+  name: beaker-vcloud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -558,11 +572,11 @@ files:
 - docs/concepts/testing_beaker_itself.md
 - docs/concepts/ticket_process.md
 - docs/concepts/types_puppet_4_and_the_all_in_one_agent.md
-- docs/how_to/access_the_live_test_console_with_pry.md
 - docs/how_to/archive_sut_files.md
 - docs/how_to/change_terminal_output_coloring.md
 - docs/how_to/cloning_private_repos.md
 - docs/how_to/confine.md
+- docs/how_to/debug_beaker_tests.md
 - docs/how_to/enabling_cross_sut_access.md
 - docs/how_to/hosts/README.md
 - docs/how_to/hosts/archlinux.md
@@ -683,6 +697,7 @@ files:
 - lib/beaker/tasks/test.rb
 - lib/beaker/test_case.rb
 - lib/beaker/test_suite.rb
+- lib/beaker/test_suite_result.rb
 - lib/beaker/version.rb
 - spec/beaker/cli_spec.rb
 - spec/beaker/command_spec.rb