beaker 2.29.1 → 2.30.0

data/lib/beaker/version.rb

@@ -1,5 +1,5 @@
 module Beaker
   module Version
-    STRING = '2.29.1'
+    STRING = '2.30.0'
   end
 end

data/spec/beaker/dsl/install_utils/foss_utils_spec.rb

@@ -87,6 +87,8 @@ describe ClassMixedWithDSLInstallUtils do
 
     it 'uses aio paths for hosts with aio type (backwards compatability)' do
       hosts.each do |host|
+        host[:pe_ver] = nil
+        host[:version] = nil
         host[:type] = 'aio'
       end
       expect(subject).to receive(:add_aio_defaults_on).exactly(hosts.length).times

data/spec/beaker/dsl/install_utils/pe_utils_spec.rb

@@ -49,6 +49,8 @@ describe ClassMixedWithDSLInstallUtils do
   context '#configure_pe_defaults_on' do
     it 'uses aio paths for hosts of role aio' do
       hosts.each do |host|
+        host[:pe_ver] = nil
+        host[:version] = nil
         host[:roles] = host[:roles] | ['aio']
       end
       expect(subject).to receive(:add_pe_defaults_on).exactly(hosts.length).times
@@ -71,6 +73,8 @@ describe ClassMixedWithDSLInstallUtils do
 
     it 'uses aio paths for hosts of type aio' do
       hosts.each do |host|
+        host[:pe_ver] = nil
+        host[:version] = nil
         host[:type] = 'aio'
       end
       expect(subject).to receive(:add_aio_defaults_on).exactly(hosts.length).times

data/spec/beaker/dsl/install_utils/puppet_utils_spec.rb

@@ -83,6 +83,8 @@ describe ClassMixedWithDSLInstallUtils do
 
     it "adds aio defaults to foss hosts when they have an aio foss puppet version" do
       hosts.each do |host|
+        host[:pe_ver] = nil
+        host[:version] = nil
         host['type'] = 'foss'
         host['version'] = '4.0'
       end
@@ -93,6 +95,8 @@ describe ClassMixedWithDSLInstallUtils do
 
     it "adds aio defaults to foss hosts when they have type foss-aio" do
       hosts.each do |host|
+        host[:pe_ver] = nil
+        host[:version] = nil
         host['type'] = 'foss-aio'
       end
       expect(subject).to receive(:add_foss_defaults_on).exactly(hosts.length).times
@@ -102,6 +106,8 @@ describe ClassMixedWithDSLInstallUtils do
 
     it "can set aio defaults for aio type (backwards compatability)" do
       hosts.each do |host|
+        host[:pe_ver] = nil
+        host[:version] = nil
         host['type'] = 'aio'
       end
       expect(subject).to receive(:add_aio_defaults_on).exactly(hosts.length).times

data/spec/beaker/dsl/roles_spec.rb

@@ -163,6 +163,124 @@ describe ClassMixedWithDSLRoles do
       agent1[:type] = 'aio-foss'
       expect( subject.aio_version?(agent1) ).to be === true
     end
+    it 'can take an empty string for pe_ver' do
+      agent1[:pe_ver] = ''
+      expect{ subject.aio_version?(agent1) }.not_to raise_error
+    end
+    it 'can take an empty string for FOSS version' do
+      agent1[:version] = ''
+      expect{ subject.aio_version?(agent1) }.not_to raise_error
+    end
+
+    context 'truth table-type testing' do
+
+      before :each do
+        @old_pe_ver  = agent1[:pe_ver]
+        @old_version = agent1[:version]
+        @old_roles   = agent1[:roles]
+        @old_type    = agent1[:type]
+      end
+
+      after :each do
+        agent1[:pe_ver]   = @old_pe_ver
+        agent1[:version]  = @old_version
+        agent1[:roles]    = @old_roles
+        agent1[:type]     = @old_type
+      end
+
+      context 'version values table' do
+        # pe_ver, version, answer
+        versions_table = [
+          [nil,      nil, false],
+          [nil,       '', false],
+          [nil,    '3.9', false],
+          [nil,    '4.0', true ],
+          [nil, '2015.1', true ],
+          \
+          ['',      nil, false],
+          ['',       '', false],
+          ['',    '3.9', false],
+          ['',    '4.0', true ],
+          ['', '2015.1', true ],
+          \
+          ['3.9',      nil, false],
+          ['3.9',       '', false],
+          ['3.9',    '3.9', false],
+          ['3.9',    '4.0', false],
+          ['3.9', '2015.1', false],
+          \
+          ['4.0',      nil, true],
+          ['4.0',       '', true],
+          ['4.0',    '3.9', true],
+          ['4.0',    '4.0', true],
+          ['4.0', '2015.1', true],
+          \
+          ['2015.1',      nil, true],
+          ['2015.1',       '', true],
+          ['2015.1',    '3.9', true],
+          ['2015.1',    '4.0', true],
+          ['2015.1', '2015.1', true],
+        ]
+
+        versions_table.each do |answers_row|
+          it "acts with values #{answers_row} correctly" do
+            agent1[:pe_ver]   = answers_row[0]
+            agent1[:version]  = answers_row[1]
+            agent1[:roles]    = nil
+            agent1[:type]     = nil
+            expect( subject.aio_version?(agent1) ).to be === answers_row[2]
+          end
+        end
+      end
+
+      context 'roles values table' do
+        roles_table = [
+          [nil,           false],
+          [[],            false],
+          [['aio'],       true ],
+          [['gun'],       false],
+          [['a', 'b'],    false],
+          [['c', 'aio'],  true ],
+        ]
+
+        roles_table.each do |answers_row|
+          it "acts with values #{answers_row} correctly" do
+            agent1[:pe_ver]   = nil
+            agent1[:version]  = nil
+            agent1[:roles]    = answers_row[0]
+            agent1[:type]     = nil
+            expect( subject.aio_version?(agent1) ).to be === answers_row[1]
+          end
+        end
+      end
+
+      context 'type values table' do
+        type_table = [
+          [nil,         false],
+          ['',          false],
+          ['cheese',    false],
+          ['paionts',   false],
+          ['aioch',     false],
+          ['chaio',     false],
+          ['aio',       true ],
+          ['aio-',      true ],
+          ['ew-aio-ji', true ],
+          ['id-aiot',   false],
+        ]
+
+        type_table.each do |answers_row|
+          it "acts with values #{answers_row} correctly" do
+            agent1[:pe_ver]   = nil
+            agent1[:version]  = nil
+            agent1[:roles]    = nil
+            agent1[:type]     = answers_row[0]
+            expect( subject.aio_version?(agent1) ).to be === answers_row[1]
+          end
+        end
+      end
+
+    end
+
   end
   describe '#aio_agent?' do
     it 'returns false if agent_only check doesn\'t pass' do

data/spec/beaker/hypervisor/vagrant_spec.rb

@@ -37,12 +37,13 @@ module Beaker
       vagrant.make_vfile( @hosts )
 
       vagrantfile = File.read( File.expand_path( File.join( path, "Vagrantfile")))
+      puts "file is #{path}\n"
       expect( vagrantfile ).to be === <<-EOF
 Vagrant.configure("2") do |c|
   c.ssh.insert_key = false
   c.vm.define 'vm1' do |v|
     v.vm.hostname = 'vm1'
-    v.vm.box = 'vm1_of_my_box'
+    v.vm.box = 'vm2vm1_of_my_box'
     v.vm.box_url = 'http://address.for.my.box.vm1'
     v.vm.box_check_update = 'true'
     v.vm.network :private_network, ip: "ip.address.for.vm1", :netmask => "255.255.0.0", :mac => "0123456789"
@@ -54,7 +55,7 @@ Vagrant.configure("2") do |c|
   end
   c.vm.define 'vm2' do |v|
     v.vm.hostname = 'vm2'
-    v.vm.box = 'vm2_of_my_box'
+    v.vm.box = 'vm2vm2_of_my_box'
     v.vm.box_url = 'http://address.for.my.box.vm2'
     v.vm.box_check_update = 'true'
     v.vm.network :private_network, ip: "ip.address.for.vm2", :netmask => "255.255.0.0", :mac => "0123456789"
@@ -66,7 +67,7 @@ Vagrant.configure("2") do |c|
   end
   c.vm.define 'vm3' do |v|
     v.vm.hostname = 'vm3'
-    v.vm.box = 'vm3_of_my_box'
+    v.vm.box = 'vm2vm3_of_my_box'
     v.vm.box_url = 'http://address.for.my.box.vm3'
     v.vm.box_check_update = 'true'
     v.vm.network :private_network, ip: "ip.address.for.vm3", :netmask => "255.255.0.0", :mac => "0123456789"
@@ -246,7 +247,6 @@ EOF
           expect{ vagrant.get_ip_from_vagrant_file(host.name) }.to raise_error
         end
       end
-
     end
 
     describe "provisioning and cleanup" do

data/spec/beaker/options/beaker_options_spec.rb

@@ -0,0 +1,34 @@
+require "spec_helper"
+
+# safely set values for ARGV in block, restoring original value on block leave
+def with_ARGV(value, &block)
+  if defined? ARGV
+    defined_ARGV, old_ARGV = true, ARGV
+    Object.send(:remove_const, :ARGV)
+  else
+    defined_ARGV, old_ARGV = false, nil
+  end
+
+  Object.send(:const_set, :ARGV, value)
+
+  yield
+ensure
+  Object.send(:remove_const, :ARGV)
+  Object.send(:const_set, :ARGV, old_ARGV) if defined_ARGV
+end
+
+describe "Beaker Options" do
+  let (:parser) { Beaker::Options::Parser.new }
+
+  it "defaults :runner to 'native'" do
+    with_ARGV([]) do
+      expect(parser.parse_args[:runner]).to be == "native"
+    end
+  end
+
+  it "accepts :runner from command-line" do
+    with_ARGV(["--runner", "minitest"]) do
+      expect(parser.parse_args[:runner]).to be == "minitest"
+    end
+  end
+end

data/spec/beaker/platform_spec.rb

@@ -128,6 +128,22 @@ module Beaker
 
     end
 
-  end
+    context 'round tripping from yaml', if: RUBY_VERSION =~ /^1\.9/ do
+      before do
+        @name = 'ubuntu-14.04-x86_64'
+      end
+
+      let(:round_tripped) { YAML.load(YAML.dump(platform)) }
 
+      [:variant, :arch, :version, :codename].each do |field|
+        it "deserializes the '#{field}' field" do
+          expect(round_tripped.send(field)).to eq platform.send(field)
+        end
+      end
+
+      it 'properly sets the string contents' do
+        expect(round_tripped.to_s).to eq @name
+      end
+    end
+  end
 end

data/spec/beaker/runner/native/test_case_spec.rb

@@ -0,0 +1,147 @@
+require 'spec_helper'
+
+module Beaker
+  module Runner
+    module Native
+      describe TestCase do
+        let(:logger) {  double('logger').as_null_object }
+        let(:path) { @path || '/tmp/nope' }
+        let(:testcase) { TestCase.new({}, logger, {}, path) }
+
+        context 'run_test' do
+          it 'defaults to test_status :pass on success' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write ""
+            end
+            @path = path
+            expect( testcase ).to_not receive( :log_and_fail_test )
+            testcase.run_test
+            status = testcase.instance_variable_get(:@test_status)
+            expect(status).to be === :pass
+          end
+
+          it 'updates test_status to :skip on SkipTest' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise SkipTest"
+            end
+            @path = path
+            expect( testcase ).to_not receive( :log_and_fail_test )
+            testcase.run_test
+            status = testcase.instance_variable_get(:@test_status)
+            expect(status).to be === :skip
+          end
+
+          it 'updates test_status to :pending on PendingTest' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise PendingTest"
+            end
+            @path = path
+            expect( testcase ).to_not receive( :log_and_fail_test )
+            testcase.run_test
+            status = testcase.instance_variable_get(:@test_status)
+            expect(status).to be === :pending
+          end
+
+          it 'updates test_status to :fail on FailTest' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise FailTest"
+            end
+            @path = path
+            expect( testcase ).to_not receive( :log_and_fail_test )
+            testcase.run_test
+            status = testcase.instance_variable_get(:@test_status)
+            expect(status).to be === :fail
+          end
+
+          it 'correctly handles RuntimeError' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise RuntimeError"
+            end
+            @path = path
+            expect( testcase ).to receive( :log_and_fail_test ).once.with(kind_of(RuntimeError))
+            testcase.run_test
+          end
+
+          it 'correctly handles ScriptError' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise ScriptError"
+            end
+            @path = path
+            expect( testcase ).to receive( :log_and_fail_test ).once.with(kind_of(ScriptError))
+            testcase.run_test
+          end
+
+          it 'correctly handles Timeout::Error' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise Timeout::Error"
+            end
+            @path = path
+            expect( testcase ).to receive( :log_and_fail_test ).once.with(kind_of(Timeout::Error))
+            testcase.run_test
+          end
+
+          it 'correctly handles CommandFailure' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write "raise Host::CommandFailure"
+            end
+            @path = path
+            expect( testcase ).to receive( :log_and_fail_test ).once.with(kind_of(Host::CommandFailure))
+            testcase.run_test
+          end
+
+          it 'records a test failure if an assertion fails in a teardown block' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write <<-EOF
+                teardown do
+                  assert_equal(1, 2, 'Oh noes!')
+                end
+              EOF
+            end
+            @path = path
+            expect( testcase ).to receive( :log_and_fail_test ).once.with(kind_of(Minitest::Assertion))
+            testcase.run_test
+          end
+        end
+
+        context 'metadata' do
+          it 'sets the filename correctly from the path' do
+            answer = 'jacket'
+            path = "#{answer}.rb"
+            File.open(path, 'w') do |f|
+              f.write ""
+            end
+            @path = path
+            testcase.run_test
+            metadata = testcase.instance_variable_get(:@metadata)
+            expect(metadata[:case][:file_name]).to be === answer
+          end
+
+          it 'resets the step name' do
+            path = 'test.rb'
+            File.open(path, 'w') do |f|
+              f.write ""
+            end
+            @path = path
+            # we have to create a TestCase by hand, so that we can set old
+            tc = TestCase.new({}, logger, {}, path)
+            # metadata on it, so that we can test that it's being reset correctly
+            old_metadata = { :step => { :name => 'CharlieBrown' } }
+            tc.instance_variable_set(:@metadata, old_metadata)
+            tc.run_test
+            metadata = tc.instance_variable_get(:@metadata)
+            expect(metadata[:step][:name]).to be_nil
+          end
+        end
+      end
+    end
+  end
+end

data/spec/beaker/runner/native/test_suite_spec.rb

@@ -0,0 +1,303 @@
+require 'spec_helper'
+require 'fileutils'
+
+module Beaker
+  module Runner
+    module Native
+      describe TestSuite do
+        context 'new' do
+          let(:test_dir) { 'tmp/tests' }
+          let(:options)  { {'name' => create_files(@files)} }
+          let(:rb_test)  { File.expand_path(test_dir + '/my_ruby_file.rb')    }
+          let(:pl_test)  { File.expand_path(test_dir + '/my_perl_file.pl')    }
+          let(:sh_test)  { File.expand_path(test_dir + '/my_shell_file.sh')   }
+
+          it 'fails without test files' do
+            expect { Beaker::Runner::Native::TestSuite.new('name', 'hosts', Hash.new, Time.now, :stop_on_error) }.to raise_error
+          end
+
+          it 'includes specific files as test file when explicitly passed' do
+            @files = [ rb_test ]
+            ts = Beaker::Runner::Native::TestSuite.new('name', 'hosts', options, Time.now, :stop_on_error)
+
+            tfs = ts.instance_variable_get(:@test_files)
+            expect(tfs).to include rb_test
+          end
+
+          it 'defaults to :slow fail_mode if not provided through parameter or options' do
+            @files = [ rb_test ]
+            ts = Beaker::Runner::Native::TestSuite.new('name', 'hosts', options, Time.now)
+            tfm = ts.instance_variable_get(:@fail_mode)
+            expect(tfm).to be == :slow
+          end
+
+          it 'uses provided parameter fail_mode' do
+            @files = [ rb_test ]
+            ts = Beaker::Runner::Native::TestSuite.new('name', 'hosts', options, Time.now, :fast)
+            tfm = ts.instance_variable_get(:@fail_mode)
+            expect(tfm).to be == :fast
+          end
+
+          it 'uses options fail_mode if fail_mode parameter is not provided' do
+            @files = [ rb_test ]
+            options[:fail_mode] = :fast
+            ts = Beaker::Runner::Native::TestSuite.new('name', 'hosts', options, Time.now)
+            tfm = ts.instance_variable_get(:@fail_mode)
+            expect(tfm).to be == :fast
+          end
+        end
+
+        context 'run' do
+          let( :options )     { make_opts.merge({ :logger => double().as_null_object, 'name' => create_files(@files), :log_dated_dir => '.', :xml_dated_dir => '.'}) }
+          let(:broken_script) { "raise RuntimeError" }
+          let(:fail_script)   { "raise Beaker::DSL::Outcomes::FailTest" }
+          let(:okay_script)   { "true" }
+          let(:rb_test)       { 'my_ruby_file.rb'     }
+          let(:pl_test)       { '/my_perl_file.pl'    }
+          let(:sh_test)       { '/my_shell_file.sh'   }
+          let(:hosts)         { make_hosts() }
+
+          it 'fails fast if fail_mode != :slow and runtime error is raised' do
+            allow( Logger ).to receive('new')
+            @files = [ rb_test, pl_test, sh_test]
+            File.open(rb_test, 'w') { |file| file.write(broken_script) }
+            File.open(pl_test, 'w') { |file| file.write(okay_script) }
+            File.open(sh_test, 'w') { |file| file.write(okay_script) }
+
+            ts = Beaker::Runner::Native::TestSuite.new( 'name', hosts, options, Time.now, :stop )
+            tsr = ts.instance_variable_get( :@test_suite_results )
+            allow( tsr ).to receive(:write_junit_xml).and_return( true )
+            allow( tsr ).to receive(:summarize).and_return( true )
+
+            ts.run
+            expect( tsr.errored_tests ).to be === 1
+            expect( tsr.failed_tests ).to be === 0
+            expect( tsr.test_count ).to be === 1
+            expect( tsr.passed_tests).to be === 0
+          end
+
+          it 'fails fast if fail_mode != :slow and fail test is raised' do
+            allow( Logger ).to receive('new')
+            @files = [ rb_test, pl_test, sh_test]
+            File.open(rb_test, 'w') { |file| file.write(fail_script) }
+            File.open(pl_test, 'w') { |file| file.write(okay_script) }
+            File.open(sh_test, 'w') { |file| file.write(okay_script) }
+
+            ts = Beaker::Runner::Native::TestSuite.new( 'name', hosts, options, Time.now, :stop )
+            tsr = ts.instance_variable_get( :@test_suite_results )
+            allow( tsr ).to receive(:write_junit_xml).and_return( true )
+            allow( tsr ).to receive(:summarize).and_return( true )
+
+            ts.run
+            expect( tsr.errored_tests ).to be === 0
+            expect( tsr.failed_tests ).to be === 1
+            expect( tsr.test_count ).to be === 1
+            expect( tsr.passed_tests).to be === 0
+          end
+
+          it 'fails slow if fail_mode = :slow, even if a test fails and there is a runtime error' do
+            allow( Logger ).to receive('new')
+            @files = [ rb_test, pl_test, sh_test]
+            File.open(rb_test, 'w') { |file| file.write(broken_script) }
+            File.open(pl_test, 'w') { |file| file.write(fail_script) }
+            File.open(sh_test, 'w') { |file| file.write(okay_script) }
+
+            ts = Beaker::Runner::Native::TestSuite.new( 'name', hosts, options, Time.now, :slow )
+            tsr = ts.instance_variable_get( :@test_suite_results )
+            allow( tsr ).to receive(:write_junit_xml).and_return( true )
+            allow( tsr ).to receive(:summarize).and_return( true )
+
+            ts.run
+            expect( tsr.errored_tests ).to be === 1
+            expect( tsr.failed_tests ).to be === 1
+            expect( tsr.test_count ).to be === 3
+            expect( tsr.passed_tests).to be === 1
+          end
+        end
+
+        describe TestSuite::TestSuiteResult do
+          let( :options )           { make_opts.merge({ :logger => double().as_null_object }) }
+          let( :hosts )             { make_hosts() }
+          let( :testcase1 )         { Beaker::Runner::Native::TestCase.new( hosts, options[:logger], options) }
+          let( :testcase2 )         { Beaker::Runner::Native::TestCase.new( hosts, options[:logger], options) }
+          let( :testcase3 )         { Beaker::Runner::Native::TestCase.new( hosts, options[:logger], options) }
+          let( :test_suite_result ) { TestSuite::TestSuiteResult.new( options, "my_suite") }
+
+          it 'supports adding test cases' do
+            expect( test_suite_result.test_count ).to be === 0
+            test_suite_result.add_test_case( testcase1 )
+            expect( test_suite_result.test_count ).to be === 1
+          end
+
+          it 'calculates passed tests' do
+            testcase1.instance_variable_set(:@test_status, :pass)
+            testcase2.instance_variable_set(:@test_status, :pass)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.passed_tests ).to be == 2
+          end
+
+          it 'calculates failed tests' do
+            testcase1.instance_variable_set(:@test_status, :pass)
+            testcase2.instance_variable_set(:@test_status, :pass)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.failed_tests ).to be == 1
+          end
+
+          it 'calculates errored tests' do
+            testcase1.instance_variable_set(:@test_status, :error)
+            testcase2.instance_variable_set(:@test_status, :pass)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.errored_tests ).to be == 1
+          end
+
+          it 'calculates skipped tests' do
+            testcase1.instance_variable_set(:@test_status, :error)
+            testcase2.instance_variable_set(:@test_status, :skip)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.skipped_tests ).to be == 1
+          end
+
+          it 'calculates pending tests' do
+            testcase1.instance_variable_set(:@test_status, :error)
+            testcase2.instance_variable_set(:@test_status, :pending)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.pending_tests ).to be == 1
+          end
+
+          it 'calculates sum_failed as a sum of errored and failed TestCases' do
+            testcase1.instance_variable_set(:@test_status, :error)
+            testcase2.instance_variable_set(:@test_status, :pending)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.sum_failed ).to be == 2
+          end
+
+          it 'reports success with no errors/failures' do
+            testcase1.instance_variable_set(:@test_status, :pass)
+            testcase2.instance_variable_set(:@test_status, :pending)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.success? ).to be == false
+          end
+
+          it 'reports failed if any tests error/fail' do
+            testcase1.instance_variable_set(:@test_status, :pass)
+            testcase2.instance_variable_set(:@test_status, :pending)
+            testcase3.instance_variable_set(:@test_status, :fail)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.failed? ).to be == true
+          end
+
+          it 'can calculate the sum of all TestCase runtimes' do
+            testcase1.instance_variable_set(:@runtime, 1)
+            testcase2.instance_variable_set(:@runtime, 10)
+            testcase3.instance_variable_set(:@runtime, 100)
+            test_suite_result.add_test_case( testcase1 )
+            test_suite_result.add_test_case( testcase2 )
+            test_suite_result.add_test_case( testcase3 )
+            expect( test_suite_result.elapsed_time ).to be == 111
+          end
+
+          describe '#write_junit_xml' do
+            let( :options )     { make_opts.merge({ :logger => double().as_null_object, 'name' => create_files(@files), :log_dated_dir => '.', :xml_dated_dir => '.'}) }
+            let(:rb_test)       { 'my_ruby_file.rb'     }
+
+            it 'doesn\'t re-order test cases themselves on time_sort' do
+              nokogiri_mock = Hash.new
+              allow( nokogiri_mock ).to receive( :add_child )
+              allow( Nokogiri::XML::Node ).to receive( :new ) { nokogiri_mock }
+              allow( LoggerJunit ).to receive( :write_xml ).and_yield( Object.new, nokogiri_mock )
+
+              @files = [ rb_test, rb_test, rb_test]
+              ts = Beaker::Runner::Native::TestSuite.new( 'name', hosts, options, Time.now, :fast )
+              tsr = ts.instance_variable_get( :@test_suite_results )
+
+              allow( tsr ).to receive( :start_time ).and_return(0)
+              allow( tsr ).to receive( :stop_time ).and_return(10)
+              expect( tsr.instance_variable_get( :@logger ) ).to receive( :error ).never
+
+              test_cases = []
+              3.times do
+                tc = Beaker::Runner::Native::TestCase.new( hosts, options[:logger], options, rb_test)
+                allow( tc ).to receive( :sublog ).and_return( false )
+                test_cases << tc
+              end
+              test_cases[0].instance_variable_set(:@runtime, 3)
+              test_cases[1].instance_variable_set(:@runtime, 301)
+              test_cases[2].instance_variable_set(:@runtime, 101)
+              test_cases.map { |tc| tsr.add_test_case( tc ) }
+
+              original_testcase_order = test_suite_result.instance_variable_get( :@test_cases ).dup
+              tsr.write_junit_xml( 'fakeFilePath07', 'fakeFileToLink09', true )
+              after_testcase_order = test_suite_result.instance_variable_get( :@test_cases ).dup
+              expect( after_testcase_order ).to be === original_testcase_order
+            end
+          end
+        end
+
+        describe '#log_path' do
+          let( :sh_test   ) { '/my_shell_file.sh'   }
+          let( :files     ) { @files ? @files : [sh_test] }
+          let( :options   ) { make_opts.merge({ :logger => double().as_null_object, 'name' => create_files(files) }) }
+          let( :hosts     ) { make_hosts() }
+          let( :testsuite ) { Beaker::Runner::Native::TestSuite.new( 'name', hosts, options, Time.now, :stop ) }
+
+          it 'returns the simple joining of the log dir & file as required' do
+            expect(testsuite.log_path('foo.txt', 'man/date')).to be === 'man/date/foo.txt'
+          end
+
+          describe 'builds the base directory correctly' do
+            # the base directory is where the latest symlink itself should live
+            it 'in the usual case' do
+              expect( File.symlink?('man/latest') ).to be_falsy
+              testsuite.log_path('foo.txt', 'man/date')
+              expect( File.symlink?('man/latest') ).to be_truthy
+            end
+
+            it 'if given a nested directory' do
+              expect( File.symlink?('a/latest') ).to be_falsy
+              testsuite.log_path('foo.txt', 'a/b/c/d/e/f')
+              expect( File.symlink?('a/latest') ).to be_truthy
+            end
+          end
+
+          describe 'builds the symlink directory correctly' do
+            # the symlink directory is where the symlink points to
+            it 'in the usual case' do
+              expect( File.symlink?('d/latest') ).to be_falsy
+              testsuite.log_path('foo.txt', 'd/e')
+              expect( File.readlink('d/latest') ).to be === 'e'
+            end
+
+            it 'if given a nested directory' do
+              expect( File.symlink?('f/latest') ).to be_falsy
+              testsuite.log_path('foo.txt', 'f/g/h/i/j/k')
+              expect( File.readlink('f/latest') ).to be === 'g/h/i/j/k'
+            end
+          end
+        end
+      end
+    end
+  end
+end