beaker 3.22.0 → 3.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,256 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'fileutils'
3
+ [ 'test_case', 'logger' , 'test_suite', 'logger_junit'].each do |lib|
4
+ require "beaker/#{lib}"
5
+ end
6
+
7
+ module Beaker
8
+ #Holds the output of a test suite, formats in plain text or xml
9
+ class TestSuiteResult
10
+ attr_accessor :start_time, :stop_time, :total_tests
11
+
12
+ #Create a {TestSuiteResult} instance.
13
+ #@param [Hash{Symbol=>String}] options Options for this object
14
+ #@option options [Logger] :logger The Logger object to report information to
15
+ #@param [String] name The name of the {TestSuite} that the results are for
16
+ def initialize( options, name )
17
+ @options = options
18
+ @logger = options[:logger]
19
+ @name = name
20
+ @test_cases = []
21
+ #Set some defaults, just in case you attempt to print without including them
22
+ start_time = Time.at(0)
23
+ stop_time = Time.at(1)
24
+ end
25
+
26
+ #Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
27
+ #@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
28
+ def add_test_case( test_case )
29
+ @test_cases << test_case
30
+ end
31
+
32
+ #How many {TestCase} instances are in this {TestSuiteResult}
33
+ def test_count
34
+ @test_cases.length
35
+ end
36
+
37
+ #How many passed {TestCase} instances are in this {TestSuiteResult}
38
+ def passed_tests
39
+ @test_cases.select { |c| c.test_status == :pass }.length
40
+ end
41
+
42
+ #How many errored {TestCase} instances are in this {TestSuiteResult}
43
+ def errored_tests
44
+ @test_cases.select { |c| c.test_status == :error }.length
45
+ end
46
+
47
+ #How many failed {TestCase} instances are in this {TestSuiteResult}
48
+ def failed_tests
49
+ @test_cases.select { |c| c.test_status == :fail }.length
50
+ end
51
+
52
+ #How many skipped {TestCase} instances are in this {TestSuiteResult}
53
+ def skipped_tests
54
+ @test_cases.select { |c| c.test_status == :skip }.length
55
+ end
56
+
57
+ #How many pending {TestCase} instances are in this {TestSuiteResult}
58
+ def pending_tests
59
+ @test_cases.select {|c| c.test_status == :pending}.length
60
+ end
61
+
62
+ #How many {TestCase} instances failed in this {TestSuiteResult}
63
+ def sum_failed
64
+ failed_tests + errored_tests
65
+ end
66
+
67
+ #Did all the {TestCase} instances in this {TestSuiteResult} pass?
68
+ def success?
69
+ sum_failed == 0
70
+ end
71
+
72
+ #Did one or more {TestCase} instances in this {TestSuiteResult} fail?
73
+ def failed?
74
+ !success?
75
+ end
76
+
77
+ #The sum of all {TestCase} runtimes in this {TestSuiteResult}
78
+ def elapsed_time
79
+ @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
80
+ end
81
+
82
+ #Plain text summay of test suite
83
+ #@param [Logger] summary_logger The logger we will print the summary to
84
+ def summarize(summary_logger)
85
+
86
+ summary_logger.notify <<-HEREDOC
87
+ Test Suite: #{@name} @ #{start_time}
88
+
89
+ - Host Configuration Summary -
90
+ HEREDOC
91
+
92
+ average_test_time = elapsed_time / test_count
93
+
94
+ summary_logger.notify %Q[
95
+
96
+ - Test Case Summary for suite '#{@name}' -
97
+ Total Suite Time: %.2f seconds
98
+ Average Test Time: %.2f seconds
99
+ Attempted: #{test_count}
100
+ Passed: #{passed_tests}
101
+ Failed: #{failed_tests}
102
+ Errored: #{errored_tests}
103
+ Skipped: #{skipped_tests}
104
+ Pending: #{pending_tests}
105
+ Total: #{@total_tests}
106
+
107
+ - Specific Test Case Status -
108
+ ] % [elapsed_time, average_test_time]
109
+
110
+ grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
111
+
112
+ summary_logger.notify "Failed Tests Cases:"
113
+ (grouped_summary[:fail] || []).each do |test_case|
114
+ summary_logger.notify print_test_result(test_case)
115
+ end
116
+
117
+ summary_logger.notify "Errored Tests Cases:"
118
+ (grouped_summary[:error] || []).each do |test_case|
119
+ summary_logger.notify print_test_result(test_case)
120
+ end
121
+
122
+ summary_logger.notify "Skipped Tests Cases:"
123
+ (grouped_summary[:skip] || []).each do |test_case|
124
+ summary_logger.notify print_test_result(test_case)
125
+ end
126
+
127
+ summary_logger.notify "Pending Tests Cases:"
128
+ (grouped_summary[:pending] || []).each do |test_case|
129
+ summary_logger.notify print_test_result(test_case)
130
+ end
131
+
132
+ summary_logger.notify("\n\n")
133
+ end
134
+
135
+ #A convenience method for printing the results of a {TestCase}
136
+ #@param [TestCase] test_case The {TestCase} to examine and print results for
137
+ def print_test_result(test_case)
138
+ if test_case.exception
139
+ test_file_trace = ""
140
+ test_case.exception.backtrace.each do |line|
141
+ if line.include?(test_case.path)
142
+ test_file_trace = "\r\n Test line: #{line}"
143
+ break
144
+ end
145
+ end if test_case.exception.backtrace && test_case.path
146
+ test_reported = "reported: #{test_case.exception.inspect}#{test_file_trace}"
147
+ else
148
+ test_case.test_status
149
+ end
150
+ " Test Case #{test_case.path} #{test_reported}"
151
+ end
152
+
153
+ # Writes Junit XML of this {TestSuiteResult}
154
+ #
155
+ # @param [String] xml_file Path to the XML file (from Beaker's running directory)
156
+ # @param [String] file_to_link Path to the paired file that should be linked
157
+ # from this one (this is relative to the XML
158
+ # file itself, so it would just be the different
159
+ # file name if they're in the same directory)
160
+ # @param [Boolean] time_sort Whether the test results should be output in
161
+ # order of time spent in the test, or in the
162
+ # order of test execution (default)
163
+ #
164
+ # @return nil
165
+ # @api private
166
+ def write_junit_xml(xml_file, file_to_link = nil, time_sort = false)
167
+ stylesheet = File.join(@options[:project_root], @options[:xml_stylesheet])
168
+
169
+ begin
170
+ LoggerJunit.write_xml(xml_file, stylesheet) do |doc, suites|
171
+
172
+ meta_info = suites.add_element(REXML::Element.new('meta_test_info'))
173
+ unless file_to_link.nil?
174
+ time_sort ? meta_info.add_attribute('page_active', 'performance') : meta_info.add_attribute('page_active', 'execution')
175
+ meta_info.add_attribute('link_url', file_to_link)
176
+ else
177
+ meta_info.add_attribute('page_active', 'no-links')
178
+ meta_info.add_attribute('link_url', '')
179
+ end
180
+
181
+ suite = suites.add_element(REXML::Element.new('testsuite'))
182
+ suite.add_attributes(
183
+ [
184
+ ['name' , @name],
185
+ ['tests', test_count],
186
+ ['errors', errored_tests],
187
+ ['failures', failed_tests],
188
+ ['skipped', skipped_tests],
189
+ ['pending', pending_tests],
190
+ ['total', @total_tests],
191
+ ['time', "%f" % (stop_time - start_time)]
192
+ ])
193
+ properties = suite.add_element(REXML::Element.new('properties'))
194
+ @options.each_pair do |name,value|
195
+ property = properties.add_element(REXML::Element.new('property'))
196
+ property.add_attributes([['name', name], ['value', value.to_s || '']])
197
+ end
198
+
199
+ test_cases_to_report = @test_cases
200
+ test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
201
+ test_cases_to_report.each do |test|
202
+ item = suite.add_element(REXML::Element.new('testcase'))
203
+ item.add_attributes(
204
+ [
205
+ ['classname', File.dirname(test.path)],
206
+ ['name', File.basename(test.path)],
207
+ ['time', "%f" % test.runtime]
208
+ ])
209
+
210
+ test.exports.each do |export|
211
+ export.keys.each do |key|
212
+ item.add_attribute(key.to_s.tr(" ", "_"), export[key])
213
+ end
214
+ end
215
+
216
+ #Report failures
217
+ if test.test_status == :fail || test.test_status == :error
218
+ status = item.add_element(REXML::Element.new('failure'))
219
+ status.add_attribute('type', test.test_status.to_s)
220
+ if test.exception
221
+ status.add_attribute('message', test.exception.to_s.gsub(/\e/,''))
222
+ data = LoggerJunit.format_cdata(test.exception.backtrace.join('\n'))
223
+ REXML::CData.new(data, true, status)
224
+ end
225
+ end
226
+
227
+ if test.test_status == :skip
228
+ status = item.add_element(REXML::Element.new('skipped'))
229
+ status.add_attribute('type', test.test_status.to_s)
230
+ end
231
+
232
+ if test.test_status == :pending
233
+ status = item.add_element(REXML::Element.new('pending'))
234
+ status.add_attribute('type', test.test_status.to_s)
235
+ end
236
+
237
+ if test.sublog
238
+ stdout = item.add_element(REXML::Element.new('system-out'))
239
+ data = LoggerJunit.format_cdata(test.sublog)
240
+ REXML::CData.new(data, true, stdout)
241
+ end
242
+
243
+ if test.last_result and test.last_result.stderr and not test.last_result.stderr.empty?
244
+ stderr = item.add_element('system-err')
245
+ data = LoggerJunit.format_cdata(test.last_result.stderr)
246
+ REXML::CData.new(data, true, stderr)
247
+ end
248
+ end
249
+ end
250
+ rescue Exception => e
251
+ @logger.error "failure in XML output: \n#{e.to_s}" + e.backtrace.join("\n")
252
+ end
253
+ end
254
+
255
+ end
256
+ end
@@ -1,5 +1,5 @@
1
1
  module Beaker
2
2
  module Version
3
- STRING = '3.22.0'
3
+ STRING = '3.23.0'
4
4
  end
5
5
  end
@@ -61,10 +61,8 @@ module Beaker
61
61
  it 'opens the given file for writing, and writes the doc to it' do
62
62
  mock_doc = Object.new
63
63
  doc_xml = 'flibbity-floo'
64
- allow( mock_doc ).to receive( :to_xml ) { doc_xml }
65
- mock_file_handle = Object.new
66
- expect( mock_file_handle ).to receive( :write ).with( doc_xml )
67
- expect( File ).to receive( :open ).with( xml_file, 'w' ).and_yield( mock_file_handle )
64
+ allow( mock_doc ).to receive( :write ).with(File, 2)
65
+ expect( File ).to receive( :open ).with( xml_file, 'w' )
68
66
  LoggerJunit.finish(mock_doc, xml_file)
69
67
  end
70
68
 
@@ -90,4 +88,4 @@ module Beaker
90
88
 
91
89
  end
92
90
  end
93
- end
91
+ end
@@ -117,18 +117,16 @@ module Beaker
117
117
  expect( tsr.passed_tests).to be === 1
118
118
 
119
119
  end
120
-
121
-
122
120
  end
123
121
 
124
- describe TestSuite::TestSuiteResult do
122
+ describe TestSuiteResult do
125
123
 
126
124
  let( :options ) { make_opts.merge({ :logger => double().as_null_object }) }
127
125
  let( :hosts ) { make_hosts() }
128
126
  let( :testcase1 ) { Beaker::TestCase.new( hosts, options[:logger], options) }
129
127
  let( :testcase2 ) { Beaker::TestCase.new( hosts, options[:logger], options) }
130
128
  let( :testcase3 ) { Beaker::TestCase.new( hosts, options[:logger], options) }
131
- let( :test_suite_result ) { TestSuite::TestSuiteResult.new( options, "my_suite") }
129
+ let( :test_suite_result ) { TestSuiteResult.new( options, "my_suite") }
132
130
 
133
131
  it 'supports adding test cases' do
134
132
  expect( test_suite_result.test_count ).to be === 0
@@ -254,11 +252,8 @@ module Beaker
254
252
  :log_dated_dir => '.',
255
253
  :xml_dated_dir => '.'}) }
256
254
  let(:rb_test) { 'my_ruby_file.rb' }
255
+
257
256
  before(:each) do
258
- @nokogiri_mock = Hash.new
259
- allow( @nokogiri_mock ).to receive( :add_child )
260
- allow( Nokogiri::XML::Node ).to receive( :new ) { @nokogiri_mock }
261
- allow( LoggerJunit ).to receive( :write_xml ).and_yield( Object.new, @nokogiri_mock )
262
257
  @files = [ rb_test, rb_test, rb_test]
263
258
  @ts = Beaker::TestSuite.new( 'name', hosts, options, Time.now, :fast )
264
259
  @tsr = @ts.instance_variable_get( :@test_suite_results )
@@ -270,6 +265,9 @@ module Beaker
270
265
  allow( tc ).to receive( :sublog ).and_return( false )
271
266
  @test_cases << tc
272
267
  end
268
+ @rexml_mock = REXML::Element.new("testsuites")
269
+ allow(REXML::Element).to receive( :add_element ).and_call_original
270
+ allow( LoggerJunit ).to receive( :write_xml ).and_yield( Object.new, @rexml_mock )
273
271
  end
274
272
 
275
273
  it 'doesn\'t re-order test cases themselves on time_sort' do
@@ -292,12 +290,14 @@ module Beaker
292
290
  inner_value = {'second' => '2nd'}
293
291
  @test_cases.each do |tc|
294
292
  tc.instance_variable_set(:@runtime, 0)
295
- tc.instance_variable_set(:@exports, [{'oh' => 'hai', 'first' => inner_value}])
293
+ tc.instance_variable_set(:@exports, [{'oh hey' => 'hai', 'first' => inner_value}])
296
294
  @tsr.add_test_case( tc )
297
295
  end
298
- @tsr.write_junit_xml( 'fakeFilePath08' )
299
- expect( @nokogiri_mock['oh'] ).to eq('hai')
300
- expect( @nokogiri_mock['first'] ).to eq(inner_value)
296
+ @tsr.write_junit_xml( 'fakeFilePath08')
297
+ @rexml_mock.elements.each("//testcase") do |e|
298
+ expect(e.attributes["oh_hey"].to_s).to eq('hai')
299
+ expect(e.attributes["first"]).to eq(inner_value.to_s)
300
+ end
301
301
  end
302
302
 
303
303
  it 'writes @export array of hashes properly' do
@@ -308,14 +308,12 @@ module Beaker
308
308
  @tsr.add_test_case( tc )
309
309
  end
310
310
  @tsr.write_junit_xml( 'fakeFilePath08' )
311
- expect( @nokogiri_mock[:yes] ).to eq('hello')
312
- expect( @nokogiri_mock[:uh] ).to eq('sher')
311
+ @rexml_mock.elements.each("//testcase") do |e|
312
+ expect(e.attributes["yes"].to_s).to eq('hello')
313
+ expect(e.attributes["uh"].to_s).to eq('sher')
314
+ end
313
315
  end
314
316
 
315
- # this isn't the best test as the @nokogiri_mock is a single hash.
316
- # it really should be an array of hashes or nested, to ensure each case
317
- # gets the correct key/value. But i could not get it to work properly with
318
- # the various calls to ::Node and #add_child
319
317
  it 'writes @export hashes per test case properly' do
320
318
  expect( @tsr.instance_variable_get( :@logger ) ).to receive( :error ).never
321
319
  @test_cases.each_with_index do |tc,index|
@@ -324,14 +322,14 @@ module Beaker
324
322
  @tsr.add_test_case( tc )
325
323
  end
326
324
  @tsr.write_junit_xml( 'fakeFilePath08' )
327
- @test_cases.each_with_index do |tc,index|
328
- expect( @nokogiri_mock["yes_#{index}"] ).to eq("hello#{index}")
325
+ index = 0
326
+ @rexml_mock.elements.each("//testcase") do |e|
327
+ expect(e.attributes["yes_#{index}"]).to eq("hello#{index}")
328
+ index += 1
329
329
  end
330
330
  end
331
-
332
331
  end
333
332
 
334
-
335
333
  end
336
334
 
337
335
  describe '#log_path' do
@@ -359,7 +357,6 @@ module Beaker
359
357
  testsuite.log_path('foo.txt', 'a/b/c/d/e/f')
360
358
  expect( File.symlink?('a/latest') ).to be_truthy
361
359
  end
362
-
363
360
  end
364
361
 
365
362
  describe 'builds the symlink directory correctly' do
@@ -376,7 +373,6 @@ module Beaker
376
373
  testsuite.log_path('foo.txt', 'f/g/h/i/j/k')
377
374
  expect( File.readlink('f/latest') ).to be === 'g/h/i/j/k'
378
375
  end
379
-
380
376
  end
381
377
 
382
378
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: beaker
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.22.0
4
+ version: 3.23.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Puppet
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-09 00:00:00.000000000 Z
11
+ date: 2017-08-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -66,20 +66,6 @@ dependencies:
66
66
  - - ! '>='
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
- - !ruby/object:Gem::Dependency
70
- name: pry
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - ~>
74
- - !ruby/object:Gem::Version
75
- version: '0.10'
76
- type: :development
77
- prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - ~>
81
- - !ruby/object:Gem::Version
82
- version: '0.10'
83
69
  - !ruby/object:Gem::Dependency
84
70
  name: rake
85
71
  requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,34 @@ dependencies:
136
122
  - - ~>
137
123
  - !ruby/object:Gem::Version
138
124
  version: '0.6'
125
+ - !ruby/object:Gem::Dependency
126
+ name: pry-byebug
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ~>
130
+ - !ruby/object:Gem::Version
131
+ version: 3.4.2
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ~>
137
+ - !ruby/object:Gem::Version
138
+ version: 3.4.2
139
+ - !ruby/object:Gem::Dependency
140
+ name: rb-readline
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ~>
144
+ - !ruby/object:Gem::Version
145
+ version: 0.5.3
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ~>
151
+ - !ruby/object:Gem::Version
152
+ version: 0.5.3
139
153
  - !ruby/object:Gem::Dependency
140
154
  name: hocon
141
155
  requirement: !ruby/object:Gem::Requirement
@@ -305,21 +319,21 @@ dependencies:
305
319
  - !ruby/object:Gem::Version
306
320
  version: '0.0'
307
321
  - !ruby/object:Gem::Dependency
308
- name: nokogiri
322
+ name: beaker-docker
309
323
  requirement: !ruby/object:Gem::Requirement
310
324
  requirements:
311
325
  - - ~>
312
326
  - !ruby/object:Gem::Version
313
- version: 1.8.0
327
+ version: '0.1'
314
328
  type: :runtime
315
329
  prerelease: false
316
330
  version_requirements: !ruby/object:Gem::Requirement
317
331
  requirements:
318
332
  - - ~>
319
333
  - !ruby/object:Gem::Version
320
- version: 1.8.0
334
+ version: '0.1'
321
335
  - !ruby/object:Gem::Dependency
322
- name: beaker-docker
336
+ name: beaker-aws
323
337
  requirement: !ruby/object:Gem::Requirement
324
338
  requirements:
325
339
  - - ~>
@@ -333,21 +347,21 @@ dependencies:
333
347
  - !ruby/object:Gem::Version
334
348
  version: '0.1'
335
349
  - !ruby/object:Gem::Dependency
336
- name: beaker-aws
350
+ name: beaker-vmpooler
337
351
  requirement: !ruby/object:Gem::Requirement
338
352
  requirements:
339
353
  - - ~>
340
354
  - !ruby/object:Gem::Version
341
- version: '0.1'
355
+ version: '1.0'
342
356
  type: :runtime
343
357
  prerelease: false
344
358
  version_requirements: !ruby/object:Gem::Requirement
345
359
  requirements:
346
360
  - - ~>
347
361
  - !ruby/object:Gem::Version
348
- version: '0.1'
362
+ version: '1.0'
349
363
  - !ruby/object:Gem::Dependency
350
- name: beaker-vmpooler
364
+ name: beaker-google
351
365
  requirement: !ruby/object:Gem::Requirement
352
366
  requirements:
353
367
  - - ~>
@@ -361,7 +375,7 @@ dependencies:
361
375
  - !ruby/object:Gem::Version
362
376
  version: '0.1'
363
377
  - !ruby/object:Gem::Dependency
364
- name: beaker-google
378
+ name: beaker-vagrant
365
379
  requirement: !ruby/object:Gem::Requirement
366
380
  requirements:
367
381
  - - ~>
@@ -375,7 +389,7 @@ dependencies:
375
389
  - !ruby/object:Gem::Version
376
390
  version: '0.1'
377
391
  - !ruby/object:Gem::Dependency
378
- name: beaker-vagrant
392
+ name: beaker-vmware
379
393
  requirement: !ruby/object:Gem::Requirement
380
394
  requirements:
381
395
  - - ~>
@@ -389,7 +403,7 @@ dependencies:
389
403
  - !ruby/object:Gem::Version
390
404
  version: '0.1'
391
405
  - !ruby/object:Gem::Dependency
392
- name: beaker-vmware
406
+ name: beaker-openstack
393
407
  requirement: !ruby/object:Gem::Requirement
394
408
  requirements:
395
409
  - - ~>
@@ -403,7 +417,7 @@ dependencies:
403
417
  - !ruby/object:Gem::Version
404
418
  version: '0.1'
405
419
  - !ruby/object:Gem::Dependency
406
- name: beaker-openstack
420
+ name: beaker-vcloud
407
421
  requirement: !ruby/object:Gem::Requirement
408
422
  requirements:
409
423
  - - ~>
@@ -558,11 +572,11 @@ files:
558
572
  - docs/concepts/testing_beaker_itself.md
559
573
  - docs/concepts/ticket_process.md
560
574
  - docs/concepts/types_puppet_4_and_the_all_in_one_agent.md
561
- - docs/how_to/access_the_live_test_console_with_pry.md
562
575
  - docs/how_to/archive_sut_files.md
563
576
  - docs/how_to/change_terminal_output_coloring.md
564
577
  - docs/how_to/cloning_private_repos.md
565
578
  - docs/how_to/confine.md
579
+ - docs/how_to/debug_beaker_tests.md
566
580
  - docs/how_to/enabling_cross_sut_access.md
567
581
  - docs/how_to/hosts/README.md
568
582
  - docs/how_to/hosts/archlinux.md
@@ -683,6 +697,7 @@ files:
683
697
  - lib/beaker/tasks/test.rb
684
698
  - lib/beaker/test_case.rb
685
699
  - lib/beaker/test_suite.rb
700
+ - lib/beaker/test_suite_result.rb
686
701
  - lib/beaker/version.rb
687
702
  - spec/beaker/cli_spec.rb
688
703
  - spec/beaker/command_spec.rb