beaker 1.12.2 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -111,12 +111,6 @@ module Beaker
111
111
  @cmd_options[:quiet] = bool
112
112
  end
113
113
 
114
- opts.on '-x', '--[no-]xml',
115
- 'Emit JUnit XML reports on tests',
116
- '(default: false)' do |bool|
117
- @cmd_options[:xml] = bool
118
- end
119
-
120
114
  opts.on '--[no-]color',
121
115
  'Do not display color in log output',
122
116
  '(default: true)' do |bool|
@@ -170,6 +164,10 @@ module Beaker
170
164
  @cmd_options[:add_el_extras] = true
171
165
  end
172
166
 
167
+ opts.on '--package-proxy URL', 'Set proxy url for package managers (yum and apt)' do |value|
168
+ @cmd_options[:package_proxy] = value
169
+ end
170
+
173
171
  opts.on '--[no-]validate',
174
172
  'Validate that SUTs are correctly configured before running tests',
175
173
  '(default: true)' do |bool|
@@ -198,9 +196,11 @@ module Beaker
198
196
  @cmd_options[:log_level] = bool ? 'debug' : 'info'
199
197
  end
200
198
 
201
- opts.on '--package-proxy URL', 'Set proxy url for package managers (yum and apt)' do |value|
202
- @cmd_options[:package_proxy] = value
199
+ opts.on '-x', '--[no-]xml',
200
+ 'DEPRECATED - JUnit XML now generated by default' do
201
+ #noop
203
202
  end
203
+
204
204
  end
205
205
 
206
206
  end
@@ -67,7 +67,11 @@ module Beaker
67
67
  :preserve_hosts => 'never',
68
68
  :root_keys => false,
69
69
  :quiet => false,
70
- :xml => false,
70
+ :project_root => File.expand_path(File.join(File.dirname(__FILE__), "../")),
71
+ :xml_dir => 'junit',
72
+ :xml_file => 'beaker_junit.xml',
73
+ :xml_stylesheet => 'junit.xsl',
74
+ :log_dir => 'log',
71
75
  :color => true,
72
76
  :dry_run => false,
73
77
  :timeout => 300,
@@ -19,6 +19,7 @@ module Beaker
19
19
 
20
20
  #Creates the Platform object. Checks to ensure that the platform String provided meets the platform
21
21
  #formatting rules. Platforms name must be of the format /^OSFAMILY-VERSION-ARCH.*$/ where OSFAMILY is one of:
22
+ # * osx
22
23
  # * centos
23
24
  # * fedora
24
25
  # * debian
@@ -5,6 +5,8 @@ require 'net/scp'
5
5
  module Beaker
6
6
  class SshConnection
7
7
 
8
+ attr_accessor :logger
9
+
8
10
  RETRYABLE_EXCEPTIONS = [
9
11
  SocketError,
10
12
  Timeout::Error,
@@ -18,14 +20,15 @@ module Beaker
18
20
  Net::SSH::AuthenticationFailed,
19
21
  ]
20
22
 
21
- def initialize hostname, user = nil, options = {}
23
+ def initialize hostname, user = nil, ssh_opts = {}, options = {}
22
24
  @hostname = hostname
23
25
  @user = user
24
- @options = options
26
+ @ssh_opts = ssh_opts
27
+ @logger = options[:logger]
25
28
  end
26
29
 
27
- def self.connect hostname, user = 'root', options = {}
28
- connection = new hostname, user, options
30
+ def self.connect hostname, user = 'root', ssh_opts = {}, options = {}
31
+ connection = new hostname, user, ssh_opts, options
29
32
  connection.connect
30
33
  connection
31
34
  end
@@ -35,21 +38,22 @@ module Beaker
35
38
  last_wait = 0
36
39
  wait = 1
37
40
  @ssh ||= begin
38
- Net::SSH.start(@hostname, @user, @options)
41
+ Net::SSH.start(@hostname, @user, @ssh_opts)
39
42
  rescue *RETRYABLE_EXCEPTIONS => e
40
43
  if try <= 11
41
- puts "Try #{try} -- Host #{@hostname} unreachable: #{e.message}"
42
- puts "Trying again in #{wait} seconds"
44
+ @logger.warn "Try #{try} -- Host #{@hostname} unreachable: #{e.message}"
45
+ @logger.warn "Trying again in #{wait} seconds"
43
46
  sleep wait
44
- (last_wait, wait) = wait, last_wait + wait
47
+ (last_wait, wait) = wait, last_wait + wait
45
48
  try += 1
46
49
  retry
47
50
  else
48
51
  # why is the logger not passed into this class?
49
- puts "Failed to connect to #{@hostname}"
52
+ @logger.error "Failed to connect to #{@hostname}"
50
53
  raise
51
54
  end
52
55
  end
56
+ @logger.debug "Created ssh connection to #{@hostname}, user: #{@user}, opts: #{@ssh_opts}"
53
57
  self
54
58
  end
55
59
 
@@ -101,7 +105,7 @@ module Beaker
101
105
  rescue *RETRYABLE_EXCEPTIONS => e
102
106
  if attempt
103
107
  attempt = false
104
- puts "Command execution failed, attempting to reconnect to #{@hostname}"
108
+ @logger.error "Command execution failed, attempting to reconnect to #{@hostname}"
105
109
  close
106
110
  connect
107
111
  retry
@@ -116,7 +120,7 @@ module Beaker
116
120
  def request_terminal_for channel, command
117
121
  channel.request_pty do |ch, success|
118
122
  if success
119
- puts "Allocated a PTY on #{@hostname} for #{command.inspect}"
123
+ @logger.info "Allocated a PTY on #{@hostname} for #{command.inspect}"
120
124
  else
121
125
  abort "FAILED: could not allocate a pty when requested on " +
122
126
  "#{@hostname} for #{command.inspect}"
@@ -0,0 +1,114 @@
1
+ require 'rake/task_arguments'
2
+ require 'rake/tasklib'
3
+ require 'rake'
4
+ require 'beaker'
5
+
6
+
7
+ module Beaker
8
+ module Tasks
9
+ class RakeTask < ::Rake::TaskLib
10
+ include ::Rake::DSL if defined?(::Rake::DSL)
11
+
12
+ DEFAULT_ACCEPTANCE_ROOT = "./acceptance"
13
+
14
+ COMMAND_OPTIONS = [:fail_mode,
15
+ :hosts,
16
+ :helper,
17
+ :keyfile,
18
+ :log_level,
19
+ :options_file,
20
+ :preserve_hosts,
21
+ :tests,
22
+ :type,
23
+ :acceptance_root,
24
+ :name]
25
+ # iterates of acceptable params
26
+ COMMAND_OPTIONS.each do |sym|
27
+ attr_accessor(sym.to_sym)
28
+ end
29
+
30
+ # Sets up the predefine task checking
31
+ # @param args [Array] First argument is always the name of the task
32
+ # if no additonal arguments are defined such as parameters it will default to [:hosts,:type]
33
+ def initialize(*args, &task_block)
34
+ @name = args.shift || 'beaker:test'
35
+ if args.empty?
36
+ args = [:hosts,:type]
37
+ end
38
+ @acceptance_root = DEFAULT_ACCEPTANCE_ROOT
39
+ @options_file = nil
40
+ define(args, &task_block)
41
+ end
42
+
43
+ private
44
+ # Run the task provided, implements the rake task interface
45
+ #
46
+ # @param verbose [bool] Defines wether to run in verbose mode or not
47
+ def run_task(verbose)
48
+ puts "Running task"
49
+
50
+ check_for_beaker_type_config
51
+ command = beaker_command
52
+
53
+ begin
54
+ puts command if verbose
55
+ success = system(command)
56
+ preserve_configuration(@options_file)
57
+ rescue
58
+ puts failure_message if failure_message
59
+ end
60
+ if fail_mode == "fast" && !success
61
+ $stderr.puts "#{command} failed"
62
+ exit $?.exitstatus
63
+ end
64
+ end
65
+
66
+ # @private
67
+ def define(args, &task_block)
68
+ desc "Run Beaker Acceptance" unless ::Rake.application.last_comment
69
+ task name, *args do |_, task_args|
70
+ RakeFileUtils.__send__(:verbose, verbose) do
71
+ task_block.call(*[self, task_args].slice(0, task_block.arity)) if task_block
72
+ run_task verbose
73
+ end
74
+ end
75
+ end
76
+
77
+ #
78
+ # If an options file exists in the acceptance path for the type given use it as a default options file
79
+ # if no other options file is provided
80
+ #
81
+ def check_for_beaker_type_config
82
+ if !@options_file && File.exists?("#{@acceptance_root}/.beaker-#{@type}.cfg")
83
+ @options_file = File.join(@acceptance_root, ".beaker-#{@type}.cfg")
84
+ end
85
+ end
86
+
87
+ #
88
+ # Check for existence of ENV variables for test if !@tests is undef
89
+ #
90
+ def check_env_variables
91
+ if File.exists?(File.join(DEFAULT_ACCEPTANCE_ROOT, 'tests'))
92
+ @tests = File.join(DEFAULT_ACCEPTANCE_ROOT, 'tests')
93
+ end
94
+ @tests = ENV['TESTS'] || ENV['TEST'] if !@tests
95
+ end
96
+
97
+ #
98
+ # Generate the beaker command to run beaker with all possible options passed
99
+ #
100
+ def beaker_command
101
+ cmd_parts = []
102
+ cmd_parts << "beaker"
103
+ cmd_parts << "--keyfile #{@keyfile}" if @keyfile
104
+ cmd_parts << "--hosts #{@hosts}" if @hosts
105
+ cmd_parts << "--tests #{tests}" if @tests
106
+ cmd_parts << "--options-file #{@options_file}" if @options_file
107
+ cmd_parts << "--type #{@type}" if @type
108
+ cmd_parts << "--helper #{@helper}" if @helper
109
+ cmd_parts << "--fail-mode #{@fail_mode}" if @fail_mode
110
+ cmd_parts.flatten.join(" ")
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,18 @@
1
+ require 'beaker/tasks/rake_task'
2
+
3
+ Beaker::Tasks::RakeTask.new do |t,args|
4
+ t.type = args[:type]
5
+ t.hosts = args[:hosts]
6
+ end
7
+
8
+ desc "Run Beaker PE tests"
9
+ Beaker::Tasks::RakeTask.new("beaker:test:pe",:hosts) do |t,args|
10
+ t.type = 'pe'
11
+ t.hosts = args[:hosts]
12
+ end
13
+
14
+ desc "Run Beaker Git tests"
15
+ Beaker::Tasks::RakeTask.new("beaker:test:git",:hosts) do |t,args|
16
+ t.type = 'git'
17
+ t.hosts = args[:hosts]
18
+ end
@@ -52,6 +52,9 @@ module Beaker
52
52
  # an instance of {Beaker::Logger}.
53
53
  attr_accessor :logger
54
54
 
55
+ #The full log for this test
56
+ attr_accessor :sublog
57
+
55
58
  # A Hash of 'product name' => 'version installed', only set when
56
59
  # products are installed via git or PE install steps. See the 'git' or
57
60
  # 'pe' directories within 'ROOT/setup' for examples.
@@ -102,6 +105,7 @@ module Beaker
102
105
  def initialize(these_hosts, logger, options={}, path=nil)
103
106
  @hosts = these_hosts
104
107
  @logger = logger
108
+ @sublog = ""
105
109
  @options = options
106
110
  @path = path
107
111
  @usr_home = options[:home]
@@ -116,6 +120,7 @@ module Beaker
116
120
  # defined in the tests don't leak out to other tests.
117
121
  class << self
118
122
  def run_test
123
+ @logger.start_sublog
119
124
  @runtime = Benchmark.realtime do
120
125
  begin
121
126
  test = File.read(path)
@@ -139,6 +144,7 @@ module Beaker
139
144
  end
140
145
  end
141
146
  end
147
+ @sublog = @logger.get_sublog
142
148
  return self
143
149
  end
144
150
 
@@ -1,20 +1,279 @@
1
1
  # -*- coding: utf-8 -*-
2
- require 'rexml/document'
2
+ require 'nokogiri'
3
3
  require 'fileutils'
4
4
  [ 'test_case', 'logger' ].each do |lib|
5
5
  require "beaker/#{lib}"
6
6
  end
7
7
 
8
8
  module Beaker
9
- # This Class is in need of some cleaning up beyond what can be quickly done.
10
- # Things to keep in mind:
11
- # * Global State Change
12
- # * File Creation Relative to CWD -- Should be a config option
13
- # * Better Method Documentation
9
+ #A collection of {TestCase} objects are considered a {TestSuite}.
10
+ #Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
14
11
  class TestSuite
12
+
13
+ #Holds the output of a test suite, formats in plain text or xml
14
+ class TestSuiteResult
15
+ attr_accessor :start_time, :stop_time
16
+
17
+ #Create a {TestSuiteResult} instance.
18
+ #@param [Hash{Symbol=>String}] options Options for this object
19
+ #@option options [Logger] :logger The Logger object to report information to
20
+ #@param [String] name The name of the {TestSuite} that the results are for
21
+ def initialize( options, name )
22
+ @options = options
23
+ @logger = options[:logger]
24
+ @name = name
25
+ @test_cases = []
26
+ #Set some defaults, just in case you attempt to print without including them
27
+ start_time = Time.at(0)
28
+ stop_time = Time.at(1)
29
+ end
30
+
31
+ #Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
32
+ #@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
33
+ def add_test_case( test_case )
34
+ @test_cases << test_case
35
+ end
36
+
37
+ #How many {TestCase} instances are in this {TestSuiteResult}
38
+ def test_count
39
+ @test_cases.length
40
+ end
41
+
42
+ #How many passed {TestCase} instances are in this {TestSuiteResult}
43
+ def passed_tests
44
+ @test_cases.select { |c| c.test_status == :pass }.length
45
+ end
46
+
47
+ #How many errored {TestCase} instances are in this {TestSuiteResult}
48
+ def errored_tests
49
+ @test_cases.select { |c| c.test_status == :error }.length
50
+ end
51
+
52
+ #How many failed {TestCase} instances are in this {TestSuiteResult}
53
+ def failed_tests
54
+ @test_cases.select { |c| c.test_status == :fail }.length
55
+ end
56
+
57
+ #How many skipped {TestCase} instances are in this {TestSuiteResult}
58
+ def skipped_tests
59
+ @test_cases.select { |c| c.test_status == :skip }.length
60
+ end
61
+
62
+ #How many pending {TestCase} instances are in this {TestSuiteResult}
63
+ def pending_tests
64
+ @test_cases.select {|c| c.test_status == :pending}.length
65
+ end
66
+
67
+ #How many {TestCase} instances failed in this {TestSuiteResult}
68
+ def sum_failed
69
+ failed_tests + errored_tests
70
+ end
71
+
72
+ #Did all the {TestCase} instances in this {TestSuiteResult} pass?
73
+ def success?
74
+ sum_failed == 0
75
+ end
76
+
77
+ #Did one or more {TestCase} instances in this {TestSuiteResult} fail?
78
+ def failed?
79
+ !success?
80
+ end
81
+
82
+ #The sum of all {TestCase} runtimes in this {TestSuiteResult}
83
+ def elapsed_time
84
+ @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
85
+ end
86
+
87
+ #Plain text summay of test suite
88
+ #@param [Logger] summary_logger The logger we will print the summary to
89
+ def summarize(summary_logger)
90
+
91
+ summary_logger.notify <<-HEREDOC
92
+ Test Suite: #{@name} @ #{start_time}
93
+
94
+ - Host Configuration Summary -
95
+ HEREDOC
96
+
97
+ average_test_time = elapsed_time / test_count
98
+
99
+ summary_logger.notify %Q[
100
+
101
+ - Test Case Summary for suite '#{@name}' -
102
+ Total Suite Time: %.2f seconds
103
+ Average Test Time: %.2f seconds
104
+ Attempted: #{test_count}
105
+ Passed: #{passed_tests}
106
+ Failed: #{failed_tests}
107
+ Errored: #{errored_tests}
108
+ Skipped: #{skipped_tests}
109
+ Pending: #{pending_tests}
110
+
111
+ - Specific Test Case Status -
112
+ ] % [elapsed_time, average_test_time]
113
+
114
+ grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
115
+
116
+ summary_logger.notify "Failed Tests Cases:"
117
+ (grouped_summary[:fail] || []).each do |test_case|
118
+ print_test_result(test_case)
119
+ end
120
+
121
+ summary_logger.notify "Errored Tests Cases:"
122
+ (grouped_summary[:error] || []).each do |test_case|
123
+ print_test_result(test_case)
124
+ end
125
+
126
+ summary_logger.notify "Skipped Tests Cases:"
127
+ (grouped_summary[:skip] || []).each do |test_case|
128
+ print_test_result(test_case)
129
+ end
130
+
131
+ summary_logger.notify "Pending Tests Cases:"
132
+ (grouped_summary[:pending] || []).each do |test_case|
133
+ print_test_result(test_case)
134
+ end
135
+
136
+ summary_logger.notify("\n\n")
137
+ end
138
+
139
+ #A convenience method for printing the results of a {TestCase}
140
+ #@param [TestCase] test_case The {TestCase} to examine and print results for
141
+ def print_test_result(test_case)
142
+ test_reported = if test_case.exception
143
+ "reported: #{test_case.exception.inspect}"
144
+ else
145
+ test_case.test_status
146
+ end
147
+ @logger.notify " Test Case #{test_case.path} #{test_reported}"
148
+ end
149
+
150
+ #Remove color codes from provided string. Color codes are of the format /(\e\[\d\d;\d\dm)+/.
151
+ #@param [String] text The string to remove color codes from
152
+ #@return [String] The text without color codes
153
+ def strip_color_codes(text)
154
+ text.gsub(/(\e|\^\[)\[(\d*;)*\d*m/, '')
155
+ end
156
+
157
+ #Format and print the {TestSuiteResult} as JUnit XML
158
+ #@param [String] xml_file The full path to print the output to.
159
+ #@param [String] stylesheet The full path to a JUnit XML stylesheet
160
+ def write_junit_xml(xml_file, stylesheet)
161
+ begin
162
+
163
+ #copy stylesheet into xml directory
164
+ if not File.file?(File.join(File.dirname(xml_file), File.basename(stylesheet)))
165
+ FileUtils.copy(stylesheet, File.join(File.dirname(xml_file), File.basename(stylesheet)))
166
+ end
167
+ suites = nil
168
+ #check to see if an output file already exists, if it does add or replace test suite data
169
+ if File.file?(xml_file)
170
+ doc = Nokogiri::XML( File.open(xml_file, 'r') )
171
+ suites = doc.at_xpath('testsuites')
172
+ #remove old data
173
+ doc.search("//testsuite").each do |node|
174
+ if node['name'] =~ /#{@name}/
175
+ node.unlink
176
+ end
177
+ end
178
+ else
179
+ #no existing file, create a new one
180
+ doc = Nokogiri::XML::Document.new
181
+ pi = Nokogiri::XML::ProcessingInstruction.new(doc, "xml-stylesheet", "type=\"text/xsl\" href=\"#{File.basename(stylesheet)}\"")
182
+ pi.parent = doc
183
+ suites = Nokogiri::XML::Node.new('testsuites', doc)
184
+ suites.parent = doc
185
+ end
186
+
187
+ suite = Nokogiri::XML::Node.new('testsuite', doc)
188
+ suite['name'] = @name
189
+ suite['tests'] = test_count
190
+ suite['errors'] = errored_tests
191
+ suite['failures'] = failed_tests
192
+ suite['skip'] = skipped_tests
193
+ suite['pending'] = pending_tests
194
+ suite['time'] = "%f" % (stop_time - start_time)
195
+ properties = Nokogiri::XML::Node.new('properties', doc)
196
+ @options.each_pair do | name, value |
197
+ property = Nokogiri::XML::Node.new('property', doc)
198
+ property['name'] = name
199
+ property['value'] = value
200
+ properties.add_child(property)
201
+ end
202
+ suite.add_child(properties)
203
+
204
+ @test_cases.each do |test|
205
+ item = Nokogiri::XML::Node.new('testcase', doc)
206
+ item['classname'] = File.dirname(test.path)
207
+ item['name'] = File.basename(test.path)
208
+ item['time'] = "%f" % test.runtime
209
+
210
+ # Did we fail? If so, report that.
211
+ # We need to remove the escape character from colorized text, the
212
+ # substitution of other entities is handled well by Rexml
213
+ if test.test_status == :fail || test.test_status == :error then
214
+ status = Nokogiri::XML::Node.new('failure', doc)
215
+ status['type'] = test.test_status.to_s
216
+ if test.exception then
217
+ status['message'] = test.exception.to_s.gsub(/\e/, '')
218
+ status.add_child(status.document.create_cdata(test.exception.backtrace.join('\n')))
219
+ end
220
+ item.add_child(status)
221
+ end
222
+
223
+ if test.test_status == :skip
224
+ status = Nokogiri::XML::Node.new('skip', doc)
225
+ status['type'] = test.test_status.to_s
226
+ item.add_child(status)
227
+ end
228
+
229
+ if test.test_status == :pending
230
+ status = Nokogiri::XML::Node.new('pending', doc)
231
+ status['type'] = test.test_status.to_s
232
+ item.add_child(status)
233
+ end
234
+
235
+ if test.sublog then
236
+ stdout = Nokogiri::XML::Node.new('system-out', doc)
237
+ stdout.add_child(stdout.document.create_cdata(strip_color_codes(test.sublog)))
238
+ item.add_child(stdout)
239
+ end
240
+
241
+ if test.stderr then
242
+ stderr = Nokogiri::XML::Node.new('system-err', doc)
243
+ stderr.add_child(stderr.document.create_cdata(strip_color_codes(test.stderr)))
244
+ item.add_child(stderr)
245
+ end
246
+
247
+ suite.add_child(item)
248
+ end
249
+ suites.add_child(suite)
250
+
251
+ # junit/name.xml will be created in a directory relative to the CWD
252
+ # -- JLS 2/12
253
+ File.open(xml_file, 'w') { |fh| fh.write(doc.to_xml) }
254
+
255
+ rescue Exception => e
256
+ @logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
257
+ end
258
+ end
259
+
260
+ end
261
+
15
262
  attr_reader :name, :options, :fail_mode
16
263
 
17
- def initialize(name, hosts, options, fail_mode = nil)
264
+ #Create {TestSuite} instance
265
+ #@param [String] name The name of the {TestSuite}
266
+ #@param [Array<Host>] hosts An Array of Hosts to act upon.
267
+ #@param [Hash{Symbol=>String}] options Options for this object
268
+ #@option options [Logger] :logger The Logger object to report information to
269
+ #@option options [String] :log_dir The directory where text run logs will be written
270
+ #@option options [String] :xml_dir The directory where JUnit XML file will be written
271
+ #@option options [String] :xml_file The name of the JUnit XML file to be written to
272
+ #@option options [String] :project_root The full path to the Beaker lib directory
273
+ #@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
274
+ #@param [Symbol] fail_mode One of :slow, :fast
275
+ #@param [Time] timestamp Beaker execution start time
276
+ def initialize(name, hosts, options, timestamp, fail_mode = :slow)
18
277
  @logger = options[:logger]
19
278
  @test_cases = []
20
279
  @test_files = options[name]
@@ -22,7 +281,9 @@ module Beaker
22
281
  @hosts = hosts
23
282
  @run = false
24
283
  @options = options
25
- @fail_mode = options[:fail_mode] || fail_mode
284
+ @fail_mode = fail_mode
285
+ @test_suite_results = TestSuiteResult.new(@options, name)
286
+ @timestamp = timestamp
26
287
 
27
288
  report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
28
289
 
@@ -30,18 +291,30 @@ module Beaker
30
291
  report_and_raise(@logger, e, "TestSuite: initialize")
31
292
  end
32
293
 
294
+ #Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
295
+ #is reported to a newly created run log.
296
+ #Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
297
+ #after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
33
298
  def run
34
299
  @run = true
35
- @start_time = Time.now
300
+ start_time = Time.now
36
301
 
37
- configure_logging
302
+ #Create a run log for this TestSuite.
303
+ run_log = log_path(@timestamp, "#{@name}-run.log", @options[:log_dir])
304
+ @logger.add_destination(run_log)
305
+
306
+ # This is an awful hack to maintain backward compatibility until tests
307
+ # are ported to use logger. Still in use in PuppetDB tests
308
+ Beaker.const_set(:Log, @logger) unless defined?( Log )
309
+
310
+ @test_suite_results.start_time = start_time
38
311
 
39
312
  @test_files.each do |test_file|
40
- @logger.notify
41
313
  @logger.notify "Begin #{test_file}"
42
314
  start = Time.now
43
315
  test_case = TestCase.new(@hosts, @logger, options, test_file).run_test
44
316
  duration = Time.now - start
317
+ @test_suite_results.add_test_case(test_case)
45
318
  @test_cases << test_case
46
319
 
47
320
  state = test_case.test_status == :skip ? 'skipp' : test_case.test_status
@@ -53,27 +326,33 @@ module Beaker
53
326
  @logger.debug msg
54
327
  when :fail
55
328
  @logger.error msg
56
- break if fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
329
+ break if @fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
57
330
  when :error
58
331
  @logger.warn msg
59
- break if fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
332
+ break if @fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
60
333
  end
61
334
  end
335
+ @test_suite_results.stop_time = Time.now
62
336
 
63
337
  # REVISIT: This changes global state, breaking logging in any future runs
64
338
  # of the suite – or, at least, making them highly confusing for anyone who
65
339
  # has not studied the implementation in detail. --daniel 2011-03-14
66
- summarize
67
- write_junit_xml if options[:xml]
340
+ @test_suite_results.summarize( Logger.new(log_path(@timestamp, "#{name}-summary.txt", @options[:log_dir]), STDOUT) )
341
+ @test_suite_results.write_junit_xml( log_path(@timestamp, @options[:xml_file], @options[:xml_dir]), File.join(@options[:project_root], @options[:xml_stylesheet]) )
342
+
343
+ #All done with this run, remove run log
344
+ @logger.remove_destination(run_log)
68
345
 
69
346
  # Allow chaining operations...
70
347
  return self
71
348
  end
72
349
 
350
+ #Execute all the TestCases in this suite.
351
+ #This is a wrapper that catches any failures generated during TestSuite::run.
73
352
  def run_and_raise_on_failure
74
353
  begin
75
354
  run
76
- return self if success?
355
+ return self if @test_suite_results.success?
77
356
  rescue => e
78
357
  #failed during run
79
358
  report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
@@ -83,187 +362,33 @@ module Beaker
83
362
  end
84
363
  end
85
364
 
86
- def fail_without_test_run
87
- report_and_raise(@logger, RuntimeError.new("#{@name}: you have not run the tests yet"), "TestSuite: fail_without_test_run") unless @run
88
- end
89
-
90
- def success?
91
- fail_without_test_run
92
- sum_failed == 0
93
- end
94
-
95
- def failed?
96
- !success?
97
- end
98
-
99
- def test_count
100
- @test_count ||= @test_cases.length
101
- end
102
-
103
- def passed_tests
104
- @passed_tests ||= @test_cases.select { |c| c.test_status == :pass }.length
105
- end
106
-
107
- def errored_tests
108
- @errored_tests ||= @test_cases.select { |c| c.test_status == :error }.length
109
- end
110
-
111
- def failed_tests
112
- @failed_tests ||= @test_cases.select { |c| c.test_status == :fail }.length
113
- end
114
-
115
- def skipped_tests
116
- @skipped_tests ||= @test_cases.select { |c| c.test_status == :skip }.length
117
- end
118
-
119
- def pending_tests
120
- @pending_tests ||= @test_cases.select {|c| c.test_status == :pending}.length
121
- end
122
-
123
- private
124
-
125
- def sum_failed
126
- @sum_failed ||= failed_tests + errored_tests
127
- end
128
-
129
- def write_junit_xml
130
- # This should be a configuration option
131
- File.directory?('junit') or FileUtils.mkdir('junit')
132
-
133
- begin
134
- doc = REXML::Document.new
135
- doc.add(REXML::XMLDecl.new(1.0))
136
-
137
- suite = REXML::Element.new('testsuite', doc)
138
- suite.add_attribute('name', name)
139
- suite.add_attribute('tests', test_count)
140
- suite.add_attribute('errors', errored_tests)
141
- suite.add_attribute('failures', failed_tests)
142
- suite.add_attribute('skip', skipped_tests)
143
- suite.add_attribute('pending', pending_tests)
144
-
145
- @test_cases.each do |test|
146
- item = REXML::Element.new('testcase', suite)
147
- item.add_attribute('classname', File.dirname(test.path))
148
- item.add_attribute('name', File.basename(test.path))
149
- item.add_attribute('time', test.runtime)
150
-
151
- # Did we fail? If so, report that.
152
- # We need to remove the escape character from colorized text, the
153
- # substitution of other entities is handled well by Rexml
154
- if test.test_status == :fail || test.test_status == :error then
155
- status = REXML::Element.new('failure', item)
156
- status.add_attribute('type', test.test_status.to_s)
157
- if test.exception then
158
- status.add_attribute('message', test.exception.to_s.gsub(/\e/, ''))
159
- status.text = test.exception.backtrace.join("\n")
160
- end
161
- end
162
-
163
- if test.stdout then
164
- REXML::Element.new('system-out', item).text =
165
- test.stdout.gsub(/\e/, '')
166
- end
167
-
168
- if test.stderr then
169
- text = REXML::Element.new('system-err', item)
170
- text.text = test.stderr.gsub(/\e/, '')
171
- end
172
- end
173
-
174
- # junit/name.xml will be created in a directory relative to the CWD
175
- # -- JLS 2/12
176
- File.open("junit/#{name}.xml", 'w') { |fh| doc.write(fh) }
177
- rescue Exception => e
178
- @logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
179
- end
180
- end
181
-
182
- def summarize
183
- fail_without_test_run
184
-
185
- summary_logger = Logger.new(log_path("#{name}-summary.txt"), STDOUT)
186
-
187
- summary_logger.notify <<-HEREDOC
188
- Test Suite: #{name} @ #{@start_time}
189
-
190
- - Host Configuration Summary -
191
- HEREDOC
192
-
193
- elapsed_time = @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
194
- average_test_time = elapsed_time / test_count
195
-
196
- summary_logger.notify %Q[
197
-
198
- - Test Case Summary for suite '#{name}' -
199
- Total Suite Time: %.2f seconds
200
- Average Test Time: %.2f seconds
201
- Attempted: #{test_count}
202
- Passed: #{passed_tests}
203
- Failed: #{failed_tests}
204
- Errored: #{errored_tests}
205
- Skipped: #{skipped_tests}
206
- Pending: #{pending_tests}
207
-
208
- - Specific Test Case Status -
209
- ] % [elapsed_time, average_test_time]
210
-
211
- grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
212
-
213
- summary_logger.notify "Failed Tests Cases:"
214
- (grouped_summary[:fail] || []).each do |test_case|
215
- print_test_failure(test_case)
216
- end
217
-
218
- summary_logger.notify "Errored Tests Cases:"
219
- (grouped_summary[:error] || []).each do |test_case|
220
- print_test_failure(test_case)
221
- end
222
-
223
- summary_logger.notify "Skipped Tests Cases:"
224
- (grouped_summary[:skip] || []).each do |test_case|
225
- print_test_failure(test_case)
226
- end
227
-
228
- summary_logger.notify "Pending Tests Cases:"
229
- (grouped_summary[:pending] || []).each do |test_case|
230
- print_test_failure(test_case)
231
- end
232
-
233
- summary_logger.notify("\n\n")
234
- end
235
-
236
- def print_test_failure(test_case)
237
- test_reported = if test_case.exception
238
- "reported: #{test_case.exception.inspect}"
239
- else
240
- test_case.test_status
241
- end
242
- @logger.notify " Test Case #{test_case.path} #{test_reported}"
243
- end
244
-
245
- def log_path(name)
246
- @@log_dir ||= File.join("log", @start_time.strftime("%F_%H_%M_%S"))
247
- unless File.directory?(@@log_dir) then
248
- FileUtils.mkdir_p(@@log_dir)
249
-
250
- latest = File.join("log", "latest")
365
+ #Create a full file path for output to be written to, using the provided timestamp, name and output directory.
366
+ #@param [Time] timestamp The time that we are making this path with
367
+ #@param [String] name The file name that we want to write to.
368
+ #@param [String] basedir The desired output directory. A subdirectory tagged with the time will be created which will contain
369
+ # the output file (./basedir/timestamp/).
370
+ # A symlink will be made from that to ./basedir/timestamp/latest.
371
+ #@example
372
+ # log_path('2014-06-02 16:31:22 -0700','output.txt', 'log')
373
+ #
374
+ # This will create the structure:
375
+ #
376
+ # ./log/2014-06-02_16_31_22/output.txt
377
+ # ./log/latest -> 2014-06-02_16_31_22
378
+ def log_path(timestamp, name, basedir)
379
+ log_dir = File.join(basedir, timestamp.strftime("%F_%H_%M_%S"))
380
+ unless File.directory?(log_dir) then
381
+ FileUtils.mkdir_p(log_dir)
382
+
383
+ latest = File.join(basedir, "latest")
251
384
  if !File.exist?(latest) or File.symlink?(latest) then
252
385
  File.delete(latest) if File.exist?(latest)
253
- File.symlink(File.basename(@@log_dir), latest)
386
+ File.symlink(File.basename(log_dir), latest)
254
387
  end
255
388
  end
256
389
 
257
- File.join('log', 'latest', name)
390
+ File.join(basedir, 'latest', name)
258
391
  end
259
392
 
260
- # Setup log dir
261
- def configure_logging
262
- @logger.add_destination(log_path("#{@name}-run.log"))
263
- #
264
- # This is an awful hack to maintain backward compatibility until tests
265
- # are ported to use logger.
266
- Beaker.const_set(:Log, @logger) unless defined?( Log )
267
- end
268
393
  end
269
394
  end