beaker 2.29.1 → 2.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -112,7 +112,7 @@ module Beaker
112
112
  ip = ''
113
113
  if File.file?(@vagrant_file) #we should have a vagrant file available to us for reading
114
114
  f = File.read(@vagrant_file)
115
- m = /#{hostname}.*?ip:\s*('|")\s*([^'"]+)('|")/m.match(f)
115
+ m = /'#{hostname}'.*?ip:\s*('|")\s*([^'"]+)('|")/m.match(f)
116
116
  if m
117
117
  ip = m[2]
118
118
  @logger.debug("Determined existing vagrant box #{hostname} ip to be: #{ip} ")
@@ -58,7 +58,12 @@ module Beaker
58
58
  'Creation time: ' + Time.now.strftime("%Y-%m-%d %H:%M") + "\n\n" +
59
59
  'CI build link: ' + ( ENV['BUILD_URL'] || 'Deployed independently of CI' ) +
60
60
  'department: ' + @options[:department] +
61
- 'project: ' + @options[:project]
61
+ 'project: ' + @options[:project],
62
+ :extraConfig => [
63
+ { :key => 'guestinfo.hostname',
64
+ :value => host['vmhostname']
65
+ }
66
+ ]
62
67
  )
63
68
 
64
69
  # Are we using a customization spec?
@@ -98,6 +98,12 @@ module Beaker
98
98
  @cmd_options[:timeout] = value
99
99
  end
100
100
 
101
+ opts.on '-r RUNNER', '--runner RUNNER',
102
+ 'Specify which test runner to use',
103
+ 'supported runners: native' do |value|
104
+ @cmd_options[:runner] = value
105
+ end
106
+
101
107
  opts.on '-i URI', '--install URI',
102
108
  'Install a project repo/app on the SUTs',
103
109
  'Provide full git URI or use short form KEYWORD/name',
@@ -185,6 +185,7 @@ module Beaker
185
185
  :puppetdb_port_nonssl => 8080,
186
186
  :puppetserver_port => 8140,
187
187
  :nodeclassifier_port => 4433,
188
+ :runner => "native",
188
189
  :aws_keyname_modifier => rand(10 ** 10).to_s.rjust(10,'0'), # 10 digit random number string
189
190
  :ssh => {
190
191
  :config => false,
@@ -110,5 +110,13 @@ module Beaker
110
110
  [@variant, @version, @arch].join('-')
111
111
  end
112
112
 
113
+ if RUBY_VERSION =~ /^1\.9/
114
+ def init_with(coder)
115
+ coder.map.each do |ivar, value|
116
+ instance_variable_set("@#{ivar}", value)
117
+ end
118
+ replace("#{@variant}-#{@version}-#{@arch}")
119
+ end
120
+ end
113
121
  end
114
122
  end
@@ -0,0 +1,58 @@
1
+ module Beaker
2
+ module Runner
3
+ module MiniTest
4
+ class TestSuite
5
+ attr_reader :name, :options, :fail_mode # TODO ?
6
+
7
+ #Create {TestSuite} instance
8
+ #@param [String] name The name of the {TestSuite}
9
+ #@param [Array<Host>] hosts An Array of Hosts to act upon.
10
+ #@param [Hash{Symbol=>String}] options Options for this object
11
+ #@option options [Logger] :logger The Logger object to report information to
12
+ #@option options [String] :log_dir The directory where text run logs will be written
13
+ #@option options [String] :xml_dir The directory where JUnit XML file will be written
14
+ #@option options [String] :xml_file The name of the JUnit XML file to be written to
15
+ #@option options [String] :project_root The full path to the Beaker lib directory
16
+ #@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
17
+ #@param [Symbol] fail_mode One of :slow, :fast
18
+ #@param [Time] timestamp Beaker execution start time
19
+ def initialize(name, hosts, options, timestamp, fail_mode=nil)
20
+ # TODO ?
21
+ end
22
+
23
+ def run
24
+ # TODO ?
25
+ end
26
+
27
+ #Execute all the TestCases in this suite.
28
+ #This is a wrapper that catches any failures generated during TestSuite::run.
29
+ def run_and_raise_on_failure
30
+ # TODO ?
31
+ end
32
+
33
+ # Gives a full file path for output to be written to, maintaining the latest symlink
34
+ # @param [String] name The file name that we want to write to.
35
+ # @param [String] log_dir The desired output directory.
36
+ # A symlink will be made from ./basedir/latest to that.
37
+ # @example
38
+ # log_path('output.txt', 'log/2014-06-02_16_31_22')
39
+ #
40
+ # This will create the structure:
41
+ #
42
+ # ./log/2014-06-02_16_31_22/output.txt
43
+ # ./log/latest -> 2014-06-02_16_31_22
44
+ #
45
+ # @example
46
+ # log_path('foo.log', 'log/man/date')
47
+ #
48
+ # This will create the structure:
49
+ #
50
+ # ./log/man/date/foo.log
51
+ # ./log/latest -> man/date
52
+ def log_path(name, log_dir)
53
+ # TODO ?
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,193 @@
1
+ [ 'host', 'dsl' ].each do |lib|
2
+ require "beaker/#{lib}"
3
+ end
4
+
5
+ require 'tempfile'
6
+ require 'benchmark'
7
+ require 'stringio'
8
+ require 'rbconfig'
9
+
10
+ module Beaker
11
+ module Runner
12
+ module Native
13
+ # This class represents a single test case. A test case is necessarily
14
+ # contained all in one file though may have multiple dependent examples.
15
+ # They are executed in order (save for any teardown procs registered
16
+ # through {Beaker::DSL::Structure#teardown}) and once completed
17
+ # the status of the TestCase is saved. Instance readers/accessors provide
18
+ # the test case access to various details of the environment and suite
19
+ # the test case is running within.
20
+ #
21
+ # See {Beaker::DSL} for more information about writing tests
22
+ # using the DSL.
23
+ class TestCase
24
+ include ::Beaker::DSL
25
+
26
+ # The Exception raised by Ruby's STDLIB's test framework (Ruby 1.9)
27
+ TEST_EXCEPTION_CLASS = ::MiniTest::Assertion
28
+
29
+ # Necessary for implementing {Beaker::DSL::Helpers#confine}.
30
+ # Assumed to be an array of valid {Beaker::Host} objects for
31
+ # this test case.
32
+ attr_accessor :hosts
33
+
34
+ # Necessary for many methods in {Beaker::DSL}. Assumed to be
35
+ # an instance of {Beaker::Logger}.
36
+ attr_accessor :logger
37
+
38
+ # Necessary for many methods in {Beaker::DSL::Helpers}. Assumed to be
39
+ # a hash.
40
+ attr_accessor :metadata
41
+
42
+ #The full log for this test
43
+ attr_accessor :sublog
44
+
45
+ #The result for the last command run
46
+ attr_accessor :last_result
47
+
48
+ # A Hash of 'product name' => 'version installed', only set when
49
+ # products are installed via git or PE install steps. See the 'git' or
50
+ # 'pe' directories within 'ROOT/setup' for examples.
51
+ attr_reader :version
52
+
53
+ # Parsed command line options.
54
+ attr_reader :options
55
+
56
+ # The path to the file which contains this test case.
57
+ attr_reader :path
58
+
59
+ # I don't know why this is here
60
+ attr_reader :fail_flag
61
+
62
+ # The user that is running this tests home directory, needed by 'net/ssh'.
63
+ attr_reader :usr_home
64
+
65
+ # A Symbol denoting the status of this test (:fail, :pending,
66
+ # :skipped, :pass).
67
+ attr_reader :test_status
68
+
69
+ # The exception that may have stopped this test's execution.
70
+ attr_reader :exception
71
+
72
+ # @deprecated
73
+ # The amount of time taken to execute the test. Unused, probably soon
74
+ # to be removed or refactored.
75
+ attr_reader :runtime
76
+
77
+ # An Array of Procs to be called after test execution has stopped
78
+ # (whether by exception or not).
79
+ attr_reader :teardown_procs
80
+
81
+ # @deprecated
82
+ # Legacy accessor from when test files would only contain one remote
83
+ # action. Contains the Result of the last call to utilize
84
+ # {Beaker::DSL::Helpers#on}. Do not use as it is not safe
85
+ # in test files that use multiple calls to
86
+ # {Beaker::DSL::Helpers#on}.
87
+ attr_accessor :result
88
+
89
+ # @param [Hosts,Array<Host>] these_hosts The hosts to execute this test
90
+ # against/on.
91
+ # @param [Logger] logger A logger that implements
92
+ # {Beaker::Logger}'s interface.
93
+ # @param [Hash{Symbol=>String}] options Parsed command line options.
94
+ # @param [String] path The local path to a test file to be executed.
95
+ def initialize(these_hosts, logger, options={}, path=nil)
96
+ @hosts = these_hosts
97
+ @logger = logger
98
+ @sublog = ""
99
+ @options = options
100
+ @path = path
101
+ @usr_home = options[:home]
102
+ @test_status = :pass
103
+ @exception = nil
104
+ @runtime = nil
105
+ @teardown_procs = []
106
+ @metadata = {}
107
+ set_current_test_filename(@path ? File.basename(@path, '.rb') : nil)
108
+
109
+
110
+ #
111
+ # We put this on each wrapper (rather than the class) so that methods
112
+ # defined in the tests don't leak out to other tests.
113
+ class << self
114
+ def run_test
115
+ @logger.start_sublog
116
+ @logger.last_result = nil
117
+
118
+ set_current_step_name(nil)
119
+
120
+ #add arbitrary role methods
121
+ roles = []
122
+ @hosts.each do |host|
123
+ roles << host[:roles]
124
+ end
125
+ add_role_def( roles.flatten.uniq )
126
+
127
+ @runtime = Benchmark.realtime do
128
+ begin
129
+ test = File.read(path)
130
+ eval test,nil,path,1
131
+ rescue FailTest, TEST_EXCEPTION_CLASS => e
132
+ @test_status = :fail
133
+ @exception = e
134
+ rescue PendingTest
135
+ @test_status = :pending
136
+ rescue SkipTest
137
+ @test_status = :skip
138
+ rescue StandardError, ScriptError, SignalException => e
139
+ log_and_fail_test(e)
140
+ ensure
141
+ @teardown_procs.each do |teardown|
142
+ begin
143
+ teardown.call
144
+ rescue StandardError, SignalException, TEST_EXCEPTION_CLASS => e
145
+ log_and_fail_test(e)
146
+ end
147
+ end
148
+ end
149
+ end
150
+ @sublog = @logger.get_sublog
151
+ @last_result = @logger.last_result
152
+ return self
153
+ end
154
+
155
+ private
156
+
157
+ # Log an error and mark the test as failed, passing through an
158
+ # exception so it can be displayed at the end of the total run.
159
+ #
160
+ # We break out the complete exception backtrace and log each line
161
+ # individually as well.
162
+ #
163
+ # @param exception [Exception] exception to fail with
164
+ def log_and_fail_test(exception)
165
+ logger.error("#{exception.class}: #{exception.message}")
166
+ bt = exception.backtrace
167
+ logger.pretty_backtrace(bt).each_line do |line|
168
+ logger.error(line)
169
+ end
170
+ @test_status = :error
171
+ @exception = exception
172
+ end
173
+ end
174
+ end
175
+
176
+ # The TestCase as a hash
177
+ # @api public
178
+ # @note The visibility and semantics of this method are valid, but the
179
+ # structure of the Hash it returns may change without notice
180
+ #
181
+ # @return [Hash] A Hash representation of this test.
182
+ def to_hash
183
+ hash = {}
184
+ hash['HOSTS'] = {}
185
+ @hosts.each do |host|
186
+ hash['HOSTS'][host.name] = host.overrides
187
+ end
188
+ hash
189
+ end
190
+ end
191
+ end
192
+ end
193
+ end
@@ -0,0 +1,410 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'nokogiri'
3
+ require 'fileutils'
4
+ [ 'test_case', 'logger' ].each do |lib|
5
+ require "beaker/#{lib}"
6
+ end
7
+
8
+ module Beaker
9
+ module Runner
10
+ module Native
11
+ #A collection of {TestCase} objects are considered a {TestSuite}.
12
+ #Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
13
+ class TestSuite
14
+
15
+ #Holds the output of a test suite, formats in plain text or xml
16
+ class TestSuiteResult
17
+ attr_accessor :start_time, :stop_time, :total_tests
18
+
19
+ #Create a {TestSuiteResult} instance.
20
+ #@param [Hash{Symbol=>String}] options Options for this object
21
+ #@option options [Logger] :logger The Logger object to report information to
22
+ #@param [String] name The name of the {TestSuite} that the results are for
23
+ def initialize( options, name )
24
+ @options = options
25
+ @logger = options[:logger]
26
+ @name = name
27
+ @test_cases = []
28
+ #Set some defaults, just in case you attempt to print without including them
29
+ start_time = Time.at(0)
30
+ stop_time = Time.at(1)
31
+ end
32
+
33
+ #Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
34
+ #@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
35
+ def add_test_case( test_case )
36
+ @test_cases << test_case
37
+ end
38
+
39
+ #How many {TestCase} instances are in this {TestSuiteResult}
40
+ def test_count
41
+ @test_cases.length
42
+ end
43
+
44
+ #How many passed {TestCase} instances are in this {TestSuiteResult}
45
+ def passed_tests
46
+ @test_cases.select { |c| c.test_status == :pass }.length
47
+ end
48
+
49
+ #How many errored {TestCase} instances are in this {TestSuiteResult}
50
+ def errored_tests
51
+ @test_cases.select { |c| c.test_status == :error }.length
52
+ end
53
+
54
+ #How many failed {TestCase} instances are in this {TestSuiteResult}
55
+ def failed_tests
56
+ @test_cases.select { |c| c.test_status == :fail }.length
57
+ end
58
+
59
+ #How many skipped {TestCase} instances are in this {TestSuiteResult}
60
+ def skipped_tests
61
+ @test_cases.select { |c| c.test_status == :skip }.length
62
+ end
63
+
64
+ #How many pending {TestCase} instances are in this {TestSuiteResult}
65
+ def pending_tests
66
+ @test_cases.select {|c| c.test_status == :pending}.length
67
+ end
68
+
69
+ #How many {TestCase} instances failed in this {TestSuiteResult}
70
+ def sum_failed
71
+ failed_tests + errored_tests
72
+ end
73
+
74
+ #Did all the {TestCase} instances in this {TestSuiteResult} pass?
75
+ def success?
76
+ sum_failed == 0
77
+ end
78
+
79
+ #Did one or more {TestCase} instances in this {TestSuiteResult} fail?
80
+ def failed?
81
+ !success?
82
+ end
83
+
84
+ #The sum of all {TestCase} runtimes in this {TestSuiteResult}
85
+ def elapsed_time
86
+ @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
87
+ end
88
+
89
+ #Plain text summay of test suite
90
+ #@param [Logger] summary_logger The logger we will print the summary to
91
+ def summarize(summary_logger)
92
+
93
+ summary_logger.notify <<-HEREDOC
94
+ Test Suite: #{@name} @ #{start_time}
95
+
96
+ - Host Configuration Summary -
97
+ HEREDOC
98
+
99
+ average_test_time = elapsed_time / test_count
100
+
101
+ summary_logger.notify %Q[
102
+
103
+ - Test Case Summary for suite '#{@name}' -
104
+ Total Suite Time: %.2f seconds
105
+ Average Test Time: %.2f seconds
106
+ Attempted: #{test_count}
107
+ Passed: #{passed_tests}
108
+ Failed: #{failed_tests}
109
+ Errored: #{errored_tests}
110
+ Skipped: #{skipped_tests}
111
+ Pending: #{pending_tests}
112
+ Total: #{@total_tests}
113
+
114
+ - Specific Test Case Status -
115
+ ] % [elapsed_time, average_test_time]
116
+
117
+ grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
118
+
119
+ summary_logger.notify "Failed Tests Cases:"
120
+ (grouped_summary[:fail] || []).each do |test_case|
121
+ print_test_result(test_case)
122
+ end
123
+
124
+ summary_logger.notify "Errored Tests Cases:"
125
+ (grouped_summary[:error] || []).each do |test_case|
126
+ print_test_result(test_case)
127
+ end
128
+
129
+ summary_logger.notify "Skipped Tests Cases:"
130
+ (grouped_summary[:skip] || []).each do |test_case|
131
+ print_test_result(test_case)
132
+ end
133
+
134
+ summary_logger.notify "Pending Tests Cases:"
135
+ (grouped_summary[:pending] || []).each do |test_case|
136
+ print_test_result(test_case)
137
+ end
138
+
139
+ summary_logger.notify("\n\n")
140
+ end
141
+
142
+ #A convenience method for printing the results of a {TestCase}
143
+ #@param [TestCase] test_case The {TestCase} to examine and print results for
144
+ def print_test_result(test_case)
145
+ test_reported = if test_case.exception
146
+ "reported: #{test_case.exception.inspect}"
147
+ else
148
+ test_case.test_status
149
+ end
150
+ @logger.notify " Test Case #{test_case.path} #{test_reported}"
151
+ end
152
+
153
+ # Writes Junit XML of this {TestSuiteResult}
154
+ #
155
+ # @param [String] xml_file Path to the XML file (from Beaker's running directory)
156
+ # @param [String] file_to_link Path to the paired file that should be linked
157
+ # from this one (this is relative to the XML
158
+ # file itself, so it would just be the different
159
+ # file name if they're in the same directory)
160
+ # @param [Boolean] time_sort Whether the test results should be output in
161
+ # order of time spent in the test, or in the
162
+ # order of test execution (default)
163
+ #
164
+ # @return nil
165
+ # @api private
166
+ def write_junit_xml(xml_file, file_to_link = nil, time_sort = false)
167
+ stylesheet = File.join(@options[:project_root], @options[:xml_stylesheet])
168
+
169
+ begin
170
+ LoggerJunit.write_xml(xml_file, stylesheet) do |doc, suites|
171
+
172
+ meta_info = Nokogiri::XML::Node.new('meta_test_info', doc)
173
+ unless file_to_link.nil?
174
+ meta_info['page_active'] = time_sort ? 'performance' : 'execution'
175
+ meta_info['link_url'] = file_to_link
176
+ else
177
+ meta_info['page_active'] = 'no-links'
178
+ meta_info['link_url'] = ''
179
+ end
180
+ suites.add_child(meta_info)
181
+
182
+ suite = Nokogiri::XML::Node.new('testsuite', doc)
183
+ suite['name'] = @name
184
+ suite['tests'] = test_count
185
+ suite['errors'] = errored_tests
186
+ suite['failures'] = failed_tests
187
+ suite['skip'] = skipped_tests
188
+ suite['pending'] = pending_tests
189
+ suite['total'] = @total_tests
190
+ suite['time'] = "%f" % (stop_time - start_time)
191
+ properties = Nokogiri::XML::Node.new('properties', doc)
192
+ @options.each_pair do | name, value |
193
+ property = Nokogiri::XML::Node.new('property', doc)
194
+ property['name'] = name
195
+ property['value'] = value
196
+ properties.add_child(property)
197
+ end
198
+ suite.add_child(properties)
199
+
200
+ test_cases_to_report = @test_cases
201
+ test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
202
+ test_cases_to_report.each do |test|
203
+ item = Nokogiri::XML::Node.new('testcase', doc)
204
+ item['classname'] = File.dirname(test.path)
205
+ item['name'] = File.basename(test.path)
206
+ item['time'] = "%f" % test.runtime
207
+
208
+ # Did we fail? If so, report that.
209
+ # We need to remove the escape character from colorized text, the
210
+ # substitution of other entities is handled well by Rexml
211
+ if test.test_status == :fail || test.test_status == :error then
212
+ status = Nokogiri::XML::Node.new('failure', doc)
213
+ status['type'] = test.test_status.to_s
214
+ if test.exception then
215
+ status['message'] = test.exception.to_s.gsub(/\e/, '')
216
+ data = LoggerJunit.format_cdata(test.exception.backtrace.join('\n'))
217
+ status.add_child(status.document.create_cdata(data))
218
+ end
219
+ item.add_child(status)
220
+ end
221
+
222
+ if test.test_status == :skip
223
+ status = Nokogiri::XML::Node.new('skip', doc)
224
+ status['type'] = test.test_status.to_s
225
+ item.add_child(status)
226
+ end
227
+
228
+ if test.test_status == :pending
229
+ status = Nokogiri::XML::Node.new('pending', doc)
230
+ status['type'] = test.test_status.to_s
231
+ item.add_child(status)
232
+ end
233
+
234
+ if test.sublog then
235
+ stdout = Nokogiri::XML::Node.new('system-out', doc)
236
+ data = LoggerJunit.format_cdata(test.sublog)
237
+ stdout.add_child(stdout.document.create_cdata(data))
238
+ item.add_child(stdout)
239
+ end
240
+
241
+ if test.last_result and test.last_result.stderr and not test.last_result.stderr.empty? then
242
+ stderr = Nokogiri::XML::Node.new('system-err', doc)
243
+ data = LoggerJunit.format_cdata(test.last_result.stderr)
244
+ stderr.add_child(stderr.document.create_cdata(data))
245
+ item.add_child(stderr)
246
+ end
247
+
248
+ suite.add_child(item)
249
+ end
250
+ suites.add_child(suite)
251
+ end
252
+ rescue Exception => e
253
+ @logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
254
+ end
255
+
256
+ end
257
+ end
258
+
259
+ attr_reader :name, :options, :fail_mode
260
+
261
+ #Create {TestSuite} instance
262
+ #@param [String] name The name of the {TestSuite}
263
+ #@param [Array<Host>] hosts An Array of Hosts to act upon.
264
+ #@param [Hash{Symbol=>String}] options Options for this object
265
+ #@option options [Logger] :logger The Logger object to report information to
266
+ #@option options [String] :log_dir The directory where text run logs will be written
267
+ #@option options [String] :xml_dir The directory where JUnit XML file will be written
268
+ #@option options [String] :xml_file The name of the JUnit XML file to be written to
269
+ #@option options [String] :project_root The full path to the Beaker lib directory
270
+ #@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
271
+ #@param [Symbol] fail_mode One of :slow, :fast
272
+ #@param [Time] timestamp Beaker execution start time
273
+ def initialize(name, hosts, options, timestamp, fail_mode=nil)
274
+ @logger = options[:logger]
275
+ @test_cases = []
276
+ @test_files = options[name]
277
+ @name = name.to_s.gsub(/\s+/, '-')
278
+ @hosts = hosts
279
+ @run = false
280
+ @options = options
281
+ @fail_mode = fail_mode || @options[:fail_mode] || :slow
282
+ @test_suite_results = TestSuiteResult.new(@options, name)
283
+ @timestamp = timestamp
284
+
285
+ report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
286
+
287
+ rescue => e
288
+ report_and_raise(@logger, e, "TestSuite: initialize")
289
+ end
290
+
291
+ #Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
292
+ #is reported to a newly created run log.
293
+ #Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
294
+ #after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
295
+ def run
296
+ @run = true
297
+ start_time = Time.now
298
+
299
+ #Create a run log for this TestSuite.
300
+ run_log = log_path("#{@name}-run.log", @options[:log_dated_dir])
301
+ @logger.add_destination(run_log)
302
+
303
+ # This is an awful hack to maintain backward compatibility until tests
304
+ # are ported to use logger. Still in use in PuppetDB tests
305
+ ::Beaker.const_set(:Log, @logger) unless defined?( ::Beaker::Log )
306
+
307
+ @test_suite_results.start_time = start_time
308
+ @test_suite_results.total_tests = @test_files.length
309
+
310
+ @test_files.each do |test_file|
311
+ @logger.info "Begin #{test_file}"
312
+ start = Time.now
313
+ test_case = TestCase.new(@hosts, @logger, options, test_file).run_test
314
+ duration = Time.now - start
315
+ @test_suite_results.add_test_case(test_case)
316
+ @test_cases << test_case
317
+
318
+ state = test_case.test_status == :skip ? 'skipp' : test_case.test_status
319
+ msg = "#{test_file} #{state}ed in %.2f seconds" % duration.to_f
320
+ case test_case.test_status
321
+ when :pass
322
+ @logger.success msg
323
+ when :skip
324
+ @logger.warn msg
325
+ when :fail
326
+ @logger.error msg
327
+ break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
328
+ when :error
329
+ @logger.warn msg
330
+ break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
331
+ end
332
+ end
333
+ @test_suite_results.stop_time = Time.now
334
+
335
+ # REVISIT: This changes global state, breaking logging in any future runs
336
+ # of the suite – or, at least, making them highly confusing for anyone who
337
+ # has not studied the implementation in detail. --daniel 2011-03-14
338
+ @test_suite_results.summarize( Logger.new(log_path("#{name}-summary.txt", @options[:log_dated_dir]), STDOUT) )
339
+
340
+ junit_file_log = log_path(@options[:xml_file], @options[:xml_dated_dir])
341
+ if @options[:xml_time_enabled]
342
+ junit_file_time = log_path(@options[:xml_time], @options[:xml_dated_dir])
343
+ @test_suite_results.write_junit_xml( junit_file_log, @options[:xml_time] )
344
+ @test_suite_results.write_junit_xml( junit_file_time, @options[:xml_file], true )
345
+ else
346
+ @test_suite_results.write_junit_xml( junit_file_log )
347
+ end
348
+ #All done with this run, remove run log
349
+ @logger.remove_destination(run_log)
350
+
351
+ # Allow chaining operations...
352
+ return self
353
+ end
354
+
355
+ #Execute all the TestCases in this suite.
356
+ #This is a wrapper that catches any failures generated during TestSuite::run.
357
+ def run_and_raise_on_failure
358
+ begin
359
+ run
360
+ return self if @test_suite_results.success?
361
+ rescue => e
362
+ #failed during run
363
+ report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
364
+ else
365
+ #failed during test
366
+ report_and_raise(@logger, RuntimeError.new("Failed while running the #{name} suite"), "TestSuite: report_and_raise_on_failure")
367
+ end
368
+ end
369
+
370
+ # Gives a full file path for output to be written to, maintaining the latest symlink
371
+ # @param [String] name The file name that we want to write to.
372
+ # @param [String] log_dir The desired output directory.
373
+ # A symlink will be made from ./basedir/latest to that.
374
+ # @example
375
+ # log_path('output.txt', 'log/2014-06-02_16_31_22')
376
+ #
377
+ # This will create the structure:
378
+ #
379
+ # ./log/2014-06-02_16_31_22/output.txt
380
+ # ./log/latest -> 2014-06-02_16_31_22
381
+ #
382
+ # @example
383
+ # log_path('foo.log', 'log/man/date')
384
+ #
385
+ # This will create the structure:
386
+ #
387
+ # ./log/man/date/foo.log
388
+ # ./log/latest -> man/date
389
+ def log_path(name, log_dir)
390
+ FileUtils.mkdir_p(log_dir) unless File.directory?(log_dir)
391
+
392
+ base_dir = log_dir
393
+ link_dir = ''
394
+ while File.dirname(base_dir) != '.' do
395
+ link_dir = link_dir == '' ? File.basename(base_dir) : File.join(File.basename(base_dir), link_dir)
396
+ base_dir = File.dirname(base_dir)
397
+ end
398
+
399
+ latest = File.join(base_dir, "latest")
400
+ if !File.exist?(latest) or File.symlink?(latest) then
401
+ File.delete(latest) if File.exist?(latest) || File.symlink?(latest)
402
+ File.symlink(link_dir, latest)
403
+ end
404
+
405
+ File.join(log_dir, name)
406
+ end
407
+ end
408
+ end
409
+ end
410
+ end