beaker 2.30.0 → 2.30.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- M2FjYzE1ZjkzNDBiNTMwOTdjZTFlOTg5OTMzOTgzNDBkMzJlMzRmMA==
4
+ YmI0MWU4ZDExZjIzYjg0OGExZDg5Mzk3NTI0ZWM2MTM1MmM4YTZkMA==
5
5
  data.tar.gz: !binary |-
6
- OTA5NTY5ZTQ5YWE0OWU2NjVlNDVhMDBjYThiZmEwYTg2ODFiMTM3OA==
6
+ ZGRlY2RhZDJkNTkxYmM0NGM0MTM1NTBmZWQ1OWU1MDJjNDU2NjdjMQ==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- YmE0NjFmMmU2ZmU5OWMxMTY2YmI2YWUxOGE3MGVjZmNlYzMzNTgxYjg2Zjc5
10
- ZTMwZGQ3OWQ5Y2Q5ZTc0NzgzY2M3NGJlZWE0ZTU4NjQxODA0MjdhNGQyNjE5
11
- NDY3YTljMzcwYjJlNTgxOGNiMTAwZDc4NjM2MDVmOGY1NzgyODA=
9
+ MWE2ODkwNWNkNWZkMDlmMGE1ZTg2N2M2ZmEzNmFlYWExMzE2ZmMwY2EyNjBm
10
+ MmE4ZjFkNmZkZDg3MjYxNjg0Y2JhMzE4Y2Y2YmE2ZmU1NTQ3ZjJhMjA2NjYx
11
+ ZmI3YjM2MGZiNGY1NmYwZjE4NTBiYjlhYmUyYjI0ODliM2VmZDg=
12
12
  data.tar.gz: !binary |-
13
- ZWJlZmY0NzBlY2Q1ZjYyNDQxMDNmYzg0ZjE0OWE0N2NmMjBjZTVjMmE5OTNh
14
- ODhjN2E5ZmYxOTg5ODYyMzQ3NjA5NTkwZDU0MmYxMTMyNTliM2YyNjNiMzUx
15
- NzZiOGQ2ZDgyYjYyMWQ2MTc1NDQzOTk2ZTJlNzQ2NGQwZTk4YmM=
13
+ M2NjZDI3MTVlMGZkZGNlYThlYWRmMTcwODlkNjhkMzJhYjAxNDIyMmNiYTFh
14
+ NGM4YWRmNTcwNTNjYjAzZmMzYWMxMWJjYzU3YWQ5YjdlNTFiYTVhYjc2NzU5
15
+ YzViMjAyNDcyOTQ2NTIxMWMzYTQ1YzFmNTgzMjRhMWFhYzgyMjA=
data/HISTORY.md CHANGED
@@ -1,6 +1,7 @@
1
1
  # default - History
2
2
  ## Tags
3
- * [LATEST - 2 Dec, 2015 (bc912e78)](#LATEST)
3
+ * [LATEST - 3 Dec, 2015 (a1ee5206)](#LATEST)
4
+ * [2.30.0 - 2 Dec, 2015 (dbb72630)](#2.30.0)
4
5
  * [2.29.1 - 23 Nov, 2015 (5d824690)](#2.29.1)
5
6
  * [2.29.0 - 18 Nov, 2015 (33fd2399)](#2.29.0)
6
7
  * [2.28.0 - 4 Nov, 2015 (89829551)](#2.28.0)
@@ -103,7 +104,23 @@
103
104
  * [pe1.2 - 6 Sep, 2011 (ba3dadd2)](#pe1.2)
104
105
 
105
106
  ## Details
106
- ### <a name = "LATEST">LATEST - 2 Dec, 2015 (bc912e78)
107
+ ### <a name = "LATEST">LATEST - 3 Dec, 2015 (a1ee5206)
108
+
109
+ * (GEM) update beaker version to 2.30.1 (a1ee5206)
110
+
111
+ * Merge pull request #1024 from puppetlabs/revert-1013-bkr-623/test-runner-reorganization (18307e09)
112
+
113
+
114
+ ```
115
+ Merge pull request #1024 from puppetlabs/revert-1013-bkr-623/test-runner-reorganization
116
+
117
+ Revert "[BKR-623] Reorganize Beaker test runner classes for introduction of minitest runner"
118
+ ```
119
+ * Revert "[BKR-623] Reorganize Beaker test runner classes for introduction of minitest runner" (979a329e)
120
+
121
+ ### <a name = "2.30.0">2.30.0 - 2 Dec, 2015 (dbb72630)
122
+
123
+ * (HISTORY) update beaker history for gem release 2.30.0 (dbb72630)
107
124
 
108
125
  * (GEM) update beaker version to 2.30.0 (bc912e78)
109
126
 
data/lib/beaker/cli.rb CHANGED
@@ -154,13 +154,7 @@ module Beaker
154
154
  @logger.notify("No tests to run for suite '#{suite_name.to_s}'")
155
155
  return
156
156
  end
157
-
158
- unless runner_class = Beaker::TestSuite.runner(@options[:runner])
159
- @logger.error "Test runner #{@options[:runner]} is unknown."
160
- exit 1
161
- end
162
-
163
- runner_class.new(
157
+ Beaker::TestSuite.new(
164
158
  suite_name, @hosts, @options, @timestamp, failure_strategy
165
159
  ).run_and_raise_on_failure
166
160
  end
@@ -98,12 +98,6 @@ module Beaker
98
98
  @cmd_options[:timeout] = value
99
99
  end
100
100
 
101
- opts.on '-r RUNNER', '--runner RUNNER',
102
- 'Specify which test runner to use',
103
- 'supported runners: native' do |value|
104
- @cmd_options[:runner] = value
105
- end
106
-
107
101
  opts.on '-i URI', '--install URI',
108
102
  'Install a project repo/app on the SUTs',
109
103
  'Provide full git URI or use short form KEYWORD/name',
@@ -185,7 +185,6 @@ module Beaker
185
185
  :puppetdb_port_nonssl => 8080,
186
186
  :puppetserver_port => 8140,
187
187
  :nodeclassifier_port => 4433,
188
- :runner => "native",
189
188
  :aws_keyname_modifier => rand(10 ** 10).to_s.rjust(10,'0'), # 10 digit random number string
190
189
  :ssh => {
191
190
  :config => false,
@@ -1,4 +1,11 @@
1
- require 'beaker/runner/native/test_case'
1
+ [ 'host', 'dsl' ].each do |lib|
2
+ require "beaker/#{lib}"
3
+ end
4
+
5
+ require 'tempfile'
6
+ require 'benchmark'
7
+ require 'stringio'
8
+ require 'rbconfig'
2
9
 
3
10
  module Beaker
4
11
  # This class represents a single test case. A test case is necessarily
@@ -11,6 +18,173 @@ module Beaker
11
18
  #
12
19
  # See {Beaker::DSL} for more information about writing tests
13
20
  # using the DSL.
14
- class TestCase < Beaker::Runner::Native::TestCase
21
+ class TestCase
22
+ include Beaker::DSL
23
+
24
+ # The Exception raised by Ruby's STDLIB's test framework (Ruby 1.9)
25
+ TEST_EXCEPTION_CLASS = ::MiniTest::Assertion
26
+
27
+ # Necessary for implementing {Beaker::DSL::Helpers#confine}.
28
+ # Assumed to be an array of valid {Beaker::Host} objects for
29
+ # this test case.
30
+ attr_accessor :hosts
31
+
32
+ # Necessary for many methods in {Beaker::DSL}. Assumed to be
33
+ # an instance of {Beaker::Logger}.
34
+ attr_accessor :logger
35
+
36
+ # Necessary for many methods in {Beaker::DSL::Helpers}. Assumed to be
37
+ # a hash.
38
+ attr_accessor :metadata
39
+
40
+ #The full log for this test
41
+ attr_accessor :sublog
42
+
43
+ #The result for the last command run
44
+ attr_accessor :last_result
45
+
46
+ # A Hash of 'product name' => 'version installed', only set when
47
+ # products are installed via git or PE install steps. See the 'git' or
48
+ # 'pe' directories within 'ROOT/setup' for examples.
49
+ attr_reader :version
50
+
51
+ # Parsed command line options.
52
+ attr_reader :options
53
+
54
+ # The path to the file which contains this test case.
55
+ attr_reader :path
56
+
57
+ # I don't know why this is here
58
+ attr_reader :fail_flag
59
+
60
+ # The user that is running this tests home directory, needed by 'net/ssh'.
61
+ attr_reader :usr_home
62
+
63
+ # A Symbol denoting the status of this test (:fail, :pending,
64
+ # :skipped, :pass).
65
+ attr_reader :test_status
66
+
67
+ # The exception that may have stopped this test's execution.
68
+ attr_reader :exception
69
+
70
+ # @deprecated
71
+ # The amount of time taken to execute the test. Unused, probably soon
72
+ # to be removed or refactored.
73
+ attr_reader :runtime
74
+
75
+ # An Array of Procs to be called after test execution has stopped
76
+ # (whether by exception or not).
77
+ attr_reader :teardown_procs
78
+
79
+ # @deprecated
80
+ # Legacy accessor from when test files would only contain one remote
81
+ # action. Contains the Result of the last call to utilize
82
+ # {Beaker::DSL::Helpers#on}. Do not use as it is not safe
83
+ # in test files that use multiple calls to
84
+ # {Beaker::DSL::Helpers#on}.
85
+ attr_accessor :result
86
+
87
+ # @param [Hosts,Array<Host>] these_hosts The hosts to execute this test
88
+ # against/on.
89
+ # @param [Logger] logger A logger that implements
90
+ # {Beaker::Logger}'s interface.
91
+ # @param [Hash{Symbol=>String}] options Parsed command line options.
92
+ # @param [String] path The local path to a test file to be executed.
93
+ def initialize(these_hosts, logger, options={}, path=nil)
94
+ @hosts = these_hosts
95
+ @logger = logger
96
+ @sublog = ""
97
+ @options = options
98
+ @path = path
99
+ @usr_home = options[:home]
100
+ @test_status = :pass
101
+ @exception = nil
102
+ @runtime = nil
103
+ @teardown_procs = []
104
+ @metadata = {}
105
+ set_current_test_filename(@path ? File.basename(@path, '.rb') : nil)
106
+
107
+
108
+ #
109
+ # We put this on each wrapper (rather than the class) so that methods
110
+ # defined in the tests don't leak out to other tests.
111
+ class << self
112
+ def run_test
113
+ @logger.start_sublog
114
+ @logger.last_result = nil
115
+
116
+ set_current_step_name(nil)
117
+
118
+ #add arbitrary role methods
119
+ roles = []
120
+ @hosts.each do |host|
121
+ roles << host[:roles]
122
+ end
123
+ add_role_def( roles.flatten.uniq )
124
+
125
+ @runtime = Benchmark.realtime do
126
+ begin
127
+ test = File.read(path)
128
+ eval test,nil,path,1
129
+ rescue FailTest, TEST_EXCEPTION_CLASS => e
130
+ @test_status = :fail
131
+ @exception = e
132
+ rescue PendingTest
133
+ @test_status = :pending
134
+ rescue SkipTest
135
+ @test_status = :skip
136
+ rescue StandardError, ScriptError, SignalException => e
137
+ log_and_fail_test(e)
138
+ ensure
139
+ @teardown_procs.each do |teardown|
140
+ begin
141
+ teardown.call
142
+ rescue StandardError, SignalException, TEST_EXCEPTION_CLASS => e
143
+ log_and_fail_test(e)
144
+ end
145
+ end
146
+ end
147
+ end
148
+ @sublog = @logger.get_sublog
149
+ @last_result = @logger.last_result
150
+ return self
151
+ end
152
+
153
+ private
154
+
155
+ # Log an error and mark the test as failed, passing through an
156
+ # exception so it can be displayed at the end of the total run.
157
+ #
158
+ # We break out the complete exception backtrace and log each line
159
+ # individually as well.
160
+ #
161
+ # @param exception [Exception] exception to fail with
162
+ def log_and_fail_test(exception)
163
+ logger.error("#{exception.class}: #{exception.message}")
164
+ bt = exception.backtrace
165
+ logger.pretty_backtrace(bt).each_line do |line|
166
+ logger.error(line)
167
+ end
168
+ @test_status = :error
169
+ @exception = exception
170
+ end
171
+ end
172
+ end
173
+
174
+ # The TestCase as a hash
175
+ # @api public
176
+ # @note The visibility and semantics of this method are valid, but the
177
+ # structure of the Hash it returns may change without notice
178
+ #
179
+ # @return [Hash] A Hash representation of this test.
180
+ def to_hash
181
+ hash = {}
182
+ hash['HOSTS'] = {}
183
+ @hosts.each do |host|
184
+ hash['HOSTS'][host.name] = host.overrides
185
+ end
186
+ hash
187
+ end
188
+
15
189
  end
16
190
  end
@@ -1,17 +1,407 @@
1
- require 'beaker/runner/native/test_suite'
2
- require 'beaker/runner/mini_test/test_suite'
1
+ # -*- coding: utf-8 -*-
2
+ require 'nokogiri'
3
+ require 'fileutils'
4
+ [ 'test_case', 'logger' ].each do |lib|
5
+ require "beaker/#{lib}"
6
+ end
3
7
 
4
8
  module Beaker
9
+ #A collection of {TestCase} objects are considered a {TestSuite}.
10
+ #Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
5
11
  class TestSuite
6
- def self.runner(runner)
7
- case runner
8
- when "native"
9
- ::Beaker::Runner::Native::TestSuite
10
- when "minitest"
11
- ::Beaker::Runner::MiniTest::TestSuite
12
+
13
+ #Holds the output of a test suite, formats in plain text or xml
14
+ class TestSuiteResult
15
+ attr_accessor :start_time, :stop_time, :total_tests
16
+
17
+ #Create a {TestSuiteResult} instance.
18
+ #@param [Hash{Symbol=>String}] options Options for this object
19
+ #@option options [Logger] :logger The Logger object to report information to
20
+ #@param [String] name The name of the {TestSuite} that the results are for
21
+ def initialize( options, name )
22
+ @options = options
23
+ @logger = options[:logger]
24
+ @name = name
25
+ @test_cases = []
26
+ #Set some defaults, just in case you attempt to print without including them
27
+ start_time = Time.at(0)
28
+ stop_time = Time.at(1)
29
+ end
30
+
31
+ #Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
32
+ #@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
33
+ def add_test_case( test_case )
34
+ @test_cases << test_case
35
+ end
36
+
37
+ #How many {TestCase} instances are in this {TestSuiteResult}
38
+ def test_count
39
+ @test_cases.length
40
+ end
41
+
42
+ #How many passed {TestCase} instances are in this {TestSuiteResult}
43
+ def passed_tests
44
+ @test_cases.select { |c| c.test_status == :pass }.length
45
+ end
46
+
47
+ #How many errored {TestCase} instances are in this {TestSuiteResult}
48
+ def errored_tests
49
+ @test_cases.select { |c| c.test_status == :error }.length
50
+ end
51
+
52
+ #How many failed {TestCase} instances are in this {TestSuiteResult}
53
+ def failed_tests
54
+ @test_cases.select { |c| c.test_status == :fail }.length
55
+ end
56
+
57
+ #How many skipped {TestCase} instances are in this {TestSuiteResult}
58
+ def skipped_tests
59
+ @test_cases.select { |c| c.test_status == :skip }.length
60
+ end
61
+
62
+ #How many pending {TestCase} instances are in this {TestSuiteResult}
63
+ def pending_tests
64
+ @test_cases.select {|c| c.test_status == :pending}.length
65
+ end
66
+
67
+ #How many {TestCase} instances failed in this {TestSuiteResult}
68
+ def sum_failed
69
+ failed_tests + errored_tests
70
+ end
71
+
72
+ #Did all the {TestCase} instances in this {TestSuiteResult} pass?
73
+ def success?
74
+ sum_failed == 0
75
+ end
76
+
77
+ #Did one or more {TestCase} instances in this {TestSuiteResult} fail?
78
+ def failed?
79
+ !success?
80
+ end
81
+
82
+ #The sum of all {TestCase} runtimes in this {TestSuiteResult}
83
+ def elapsed_time
84
+ @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
85
+ end
86
+
87
+ #Plain text summay of test suite
88
+ #@param [Logger] summary_logger The logger we will print the summary to
89
+ def summarize(summary_logger)
90
+
91
+ summary_logger.notify <<-HEREDOC
92
+ Test Suite: #{@name} @ #{start_time}
93
+
94
+ - Host Configuration Summary -
95
+ HEREDOC
96
+
97
+ average_test_time = elapsed_time / test_count
98
+
99
+ summary_logger.notify %Q[
100
+
101
+ - Test Case Summary for suite '#{@name}' -
102
+ Total Suite Time: %.2f seconds
103
+ Average Test Time: %.2f seconds
104
+ Attempted: #{test_count}
105
+ Passed: #{passed_tests}
106
+ Failed: #{failed_tests}
107
+ Errored: #{errored_tests}
108
+ Skipped: #{skipped_tests}
109
+ Pending: #{pending_tests}
110
+ Total: #{@total_tests}
111
+
112
+ - Specific Test Case Status -
113
+ ] % [elapsed_time, average_test_time]
114
+
115
+ grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
116
+
117
+ summary_logger.notify "Failed Tests Cases:"
118
+ (grouped_summary[:fail] || []).each do |test_case|
119
+ print_test_result(test_case)
120
+ end
121
+
122
+ summary_logger.notify "Errored Tests Cases:"
123
+ (grouped_summary[:error] || []).each do |test_case|
124
+ print_test_result(test_case)
125
+ end
126
+
127
+ summary_logger.notify "Skipped Tests Cases:"
128
+ (grouped_summary[:skip] || []).each do |test_case|
129
+ print_test_result(test_case)
130
+ end
131
+
132
+ summary_logger.notify "Pending Tests Cases:"
133
+ (grouped_summary[:pending] || []).each do |test_case|
134
+ print_test_result(test_case)
135
+ end
136
+
137
+ summary_logger.notify("\n\n")
138
+ end
139
+
140
+ #A convenience method for printing the results of a {TestCase}
141
+ #@param [TestCase] test_case The {TestCase} to examine and print results for
142
+ def print_test_result(test_case)
143
+ test_reported = if test_case.exception
144
+ "reported: #{test_case.exception.inspect}"
145
+ else
146
+ test_case.test_status
147
+ end
148
+ @logger.notify " Test Case #{test_case.path} #{test_reported}"
149
+ end
150
+
151
+ # Writes Junit XML of this {TestSuiteResult}
152
+ #
153
+ # @param [String] xml_file Path to the XML file (from Beaker's running directory)
154
+ # @param [String] file_to_link Path to the paired file that should be linked
155
+ # from this one (this is relative to the XML
156
+ # file itself, so it would just be the different
157
+ # file name if they're in the same directory)
158
+ # @param [Boolean] time_sort Whether the test results should be output in
159
+ # order of time spent in the test, or in the
160
+ # order of test execution (default)
161
+ #
162
+ # @return nil
163
+ # @api private
164
+ def write_junit_xml(xml_file, file_to_link = nil, time_sort = false)
165
+ stylesheet = File.join(@options[:project_root], @options[:xml_stylesheet])
166
+
167
+ begin
168
+ LoggerJunit.write_xml(xml_file, stylesheet) do |doc, suites|
169
+
170
+ meta_info = Nokogiri::XML::Node.new('meta_test_info', doc)
171
+ unless file_to_link.nil?
172
+ meta_info['page_active'] = time_sort ? 'performance' : 'execution'
173
+ meta_info['link_url'] = file_to_link
174
+ else
175
+ meta_info['page_active'] = 'no-links'
176
+ meta_info['link_url'] = ''
177
+ end
178
+ suites.add_child(meta_info)
179
+
180
+ suite = Nokogiri::XML::Node.new('testsuite', doc)
181
+ suite['name'] = @name
182
+ suite['tests'] = test_count
183
+ suite['errors'] = errored_tests
184
+ suite['failures'] = failed_tests
185
+ suite['skip'] = skipped_tests
186
+ suite['pending'] = pending_tests
187
+ suite['total'] = @total_tests
188
+ suite['time'] = "%f" % (stop_time - start_time)
189
+ properties = Nokogiri::XML::Node.new('properties', doc)
190
+ @options.each_pair do | name, value |
191
+ property = Nokogiri::XML::Node.new('property', doc)
192
+ property['name'] = name
193
+ property['value'] = value
194
+ properties.add_child(property)
195
+ end
196
+ suite.add_child(properties)
197
+
198
+ test_cases_to_report = @test_cases
199
+ test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
200
+ test_cases_to_report.each do |test|
201
+ item = Nokogiri::XML::Node.new('testcase', doc)
202
+ item['classname'] = File.dirname(test.path)
203
+ item['name'] = File.basename(test.path)
204
+ item['time'] = "%f" % test.runtime
205
+
206
+ # Did we fail? If so, report that.
207
+ # We need to remove the escape character from colorized text, the
208
+ # substitution of other entities is handled well by Rexml
209
+ if test.test_status == :fail || test.test_status == :error then
210
+ status = Nokogiri::XML::Node.new('failure', doc)
211
+ status['type'] = test.test_status.to_s
212
+ if test.exception then
213
+ status['message'] = test.exception.to_s.gsub(/\e/, '')
214
+ data = LoggerJunit.format_cdata(test.exception.backtrace.join('\n'))
215
+ status.add_child(status.document.create_cdata(data))
216
+ end
217
+ item.add_child(status)
218
+ end
219
+
220
+ if test.test_status == :skip
221
+ status = Nokogiri::XML::Node.new('skip', doc)
222
+ status['type'] = test.test_status.to_s
223
+ item.add_child(status)
224
+ end
225
+
226
+ if test.test_status == :pending
227
+ status = Nokogiri::XML::Node.new('pending', doc)
228
+ status['type'] = test.test_status.to_s
229
+ item.add_child(status)
230
+ end
231
+
232
+ if test.sublog then
233
+ stdout = Nokogiri::XML::Node.new('system-out', doc)
234
+ data = LoggerJunit.format_cdata(test.sublog)
235
+ stdout.add_child(stdout.document.create_cdata(data))
236
+ item.add_child(stdout)
237
+ end
238
+
239
+ if test.last_result and test.last_result.stderr and not test.last_result.stderr.empty? then
240
+ stderr = Nokogiri::XML::Node.new('system-err', doc)
241
+ data = LoggerJunit.format_cdata(test.last_result.stderr)
242
+ stderr.add_child(stderr.document.create_cdata(data))
243
+ item.add_child(stderr)
244
+ end
245
+
246
+ suite.add_child(item)
247
+ end
248
+ suites.add_child(suite)
249
+ end
250
+ rescue Exception => e
251
+ @logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
252
+ end
253
+
254
+ end
255
+ end
256
+
257
+ attr_reader :name, :options, :fail_mode
258
+
259
+ #Create {TestSuite} instance
260
+ #@param [String] name The name of the {TestSuite}
261
+ #@param [Array<Host>] hosts An Array of Hosts to act upon.
262
+ #@param [Hash{Symbol=>String}] options Options for this object
263
+ #@option options [Logger] :logger The Logger object to report information to
264
+ #@option options [String] :log_dir The directory where text run logs will be written
265
+ #@option options [String] :xml_dir The directory where JUnit XML file will be written
266
+ #@option options [String] :xml_file The name of the JUnit XML file to be written to
267
+ #@option options [String] :project_root The full path to the Beaker lib directory
268
+ #@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
269
+ #@param [Symbol] fail_mode One of :slow, :fast
270
+ #@param [Time] timestamp Beaker execution start time
271
+ def initialize(name, hosts, options, timestamp, fail_mode=nil)
272
+ @logger = options[:logger]
273
+ @test_cases = []
274
+ @test_files = options[name]
275
+ @name = name.to_s.gsub(/\s+/, '-')
276
+ @hosts = hosts
277
+ @run = false
278
+ @options = options
279
+ @fail_mode = fail_mode || @options[:fail_mode] || :slow
280
+ @test_suite_results = TestSuiteResult.new(@options, name)
281
+ @timestamp = timestamp
282
+
283
+ report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
284
+
285
+ rescue => e
286
+ report_and_raise(@logger, e, "TestSuite: initialize")
287
+ end
288
+
289
+ #Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
290
+ #is reported to a newly created run log.
291
+ #Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
292
+ #after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
293
+ def run
294
+ @run = true
295
+ start_time = Time.now
296
+
297
+ #Create a run log for this TestSuite.
298
+ run_log = log_path("#{@name}-run.log", @options[:log_dated_dir])
299
+ @logger.add_destination(run_log)
300
+
301
+ # This is an awful hack to maintain backward compatibility until tests
302
+ # are ported to use logger. Still in use in PuppetDB tests
303
+ Beaker.const_set(:Log, @logger) unless defined?( Log )
304
+
305
+ @test_suite_results.start_time = start_time
306
+ @test_suite_results.total_tests = @test_files.length
307
+
308
+ @test_files.each do |test_file|
309
+ @logger.info "Begin #{test_file}"
310
+ start = Time.now
311
+ test_case = TestCase.new(@hosts, @logger, options, test_file).run_test
312
+ duration = Time.now - start
313
+ @test_suite_results.add_test_case(test_case)
314
+ @test_cases << test_case
315
+
316
+ state = test_case.test_status == :skip ? 'skipp' : test_case.test_status
317
+ msg = "#{test_file} #{state}ed in %.2f seconds" % duration.to_f
318
+ case test_case.test_status
319
+ when :pass
320
+ @logger.success msg
321
+ when :skip
322
+ @logger.warn msg
323
+ when :fail
324
+ @logger.error msg
325
+ break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
326
+ when :error
327
+ @logger.warn msg
328
+ break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
329
+ end
330
+ end
331
+ @test_suite_results.stop_time = Time.now
332
+
333
+ # REVISIT: This changes global state, breaking logging in any future runs
334
+ # of the suite – or, at least, making them highly confusing for anyone who
335
+ # has not studied the implementation in detail. --daniel 2011-03-14
336
+ @test_suite_results.summarize( Logger.new(log_path("#{name}-summary.txt", @options[:log_dated_dir]), STDOUT) )
337
+
338
+ junit_file_log = log_path(@options[:xml_file], @options[:xml_dated_dir])
339
+ if @options[:xml_time_enabled]
340
+ junit_file_time = log_path(@options[:xml_time], @options[:xml_dated_dir])
341
+ @test_suite_results.write_junit_xml( junit_file_log, @options[:xml_time] )
342
+ @test_suite_results.write_junit_xml( junit_file_time, @options[:xml_file], true )
343
+ else
344
+ @test_suite_results.write_junit_xml( junit_file_log )
345
+ end
346
+ #All done with this run, remove run log
347
+ @logger.remove_destination(run_log)
348
+
349
+ # Allow chaining operations...
350
+ return self
351
+ end
352
+
353
+ #Execute all the TestCases in this suite.
354
+ #This is a wrapper that catches any failures generated during TestSuite::run.
355
+ def run_and_raise_on_failure
356
+ begin
357
+ run
358
+ return self if @test_suite_results.success?
359
+ rescue => e
360
+ #failed during run
361
+ report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
12
362
  else
13
- nil
363
+ #failed during test
364
+ report_and_raise(@logger, RuntimeError.new("Failed while running the #{name} suite"), "TestSuite: report_and_raise_on_failure")
365
+ end
366
+ end
367
+
368
+ # Gives a full file path for output to be written to, maintaining the latest symlink
369
+ # @param [String] name The file name that we want to write to.
370
+ # @param [String] log_dir The desired output directory.
371
+ # A symlink will be made from ./basedir/latest to that.
372
+ # @example
373
+ # log_path('output.txt', 'log/2014-06-02_16_31_22')
374
+ #
375
+ # This will create the structure:
376
+ #
377
+ # ./log/2014-06-02_16_31_22/output.txt
378
+ # ./log/latest -> 2014-06-02_16_31_22
379
+ #
380
+ # @example
381
+ # log_path('foo.log', 'log/man/date')
382
+ #
383
+ # This will create the structure:
384
+ #
385
+ # ./log/man/date/foo.log
386
+ # ./log/latest -> man/date
387
+ def log_path(name, log_dir)
388
+ FileUtils.mkdir_p(log_dir) unless File.directory?(log_dir)
389
+
390
+ base_dir = log_dir
391
+ link_dir = ''
392
+ while File.dirname(base_dir) != '.' do
393
+ link_dir = link_dir == '' ? File.basename(base_dir) : File.join(File.basename(base_dir), link_dir)
394
+ base_dir = File.dirname(base_dir)
395
+ end
396
+
397
+ latest = File.join(base_dir, "latest")
398
+ if !File.exist?(latest) or File.symlink?(latest) then
399
+ File.delete(latest) if File.exist?(latest) || File.symlink?(latest)
400
+ File.symlink(link_dir, latest)
14
401
  end
402
+
403
+ File.join(log_dir, name)
15
404
  end
405
+
16
406
  end
17
407
  end