beaker 2.30.0 → 2.30.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/HISTORY.md +19 -2
- data/lib/beaker/cli.rb +1 -7
- data/lib/beaker/options/command_line_parser.rb +0 -6
- data/lib/beaker/options/presets.rb +0 -1
- data/lib/beaker/test_case.rb +176 -2
- data/lib/beaker/test_suite.rb +399 -9
- data/lib/beaker/version.rb +1 -1
- data/spec/beaker/test_suite_spec.rb +306 -9
- metadata +2 -8
- data/lib/beaker/runner/mini_test/test_suite.rb +0 -58
- data/lib/beaker/runner/native/test_case.rb +0 -193
- data/lib/beaker/runner/native/test_suite.rb +0 -410
- data/spec/beaker/options/beaker_options_spec.rb +0 -34
- data/spec/beaker/runner/native/test_case_spec.rb +0 -147
- data/spec/beaker/runner/native/test_suite_spec.rb +0 -303
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YmI0MWU4ZDExZjIzYjg0OGExZDg5Mzk3NTI0ZWM2MTM1MmM4YTZkMA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ZGRlY2RhZDJkNTkxYmM0NGM0MTM1NTBmZWQ1OWU1MDJjNDU2NjdjMQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MWE2ODkwNWNkNWZkMDlmMGE1ZTg2N2M2ZmEzNmFlYWExMzE2ZmMwY2EyNjBm
|
10
|
+
MmE4ZjFkNmZkZDg3MjYxNjg0Y2JhMzE4Y2Y2YmE2ZmU1NTQ3ZjJhMjA2NjYx
|
11
|
+
ZmI3YjM2MGZiNGY1NmYwZjE4NTBiYjlhYmUyYjI0ODliM2VmZDg=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
M2NjZDI3MTVlMGZkZGNlYThlYWRmMTcwODlkNjhkMzJhYjAxNDIyMmNiYTFh
|
14
|
+
NGM4YWRmNTcwNTNjYjAzZmMzYWMxMWJjYzU3YWQ5YjdlNTFiYTVhYjc2NzU5
|
15
|
+
YzViMjAyNDcyOTQ2NTIxMWMzYTQ1YzFmNTgzMjRhMWFhYzgyMjA=
|
data/HISTORY.md
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# default - History
|
2
2
|
## Tags
|
3
|
-
* [LATEST -
|
3
|
+
* [LATEST - 3 Dec, 2015 (a1ee5206)](#LATEST)
|
4
|
+
* [2.30.0 - 2 Dec, 2015 (dbb72630)](#2.30.0)
|
4
5
|
* [2.29.1 - 23 Nov, 2015 (5d824690)](#2.29.1)
|
5
6
|
* [2.29.0 - 18 Nov, 2015 (33fd2399)](#2.29.0)
|
6
7
|
* [2.28.0 - 4 Nov, 2015 (89829551)](#2.28.0)
|
@@ -103,7 +104,23 @@
|
|
103
104
|
* [pe1.2 - 6 Sep, 2011 (ba3dadd2)](#pe1.2)
|
104
105
|
|
105
106
|
## Details
|
106
|
-
### <a name = "LATEST">LATEST -
|
107
|
+
### <a name = "LATEST">LATEST - 3 Dec, 2015 (a1ee5206)
|
108
|
+
|
109
|
+
* (GEM) update beaker version to 2.30.1 (a1ee5206)
|
110
|
+
|
111
|
+
* Merge pull request #1024 from puppetlabs/revert-1013-bkr-623/test-runner-reorganization (18307e09)
|
112
|
+
|
113
|
+
|
114
|
+
```
|
115
|
+
Merge pull request #1024 from puppetlabs/revert-1013-bkr-623/test-runner-reorganization
|
116
|
+
|
117
|
+
Revert "[BKR-623] Reorganize Beaker test runner classes for introduction of minitest runner"
|
118
|
+
```
|
119
|
+
* Revert "[BKR-623] Reorganize Beaker test runner classes for introduction of minitest runner" (979a329e)
|
120
|
+
|
121
|
+
### <a name = "2.30.0">2.30.0 - 2 Dec, 2015 (dbb72630)
|
122
|
+
|
123
|
+
* (HISTORY) update beaker history for gem release 2.30.0 (dbb72630)
|
107
124
|
|
108
125
|
* (GEM) update beaker version to 2.30.0 (bc912e78)
|
109
126
|
|
data/lib/beaker/cli.rb
CHANGED
@@ -154,13 +154,7 @@ module Beaker
|
|
154
154
|
@logger.notify("No tests to run for suite '#{suite_name.to_s}'")
|
155
155
|
return
|
156
156
|
end
|
157
|
-
|
158
|
-
unless runner_class = Beaker::TestSuite.runner(@options[:runner])
|
159
|
-
@logger.error "Test runner #{@options[:runner]} is unknown."
|
160
|
-
exit 1
|
161
|
-
end
|
162
|
-
|
163
|
-
runner_class.new(
|
157
|
+
Beaker::TestSuite.new(
|
164
158
|
suite_name, @hosts, @options, @timestamp, failure_strategy
|
165
159
|
).run_and_raise_on_failure
|
166
160
|
end
|
@@ -98,12 +98,6 @@ module Beaker
|
|
98
98
|
@cmd_options[:timeout] = value
|
99
99
|
end
|
100
100
|
|
101
|
-
opts.on '-r RUNNER', '--runner RUNNER',
|
102
|
-
'Specify which test runner to use',
|
103
|
-
'supported runners: native' do |value|
|
104
|
-
@cmd_options[:runner] = value
|
105
|
-
end
|
106
|
-
|
107
101
|
opts.on '-i URI', '--install URI',
|
108
102
|
'Install a project repo/app on the SUTs',
|
109
103
|
'Provide full git URI or use short form KEYWORD/name',
|
@@ -185,7 +185,6 @@ module Beaker
|
|
185
185
|
:puppetdb_port_nonssl => 8080,
|
186
186
|
:puppetserver_port => 8140,
|
187
187
|
:nodeclassifier_port => 4433,
|
188
|
-
:runner => "native",
|
189
188
|
:aws_keyname_modifier => rand(10 ** 10).to_s.rjust(10,'0'), # 10 digit random number string
|
190
189
|
:ssh => {
|
191
190
|
:config => false,
|
data/lib/beaker/test_case.rb
CHANGED
@@ -1,4 +1,11 @@
|
|
1
|
-
|
1
|
+
[ 'host', 'dsl' ].each do |lib|
|
2
|
+
require "beaker/#{lib}"
|
3
|
+
end
|
4
|
+
|
5
|
+
require 'tempfile'
|
6
|
+
require 'benchmark'
|
7
|
+
require 'stringio'
|
8
|
+
require 'rbconfig'
|
2
9
|
|
3
10
|
module Beaker
|
4
11
|
# This class represents a single test case. A test case is necessarily
|
@@ -11,6 +18,173 @@ module Beaker
|
|
11
18
|
#
|
12
19
|
# See {Beaker::DSL} for more information about writing tests
|
13
20
|
# using the DSL.
|
14
|
-
class TestCase
|
21
|
+
class TestCase
|
22
|
+
include Beaker::DSL
|
23
|
+
|
24
|
+
# The Exception raised by Ruby's STDLIB's test framework (Ruby 1.9)
|
25
|
+
TEST_EXCEPTION_CLASS = ::MiniTest::Assertion
|
26
|
+
|
27
|
+
# Necessary for implementing {Beaker::DSL::Helpers#confine}.
|
28
|
+
# Assumed to be an array of valid {Beaker::Host} objects for
|
29
|
+
# this test case.
|
30
|
+
attr_accessor :hosts
|
31
|
+
|
32
|
+
# Necessary for many methods in {Beaker::DSL}. Assumed to be
|
33
|
+
# an instance of {Beaker::Logger}.
|
34
|
+
attr_accessor :logger
|
35
|
+
|
36
|
+
# Necessary for many methods in {Beaker::DSL::Helpers}. Assumed to be
|
37
|
+
# a hash.
|
38
|
+
attr_accessor :metadata
|
39
|
+
|
40
|
+
#The full log for this test
|
41
|
+
attr_accessor :sublog
|
42
|
+
|
43
|
+
#The result for the last command run
|
44
|
+
attr_accessor :last_result
|
45
|
+
|
46
|
+
# A Hash of 'product name' => 'version installed', only set when
|
47
|
+
# products are installed via git or PE install steps. See the 'git' or
|
48
|
+
# 'pe' directories within 'ROOT/setup' for examples.
|
49
|
+
attr_reader :version
|
50
|
+
|
51
|
+
# Parsed command line options.
|
52
|
+
attr_reader :options
|
53
|
+
|
54
|
+
# The path to the file which contains this test case.
|
55
|
+
attr_reader :path
|
56
|
+
|
57
|
+
# I don't know why this is here
|
58
|
+
attr_reader :fail_flag
|
59
|
+
|
60
|
+
# The user that is running this tests home directory, needed by 'net/ssh'.
|
61
|
+
attr_reader :usr_home
|
62
|
+
|
63
|
+
# A Symbol denoting the status of this test (:fail, :pending,
|
64
|
+
# :skipped, :pass).
|
65
|
+
attr_reader :test_status
|
66
|
+
|
67
|
+
# The exception that may have stopped this test's execution.
|
68
|
+
attr_reader :exception
|
69
|
+
|
70
|
+
# @deprecated
|
71
|
+
# The amount of time taken to execute the test. Unused, probably soon
|
72
|
+
# to be removed or refactored.
|
73
|
+
attr_reader :runtime
|
74
|
+
|
75
|
+
# An Array of Procs to be called after test execution has stopped
|
76
|
+
# (whether by exception or not).
|
77
|
+
attr_reader :teardown_procs
|
78
|
+
|
79
|
+
# @deprecated
|
80
|
+
# Legacy accessor from when test files would only contain one remote
|
81
|
+
# action. Contains the Result of the last call to utilize
|
82
|
+
# {Beaker::DSL::Helpers#on}. Do not use as it is not safe
|
83
|
+
# in test files that use multiple calls to
|
84
|
+
# {Beaker::DSL::Helpers#on}.
|
85
|
+
attr_accessor :result
|
86
|
+
|
87
|
+
# @param [Hosts,Array<Host>] these_hosts The hosts to execute this test
|
88
|
+
# against/on.
|
89
|
+
# @param [Logger] logger A logger that implements
|
90
|
+
# {Beaker::Logger}'s interface.
|
91
|
+
# @param [Hash{Symbol=>String}] options Parsed command line options.
|
92
|
+
# @param [String] path The local path to a test file to be executed.
|
93
|
+
def initialize(these_hosts, logger, options={}, path=nil)
|
94
|
+
@hosts = these_hosts
|
95
|
+
@logger = logger
|
96
|
+
@sublog = ""
|
97
|
+
@options = options
|
98
|
+
@path = path
|
99
|
+
@usr_home = options[:home]
|
100
|
+
@test_status = :pass
|
101
|
+
@exception = nil
|
102
|
+
@runtime = nil
|
103
|
+
@teardown_procs = []
|
104
|
+
@metadata = {}
|
105
|
+
set_current_test_filename(@path ? File.basename(@path, '.rb') : nil)
|
106
|
+
|
107
|
+
|
108
|
+
#
|
109
|
+
# We put this on each wrapper (rather than the class) so that methods
|
110
|
+
# defined in the tests don't leak out to other tests.
|
111
|
+
class << self
|
112
|
+
def run_test
|
113
|
+
@logger.start_sublog
|
114
|
+
@logger.last_result = nil
|
115
|
+
|
116
|
+
set_current_step_name(nil)
|
117
|
+
|
118
|
+
#add arbitrary role methods
|
119
|
+
roles = []
|
120
|
+
@hosts.each do |host|
|
121
|
+
roles << host[:roles]
|
122
|
+
end
|
123
|
+
add_role_def( roles.flatten.uniq )
|
124
|
+
|
125
|
+
@runtime = Benchmark.realtime do
|
126
|
+
begin
|
127
|
+
test = File.read(path)
|
128
|
+
eval test,nil,path,1
|
129
|
+
rescue FailTest, TEST_EXCEPTION_CLASS => e
|
130
|
+
@test_status = :fail
|
131
|
+
@exception = e
|
132
|
+
rescue PendingTest
|
133
|
+
@test_status = :pending
|
134
|
+
rescue SkipTest
|
135
|
+
@test_status = :skip
|
136
|
+
rescue StandardError, ScriptError, SignalException => e
|
137
|
+
log_and_fail_test(e)
|
138
|
+
ensure
|
139
|
+
@teardown_procs.each do |teardown|
|
140
|
+
begin
|
141
|
+
teardown.call
|
142
|
+
rescue StandardError, SignalException, TEST_EXCEPTION_CLASS => e
|
143
|
+
log_and_fail_test(e)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
@sublog = @logger.get_sublog
|
149
|
+
@last_result = @logger.last_result
|
150
|
+
return self
|
151
|
+
end
|
152
|
+
|
153
|
+
private
|
154
|
+
|
155
|
+
# Log an error and mark the test as failed, passing through an
|
156
|
+
# exception so it can be displayed at the end of the total run.
|
157
|
+
#
|
158
|
+
# We break out the complete exception backtrace and log each line
|
159
|
+
# individually as well.
|
160
|
+
#
|
161
|
+
# @param exception [Exception] exception to fail with
|
162
|
+
def log_and_fail_test(exception)
|
163
|
+
logger.error("#{exception.class}: #{exception.message}")
|
164
|
+
bt = exception.backtrace
|
165
|
+
logger.pretty_backtrace(bt).each_line do |line|
|
166
|
+
logger.error(line)
|
167
|
+
end
|
168
|
+
@test_status = :error
|
169
|
+
@exception = exception
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
# The TestCase as a hash
|
175
|
+
# @api public
|
176
|
+
# @note The visibility and semantics of this method are valid, but the
|
177
|
+
# structure of the Hash it returns may change without notice
|
178
|
+
#
|
179
|
+
# @return [Hash] A Hash representation of this test.
|
180
|
+
def to_hash
|
181
|
+
hash = {}
|
182
|
+
hash['HOSTS'] = {}
|
183
|
+
@hosts.each do |host|
|
184
|
+
hash['HOSTS'][host.name] = host.overrides
|
185
|
+
end
|
186
|
+
hash
|
187
|
+
end
|
188
|
+
|
15
189
|
end
|
16
190
|
end
|
data/lib/beaker/test_suite.rb
CHANGED
@@ -1,17 +1,407 @@
|
|
1
|
-
|
2
|
-
require '
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'fileutils'
|
4
|
+
[ 'test_case', 'logger' ].each do |lib|
|
5
|
+
require "beaker/#{lib}"
|
6
|
+
end
|
3
7
|
|
4
8
|
module Beaker
|
9
|
+
#A collection of {TestCase} objects are considered a {TestSuite}.
|
10
|
+
#Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
|
5
11
|
class TestSuite
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
+
|
13
|
+
#Holds the output of a test suite, formats in plain text or xml
|
14
|
+
class TestSuiteResult
|
15
|
+
attr_accessor :start_time, :stop_time, :total_tests
|
16
|
+
|
17
|
+
#Create a {TestSuiteResult} instance.
|
18
|
+
#@param [Hash{Symbol=>String}] options Options for this object
|
19
|
+
#@option options [Logger] :logger The Logger object to report information to
|
20
|
+
#@param [String] name The name of the {TestSuite} that the results are for
|
21
|
+
def initialize( options, name )
|
22
|
+
@options = options
|
23
|
+
@logger = options[:logger]
|
24
|
+
@name = name
|
25
|
+
@test_cases = []
|
26
|
+
#Set some defaults, just in case you attempt to print without including them
|
27
|
+
start_time = Time.at(0)
|
28
|
+
stop_time = Time.at(1)
|
29
|
+
end
|
30
|
+
|
31
|
+
#Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
|
32
|
+
#@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
|
33
|
+
def add_test_case( test_case )
|
34
|
+
@test_cases << test_case
|
35
|
+
end
|
36
|
+
|
37
|
+
#How many {TestCase} instances are in this {TestSuiteResult}
|
38
|
+
def test_count
|
39
|
+
@test_cases.length
|
40
|
+
end
|
41
|
+
|
42
|
+
#How many passed {TestCase} instances are in this {TestSuiteResult}
|
43
|
+
def passed_tests
|
44
|
+
@test_cases.select { |c| c.test_status == :pass }.length
|
45
|
+
end
|
46
|
+
|
47
|
+
#How many errored {TestCase} instances are in this {TestSuiteResult}
|
48
|
+
def errored_tests
|
49
|
+
@test_cases.select { |c| c.test_status == :error }.length
|
50
|
+
end
|
51
|
+
|
52
|
+
#How many failed {TestCase} instances are in this {TestSuiteResult}
|
53
|
+
def failed_tests
|
54
|
+
@test_cases.select { |c| c.test_status == :fail }.length
|
55
|
+
end
|
56
|
+
|
57
|
+
#How many skipped {TestCase} instances are in this {TestSuiteResult}
|
58
|
+
def skipped_tests
|
59
|
+
@test_cases.select { |c| c.test_status == :skip }.length
|
60
|
+
end
|
61
|
+
|
62
|
+
#How many pending {TestCase} instances are in this {TestSuiteResult}
|
63
|
+
def pending_tests
|
64
|
+
@test_cases.select {|c| c.test_status == :pending}.length
|
65
|
+
end
|
66
|
+
|
67
|
+
#How many {TestCase} instances failed in this {TestSuiteResult}
|
68
|
+
def sum_failed
|
69
|
+
failed_tests + errored_tests
|
70
|
+
end
|
71
|
+
|
72
|
+
#Did all the {TestCase} instances in this {TestSuiteResult} pass?
|
73
|
+
def success?
|
74
|
+
sum_failed == 0
|
75
|
+
end
|
76
|
+
|
77
|
+
#Did one or more {TestCase} instances in this {TestSuiteResult} fail?
|
78
|
+
def failed?
|
79
|
+
!success?
|
80
|
+
end
|
81
|
+
|
82
|
+
#The sum of all {TestCase} runtimes in this {TestSuiteResult}
|
83
|
+
def elapsed_time
|
84
|
+
@test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
|
85
|
+
end
|
86
|
+
|
87
|
+
#Plain text summay of test suite
|
88
|
+
#@param [Logger] summary_logger The logger we will print the summary to
|
89
|
+
def summarize(summary_logger)
|
90
|
+
|
91
|
+
summary_logger.notify <<-HEREDOC
|
92
|
+
Test Suite: #{@name} @ #{start_time}
|
93
|
+
|
94
|
+
- Host Configuration Summary -
|
95
|
+
HEREDOC
|
96
|
+
|
97
|
+
average_test_time = elapsed_time / test_count
|
98
|
+
|
99
|
+
summary_logger.notify %Q[
|
100
|
+
|
101
|
+
- Test Case Summary for suite '#{@name}' -
|
102
|
+
Total Suite Time: %.2f seconds
|
103
|
+
Average Test Time: %.2f seconds
|
104
|
+
Attempted: #{test_count}
|
105
|
+
Passed: #{passed_tests}
|
106
|
+
Failed: #{failed_tests}
|
107
|
+
Errored: #{errored_tests}
|
108
|
+
Skipped: #{skipped_tests}
|
109
|
+
Pending: #{pending_tests}
|
110
|
+
Total: #{@total_tests}
|
111
|
+
|
112
|
+
- Specific Test Case Status -
|
113
|
+
] % [elapsed_time, average_test_time]
|
114
|
+
|
115
|
+
grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
|
116
|
+
|
117
|
+
summary_logger.notify "Failed Tests Cases:"
|
118
|
+
(grouped_summary[:fail] || []).each do |test_case|
|
119
|
+
print_test_result(test_case)
|
120
|
+
end
|
121
|
+
|
122
|
+
summary_logger.notify "Errored Tests Cases:"
|
123
|
+
(grouped_summary[:error] || []).each do |test_case|
|
124
|
+
print_test_result(test_case)
|
125
|
+
end
|
126
|
+
|
127
|
+
summary_logger.notify "Skipped Tests Cases:"
|
128
|
+
(grouped_summary[:skip] || []).each do |test_case|
|
129
|
+
print_test_result(test_case)
|
130
|
+
end
|
131
|
+
|
132
|
+
summary_logger.notify "Pending Tests Cases:"
|
133
|
+
(grouped_summary[:pending] || []).each do |test_case|
|
134
|
+
print_test_result(test_case)
|
135
|
+
end
|
136
|
+
|
137
|
+
summary_logger.notify("\n\n")
|
138
|
+
end
|
139
|
+
|
140
|
+
#A convenience method for printing the results of a {TestCase}
|
141
|
+
#@param [TestCase] test_case The {TestCase} to examine and print results for
|
142
|
+
def print_test_result(test_case)
|
143
|
+
test_reported = if test_case.exception
|
144
|
+
"reported: #{test_case.exception.inspect}"
|
145
|
+
else
|
146
|
+
test_case.test_status
|
147
|
+
end
|
148
|
+
@logger.notify " Test Case #{test_case.path} #{test_reported}"
|
149
|
+
end
|
150
|
+
|
151
|
+
# Writes Junit XML of this {TestSuiteResult}
|
152
|
+
#
|
153
|
+
# @param [String] xml_file Path to the XML file (from Beaker's running directory)
|
154
|
+
# @param [String] file_to_link Path to the paired file that should be linked
|
155
|
+
# from this one (this is relative to the XML
|
156
|
+
# file itself, so it would just be the different
|
157
|
+
# file name if they're in the same directory)
|
158
|
+
# @param [Boolean] time_sort Whether the test results should be output in
|
159
|
+
# order of time spent in the test, or in the
|
160
|
+
# order of test execution (default)
|
161
|
+
#
|
162
|
+
# @return nil
|
163
|
+
# @api private
|
164
|
+
def write_junit_xml(xml_file, file_to_link = nil, time_sort = false)
|
165
|
+
stylesheet = File.join(@options[:project_root], @options[:xml_stylesheet])
|
166
|
+
|
167
|
+
begin
|
168
|
+
LoggerJunit.write_xml(xml_file, stylesheet) do |doc, suites|
|
169
|
+
|
170
|
+
meta_info = Nokogiri::XML::Node.new('meta_test_info', doc)
|
171
|
+
unless file_to_link.nil?
|
172
|
+
meta_info['page_active'] = time_sort ? 'performance' : 'execution'
|
173
|
+
meta_info['link_url'] = file_to_link
|
174
|
+
else
|
175
|
+
meta_info['page_active'] = 'no-links'
|
176
|
+
meta_info['link_url'] = ''
|
177
|
+
end
|
178
|
+
suites.add_child(meta_info)
|
179
|
+
|
180
|
+
suite = Nokogiri::XML::Node.new('testsuite', doc)
|
181
|
+
suite['name'] = @name
|
182
|
+
suite['tests'] = test_count
|
183
|
+
suite['errors'] = errored_tests
|
184
|
+
suite['failures'] = failed_tests
|
185
|
+
suite['skip'] = skipped_tests
|
186
|
+
suite['pending'] = pending_tests
|
187
|
+
suite['total'] = @total_tests
|
188
|
+
suite['time'] = "%f" % (stop_time - start_time)
|
189
|
+
properties = Nokogiri::XML::Node.new('properties', doc)
|
190
|
+
@options.each_pair do | name, value |
|
191
|
+
property = Nokogiri::XML::Node.new('property', doc)
|
192
|
+
property['name'] = name
|
193
|
+
property['value'] = value
|
194
|
+
properties.add_child(property)
|
195
|
+
end
|
196
|
+
suite.add_child(properties)
|
197
|
+
|
198
|
+
test_cases_to_report = @test_cases
|
199
|
+
test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
|
200
|
+
test_cases_to_report.each do |test|
|
201
|
+
item = Nokogiri::XML::Node.new('testcase', doc)
|
202
|
+
item['classname'] = File.dirname(test.path)
|
203
|
+
item['name'] = File.basename(test.path)
|
204
|
+
item['time'] = "%f" % test.runtime
|
205
|
+
|
206
|
+
# Did we fail? If so, report that.
|
207
|
+
# We need to remove the escape character from colorized text, the
|
208
|
+
# substitution of other entities is handled well by Rexml
|
209
|
+
if test.test_status == :fail || test.test_status == :error then
|
210
|
+
status = Nokogiri::XML::Node.new('failure', doc)
|
211
|
+
status['type'] = test.test_status.to_s
|
212
|
+
if test.exception then
|
213
|
+
status['message'] = test.exception.to_s.gsub(/\e/, '')
|
214
|
+
data = LoggerJunit.format_cdata(test.exception.backtrace.join('\n'))
|
215
|
+
status.add_child(status.document.create_cdata(data))
|
216
|
+
end
|
217
|
+
item.add_child(status)
|
218
|
+
end
|
219
|
+
|
220
|
+
if test.test_status == :skip
|
221
|
+
status = Nokogiri::XML::Node.new('skip', doc)
|
222
|
+
status['type'] = test.test_status.to_s
|
223
|
+
item.add_child(status)
|
224
|
+
end
|
225
|
+
|
226
|
+
if test.test_status == :pending
|
227
|
+
status = Nokogiri::XML::Node.new('pending', doc)
|
228
|
+
status['type'] = test.test_status.to_s
|
229
|
+
item.add_child(status)
|
230
|
+
end
|
231
|
+
|
232
|
+
if test.sublog then
|
233
|
+
stdout = Nokogiri::XML::Node.new('system-out', doc)
|
234
|
+
data = LoggerJunit.format_cdata(test.sublog)
|
235
|
+
stdout.add_child(stdout.document.create_cdata(data))
|
236
|
+
item.add_child(stdout)
|
237
|
+
end
|
238
|
+
|
239
|
+
if test.last_result and test.last_result.stderr and not test.last_result.stderr.empty? then
|
240
|
+
stderr = Nokogiri::XML::Node.new('system-err', doc)
|
241
|
+
data = LoggerJunit.format_cdata(test.last_result.stderr)
|
242
|
+
stderr.add_child(stderr.document.create_cdata(data))
|
243
|
+
item.add_child(stderr)
|
244
|
+
end
|
245
|
+
|
246
|
+
suite.add_child(item)
|
247
|
+
end
|
248
|
+
suites.add_child(suite)
|
249
|
+
end
|
250
|
+
rescue Exception => e
|
251
|
+
@logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
|
252
|
+
end
|
253
|
+
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
attr_reader :name, :options, :fail_mode
|
258
|
+
|
259
|
+
#Create {TestSuite} instance
|
260
|
+
#@param [String] name The name of the {TestSuite}
|
261
|
+
#@param [Array<Host>] hosts An Array of Hosts to act upon.
|
262
|
+
#@param [Hash{Symbol=>String}] options Options for this object
|
263
|
+
#@option options [Logger] :logger The Logger object to report information to
|
264
|
+
#@option options [String] :log_dir The directory where text run logs will be written
|
265
|
+
#@option options [String] :xml_dir The directory where JUnit XML file will be written
|
266
|
+
#@option options [String] :xml_file The name of the JUnit XML file to be written to
|
267
|
+
#@option options [String] :project_root The full path to the Beaker lib directory
|
268
|
+
#@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
|
269
|
+
#@param [Symbol] fail_mode One of :slow, :fast
|
270
|
+
#@param [Time] timestamp Beaker execution start time
|
271
|
+
def initialize(name, hosts, options, timestamp, fail_mode=nil)
|
272
|
+
@logger = options[:logger]
|
273
|
+
@test_cases = []
|
274
|
+
@test_files = options[name]
|
275
|
+
@name = name.to_s.gsub(/\s+/, '-')
|
276
|
+
@hosts = hosts
|
277
|
+
@run = false
|
278
|
+
@options = options
|
279
|
+
@fail_mode = fail_mode || @options[:fail_mode] || :slow
|
280
|
+
@test_suite_results = TestSuiteResult.new(@options, name)
|
281
|
+
@timestamp = timestamp
|
282
|
+
|
283
|
+
report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
|
284
|
+
|
285
|
+
rescue => e
|
286
|
+
report_and_raise(@logger, e, "TestSuite: initialize")
|
287
|
+
end
|
288
|
+
|
289
|
+
#Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
|
290
|
+
#is reported to a newly created run log.
|
291
|
+
#Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
|
292
|
+
#after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
|
293
|
+
def run
|
294
|
+
@run = true
|
295
|
+
start_time = Time.now
|
296
|
+
|
297
|
+
#Create a run log for this TestSuite.
|
298
|
+
run_log = log_path("#{@name}-run.log", @options[:log_dated_dir])
|
299
|
+
@logger.add_destination(run_log)
|
300
|
+
|
301
|
+
# This is an awful hack to maintain backward compatibility until tests
|
302
|
+
# are ported to use logger. Still in use in PuppetDB tests
|
303
|
+
Beaker.const_set(:Log, @logger) unless defined?( Log )
|
304
|
+
|
305
|
+
@test_suite_results.start_time = start_time
|
306
|
+
@test_suite_results.total_tests = @test_files.length
|
307
|
+
|
308
|
+
@test_files.each do |test_file|
|
309
|
+
@logger.info "Begin #{test_file}"
|
310
|
+
start = Time.now
|
311
|
+
test_case = TestCase.new(@hosts, @logger, options, test_file).run_test
|
312
|
+
duration = Time.now - start
|
313
|
+
@test_suite_results.add_test_case(test_case)
|
314
|
+
@test_cases << test_case
|
315
|
+
|
316
|
+
state = test_case.test_status == :skip ? 'skipp' : test_case.test_status
|
317
|
+
msg = "#{test_file} #{state}ed in %.2f seconds" % duration.to_f
|
318
|
+
case test_case.test_status
|
319
|
+
when :pass
|
320
|
+
@logger.success msg
|
321
|
+
when :skip
|
322
|
+
@logger.warn msg
|
323
|
+
when :fail
|
324
|
+
@logger.error msg
|
325
|
+
break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
|
326
|
+
when :error
|
327
|
+
@logger.warn msg
|
328
|
+
break if @fail_mode.to_s !~ /slow/ #all failure modes except slow cause us to kick out early on failure
|
329
|
+
end
|
330
|
+
end
|
331
|
+
@test_suite_results.stop_time = Time.now
|
332
|
+
|
333
|
+
# REVISIT: This changes global state, breaking logging in any future runs
|
334
|
+
# of the suite – or, at least, making them highly confusing for anyone who
|
335
|
+
# has not studied the implementation in detail. --daniel 2011-03-14
|
336
|
+
@test_suite_results.summarize( Logger.new(log_path("#{name}-summary.txt", @options[:log_dated_dir]), STDOUT) )
|
337
|
+
|
338
|
+
junit_file_log = log_path(@options[:xml_file], @options[:xml_dated_dir])
|
339
|
+
if @options[:xml_time_enabled]
|
340
|
+
junit_file_time = log_path(@options[:xml_time], @options[:xml_dated_dir])
|
341
|
+
@test_suite_results.write_junit_xml( junit_file_log, @options[:xml_time] )
|
342
|
+
@test_suite_results.write_junit_xml( junit_file_time, @options[:xml_file], true )
|
343
|
+
else
|
344
|
+
@test_suite_results.write_junit_xml( junit_file_log )
|
345
|
+
end
|
346
|
+
#All done with this run, remove run log
|
347
|
+
@logger.remove_destination(run_log)
|
348
|
+
|
349
|
+
# Allow chaining operations...
|
350
|
+
return self
|
351
|
+
end
|
352
|
+
|
353
|
+
#Execute all the TestCases in this suite.
|
354
|
+
#This is a wrapper that catches any failures generated during TestSuite::run.
|
355
|
+
def run_and_raise_on_failure
|
356
|
+
begin
|
357
|
+
run
|
358
|
+
return self if @test_suite_results.success?
|
359
|
+
rescue => e
|
360
|
+
#failed during run
|
361
|
+
report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
|
12
362
|
else
|
13
|
-
|
363
|
+
#failed during test
|
364
|
+
report_and_raise(@logger, RuntimeError.new("Failed while running the #{name} suite"), "TestSuite: report_and_raise_on_failure")
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
# Gives a full file path for output to be written to, maintaining the latest symlink
|
369
|
+
# @param [String] name The file name that we want to write to.
|
370
|
+
# @param [String] log_dir The desired output directory.
|
371
|
+
# A symlink will be made from ./basedir/latest to that.
|
372
|
+
# @example
|
373
|
+
# log_path('output.txt', 'log/2014-06-02_16_31_22')
|
374
|
+
#
|
375
|
+
# This will create the structure:
|
376
|
+
#
|
377
|
+
# ./log/2014-06-02_16_31_22/output.txt
|
378
|
+
# ./log/latest -> 2014-06-02_16_31_22
|
379
|
+
#
|
380
|
+
# @example
|
381
|
+
# log_path('foo.log', 'log/man/date')
|
382
|
+
#
|
383
|
+
# This will create the structure:
|
384
|
+
#
|
385
|
+
# ./log/man/date/foo.log
|
386
|
+
# ./log/latest -> man/date
|
387
|
+
def log_path(name, log_dir)
|
388
|
+
FileUtils.mkdir_p(log_dir) unless File.directory?(log_dir)
|
389
|
+
|
390
|
+
base_dir = log_dir
|
391
|
+
link_dir = ''
|
392
|
+
while File.dirname(base_dir) != '.' do
|
393
|
+
link_dir = link_dir == '' ? File.basename(base_dir) : File.join(File.basename(base_dir), link_dir)
|
394
|
+
base_dir = File.dirname(base_dir)
|
395
|
+
end
|
396
|
+
|
397
|
+
latest = File.join(base_dir, "latest")
|
398
|
+
if !File.exist?(latest) or File.symlink?(latest) then
|
399
|
+
File.delete(latest) if File.exist?(latest) || File.symlink?(latest)
|
400
|
+
File.symlink(link_dir, latest)
|
14
401
|
end
|
402
|
+
|
403
|
+
File.join(log_dir, name)
|
15
404
|
end
|
405
|
+
|
16
406
|
end
|
17
407
|
end
|