beaker 1.12.2 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/beaker.gemspec +1 -1
- data/lib/beaker/cli.rb +2 -1
- data/lib/beaker/dsl/helpers.rb +74 -18
- data/lib/beaker/dsl/install_utils.rb +172 -39
- data/lib/beaker/host.rb +6 -3
- data/lib/beaker/host/unix/pkg.rb +22 -1
- data/lib/beaker/host/windows.rb +1 -0
- data/lib/beaker/host/windows/pkg.rb +7 -2
- data/lib/beaker/host_prebuilt_steps.rb +22 -21
- data/lib/beaker/hypervisor/docker.rb +1 -1
- data/lib/beaker/hypervisor/vagrant.rb +13 -0
- data/lib/beaker/junit.xsl +268 -0
- data/lib/beaker/logger.rb +25 -5
- data/lib/beaker/options/command_line_parser.rb +8 -8
- data/lib/beaker/options/presets.rb +5 -1
- data/lib/beaker/platform.rb +1 -0
- data/lib/beaker/ssh_connection.rb +15 -11
- data/lib/beaker/tasks/rake_task.rb +114 -0
- data/lib/beaker/tasks/test.rb +18 -0
- data/lib/beaker/test_case.rb +6 -0
- data/lib/beaker/test_suite.rb +316 -191
- data/lib/beaker/version.rb +1 -1
- data/spec/beaker/dsl/helpers_spec.rb +98 -25
- data/spec/beaker/dsl/install_utils_spec.rb +26 -12
- data/spec/beaker/host/unix/pkg_spec.rb +109 -0
- data/spec/beaker/host_prebuilt_steps_spec.rb +2 -18
- data/spec/beaker/host_spec.rb +3 -2
- data/spec/beaker/hypervisor/docker_spec.rb +1 -1
- data/spec/beaker/options/command_line_parser_spec.rb +2 -2
- data/spec/beaker/puppet_command_spec.rb +2 -2
- data/spec/beaker/ssh_connection_spec.rb +9 -8
- data/spec/beaker/test_suite_spec.rb +108 -0
- metadata +9 -4
@@ -111,12 +111,6 @@ module Beaker
|
|
111
111
|
@cmd_options[:quiet] = bool
|
112
112
|
end
|
113
113
|
|
114
|
-
opts.on '-x', '--[no-]xml',
|
115
|
-
'Emit JUnit XML reports on tests',
|
116
|
-
'(default: false)' do |bool|
|
117
|
-
@cmd_options[:xml] = bool
|
118
|
-
end
|
119
|
-
|
120
114
|
opts.on '--[no-]color',
|
121
115
|
'Do not display color in log output',
|
122
116
|
'(default: true)' do |bool|
|
@@ -170,6 +164,10 @@ module Beaker
|
|
170
164
|
@cmd_options[:add_el_extras] = true
|
171
165
|
end
|
172
166
|
|
167
|
+
opts.on '--package-proxy URL', 'Set proxy url for package managers (yum and apt)' do |value|
|
168
|
+
@cmd_options[:package_proxy] = value
|
169
|
+
end
|
170
|
+
|
173
171
|
opts.on '--[no-]validate',
|
174
172
|
'Validate that SUTs are correctly configured before running tests',
|
175
173
|
'(default: true)' do |bool|
|
@@ -198,9 +196,11 @@ module Beaker
|
|
198
196
|
@cmd_options[:log_level] = bool ? 'debug' : 'info'
|
199
197
|
end
|
200
198
|
|
201
|
-
opts.on '
|
202
|
-
|
199
|
+
opts.on '-x', '--[no-]xml',
|
200
|
+
'DEPRECATED - JUnit XML now generated by default' do
|
201
|
+
#noop
|
203
202
|
end
|
203
|
+
|
204
204
|
end
|
205
205
|
|
206
206
|
end
|
@@ -67,7 +67,11 @@ module Beaker
|
|
67
67
|
:preserve_hosts => 'never',
|
68
68
|
:root_keys => false,
|
69
69
|
:quiet => false,
|
70
|
-
:
|
70
|
+
:project_root => File.expand_path(File.join(File.dirname(__FILE__), "../")),
|
71
|
+
:xml_dir => 'junit',
|
72
|
+
:xml_file => 'beaker_junit.xml',
|
73
|
+
:xml_stylesheet => 'junit.xsl',
|
74
|
+
:log_dir => 'log',
|
71
75
|
:color => true,
|
72
76
|
:dry_run => false,
|
73
77
|
:timeout => 300,
|
data/lib/beaker/platform.rb
CHANGED
@@ -19,6 +19,7 @@ module Beaker
|
|
19
19
|
|
20
20
|
#Creates the Platform object. Checks to ensure that the platform String provided meets the platform
|
21
21
|
#formatting rules. Platforms name must be of the format /^OSFAMILY-VERSION-ARCH.*$/ where OSFAMILY is one of:
|
22
|
+
# * osx
|
22
23
|
# * centos
|
23
24
|
# * fedora
|
24
25
|
# * debian
|
@@ -5,6 +5,8 @@ require 'net/scp'
|
|
5
5
|
module Beaker
|
6
6
|
class SshConnection
|
7
7
|
|
8
|
+
attr_accessor :logger
|
9
|
+
|
8
10
|
RETRYABLE_EXCEPTIONS = [
|
9
11
|
SocketError,
|
10
12
|
Timeout::Error,
|
@@ -18,14 +20,15 @@ module Beaker
|
|
18
20
|
Net::SSH::AuthenticationFailed,
|
19
21
|
]
|
20
22
|
|
21
|
-
def initialize hostname, user = nil, options = {}
|
23
|
+
def initialize hostname, user = nil, ssh_opts = {}, options = {}
|
22
24
|
@hostname = hostname
|
23
25
|
@user = user
|
24
|
-
@
|
26
|
+
@ssh_opts = ssh_opts
|
27
|
+
@logger = options[:logger]
|
25
28
|
end
|
26
29
|
|
27
|
-
def self.connect hostname, user = 'root', options = {}
|
28
|
-
connection = new hostname, user, options
|
30
|
+
def self.connect hostname, user = 'root', ssh_opts = {}, options = {}
|
31
|
+
connection = new hostname, user, ssh_opts, options
|
29
32
|
connection.connect
|
30
33
|
connection
|
31
34
|
end
|
@@ -35,21 +38,22 @@ module Beaker
|
|
35
38
|
last_wait = 0
|
36
39
|
wait = 1
|
37
40
|
@ssh ||= begin
|
38
|
-
Net::SSH.start(@hostname, @user, @
|
41
|
+
Net::SSH.start(@hostname, @user, @ssh_opts)
|
39
42
|
rescue *RETRYABLE_EXCEPTIONS => e
|
40
43
|
if try <= 11
|
41
|
-
|
42
|
-
|
44
|
+
@logger.warn "Try #{try} -- Host #{@hostname} unreachable: #{e.message}"
|
45
|
+
@logger.warn "Trying again in #{wait} seconds"
|
43
46
|
sleep wait
|
44
|
-
|
47
|
+
(last_wait, wait) = wait, last_wait + wait
|
45
48
|
try += 1
|
46
49
|
retry
|
47
50
|
else
|
48
51
|
# why is the logger not passed into this class?
|
49
|
-
|
52
|
+
@logger.error "Failed to connect to #{@hostname}"
|
50
53
|
raise
|
51
54
|
end
|
52
55
|
end
|
56
|
+
@logger.debug "Created ssh connection to #{@hostname}, user: #{@user}, opts: #{@ssh_opts}"
|
53
57
|
self
|
54
58
|
end
|
55
59
|
|
@@ -101,7 +105,7 @@ module Beaker
|
|
101
105
|
rescue *RETRYABLE_EXCEPTIONS => e
|
102
106
|
if attempt
|
103
107
|
attempt = false
|
104
|
-
|
108
|
+
@logger.error "Command execution failed, attempting to reconnect to #{@hostname}"
|
105
109
|
close
|
106
110
|
connect
|
107
111
|
retry
|
@@ -116,7 +120,7 @@ module Beaker
|
|
116
120
|
def request_terminal_for channel, command
|
117
121
|
channel.request_pty do |ch, success|
|
118
122
|
if success
|
119
|
-
|
123
|
+
@logger.info "Allocated a PTY on #{@hostname} for #{command.inspect}"
|
120
124
|
else
|
121
125
|
abort "FAILED: could not allocate a pty when requested on " +
|
122
126
|
"#{@hostname} for #{command.inspect}"
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'rake/task_arguments'
|
2
|
+
require 'rake/tasklib'
|
3
|
+
require 'rake'
|
4
|
+
require 'beaker'
|
5
|
+
|
6
|
+
|
7
|
+
module Beaker
|
8
|
+
module Tasks
|
9
|
+
class RakeTask < ::Rake::TaskLib
|
10
|
+
include ::Rake::DSL if defined?(::Rake::DSL)
|
11
|
+
|
12
|
+
DEFAULT_ACCEPTANCE_ROOT = "./acceptance"
|
13
|
+
|
14
|
+
COMMAND_OPTIONS = [:fail_mode,
|
15
|
+
:hosts,
|
16
|
+
:helper,
|
17
|
+
:keyfile,
|
18
|
+
:log_level,
|
19
|
+
:options_file,
|
20
|
+
:preserve_hosts,
|
21
|
+
:tests,
|
22
|
+
:type,
|
23
|
+
:acceptance_root,
|
24
|
+
:name]
|
25
|
+
# iterates of acceptable params
|
26
|
+
COMMAND_OPTIONS.each do |sym|
|
27
|
+
attr_accessor(sym.to_sym)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Sets up the predefine task checking
|
31
|
+
# @param args [Array] First argument is always the name of the task
|
32
|
+
# if no additonal arguments are defined such as parameters it will default to [:hosts,:type]
|
33
|
+
def initialize(*args, &task_block)
|
34
|
+
@name = args.shift || 'beaker:test'
|
35
|
+
if args.empty?
|
36
|
+
args = [:hosts,:type]
|
37
|
+
end
|
38
|
+
@acceptance_root = DEFAULT_ACCEPTANCE_ROOT
|
39
|
+
@options_file = nil
|
40
|
+
define(args, &task_block)
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
# Run the task provided, implements the rake task interface
|
45
|
+
#
|
46
|
+
# @param verbose [bool] Defines wether to run in verbose mode or not
|
47
|
+
def run_task(verbose)
|
48
|
+
puts "Running task"
|
49
|
+
|
50
|
+
check_for_beaker_type_config
|
51
|
+
command = beaker_command
|
52
|
+
|
53
|
+
begin
|
54
|
+
puts command if verbose
|
55
|
+
success = system(command)
|
56
|
+
preserve_configuration(@options_file)
|
57
|
+
rescue
|
58
|
+
puts failure_message if failure_message
|
59
|
+
end
|
60
|
+
if fail_mode == "fast" && !success
|
61
|
+
$stderr.puts "#{command} failed"
|
62
|
+
exit $?.exitstatus
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# @private
|
67
|
+
def define(args, &task_block)
|
68
|
+
desc "Run Beaker Acceptance" unless ::Rake.application.last_comment
|
69
|
+
task name, *args do |_, task_args|
|
70
|
+
RakeFileUtils.__send__(:verbose, verbose) do
|
71
|
+
task_block.call(*[self, task_args].slice(0, task_block.arity)) if task_block
|
72
|
+
run_task verbose
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
#
|
78
|
+
# If an options file exists in the acceptance path for the type given use it as a default options file
|
79
|
+
# if no other options file is provided
|
80
|
+
#
|
81
|
+
def check_for_beaker_type_config
|
82
|
+
if !@options_file && File.exists?("#{@acceptance_root}/.beaker-#{@type}.cfg")
|
83
|
+
@options_file = File.join(@acceptance_root, ".beaker-#{@type}.cfg")
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Check for existence of ENV variables for test if !@tests is undef
|
89
|
+
#
|
90
|
+
def check_env_variables
|
91
|
+
if File.exists?(File.join(DEFAULT_ACCEPTANCE_ROOT, 'tests'))
|
92
|
+
@tests = File.join(DEFAULT_ACCEPTANCE_ROOT, 'tests')
|
93
|
+
end
|
94
|
+
@tests = ENV['TESTS'] || ENV['TEST'] if !@tests
|
95
|
+
end
|
96
|
+
|
97
|
+
#
|
98
|
+
# Generate the beaker command to run beaker with all possible options passed
|
99
|
+
#
|
100
|
+
def beaker_command
|
101
|
+
cmd_parts = []
|
102
|
+
cmd_parts << "beaker"
|
103
|
+
cmd_parts << "--keyfile #{@keyfile}" if @keyfile
|
104
|
+
cmd_parts << "--hosts #{@hosts}" if @hosts
|
105
|
+
cmd_parts << "--tests #{tests}" if @tests
|
106
|
+
cmd_parts << "--options-file #{@options_file}" if @options_file
|
107
|
+
cmd_parts << "--type #{@type}" if @type
|
108
|
+
cmd_parts << "--helper #{@helper}" if @helper
|
109
|
+
cmd_parts << "--fail-mode #{@fail_mode}" if @fail_mode
|
110
|
+
cmd_parts.flatten.join(" ")
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'beaker/tasks/rake_task'
|
2
|
+
|
3
|
+
Beaker::Tasks::RakeTask.new do |t,args|
|
4
|
+
t.type = args[:type]
|
5
|
+
t.hosts = args[:hosts]
|
6
|
+
end
|
7
|
+
|
8
|
+
desc "Run Beaker PE tests"
|
9
|
+
Beaker::Tasks::RakeTask.new("beaker:test:pe",:hosts) do |t,args|
|
10
|
+
t.type = 'pe'
|
11
|
+
t.hosts = args[:hosts]
|
12
|
+
end
|
13
|
+
|
14
|
+
desc "Run Beaker Git tests"
|
15
|
+
Beaker::Tasks::RakeTask.new("beaker:test:git",:hosts) do |t,args|
|
16
|
+
t.type = 'git'
|
17
|
+
t.hosts = args[:hosts]
|
18
|
+
end
|
data/lib/beaker/test_case.rb
CHANGED
@@ -52,6 +52,9 @@ module Beaker
|
|
52
52
|
# an instance of {Beaker::Logger}.
|
53
53
|
attr_accessor :logger
|
54
54
|
|
55
|
+
#The full log for this test
|
56
|
+
attr_accessor :sublog
|
57
|
+
|
55
58
|
# A Hash of 'product name' => 'version installed', only set when
|
56
59
|
# products are installed via git or PE install steps. See the 'git' or
|
57
60
|
# 'pe' directories within 'ROOT/setup' for examples.
|
@@ -102,6 +105,7 @@ module Beaker
|
|
102
105
|
def initialize(these_hosts, logger, options={}, path=nil)
|
103
106
|
@hosts = these_hosts
|
104
107
|
@logger = logger
|
108
|
+
@sublog = ""
|
105
109
|
@options = options
|
106
110
|
@path = path
|
107
111
|
@usr_home = options[:home]
|
@@ -116,6 +120,7 @@ module Beaker
|
|
116
120
|
# defined in the tests don't leak out to other tests.
|
117
121
|
class << self
|
118
122
|
def run_test
|
123
|
+
@logger.start_sublog
|
119
124
|
@runtime = Benchmark.realtime do
|
120
125
|
begin
|
121
126
|
test = File.read(path)
|
@@ -139,6 +144,7 @@ module Beaker
|
|
139
144
|
end
|
140
145
|
end
|
141
146
|
end
|
147
|
+
@sublog = @logger.get_sublog
|
142
148
|
return self
|
143
149
|
end
|
144
150
|
|
data/lib/beaker/test_suite.rb
CHANGED
@@ -1,20 +1,279 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
|
-
require '
|
2
|
+
require 'nokogiri'
|
3
3
|
require 'fileutils'
|
4
4
|
[ 'test_case', 'logger' ].each do |lib|
|
5
5
|
require "beaker/#{lib}"
|
6
6
|
end
|
7
7
|
|
8
8
|
module Beaker
|
9
|
-
#
|
10
|
-
#
|
11
|
-
# * Global State Change
|
12
|
-
# * File Creation Relative to CWD -- Should be a config option
|
13
|
-
# * Better Method Documentation
|
9
|
+
#A collection of {TestCase} objects are considered a {TestSuite}.
|
10
|
+
#Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
|
14
11
|
class TestSuite
|
12
|
+
|
13
|
+
#Holds the output of a test suite, formats in plain text or xml
|
14
|
+
class TestSuiteResult
|
15
|
+
attr_accessor :start_time, :stop_time
|
16
|
+
|
17
|
+
#Create a {TestSuiteResult} instance.
|
18
|
+
#@param [Hash{Symbol=>String}] options Options for this object
|
19
|
+
#@option options [Logger] :logger The Logger object to report information to
|
20
|
+
#@param [String] name The name of the {TestSuite} that the results are for
|
21
|
+
def initialize( options, name )
|
22
|
+
@options = options
|
23
|
+
@logger = options[:logger]
|
24
|
+
@name = name
|
25
|
+
@test_cases = []
|
26
|
+
#Set some defaults, just in case you attempt to print without including them
|
27
|
+
start_time = Time.at(0)
|
28
|
+
stop_time = Time.at(1)
|
29
|
+
end
|
30
|
+
|
31
|
+
#Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
|
32
|
+
#@param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
|
33
|
+
def add_test_case( test_case )
|
34
|
+
@test_cases << test_case
|
35
|
+
end
|
36
|
+
|
37
|
+
#How many {TestCase} instances are in this {TestSuiteResult}
|
38
|
+
def test_count
|
39
|
+
@test_cases.length
|
40
|
+
end
|
41
|
+
|
42
|
+
#How many passed {TestCase} instances are in this {TestSuiteResult}
|
43
|
+
def passed_tests
|
44
|
+
@test_cases.select { |c| c.test_status == :pass }.length
|
45
|
+
end
|
46
|
+
|
47
|
+
#How many errored {TestCase} instances are in this {TestSuiteResult}
|
48
|
+
def errored_tests
|
49
|
+
@test_cases.select { |c| c.test_status == :error }.length
|
50
|
+
end
|
51
|
+
|
52
|
+
#How many failed {TestCase} instances are in this {TestSuiteResult}
|
53
|
+
def failed_tests
|
54
|
+
@test_cases.select { |c| c.test_status == :fail }.length
|
55
|
+
end
|
56
|
+
|
57
|
+
#How many skipped {TestCase} instances are in this {TestSuiteResult}
|
58
|
+
def skipped_tests
|
59
|
+
@test_cases.select { |c| c.test_status == :skip }.length
|
60
|
+
end
|
61
|
+
|
62
|
+
#How many pending {TestCase} instances are in this {TestSuiteResult}
|
63
|
+
def pending_tests
|
64
|
+
@test_cases.select {|c| c.test_status == :pending}.length
|
65
|
+
end
|
66
|
+
|
67
|
+
#How many {TestCase} instances failed in this {TestSuiteResult}
|
68
|
+
def sum_failed
|
69
|
+
failed_tests + errored_tests
|
70
|
+
end
|
71
|
+
|
72
|
+
#Did all the {TestCase} instances in this {TestSuiteResult} pass?
|
73
|
+
def success?
|
74
|
+
sum_failed == 0
|
75
|
+
end
|
76
|
+
|
77
|
+
#Did one or more {TestCase} instances in this {TestSuiteResult} fail?
|
78
|
+
def failed?
|
79
|
+
!success?
|
80
|
+
end
|
81
|
+
|
82
|
+
#The sum of all {TestCase} runtimes in this {TestSuiteResult}
|
83
|
+
def elapsed_time
|
84
|
+
@test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
|
85
|
+
end
|
86
|
+
|
87
|
+
#Plain text summay of test suite
|
88
|
+
#@param [Logger] summary_logger The logger we will print the summary to
|
89
|
+
def summarize(summary_logger)
|
90
|
+
|
91
|
+
summary_logger.notify <<-HEREDOC
|
92
|
+
Test Suite: #{@name} @ #{start_time}
|
93
|
+
|
94
|
+
- Host Configuration Summary -
|
95
|
+
HEREDOC
|
96
|
+
|
97
|
+
average_test_time = elapsed_time / test_count
|
98
|
+
|
99
|
+
summary_logger.notify %Q[
|
100
|
+
|
101
|
+
- Test Case Summary for suite '#{@name}' -
|
102
|
+
Total Suite Time: %.2f seconds
|
103
|
+
Average Test Time: %.2f seconds
|
104
|
+
Attempted: #{test_count}
|
105
|
+
Passed: #{passed_tests}
|
106
|
+
Failed: #{failed_tests}
|
107
|
+
Errored: #{errored_tests}
|
108
|
+
Skipped: #{skipped_tests}
|
109
|
+
Pending: #{pending_tests}
|
110
|
+
|
111
|
+
- Specific Test Case Status -
|
112
|
+
] % [elapsed_time, average_test_time]
|
113
|
+
|
114
|
+
grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
|
115
|
+
|
116
|
+
summary_logger.notify "Failed Tests Cases:"
|
117
|
+
(grouped_summary[:fail] || []).each do |test_case|
|
118
|
+
print_test_result(test_case)
|
119
|
+
end
|
120
|
+
|
121
|
+
summary_logger.notify "Errored Tests Cases:"
|
122
|
+
(grouped_summary[:error] || []).each do |test_case|
|
123
|
+
print_test_result(test_case)
|
124
|
+
end
|
125
|
+
|
126
|
+
summary_logger.notify "Skipped Tests Cases:"
|
127
|
+
(grouped_summary[:skip] || []).each do |test_case|
|
128
|
+
print_test_result(test_case)
|
129
|
+
end
|
130
|
+
|
131
|
+
summary_logger.notify "Pending Tests Cases:"
|
132
|
+
(grouped_summary[:pending] || []).each do |test_case|
|
133
|
+
print_test_result(test_case)
|
134
|
+
end
|
135
|
+
|
136
|
+
summary_logger.notify("\n\n")
|
137
|
+
end
|
138
|
+
|
139
|
+
#A convenience method for printing the results of a {TestCase}
|
140
|
+
#@param [TestCase] test_case The {TestCase} to examine and print results for
|
141
|
+
def print_test_result(test_case)
|
142
|
+
test_reported = if test_case.exception
|
143
|
+
"reported: #{test_case.exception.inspect}"
|
144
|
+
else
|
145
|
+
test_case.test_status
|
146
|
+
end
|
147
|
+
@logger.notify " Test Case #{test_case.path} #{test_reported}"
|
148
|
+
end
|
149
|
+
|
150
|
+
#Remove color codes from provided string. Color codes are of the format /(\e\[\d\d;\d\dm)+/.
|
151
|
+
#@param [String] text The string to remove color codes from
|
152
|
+
#@return [String] The text without color codes
|
153
|
+
def strip_color_codes(text)
|
154
|
+
text.gsub(/(\e|\^\[)\[(\d*;)*\d*m/, '')
|
155
|
+
end
|
156
|
+
|
157
|
+
#Format and print the {TestSuiteResult} as JUnit XML
|
158
|
+
#@param [String] xml_file The full path to print the output to.
|
159
|
+
#@param [String] stylesheet The full path to a JUnit XML stylesheet
|
160
|
+
def write_junit_xml(xml_file, stylesheet)
|
161
|
+
begin
|
162
|
+
|
163
|
+
#copy stylesheet into xml directory
|
164
|
+
if not File.file?(File.join(File.dirname(xml_file), File.basename(stylesheet)))
|
165
|
+
FileUtils.copy(stylesheet, File.join(File.dirname(xml_file), File.basename(stylesheet)))
|
166
|
+
end
|
167
|
+
suites = nil
|
168
|
+
#check to see if an output file already exists, if it does add or replace test suite data
|
169
|
+
if File.file?(xml_file)
|
170
|
+
doc = Nokogiri::XML( File.open(xml_file, 'r') )
|
171
|
+
suites = doc.at_xpath('testsuites')
|
172
|
+
#remove old data
|
173
|
+
doc.search("//testsuite").each do |node|
|
174
|
+
if node['name'] =~ /#{@name}/
|
175
|
+
node.unlink
|
176
|
+
end
|
177
|
+
end
|
178
|
+
else
|
179
|
+
#no existing file, create a new one
|
180
|
+
doc = Nokogiri::XML::Document.new
|
181
|
+
pi = Nokogiri::XML::ProcessingInstruction.new(doc, "xml-stylesheet", "type=\"text/xsl\" href=\"#{File.basename(stylesheet)}\"")
|
182
|
+
pi.parent = doc
|
183
|
+
suites = Nokogiri::XML::Node.new('testsuites', doc)
|
184
|
+
suites.parent = doc
|
185
|
+
end
|
186
|
+
|
187
|
+
suite = Nokogiri::XML::Node.new('testsuite', doc)
|
188
|
+
suite['name'] = @name
|
189
|
+
suite['tests'] = test_count
|
190
|
+
suite['errors'] = errored_tests
|
191
|
+
suite['failures'] = failed_tests
|
192
|
+
suite['skip'] = skipped_tests
|
193
|
+
suite['pending'] = pending_tests
|
194
|
+
suite['time'] = "%f" % (stop_time - start_time)
|
195
|
+
properties = Nokogiri::XML::Node.new('properties', doc)
|
196
|
+
@options.each_pair do | name, value |
|
197
|
+
property = Nokogiri::XML::Node.new('property', doc)
|
198
|
+
property['name'] = name
|
199
|
+
property['value'] = value
|
200
|
+
properties.add_child(property)
|
201
|
+
end
|
202
|
+
suite.add_child(properties)
|
203
|
+
|
204
|
+
@test_cases.each do |test|
|
205
|
+
item = Nokogiri::XML::Node.new('testcase', doc)
|
206
|
+
item['classname'] = File.dirname(test.path)
|
207
|
+
item['name'] = File.basename(test.path)
|
208
|
+
item['time'] = "%f" % test.runtime
|
209
|
+
|
210
|
+
# Did we fail? If so, report that.
|
211
|
+
# We need to remove the escape character from colorized text, the
|
212
|
+
# substitution of other entities is handled well by Rexml
|
213
|
+
if test.test_status == :fail || test.test_status == :error then
|
214
|
+
status = Nokogiri::XML::Node.new('failure', doc)
|
215
|
+
status['type'] = test.test_status.to_s
|
216
|
+
if test.exception then
|
217
|
+
status['message'] = test.exception.to_s.gsub(/\e/, '')
|
218
|
+
status.add_child(status.document.create_cdata(test.exception.backtrace.join('\n')))
|
219
|
+
end
|
220
|
+
item.add_child(status)
|
221
|
+
end
|
222
|
+
|
223
|
+
if test.test_status == :skip
|
224
|
+
status = Nokogiri::XML::Node.new('skip', doc)
|
225
|
+
status['type'] = test.test_status.to_s
|
226
|
+
item.add_child(status)
|
227
|
+
end
|
228
|
+
|
229
|
+
if test.test_status == :pending
|
230
|
+
status = Nokogiri::XML::Node.new('pending', doc)
|
231
|
+
status['type'] = test.test_status.to_s
|
232
|
+
item.add_child(status)
|
233
|
+
end
|
234
|
+
|
235
|
+
if test.sublog then
|
236
|
+
stdout = Nokogiri::XML::Node.new('system-out', doc)
|
237
|
+
stdout.add_child(stdout.document.create_cdata(strip_color_codes(test.sublog)))
|
238
|
+
item.add_child(stdout)
|
239
|
+
end
|
240
|
+
|
241
|
+
if test.stderr then
|
242
|
+
stderr = Nokogiri::XML::Node.new('system-err', doc)
|
243
|
+
stderr.add_child(stderr.document.create_cdata(strip_color_codes(test.stderr)))
|
244
|
+
item.add_child(stderr)
|
245
|
+
end
|
246
|
+
|
247
|
+
suite.add_child(item)
|
248
|
+
end
|
249
|
+
suites.add_child(suite)
|
250
|
+
|
251
|
+
# junit/name.xml will be created in a directory relative to the CWD
|
252
|
+
# -- JLS 2/12
|
253
|
+
File.open(xml_file, 'w') { |fh| fh.write(doc.to_xml) }
|
254
|
+
|
255
|
+
rescue Exception => e
|
256
|
+
@logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
end
|
261
|
+
|
15
262
|
attr_reader :name, :options, :fail_mode
|
16
263
|
|
17
|
-
|
264
|
+
#Create {TestSuite} instance
|
265
|
+
#@param [String] name The name of the {TestSuite}
|
266
|
+
#@param [Array<Host>] hosts An Array of Hosts to act upon.
|
267
|
+
#@param [Hash{Symbol=>String}] options Options for this object
|
268
|
+
#@option options [Logger] :logger The Logger object to report information to
|
269
|
+
#@option options [String] :log_dir The directory where text run logs will be written
|
270
|
+
#@option options [String] :xml_dir The directory where JUnit XML file will be written
|
271
|
+
#@option options [String] :xml_file The name of the JUnit XML file to be written to
|
272
|
+
#@option options [String] :project_root The full path to the Beaker lib directory
|
273
|
+
#@option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
|
274
|
+
#@param [Symbol] fail_mode One of :slow, :fast
|
275
|
+
#@param [Time] timestamp Beaker execution start time
|
276
|
+
def initialize(name, hosts, options, timestamp, fail_mode = :slow)
|
18
277
|
@logger = options[:logger]
|
19
278
|
@test_cases = []
|
20
279
|
@test_files = options[name]
|
@@ -22,7 +281,9 @@ module Beaker
|
|
22
281
|
@hosts = hosts
|
23
282
|
@run = false
|
24
283
|
@options = options
|
25
|
-
@fail_mode =
|
284
|
+
@fail_mode = fail_mode
|
285
|
+
@test_suite_results = TestSuiteResult.new(@options, name)
|
286
|
+
@timestamp = timestamp
|
26
287
|
|
27
288
|
report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
|
28
289
|
|
@@ -30,18 +291,30 @@ module Beaker
|
|
30
291
|
report_and_raise(@logger, e, "TestSuite: initialize")
|
31
292
|
end
|
32
293
|
|
294
|
+
#Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
|
295
|
+
#is reported to a newly created run log.
|
296
|
+
#Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
|
297
|
+
#after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
|
33
298
|
def run
|
34
299
|
@run = true
|
35
|
-
|
300
|
+
start_time = Time.now
|
36
301
|
|
37
|
-
|
302
|
+
#Create a run log for this TestSuite.
|
303
|
+
run_log = log_path(@timestamp, "#{@name}-run.log", @options[:log_dir])
|
304
|
+
@logger.add_destination(run_log)
|
305
|
+
|
306
|
+
# This is an awful hack to maintain backward compatibility until tests
|
307
|
+
# are ported to use logger. Still in use in PuppetDB tests
|
308
|
+
Beaker.const_set(:Log, @logger) unless defined?( Log )
|
309
|
+
|
310
|
+
@test_suite_results.start_time = start_time
|
38
311
|
|
39
312
|
@test_files.each do |test_file|
|
40
|
-
@logger.notify
|
41
313
|
@logger.notify "Begin #{test_file}"
|
42
314
|
start = Time.now
|
43
315
|
test_case = TestCase.new(@hosts, @logger, options, test_file).run_test
|
44
316
|
duration = Time.now - start
|
317
|
+
@test_suite_results.add_test_case(test_case)
|
45
318
|
@test_cases << test_case
|
46
319
|
|
47
320
|
state = test_case.test_status == :skip ? 'skipp' : test_case.test_status
|
@@ -53,27 +326,33 @@ module Beaker
|
|
53
326
|
@logger.debug msg
|
54
327
|
when :fail
|
55
328
|
@logger.error msg
|
56
|
-
break if fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
|
329
|
+
break if @fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
|
57
330
|
when :error
|
58
331
|
@logger.warn msg
|
59
|
-
break if fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
|
332
|
+
break if @fail_mode !~ /slow/ #all failure modes except slow cause us to kick out early on failure
|
60
333
|
end
|
61
334
|
end
|
335
|
+
@test_suite_results.stop_time = Time.now
|
62
336
|
|
63
337
|
# REVISIT: This changes global state, breaking logging in any future runs
|
64
338
|
# of the suite – or, at least, making them highly confusing for anyone who
|
65
339
|
# has not studied the implementation in detail. --daniel 2011-03-14
|
66
|
-
summarize
|
67
|
-
write_junit_xml
|
340
|
+
@test_suite_results.summarize( Logger.new(log_path(@timestamp, "#{name}-summary.txt", @options[:log_dir]), STDOUT) )
|
341
|
+
@test_suite_results.write_junit_xml( log_path(@timestamp, @options[:xml_file], @options[:xml_dir]), File.join(@options[:project_root], @options[:xml_stylesheet]) )
|
342
|
+
|
343
|
+
#All done with this run, remove run log
|
344
|
+
@logger.remove_destination(run_log)
|
68
345
|
|
69
346
|
# Allow chaining operations...
|
70
347
|
return self
|
71
348
|
end
|
72
349
|
|
350
|
+
#Execute all the TestCases in this suite.
|
351
|
+
#This is a wrapper that catches any failures generated during TestSuite::run.
|
73
352
|
def run_and_raise_on_failure
|
74
353
|
begin
|
75
354
|
run
|
76
|
-
return self if success?
|
355
|
+
return self if @test_suite_results.success?
|
77
356
|
rescue => e
|
78
357
|
#failed during run
|
79
358
|
report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
|
@@ -83,187 +362,33 @@ module Beaker
|
|
83
362
|
end
|
84
363
|
end
|
85
364
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
def
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
end
|
106
|
-
|
107
|
-
def errored_tests
|
108
|
-
@errored_tests ||= @test_cases.select { |c| c.test_status == :error }.length
|
109
|
-
end
|
110
|
-
|
111
|
-
def failed_tests
|
112
|
-
@failed_tests ||= @test_cases.select { |c| c.test_status == :fail }.length
|
113
|
-
end
|
114
|
-
|
115
|
-
def skipped_tests
|
116
|
-
@skipped_tests ||= @test_cases.select { |c| c.test_status == :skip }.length
|
117
|
-
end
|
118
|
-
|
119
|
-
def pending_tests
|
120
|
-
@pending_tests ||= @test_cases.select {|c| c.test_status == :pending}.length
|
121
|
-
end
|
122
|
-
|
123
|
-
private
|
124
|
-
|
125
|
-
def sum_failed
|
126
|
-
@sum_failed ||= failed_tests + errored_tests
|
127
|
-
end
|
128
|
-
|
129
|
-
def write_junit_xml
|
130
|
-
# This should be a configuration option
|
131
|
-
File.directory?('junit') or FileUtils.mkdir('junit')
|
132
|
-
|
133
|
-
begin
|
134
|
-
doc = REXML::Document.new
|
135
|
-
doc.add(REXML::XMLDecl.new(1.0))
|
136
|
-
|
137
|
-
suite = REXML::Element.new('testsuite', doc)
|
138
|
-
suite.add_attribute('name', name)
|
139
|
-
suite.add_attribute('tests', test_count)
|
140
|
-
suite.add_attribute('errors', errored_tests)
|
141
|
-
suite.add_attribute('failures', failed_tests)
|
142
|
-
suite.add_attribute('skip', skipped_tests)
|
143
|
-
suite.add_attribute('pending', pending_tests)
|
144
|
-
|
145
|
-
@test_cases.each do |test|
|
146
|
-
item = REXML::Element.new('testcase', suite)
|
147
|
-
item.add_attribute('classname', File.dirname(test.path))
|
148
|
-
item.add_attribute('name', File.basename(test.path))
|
149
|
-
item.add_attribute('time', test.runtime)
|
150
|
-
|
151
|
-
# Did we fail? If so, report that.
|
152
|
-
# We need to remove the escape character from colorized text, the
|
153
|
-
# substitution of other entities is handled well by Rexml
|
154
|
-
if test.test_status == :fail || test.test_status == :error then
|
155
|
-
status = REXML::Element.new('failure', item)
|
156
|
-
status.add_attribute('type', test.test_status.to_s)
|
157
|
-
if test.exception then
|
158
|
-
status.add_attribute('message', test.exception.to_s.gsub(/\e/, ''))
|
159
|
-
status.text = test.exception.backtrace.join("\n")
|
160
|
-
end
|
161
|
-
end
|
162
|
-
|
163
|
-
if test.stdout then
|
164
|
-
REXML::Element.new('system-out', item).text =
|
165
|
-
test.stdout.gsub(/\e/, '')
|
166
|
-
end
|
167
|
-
|
168
|
-
if test.stderr then
|
169
|
-
text = REXML::Element.new('system-err', item)
|
170
|
-
text.text = test.stderr.gsub(/\e/, '')
|
171
|
-
end
|
172
|
-
end
|
173
|
-
|
174
|
-
# junit/name.xml will be created in a directory relative to the CWD
|
175
|
-
# -- JLS 2/12
|
176
|
-
File.open("junit/#{name}.xml", 'w') { |fh| doc.write(fh) }
|
177
|
-
rescue Exception => e
|
178
|
-
@logger.error "failure in XML output:\n#{e.to_s}\n" + e.backtrace.join("\n")
|
179
|
-
end
|
180
|
-
end
|
181
|
-
|
182
|
-
def summarize
|
183
|
-
fail_without_test_run
|
184
|
-
|
185
|
-
summary_logger = Logger.new(log_path("#{name}-summary.txt"), STDOUT)
|
186
|
-
|
187
|
-
summary_logger.notify <<-HEREDOC
|
188
|
-
Test Suite: #{name} @ #{@start_time}
|
189
|
-
|
190
|
-
- Host Configuration Summary -
|
191
|
-
HEREDOC
|
192
|
-
|
193
|
-
elapsed_time = @test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
|
194
|
-
average_test_time = elapsed_time / test_count
|
195
|
-
|
196
|
-
summary_logger.notify %Q[
|
197
|
-
|
198
|
-
- Test Case Summary for suite '#{name}' -
|
199
|
-
Total Suite Time: %.2f seconds
|
200
|
-
Average Test Time: %.2f seconds
|
201
|
-
Attempted: #{test_count}
|
202
|
-
Passed: #{passed_tests}
|
203
|
-
Failed: #{failed_tests}
|
204
|
-
Errored: #{errored_tests}
|
205
|
-
Skipped: #{skipped_tests}
|
206
|
-
Pending: #{pending_tests}
|
207
|
-
|
208
|
-
- Specific Test Case Status -
|
209
|
-
] % [elapsed_time, average_test_time]
|
210
|
-
|
211
|
-
grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
|
212
|
-
|
213
|
-
summary_logger.notify "Failed Tests Cases:"
|
214
|
-
(grouped_summary[:fail] || []).each do |test_case|
|
215
|
-
print_test_failure(test_case)
|
216
|
-
end
|
217
|
-
|
218
|
-
summary_logger.notify "Errored Tests Cases:"
|
219
|
-
(grouped_summary[:error] || []).each do |test_case|
|
220
|
-
print_test_failure(test_case)
|
221
|
-
end
|
222
|
-
|
223
|
-
summary_logger.notify "Skipped Tests Cases:"
|
224
|
-
(grouped_summary[:skip] || []).each do |test_case|
|
225
|
-
print_test_failure(test_case)
|
226
|
-
end
|
227
|
-
|
228
|
-
summary_logger.notify "Pending Tests Cases:"
|
229
|
-
(grouped_summary[:pending] || []).each do |test_case|
|
230
|
-
print_test_failure(test_case)
|
231
|
-
end
|
232
|
-
|
233
|
-
summary_logger.notify("\n\n")
|
234
|
-
end
|
235
|
-
|
236
|
-
def print_test_failure(test_case)
|
237
|
-
test_reported = if test_case.exception
|
238
|
-
"reported: #{test_case.exception.inspect}"
|
239
|
-
else
|
240
|
-
test_case.test_status
|
241
|
-
end
|
242
|
-
@logger.notify " Test Case #{test_case.path} #{test_reported}"
|
243
|
-
end
|
244
|
-
|
245
|
-
def log_path(name)
|
246
|
-
@@log_dir ||= File.join("log", @start_time.strftime("%F_%H_%M_%S"))
|
247
|
-
unless File.directory?(@@log_dir) then
|
248
|
-
FileUtils.mkdir_p(@@log_dir)
|
249
|
-
|
250
|
-
latest = File.join("log", "latest")
|
365
|
+
#Create a full file path for output to be written to, using the provided timestamp, name and output directory.
|
366
|
+
#@param [Time] timestamp The time that we are making this path with
|
367
|
+
#@param [String] name The file name that we want to write to.
|
368
|
+
#@param [String] basedir The desired output directory. A subdirectory tagged with the time will be created which will contain
|
369
|
+
# the output file (./basedir/timestamp/).
|
370
|
+
# A symlink will be made from that to ./basedir/timestamp/latest.
|
371
|
+
#@example
|
372
|
+
# log_path('2014-06-02 16:31:22 -0700','output.txt', 'log')
|
373
|
+
#
|
374
|
+
# This will create the structure:
|
375
|
+
#
|
376
|
+
# ./log/2014-06-02_16_31_22/output.txt
|
377
|
+
# ./log/latest -> 2014-06-02_16_31_22
|
378
|
+
def log_path(timestamp, name, basedir)
|
379
|
+
log_dir = File.join(basedir, timestamp.strftime("%F_%H_%M_%S"))
|
380
|
+
unless File.directory?(log_dir) then
|
381
|
+
FileUtils.mkdir_p(log_dir)
|
382
|
+
|
383
|
+
latest = File.join(basedir, "latest")
|
251
384
|
if !File.exist?(latest) or File.symlink?(latest) then
|
252
385
|
File.delete(latest) if File.exist?(latest)
|
253
|
-
File.symlink(File.basename(
|
386
|
+
File.symlink(File.basename(log_dir), latest)
|
254
387
|
end
|
255
388
|
end
|
256
389
|
|
257
|
-
File.join(
|
390
|
+
File.join(basedir, 'latest', name)
|
258
391
|
end
|
259
392
|
|
260
|
-
# Setup log dir
|
261
|
-
def configure_logging
|
262
|
-
@logger.add_destination(log_path("#{@name}-run.log"))
|
263
|
-
#
|
264
|
-
# This is an awful hack to maintain backward compatibility until tests
|
265
|
-
# are ported to use logger.
|
266
|
-
Beaker.const_set(:Log, @logger) unless defined?( Log )
|
267
|
-
end
|
268
393
|
end
|
269
394
|
end
|