beaker 4.40.2 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +9 -0
- data/.github/workflows/release.yml +2 -2
- data/.github/workflows/test.yml +26 -13
- data/.rubocop.yml +23 -10
- data/.rubocop_todo.yml +34 -10
- data/CHANGELOG.md +17 -28
- data/Gemfile +12 -7
- data/Rakefile +99 -111
- data/acceptance/config/acceptance-options.rb +1 -1
- data/acceptance/config/base/acceptance-options.rb +2 -2
- data/acceptance/config/hypervisor/acceptance-options.rb +2 -2
- data/acceptance/config/subcommands/acceptance-options.rb +2 -2
- data/acceptance/fixtures/module/Rakefile +1 -1
- data/acceptance/fixtures/module/spec/acceptance/demo_spec.rb +8 -12
- data/acceptance/fixtures/module/spec/classes/init_spec.rb +0 -1
- data/acceptance/fixtures/module/spec/spec_helper_acceptance.rb +3 -5
- data/acceptance/lib/helpers/test_helper.rb +6 -5
- data/acceptance/pre_suite/subcommands/05_install_ruby.rb +2 -3
- data/acceptance/pre_suite/subcommands/08_install_beaker.rb +1 -2
- data/acceptance/tests/base/dsl/helpers/configuration_test.rb +4 -4
- data/acceptance/tests/base/dsl/helpers/hocon_helpers_test.rb +1 -2
- data/acceptance/tests/base/dsl/helpers/host_helpers/add_system32_hosts_entry_test.rb +0 -3
- data/acceptance/tests/base/dsl/helpers/host_helpers/archive_file_from_test.rb +3 -1
- data/acceptance/tests/base/dsl/helpers/host_helpers/backup_the_file_test.rb +9 -9
- data/acceptance/tests/base/dsl/helpers/host_helpers/check_for_package_test.rb +0 -4
- data/acceptance/tests/base/dsl/helpers/host_helpers/create_remote_file_test.rb +19 -26
- data/acceptance/tests/base/dsl/helpers/host_helpers/curl_on_test.rb +2 -2
- data/acceptance/tests/base/dsl/helpers/host_helpers/curl_with_retries_test.rb +0 -1
- data/acceptance/tests/base/dsl/helpers/host_helpers/on_test.rb +18 -13
- data/acceptance/tests/base/dsl/helpers/host_helpers/retry_on_test.rb +3 -3
- data/acceptance/tests/base/dsl/helpers/host_helpers/rsync_to_test.rb +22 -27
- data/acceptance/tests/base/dsl/helpers/host_helpers/run_cron_on_test.rb +12 -16
- data/acceptance/tests/base/dsl/helpers/host_helpers/run_script_on_test.rb +3 -4
- data/acceptance/tests/base/dsl/helpers/host_helpers/run_script_test.rb +3 -3
- data/acceptance/tests/base/dsl/helpers/host_helpers/scp_from_test.rb +4 -4
- data/acceptance/tests/base/dsl/helpers/host_helpers/scp_to_test.rb +3 -4
- data/acceptance/tests/base/dsl/helpers/host_helpers/shell_test.rb +5 -5
- data/acceptance/tests/base/dsl/helpers/host_helpers/upgrade_package_test.rb +3 -5
- data/acceptance/tests/base/dsl/platform_tag_confiner_test.rb +14 -17
- data/acceptance/tests/base/dsl/structure_test.rb +5 -11
- data/acceptance/tests/base/host/file_test.rb +2 -2
- data/acceptance/tests/base/host/group_test.rb +0 -1
- data/acceptance/tests/base/host/host_test.rb +66 -63
- data/acceptance/tests/base/host/packages.rb +1 -2
- data/acceptance/tests/base/host/packages_unix.rb +0 -55
- data/acceptance/tests/base/host/user_test.rb +0 -1
- data/acceptance/tests/base/host_prebuilt_steps/ssh_environment_test.rb +1 -2
- data/acceptance/tests/base/test_suite/export.rb +6 -9
- data/acceptance/tests/install/from_file.rb +2 -4
- data/acceptance/tests/load_path_bootstrap.rb +1 -1
- data/acceptance/tests/subcommands/destroy.rb +19 -21
- data/acceptance/tests/subcommands/exec.rb +0 -1
- data/acceptance/tests/subcommands/init.rb +2 -3
- data/acceptance/tests/subcommands/provision.rb +0 -1
- data/beaker.gemspec +4 -7
- data/docs/concepts/argument_processing_and_precedence.md +1 -10
- data/docs/how_to/debug_beaker_tests.md +12 -12
- data/docs/how_to/hosts/eos.md +2 -12
- data/docs/how_to/install_puppet.md +0 -18
- data/docs/how_to/the_beaker_dsl.md +0 -2
- data/lib/beaker/cli.rb +59 -68
- data/lib/beaker/command.rb +20 -28
- data/lib/beaker/command_factory.rb +3 -2
- data/lib/beaker/dsl/assertions.rb +6 -18
- data/lib/beaker/dsl/helpers/hocon_helpers.rb +3 -7
- data/lib/beaker/dsl/helpers/host_helpers.rb +62 -123
- data/lib/beaker/dsl/helpers/test_helpers.rb +3 -5
- data/lib/beaker/dsl/helpers/web_helpers.rb +19 -39
- data/lib/beaker/dsl/helpers.rb +2 -4
- data/lib/beaker/dsl/outcomes.rb +13 -15
- data/lib/beaker/dsl/patterns.rb +1 -3
- data/lib/beaker/dsl/roles.rb +17 -20
- data/lib/beaker/dsl/structure.rb +53 -65
- data/lib/beaker/dsl/test_tagging.rb +7 -10
- data/lib/beaker/dsl/wrappers.rb +15 -16
- data/lib/beaker/dsl.rb +2 -3
- data/lib/beaker/host/aix/exec.rb +1 -1
- data/lib/beaker/host/aix/file.rb +0 -1
- data/lib/beaker/host/aix/group.rb +1 -1
- data/lib/beaker/host/aix/user.rb +1 -1
- data/lib/beaker/host/aix.rb +3 -4
- data/lib/beaker/host/cisco.rb +27 -39
- data/lib/beaker/host/eos.rb +4 -30
- data/lib/beaker/host/freebsd/exec.rb +1 -1
- data/lib/beaker/host/freebsd/pkg.rb +3 -3
- data/lib/beaker/host/freebsd.rb +9 -12
- data/lib/beaker/host/mac/exec.rb +4 -4
- data/lib/beaker/host/mac/group.rb +7 -7
- data/lib/beaker/host/mac/pkg.rb +3 -106
- data/lib/beaker/host/mac/user.rb +2 -2
- data/lib/beaker/host/mac.rb +8 -9
- data/lib/beaker/host/pswindows/exec.rb +66 -70
- data/lib/beaker/host/pswindows/file.rb +3 -5
- data/lib/beaker/host/pswindows/group.rb +3 -3
- data/lib/beaker/host/pswindows/pkg.rb +12 -12
- data/lib/beaker/host/pswindows/user.rb +3 -3
- data/lib/beaker/host/pswindows.rb +4 -3
- data/lib/beaker/host/unix/exec.rb +80 -82
- data/lib/beaker/host/unix/file.rb +38 -43
- data/lib/beaker/host/unix/group.rb +1 -1
- data/lib/beaker/host/unix/pkg.rb +154 -417
- data/lib/beaker/host/unix/user.rb +2 -2
- data/lib/beaker/host/unix.rb +8 -11
- data/lib/beaker/host/windows/exec.rb +17 -17
- data/lib/beaker/host/windows/file.rb +3 -3
- data/lib/beaker/host/windows/group.rb +3 -3
- data/lib/beaker/host/windows/pkg.rb +3 -54
- data/lib/beaker/host/windows/user.rb +3 -3
- data/lib/beaker/host/windows.rb +12 -12
- data/lib/beaker/host.rb +76 -133
- data/lib/beaker/host_prebuilt_steps.rb +93 -198
- data/lib/beaker/hypervisor/noop.rb +2 -4
- data/lib/beaker/hypervisor.rb +44 -61
- data/lib/beaker/local_connection.rb +2 -4
- data/lib/beaker/logger.rb +68 -76
- data/lib/beaker/logger_junit.rb +21 -25
- data/lib/beaker/network_manager.rb +39 -42
- data/lib/beaker/options/command_line_parser.rb +12 -23
- data/lib/beaker/options/hosts_file_parser.rb +16 -24
- data/lib/beaker/options/options_file_parser.rb +3 -6
- data/lib/beaker/options/options_hash.rb +2 -7
- data/lib/beaker/options/parser.rb +86 -102
- data/lib/beaker/options/presets.rb +114 -123
- data/lib/beaker/options/subcommand_options_file_parser.rb +3 -6
- data/lib/beaker/options/validator.rb +26 -31
- data/lib/beaker/perf.rb +22 -27
- data/lib/beaker/platform.rb +38 -46
- data/lib/beaker/result.rb +7 -6
- data/lib/beaker/shared/error_handler.rb +8 -10
- data/lib/beaker/shared/fog_credentials.rb +5 -9
- data/lib/beaker/shared/host_manager.rb +36 -41
- data/lib/beaker/shared/options_resolver.rb +3 -7
- data/lib/beaker/shared/repetition.rb +2 -4
- data/lib/beaker/shared/semvar.rb +37 -41
- data/lib/beaker/shared/timed.rb +0 -3
- data/lib/beaker/shared.rb +1 -1
- data/lib/beaker/ssh_connection.rb +38 -47
- data/lib/beaker/subcommand.rb +17 -24
- data/lib/beaker/subcommands/subcommand_util.rb +4 -4
- data/lib/beaker/tasks/quick_start.rb +4 -9
- data/lib/beaker/tasks/rake_task.rb +25 -27
- data/lib/beaker/tasks/test.rb +4 -4
- data/lib/beaker/test_case.rb +16 -28
- data/lib/beaker/test_suite.rb +35 -39
- data/lib/beaker/test_suite_result.rb +45 -47
- data/lib/beaker/version.rb +1 -1
- data/lib/beaker.rb +6 -7
- data/spec/beaker/cli_spec.rb +121 -142
- data/spec/beaker/command_spec.rb +55 -59
- data/spec/beaker/dsl/assertions_spec.rb +36 -36
- data/spec/beaker/dsl/helpers/host_helpers_spec.rb +110 -131
- data/spec/beaker/dsl/helpers/test_helpers_spec.rb +9 -10
- data/spec/beaker/dsl/helpers/web_helpers_spec.rb +22 -31
- data/spec/beaker/dsl/outcomes_spec.rb +14 -14
- data/spec/beaker/dsl/roles_spec.rb +125 -130
- data/spec/beaker/dsl/structure_spec.rb +172 -161
- data/spec/beaker/dsl/test_tagging_spec.rb +89 -90
- data/spec/beaker/dsl/wrappers_spec.rb +32 -33
- data/spec/beaker/host/aix_spec.rb +14 -14
- data/spec/beaker/host/cisco_spec.rb +84 -94
- data/spec/beaker/host/eos_spec.rb +15 -36
- data/spec/beaker/host/freebsd/exec_spec.rb +3 -6
- data/spec/beaker/host/freebsd/pkg_spec.rb +24 -27
- data/spec/beaker/host/mac/exec_spec.rb +2 -3
- data/spec/beaker/host/mac/group_spec.rb +48 -57
- data/spec/beaker/host/mac/user_spec.rb +54 -63
- data/spec/beaker/host/pswindows/exec_spec.rb +30 -32
- data/spec/beaker/host/pswindows/file_spec.rb +16 -16
- data/spec/beaker/host/pswindows/user_spec.rb +17 -23
- data/spec/beaker/host/pswindows_spec.rb +13 -13
- data/spec/beaker/host/unix/exec_spec.rb +75 -80
- data/spec/beaker/host/unix/file_spec.rb +66 -73
- data/spec/beaker/host/unix/pkg_spec.rb +155 -401
- data/spec/beaker/host/unix_spec.rb +11 -207
- data/spec/beaker/host/windows/exec_spec.rb +30 -32
- data/spec/beaker/host/windows/file_spec.rb +18 -19
- data/spec/beaker/host/windows/group_spec.rb +10 -12
- data/spec/beaker/host/windows/pkg_spec.rb +6 -9
- data/spec/beaker/host/windows/user_spec.rb +17 -23
- data/spec/beaker/host/windows_spec.rb +39 -39
- data/spec/beaker/host_prebuilt_steps_spec.rb +172 -326
- data/spec/beaker/host_spec.rb +204 -284
- data/spec/beaker/hypervisor/hypervisor_spec.rb +36 -61
- data/spec/beaker/localhost_connection_spec.rb +10 -11
- data/spec/beaker/logger_junit_spec.rb +19 -30
- data/spec/beaker/logger_spec.rb +153 -136
- data/spec/beaker/network_manager_spec.rb +23 -23
- data/spec/beaker/options/command_line_parser_spec.rb +20 -23
- data/spec/beaker/options/hosts_file_parser_spec.rb +30 -32
- data/spec/beaker/options/options_file_parser_spec.rb +4 -7
- data/spec/beaker/options/options_hash_spec.rb +4 -6
- data/spec/beaker/options/parser_spec.rb +167 -167
- data/spec/beaker/options/presets_spec.rb +7 -9
- data/spec/beaker/options/subcommand_options_parser_spec.rb +13 -14
- data/spec/beaker/options/validator_spec.rb +10 -11
- data/spec/beaker/perf_spec.rb +18 -21
- data/spec/beaker/platform_spec.rb +25 -36
- data/spec/beaker/shared/error_handler_spec.rb +7 -16
- data/spec/beaker/shared/fog_credentials_spec.rb +29 -29
- data/spec/beaker/shared/host_manager_spec.rb +50 -84
- data/spec/beaker/shared/options_resolver_spec.rb +9 -12
- data/spec/beaker/shared/repetition_spec.rb +17 -24
- data/spec/beaker/shared/semvar_spec.rb +21 -26
- data/spec/beaker/ssh_connection_spec.rb +76 -83
- data/spec/beaker/subcommand/subcommand_util_spec.rb +31 -33
- data/spec/beaker/subcommand_spec.rb +75 -77
- data/spec/beaker/test_case_spec.rb +25 -50
- data/spec/beaker/test_suite_spec.rb +147 -154
- data/spec/helpers.rb +48 -53
- data/spec/matchers.rb +8 -7
- data/spec/mocks.rb +1 -6
- data/spec/spec_helper.rb +0 -1
- metadata +7 -24
- data/acceptance/tests/base/dsl/helpers/host_helpers/deploy_package_repo_test.rb +0 -142
- data/acceptance/tests/base/external_resources_test.rb +0 -31
- data/spec/beaker/host/mac_spec.rb +0 -113
data/lib/beaker/test_suite.rb
CHANGED
|
@@ -1,29 +1,27 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
require 'fileutils'
|
|
3
|
-
[
|
|
2
|
+
%w[test_case logger test_suite_result].each do |lib|
|
|
4
3
|
require "beaker/#{lib}"
|
|
5
4
|
end
|
|
6
5
|
|
|
7
6
|
module Beaker
|
|
8
|
-
#A collection of {TestCase} objects are considered a {TestSuite}.
|
|
9
|
-
#Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
|
|
7
|
+
# A collection of {TestCase} objects are considered a {TestSuite}.
|
|
8
|
+
# Handles executing the set of {TestCase} instances and reporting results as post summary text and JUnit XML.
|
|
10
9
|
class TestSuite
|
|
11
|
-
|
|
12
10
|
attr_reader :name, :options, :fail_mode
|
|
13
11
|
|
|
14
|
-
#Create {TestSuite} instance
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def initialize(name, hosts, options, timestamp, fail_mode=nil)
|
|
12
|
+
# Create {TestSuite} instance
|
|
13
|
+
# @param [String] name The name of the {TestSuite}
|
|
14
|
+
# @param [Array<Host>] hosts An Array of Hosts to act upon.
|
|
15
|
+
# @param [Hash{Symbol=>String}] options Options for this object
|
|
16
|
+
# @option options [Logger] :logger The Logger object to report information to
|
|
17
|
+
# @option options [String] :log_dir The directory where text run logs will be written
|
|
18
|
+
# @option options [String] :xml_dir The directory where JUnit XML file will be written
|
|
19
|
+
# @option options [String] :xml_file The name of the JUnit XML file to be written to
|
|
20
|
+
# @option options [String] :project_root The full path to the Beaker lib directory
|
|
21
|
+
# @option options [String] :xml_stylesheet The path to a stylesheet to be applied to the generated XML output
|
|
22
|
+
# @param [Symbol] fail_mode One of :slow, :fast
|
|
23
|
+
# @param [Time] timestamp Beaker execution start time
|
|
24
|
+
def initialize(name, hosts, options, timestamp, fail_mode = nil)
|
|
27
25
|
@logger = options[:logger]
|
|
28
26
|
@test_cases = []
|
|
29
27
|
@test_files = options[name]
|
|
@@ -36,26 +34,25 @@ module Beaker
|
|
|
36
34
|
@timestamp = timestamp
|
|
37
35
|
|
|
38
36
|
report_and_raise(@logger, RuntimeError.new("#{@name}: no test files found..."), "TestSuite: initialize") if @test_files.empty?
|
|
39
|
-
|
|
40
37
|
rescue => e
|
|
41
38
|
report_and_raise(@logger, e, "TestSuite: initialize")
|
|
42
39
|
end
|
|
43
40
|
|
|
44
|
-
#Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
|
|
45
|
-
#is reported to a newly created run log.
|
|
46
|
-
#Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
|
|
47
|
-
#after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
|
|
41
|
+
# Execute all the {TestCase} instances and then report the results as both plain text and xml. The text result
|
|
42
|
+
# is reported to a newly created run log.
|
|
43
|
+
# Execution is dependent upon the fail_mode. If mode is :fast then stop running any additional {TestCase} instances
|
|
44
|
+
# after first failure, if mode is :slow continue execution no matter what {TestCase} results are.
|
|
48
45
|
def run
|
|
49
46
|
@run = true
|
|
50
47
|
start_time = Time.now
|
|
51
48
|
|
|
52
|
-
#Create a run log for this TestSuite.
|
|
49
|
+
# Create a run log for this TestSuite.
|
|
53
50
|
run_log = log_path("#{@name}-run.log", @options[:log_dated_dir])
|
|
54
51
|
@logger.add_destination(run_log)
|
|
55
52
|
|
|
56
53
|
# This is an awful hack to maintain backward compatibility until tests
|
|
57
54
|
# are ported to use logger. Still in use in PuppetDB tests
|
|
58
|
-
Beaker.const_set(:Log, @logger) unless defined?(
|
|
55
|
+
Beaker.const_set(:Log, @logger) unless defined?(Log)
|
|
59
56
|
|
|
60
57
|
@test_suite_results.start_time = start_time
|
|
61
58
|
@test_suite_results.total_tests = @test_files.length
|
|
@@ -77,10 +74,10 @@ module Beaker
|
|
|
77
74
|
@logger.warn msg
|
|
78
75
|
when :fail
|
|
79
76
|
@logger.error msg
|
|
80
|
-
break if !@fail_mode.to_s.include?('slow') #all failure modes except slow cause us to kick out early on failure
|
|
77
|
+
break if !@fail_mode.to_s.include?('slow') # all failure modes except slow cause us to kick out early on failure
|
|
81
78
|
when :error
|
|
82
79
|
@logger.warn msg
|
|
83
|
-
break if !@fail_mode.to_s.include?('slow') #all failure modes except slow cause us to kick out early on failure
|
|
80
|
+
break if !@fail_mode.to_s.include?('slow') # all failure modes except slow cause us to kick out early on failure
|
|
84
81
|
end
|
|
85
82
|
end
|
|
86
83
|
@test_suite_results.stop_time = Time.now
|
|
@@ -88,37 +85,37 @@ module Beaker
|
|
|
88
85
|
# REVISIT: This changes global state, breaking logging in any future runs
|
|
89
86
|
# of the suite – or, at least, making them highly confusing for anyone who
|
|
90
87
|
# has not studied the implementation in detail. --daniel 2011-03-14
|
|
91
|
-
@test_suite_results.summarize(
|
|
88
|
+
@test_suite_results.summarize(Logger.new(log_path("#{name}-summary.txt", @options[:log_dated_dir]), STDOUT))
|
|
92
89
|
|
|
93
|
-
junit_file_log
|
|
90
|
+
junit_file_log = log_path(@options[:xml_file], @options[:xml_dated_dir])
|
|
94
91
|
if @options[:xml_time_enabled]
|
|
95
92
|
junit_file_time = log_path(@options[:xml_time], @options[:xml_dated_dir])
|
|
96
|
-
@test_suite_results.write_junit_xml(
|
|
97
|
-
@test_suite_results.write_junit_xml(
|
|
93
|
+
@test_suite_results.write_junit_xml(junit_file_log, @options[:xml_time])
|
|
94
|
+
@test_suite_results.write_junit_xml(junit_file_time, @options[:xml_file], true)
|
|
98
95
|
else
|
|
99
|
-
@test_suite_results.write_junit_xml(
|
|
96
|
+
@test_suite_results.write_junit_xml(junit_file_log)
|
|
100
97
|
end
|
|
101
98
|
|
|
102
99
|
@test_suite_results.persist_test_results(@options[:test_results_file])
|
|
103
100
|
|
|
104
|
-
#All done with this run, remove run log
|
|
101
|
+
# All done with this run, remove run log
|
|
105
102
|
@logger.remove_destination(run_log)
|
|
106
103
|
|
|
107
104
|
# Allow chaining operations...
|
|
108
105
|
return self
|
|
109
106
|
end
|
|
110
107
|
|
|
111
|
-
#Execute all the TestCases in this suite.
|
|
112
|
-
#This is a wrapper that catches any failures generated during TestSuite::run.
|
|
108
|
+
# Execute all the TestCases in this suite.
|
|
109
|
+
# This is a wrapper that catches any failures generated during TestSuite::run.
|
|
113
110
|
def run_and_raise_on_failure
|
|
114
111
|
begin
|
|
115
112
|
run
|
|
116
113
|
return self if @test_suite_results.success?
|
|
117
114
|
rescue => e
|
|
118
|
-
#failed during run
|
|
115
|
+
# failed during run
|
|
119
116
|
report_and_raise(@logger, e, "TestSuite :run_and_raise_on_failure")
|
|
120
117
|
else
|
|
121
|
-
#failed during test
|
|
118
|
+
# failed during test
|
|
122
119
|
report_and_raise(@logger, RuntimeError.new("Failed while running the #{name} suite"), "TestSuite: report_and_raise_on_failure")
|
|
123
120
|
end
|
|
124
121
|
end
|
|
@@ -147,7 +144,7 @@ module Beaker
|
|
|
147
144
|
|
|
148
145
|
base_dir = log_dir
|
|
149
146
|
link_dir = ''
|
|
150
|
-
while File.dirname(base_dir) != '.'
|
|
147
|
+
while File.dirname(base_dir) != '.'
|
|
151
148
|
link_dir = link_dir == '' ? File.basename(base_dir) : File.join(File.basename(base_dir), link_dir)
|
|
152
149
|
base_dir = File.dirname(base_dir)
|
|
153
150
|
end
|
|
@@ -160,6 +157,5 @@ module Beaker
|
|
|
160
157
|
|
|
161
158
|
File.join(log_dir, name)
|
|
162
159
|
end
|
|
163
|
-
|
|
164
160
|
end
|
|
165
161
|
end
|
|
@@ -1,94 +1,92 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
require 'fileutils'
|
|
3
|
-
[
|
|
2
|
+
%w[test_case logger test_suite logger_junit].each do |lib|
|
|
4
3
|
require "beaker/#{lib}"
|
|
5
4
|
end
|
|
6
5
|
|
|
7
6
|
module Beaker
|
|
8
|
-
#Holds the output of a test suite, formats in plain text or xml
|
|
7
|
+
# Holds the output of a test suite, formats in plain text or xml
|
|
9
8
|
class TestSuiteResult
|
|
10
9
|
attr_accessor :start_time, :stop_time, :total_tests
|
|
11
10
|
|
|
12
|
-
#Create a {TestSuiteResult} instance.
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def initialize(
|
|
11
|
+
# Create a {TestSuiteResult} instance.
|
|
12
|
+
# @param [Hash{Symbol=>String}] options Options for this object
|
|
13
|
+
# @option options [Logger] :logger The Logger object to report information to
|
|
14
|
+
# @param [String] name The name of the {TestSuite} that the results are for
|
|
15
|
+
def initialize(options, name)
|
|
17
16
|
@options = options
|
|
18
17
|
@logger = options[:logger]
|
|
19
18
|
@name = name
|
|
20
19
|
@test_cases = []
|
|
21
20
|
end
|
|
22
21
|
|
|
23
|
-
#Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
|
|
24
|
-
|
|
25
|
-
def add_test_case(
|
|
22
|
+
# Add a {TestCase} to this {TestSuiteResult} instance, used in calculating {TestSuiteResult} data.
|
|
23
|
+
# @param [TestCase] test_case An individual, completed {TestCase} to be included in this set of {TestSuiteResult}.
|
|
24
|
+
def add_test_case(test_case)
|
|
26
25
|
@test_cases << test_case
|
|
27
26
|
end
|
|
28
27
|
|
|
29
|
-
#How many {TestCase} instances are in this {TestSuiteResult}
|
|
28
|
+
# How many {TestCase} instances are in this {TestSuiteResult}
|
|
30
29
|
def test_count
|
|
31
30
|
@test_cases.length
|
|
32
31
|
end
|
|
33
32
|
|
|
34
|
-
#How many passed {TestCase} instances are in this {TestSuiteResult}
|
|
33
|
+
# How many passed {TestCase} instances are in this {TestSuiteResult}
|
|
35
34
|
def passed_tests
|
|
36
35
|
@test_cases.count { |c| c.test_status == :pass }
|
|
37
36
|
end
|
|
38
37
|
|
|
39
|
-
#How many errored {TestCase} instances are in this {TestSuiteResult}
|
|
38
|
+
# How many errored {TestCase} instances are in this {TestSuiteResult}
|
|
40
39
|
def errored_tests
|
|
41
40
|
@test_cases.count { |c| c.test_status == :error }
|
|
42
41
|
end
|
|
43
42
|
|
|
44
|
-
#How many failed {TestCase} instances are in this {TestSuiteResult}
|
|
43
|
+
# How many failed {TestCase} instances are in this {TestSuiteResult}
|
|
45
44
|
def failed_tests
|
|
46
45
|
@test_cases.count { |c| c.test_status == :fail }
|
|
47
46
|
end
|
|
48
47
|
|
|
49
|
-
#How many skipped {TestCase} instances are in this {TestSuiteResult}
|
|
48
|
+
# How many skipped {TestCase} instances are in this {TestSuiteResult}
|
|
50
49
|
def skipped_tests
|
|
51
50
|
@test_cases.count { |c| c.test_status == :skip }
|
|
52
51
|
end
|
|
53
52
|
|
|
54
|
-
#How many pending {TestCase} instances are in this {TestSuiteResult}
|
|
53
|
+
# How many pending {TestCase} instances are in this {TestSuiteResult}
|
|
55
54
|
def pending_tests
|
|
56
|
-
@test_cases.count {|c| c.test_status == :pending}
|
|
55
|
+
@test_cases.count { |c| c.test_status == :pending }
|
|
57
56
|
end
|
|
58
57
|
|
|
59
|
-
#How many {TestCase} instances failed in this {TestSuiteResult}
|
|
58
|
+
# How many {TestCase} instances failed in this {TestSuiteResult}
|
|
60
59
|
def sum_failed
|
|
61
60
|
failed_tests + errored_tests
|
|
62
61
|
end
|
|
63
62
|
|
|
64
|
-
#Did all the {TestCase} instances in this {TestSuiteResult} pass?
|
|
63
|
+
# Did all the {TestCase} instances in this {TestSuiteResult} pass?
|
|
65
64
|
def success?
|
|
66
65
|
sum_failed == 0
|
|
67
66
|
end
|
|
68
67
|
|
|
69
|
-
#Did one or more {TestCase} instances in this {TestSuiteResult} fail?
|
|
68
|
+
# Did one or more {TestCase} instances in this {TestSuiteResult} fail?
|
|
70
69
|
def failed?
|
|
71
70
|
!success?
|
|
72
71
|
end
|
|
73
72
|
|
|
74
|
-
#The sum of all {TestCase} runtimes in this {TestSuiteResult}
|
|
73
|
+
# The sum of all {TestCase} runtimes in this {TestSuiteResult}
|
|
75
74
|
def elapsed_time
|
|
76
|
-
@test_cases.inject(0.0) {|r, t| r + t.runtime.to_f }
|
|
75
|
+
@test_cases.inject(0.0) { |r, t| r + t.runtime.to_f }
|
|
77
76
|
end
|
|
78
77
|
|
|
79
|
-
#Plain text summay of test suite
|
|
80
|
-
|
|
78
|
+
# Plain text summay of test suite
|
|
79
|
+
# @param [Logger] summary_logger The logger we will print the summary to
|
|
81
80
|
def summarize(summary_logger)
|
|
82
|
-
|
|
83
81
|
summary_logger.notify <<-HEREDOC
|
|
84
82
|
Test Suite: #{@name} @ #{start_time}
|
|
85
83
|
|
|
86
84
|
- Host Configuration Summary -
|
|
87
|
-
|
|
85
|
+
HEREDOC
|
|
88
86
|
|
|
89
87
|
average_test_time = elapsed_time / test_count
|
|
90
88
|
|
|
91
|
-
summary_logger.notify %
|
|
89
|
+
summary_logger.notify format(%[
|
|
92
90
|
|
|
93
91
|
- Test Case Summary for suite '#{@name}' -
|
|
94
92
|
Total Suite Time: %.2f seconds
|
|
@@ -102,9 +100,9 @@ module Beaker
|
|
|
102
100
|
Total: #{@total_tests}
|
|
103
101
|
|
|
104
102
|
- Specific Test Case Status -
|
|
105
|
-
]
|
|
103
|
+
], elapsed_time, average_test_time)
|
|
106
104
|
|
|
107
|
-
grouped_summary = @test_cases.group_by{|test_case| test_case.test_status }
|
|
105
|
+
grouped_summary = @test_cases.group_by { |test_case| test_case.test_status }
|
|
108
106
|
|
|
109
107
|
summary_logger.notify "Failed Tests Cases:"
|
|
110
108
|
(grouped_summary[:fail] || []).each do |test_case|
|
|
@@ -129,8 +127,8 @@ module Beaker
|
|
|
129
127
|
summary_logger.notify("\n\n")
|
|
130
128
|
end
|
|
131
129
|
|
|
132
|
-
#A convenience method for printing the results of a {TestCase}
|
|
133
|
-
|
|
130
|
+
# A convenience method for printing the results of a {TestCase}
|
|
131
|
+
# @param [TestCase] test_case The {TestCase} to examine and print results for
|
|
134
132
|
def print_test_result(test_case)
|
|
135
133
|
if test_case.exception
|
|
136
134
|
test_file_trace = ""
|
|
@@ -154,7 +152,7 @@ module Beaker
|
|
|
154
152
|
def persist_test_results(filepath)
|
|
155
153
|
return if filepath.empty?
|
|
156
154
|
|
|
157
|
-
results = @test_cases.select { |c| [
|
|
155
|
+
results = @test_cases.select { |c| %i[fail error].include? c.test_status }.map(&:path)
|
|
158
156
|
File.open(filepath, 'w') { |file| file.puts JSON.dump(results) }
|
|
159
157
|
end
|
|
160
158
|
|
|
@@ -176,44 +174,45 @@ module Beaker
|
|
|
176
174
|
|
|
177
175
|
begin
|
|
178
176
|
LoggerJunit.write_xml(xml_file, stylesheet) do |_doc, suites|
|
|
179
|
-
|
|
180
177
|
meta_info = suites.add_element(REXML::Element.new('meta_test_info'))
|
|
181
|
-
|
|
182
|
-
time_sort ? meta_info.add_attribute('page_active', 'performance') : meta_info.add_attribute('page_active', 'execution')
|
|
183
|
-
meta_info.add_attribute('link_url', file_to_link)
|
|
184
|
-
else
|
|
178
|
+
if file_to_link.nil?
|
|
185
179
|
meta_info.add_attribute('page_active', 'no-links')
|
|
186
180
|
meta_info.add_attribute('link_url', '')
|
|
181
|
+
else
|
|
182
|
+
time_sort ? meta_info.add_attribute('page_active', 'performance') : meta_info.add_attribute('page_active', 'execution')
|
|
183
|
+
meta_info.add_attribute('link_url', file_to_link)
|
|
187
184
|
end
|
|
188
185
|
|
|
189
186
|
suite = suites.add_element(REXML::Element.new('testsuite'))
|
|
190
187
|
suite.add_attributes(
|
|
191
188
|
[
|
|
192
|
-
['name'
|
|
189
|
+
['name', @name],
|
|
193
190
|
['tests', test_count],
|
|
194
191
|
['errors', errored_tests],
|
|
195
192
|
['failures', failed_tests],
|
|
196
193
|
['skipped', skipped_tests],
|
|
197
194
|
['pending', pending_tests],
|
|
198
195
|
['total', @total_tests],
|
|
199
|
-
['time', "%f"
|
|
200
|
-
|
|
196
|
+
['time', format("%f", (stop_time - start_time))],
|
|
197
|
+
],
|
|
198
|
+
)
|
|
201
199
|
properties = suite.add_element(REXML::Element.new('properties'))
|
|
202
|
-
@options.each_pair do |name,value|
|
|
200
|
+
@options.each_pair do |name, value|
|
|
203
201
|
property = properties.add_element(REXML::Element.new('property'))
|
|
204
202
|
property.add_attributes([['name', name], ['value', value.to_s || '']])
|
|
205
203
|
end
|
|
206
204
|
|
|
207
205
|
test_cases_to_report = @test_cases
|
|
208
|
-
test_cases_to_report = @test_cases.sort { |x,y| y.runtime <=> x.runtime } if time_sort
|
|
206
|
+
test_cases_to_report = @test_cases.sort { |x, y| y.runtime <=> x.runtime } if time_sort
|
|
209
207
|
test_cases_to_report.each do |test|
|
|
210
208
|
item = suite.add_element(REXML::Element.new('testcase'))
|
|
211
209
|
item.add_attributes(
|
|
212
210
|
[
|
|
213
211
|
['classname', File.dirname(test.path)],
|
|
214
212
|
['name', File.basename(test.path)],
|
|
215
|
-
['time', "%f" % test.runtime]
|
|
216
|
-
]
|
|
213
|
+
['time', "%f" % test.runtime],
|
|
214
|
+
],
|
|
215
|
+
)
|
|
217
216
|
|
|
218
217
|
test.exports.each do |export|
|
|
219
218
|
export.keys.each do |key|
|
|
@@ -221,7 +220,7 @@ module Beaker
|
|
|
221
220
|
end
|
|
222
221
|
end
|
|
223
222
|
|
|
224
|
-
#Report failures
|
|
223
|
+
# Report failures
|
|
225
224
|
if test.test_status == :fail || test.test_status == :error
|
|
226
225
|
status = item.add_element(REXML::Element.new('failure'))
|
|
227
226
|
status.add_attribute('type', test.test_status.to_s)
|
|
@@ -259,6 +258,5 @@ module Beaker
|
|
|
259
258
|
@logger.error "failure in XML output: \n#{e}" + e.backtrace.join("\n")
|
|
260
259
|
end
|
|
261
260
|
end
|
|
262
|
-
|
|
263
261
|
end
|
|
264
262
|
end
|
data/lib/beaker/version.rb
CHANGED
data/lib/beaker.rb
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
require 'rubygems' unless defined?(Gem)
|
|
2
2
|
module Beaker
|
|
3
|
-
|
|
4
|
-
%w( version platform test_suite test_suite_result result command options network_manager cli perf logger_junit subcommand ).each do |lib|
|
|
3
|
+
%w(version platform test_suite test_suite_result result command options network_manager cli perf logger_junit subcommand).each do |lib|
|
|
5
4
|
begin
|
|
6
5
|
require "beaker/#{lib}"
|
|
7
6
|
rescue LoadError
|
|
8
|
-
require File.expand_path(File.join(
|
|
7
|
+
require File.expand_path(File.join(__dir__, 'beaker', lib))
|
|
9
8
|
end
|
|
10
9
|
end
|
|
11
10
|
# These really are our sub-systems that live within the harness today
|
|
@@ -22,13 +21,13 @@ module Beaker
|
|
|
22
21
|
require 'beaker/hypervisor'
|
|
23
22
|
#
|
|
24
23
|
# How we manage connecting to hosts and hypervisors
|
|
25
|
-
#require 'beaker/connectivity'
|
|
24
|
+
# require 'beaker/connectivity'
|
|
26
25
|
#
|
|
27
26
|
# Our test runner, suite, test cases and steps
|
|
28
|
-
#require 'beaker/runner'
|
|
27
|
+
# require 'beaker/runner'
|
|
29
28
|
#
|
|
30
29
|
# Common setup and testing steps
|
|
31
|
-
#require 'beaker/steps'
|
|
30
|
+
# require 'beaker/steps'
|
|
32
31
|
|
|
33
32
|
# InParallel, for executing in parallel
|
|
34
33
|
require 'in_parallel'
|
|
@@ -36,6 +35,6 @@ module Beaker
|
|
|
36
35
|
# Shared methods and helpers
|
|
37
36
|
require 'beaker/shared'
|
|
38
37
|
|
|
39
|
-
#
|
|
38
|
+
# MiniTest, for including MiniTest::Assertions
|
|
40
39
|
require 'minitest/test'
|
|
41
40
|
end
|