rubytest 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.md +30 -1
- data/README.md +67 -7
- data/demo/03_runner_reliability.md +146 -0
- data/demo/04_result_pipeline.md +160 -0
- data/demo/05_builtin_reporters.md +118 -0
- data/lib/rubytest/cli.rb +6 -2
- data/lib/rubytest/config.rb +1 -1
- data/lib/rubytest/format/abstract.rb +0 -100
- data/lib/rubytest/format/abstract_hash.rb +70 -34
- data/lib/rubytest/format/dotprogress.rb +82 -56
- data/lib/rubytest/format/outline.rb +44 -0
- data/lib/rubytest/format/progress.rb +35 -0
- data/lib/rubytest/format/summary.rb +31 -0
- data/lib/rubytest/format/tap.rb +54 -0
- data/lib/rubytest/format/tapj.rb +76 -0
- data/lib/rubytest/format/tapy.rb +77 -0
- data/lib/rubytest/recorder.rb +36 -29
- data/lib/rubytest/result.rb +54 -0
- data/lib/rubytest/runner.rb +211 -77
- data/lib/rubytest.rb +2 -1
- metadata +12 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 87813efd760c9e9e813d32fe22a048da1c82e47bf580420892942931008039ba
|
|
4
|
+
data.tar.gz: 4f8c2918e8885f5a29f144d1315135ce61a44016711b4289e0419f7b8dfff02e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6d91841470661e6bfdc0131d970723154d6c6ff6eb18da980e4a4b205f43a1f5134192e73fef2430dbbe6f77b71d297d4b25db5c29ff5972a1179e0e3c155075
|
|
7
|
+
data.tar.gz: cc2829ce1797a428386e37cf2ab98a11f575ea0848e215ca3f65f19665d5e36057f0d8fd5ac488746bf43a0df1b7b9a275375791138a8693a0b18ce0021cd8e1
|
data/HISTORY.md
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# RELEASE HISTORY
|
|
2
2
|
|
|
3
|
+
## 0.10.0 / 2026-11-17
|
|
4
|
+
|
|
5
|
+
Results now go through a recorder to the reporters, and the reporters that
|
|
6
|
+
were spun off into separate gems are back in Rubytest, so rubytest-cli,
|
|
7
|
+
rubytest-tap, rubytest-summary and rubytest-progress are no longer needed.
|
|
8
|
+
Supports Ruby 2.7 and later.
|
|
9
|
+
|
|
10
|
+
Changes:
|
|
11
|
+
|
|
12
|
+
* Support Ruby 2.7 and 3.0 again (0.9.0 required 3.1).
|
|
13
|
+
* Report an error raised outside of a test, such as while a test file loads,
|
|
14
|
+
with its class, where it was raised, and a pointer to `--debug`, instead of
|
|
15
|
+
the bare message.
|
|
16
|
+
* Bring summary, outline, and progress reporters into Rubytest and add a TAP 13
|
|
17
|
+
reporter that counts completed results, including skipped and broken cases.
|
|
18
|
+
* Bring the TAP-Y and TAP-J reporters in from rubytest-tap, with their `-y` and
|
|
19
|
+
`-j` shortcuts. The TAP-Y stream now opens with a YAML document (it was
|
|
20
|
+
JSON), the TAP-J stream no longer ends with a `...` line, and both report
|
|
21
|
+
skipped tests and cases.
|
|
22
|
+
* Remove the old Recorder status callbacks and status-indexed result table.
|
|
23
|
+
* Preserve test and teardown exceptions together in a result, and turn outcome
|
|
24
|
+
hook failures into recorded errors.
|
|
25
|
+
* Route structured test results through the recorder to reporters, while
|
|
26
|
+
retaining Advice hooks and older reporter callbacks.
|
|
27
|
+
* Fail empty runs and unmatched test file requests instead of reporting success.
|
|
28
|
+
* Apply test filters at every suite level and record skipped tests and cases.
|
|
29
|
+
* Record case errors, finish suite callbacks, and restore reporter output after
|
|
30
|
+
failures.
|
|
31
|
+
* Fix the CLI's alternate configuration option and `Testfile` discovery.
|
|
32
|
+
|
|
3
33
|
## 0.9.0 / 2026-03-31
|
|
4
34
|
|
|
5
35
|
Maintenance release. Modernized project tooling and merged CLI back in.
|
|
@@ -222,4 +252,3 @@ First release of Ruby Test.
|
|
|
222
252
|
Changes:
|
|
223
253
|
|
|
224
254
|
* It's Your Birthday!
|
|
225
|
-
|
data/README.md
CHANGED
|
@@ -39,6 +39,28 @@ configured to do so.
|
|
|
39
39
|
That is the crux of Rubytest specification. Rubytest supports some
|
|
40
40
|
additional features that can make its usage even more convenient.
|
|
41
41
|
|
|
42
|
+
### Results and reporting
|
|
43
|
+
|
|
44
|
+
The runner turns each completed test into a `Test::Result` with its status,
|
|
45
|
+
exception or skip reason, elapsed time, and captured output when requested.
|
|
46
|
+
If both the test and a hook raise, the result retains both exceptions.
|
|
47
|
+
Skipped and broken cases also produce results. `Test::Recorder` keeps these
|
|
48
|
+
results, calculates the run summary, and forwards reporting events to the
|
|
49
|
+
selected reporter. Advice hooks remain with the runner, where they can affect
|
|
50
|
+
test execution.
|
|
51
|
+
|
|
52
|
+
Reporters implement `record(result)` for completed outcomes and
|
|
53
|
+
`finish(summary)` for the final totals. Suite and case boundary callbacks are
|
|
54
|
+
available for formats that show nesting.
|
|
55
|
+
|
|
56
|
+
The built-in formats are `dotprogress` (default), `summary` (one status line per
|
|
57
|
+
test), `outline` (nested cases), `progress` (running count and timing), `tap`
|
|
58
|
+
(TAP version 13), and `tapy` and `tapj` (TAP-Y and TAP-J, the structured
|
|
59
|
+
streams that [tapout](https://github.com/rubyworks/tapout) reads). Select one
|
|
60
|
+
with `rubytest -f FORMAT`; `-y` and `-j` are short for `-f tapy` and `-f tapj`.
|
|
61
|
+
Formats load only when selected. The TAP formats capture test output: TAP
|
|
62
|
+
emits it as diagnostics, and TAP-Y and TAP-J record it with each test.
|
|
63
|
+
|
|
42
64
|
|
|
43
65
|
## Installation
|
|
44
66
|
|
|
@@ -57,16 +79,54 @@ The easiest way to run tests is via the command line tool:
|
|
|
57
79
|
|
|
58
80
|
$ rubytest -Ilib test/test_*.rb
|
|
59
81
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
This would add `lib` to Ruby's `$LOAD_PATH` and then load all the
|
|
83
|
+
test files matching the glob. When running tests, be sure to load your
|
|
84
|
+
test framework or its adapter, e.g.
|
|
85
|
+
|
|
86
|
+
$ rubytest -r lemon -r ae test/test_*.rb
|
|
87
|
+
|
|
88
|
+
Rubytest exits unsuccessfully if a requested path matches no files or if the
|
|
89
|
+
selection runs no tests.
|
|
90
|
+
|
|
91
|
+
Use `-h/--help` to see all available options.
|
|
92
|
+
|
|
93
|
+
#### Configuration File
|
|
94
|
+
|
|
95
|
+
To avoid repeating options, add an `etc/test.rb` file to your project
|
|
96
|
+
with `Test.run` (or `Test.configure`) entries:
|
|
97
|
+
|
|
98
|
+
```ruby
|
|
99
|
+
Test.run do |r|
|
|
100
|
+
r.loadpath 'lib'
|
|
101
|
+
r.test_files << 'test/*_test.rb'
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
Test.run 'coverage' do |r|
|
|
105
|
+
r.loadpath 'lib'
|
|
106
|
+
r.test_files << 'test/*_test.rb'
|
|
107
|
+
r.before do
|
|
108
|
+
require 'simplecov'
|
|
109
|
+
SimpleCov.start do |s|
|
|
110
|
+
s.filter 'test/'
|
|
111
|
+
s.coverage_dir 'log/coverage'
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Now `rubytest` will use the first configuration by default. To use a
|
|
118
|
+
named profile, use the `-p/--profile` option:
|
|
119
|
+
|
|
120
|
+
$ rubytest -p coverage
|
|
121
|
+
|
|
122
|
+
The configuration file can also be at `config/test.rb`, `Testfile`,
|
|
123
|
+
or `.test` (with optional `.rb` extension).
|
|
63
124
|
|
|
64
|
-
If you are using
|
|
65
|
-
|
|
66
|
-
pristine as possible, e.g.
|
|
125
|
+
If you are using Rake, shelling out to `rubytest` keeps your test
|
|
126
|
+
environment pristine:
|
|
67
127
|
|
|
68
128
|
desc "run tests"
|
|
69
|
-
task :test
|
|
129
|
+
task :test do
|
|
70
130
|
sh "rubytest"
|
|
71
131
|
end
|
|
72
132
|
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
## Runner reliability
|
|
2
|
+
|
|
3
|
+
RubyTest reports an explicitly requested file that does not exist, rather than
|
|
4
|
+
passing an empty run.
|
|
5
|
+
|
|
6
|
+
runner = Test::Runner.new(files: ['__rubytest_missing_file__.rb'], format: 'test')
|
|
7
|
+
cleanup_ran = false
|
|
8
|
+
runner.config.after { cleanup_ran = true }
|
|
9
|
+
missing_file = begin
|
|
10
|
+
runner.run
|
|
11
|
+
nil
|
|
12
|
+
rescue ArgumentError => error
|
|
13
|
+
error
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
missing_file.class.assert == ArgumentError
|
|
17
|
+
missing_file.message.include?('__rubytest_missing_file__.rb').assert == true
|
|
18
|
+
cleanup_ran.assert == true
|
|
19
|
+
|
|
20
|
+
An empty suite also has an unsuccessful result.
|
|
21
|
+
|
|
22
|
+
empty_runner = Test::Runner.new(suite: [], format: 'test')
|
|
23
|
+
empty_runner.run.assert == false
|
|
24
|
+
|
|
25
|
+
### Selection and skips
|
|
26
|
+
|
|
27
|
+
Test descriptions can be matched at the top level and inside cases. A skipped
|
|
28
|
+
test that matches the selection is still reported.
|
|
29
|
+
|
|
30
|
+
class RunnerProbe
|
|
31
|
+
def initialize(description, skip_reason = nil, &block)
|
|
32
|
+
@description = description
|
|
33
|
+
@skip_reason = skip_reason
|
|
34
|
+
@block = block
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def call
|
|
38
|
+
@block.call
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def skip?
|
|
42
|
+
@skip_reason
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def to_s
|
|
46
|
+
@description
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
top = RunnerProbe.new('wanted top') { true }
|
|
51
|
+
other = RunnerProbe.new('other') { true }
|
|
52
|
+
nested = RunnerProbe.new('wanted nested') { true }
|
|
53
|
+
skipped = RunnerProbe.new('wanted skipped', 'later') { raise 'should not run' }
|
|
54
|
+
|
|
55
|
+
selected = Test::Runner.new(suite: [top, other, [nested, skipped]],
|
|
56
|
+
match: ['wanted'], format: 'test')
|
|
57
|
+
selected.run.assert == true
|
|
58
|
+
selected.recorder.summary.counts[:pass].assert == 2
|
|
59
|
+
selected.recorder.summary.counts[:skip].assert == 1
|
|
60
|
+
|
|
61
|
+
Skipping an entire case is recorded too.
|
|
62
|
+
|
|
63
|
+
skipped_case = Class.new(Array) do
|
|
64
|
+
def skip?
|
|
65
|
+
'later'
|
|
66
|
+
end
|
|
67
|
+
end.new
|
|
68
|
+
case_skip_runner = Test::Runner.new(suite: [skipped_case], format: 'test')
|
|
69
|
+
case_skip_runner.run.assert == true
|
|
70
|
+
case_skip_runner.recorder.summary.counts[:skip].assert == 1
|
|
71
|
+
|
|
72
|
+
### Case errors and cleanup
|
|
73
|
+
|
|
74
|
+
A case setup error is recorded, and the next test and suite cleanup still run.
|
|
75
|
+
|
|
76
|
+
broken_case = Class.new(Array) do
|
|
77
|
+
def call
|
|
78
|
+
raise 'setup failed'
|
|
79
|
+
end
|
|
80
|
+
end.new
|
|
81
|
+
following_test = RunnerProbe.new('following test') { true }
|
|
82
|
+
case_runner = Test::Runner.new(suite: [broken_case, following_test], format: 'test')
|
|
83
|
+
suite_ended = false
|
|
84
|
+
case_runner.after(:suite) { suite_ended = true }
|
|
85
|
+
|
|
86
|
+
case_runner.run.assert == false
|
|
87
|
+
case_runner.recorder.summary.counts[:error].assert == 1
|
|
88
|
+
case_runner.recorder.summary.counts[:pass].assert == 1
|
|
89
|
+
suite_ended.assert == true
|
|
90
|
+
|
|
91
|
+
An assertion failure in a hash reporter records the failure and restores
|
|
92
|
+
standard output.
|
|
93
|
+
|
|
94
|
+
original_stdout = $stdout
|
|
95
|
+
failing_test = RunnerProbe.new('failing test') { raise Assertion, 'expected failure' }
|
|
96
|
+
failure_runner = Test::Runner.new(suite: [failing_test], format: 'test')
|
|
97
|
+
|
|
98
|
+
failure_runner.run.assert == false
|
|
99
|
+
failure_runner.recorder.summary.counts[:fail].assert == 1
|
|
100
|
+
($stdout.equal?(original_stdout)).assert == true
|
|
101
|
+
|
|
102
|
+
Global assertionless mode treats a false return as a failure.
|
|
103
|
+
|
|
104
|
+
previous_assertionless = Test::Config.assertionless
|
|
105
|
+
begin
|
|
106
|
+
Test::Config.assertionless = true
|
|
107
|
+
false_test = RunnerProbe.new('false result') { false }
|
|
108
|
+
hard_runner = Test::Runner.new(suite: [false_test], format: 'test')
|
|
109
|
+
hard_runner.run.assert == false
|
|
110
|
+
hard_runner.recorder.summary.counts[:fail].assert == 1
|
|
111
|
+
ensure
|
|
112
|
+
Test::Config.assertionless = previous_assertionless
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
### CLI configuration
|
|
116
|
+
|
|
117
|
+
The `--config` option accepts an alternate configuration file.
|
|
118
|
+
|
|
119
|
+
require 'rubytest/cli'
|
|
120
|
+
cli = Test::CLI.new
|
|
121
|
+
cli.options.parse!(['--config', 'custom-test.rb'])
|
|
122
|
+
cli.config_file.assert == 'custom-test.rb'
|
|
123
|
+
|
|
124
|
+
### Errors outside of tests
|
|
125
|
+
|
|
126
|
+
An error raised outside of any test, such as one raised while a test file
|
|
127
|
+
loads, ends the run. The report names the error's class, says where it was
|
|
128
|
+
raised, and points to the full backtrace.
|
|
129
|
+
|
|
130
|
+
load_error = begin
|
|
131
|
+
''.freeze << 'x'
|
|
132
|
+
rescue FrozenError => error
|
|
133
|
+
error
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
report = Test.error_report(load_error)
|
|
137
|
+
lines = report.lines.map(&:chomp)
|
|
138
|
+
|
|
139
|
+
lines[0].assert.start_with?('ERROR: FrozenError: ')
|
|
140
|
+
lines[1].assert.start_with?(' at ')
|
|
141
|
+
lines[1].include?(load_error.backtrace.first).assert == true
|
|
142
|
+
lines[2].include?('--debug').assert == true
|
|
143
|
+
|
|
144
|
+
An error without a message or a backtrace still names its class.
|
|
145
|
+
|
|
146
|
+
Test.error_report(RuntimeError.new).lines.first.chomp.assert == 'ERROR: RuntimeError'
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
## Result pipeline
|
|
2
|
+
|
|
3
|
+
The runner sends completed outcomes to the recorder. The recorder keeps them
|
|
4
|
+
and forwards the same result objects, along with case boundaries, to a reporter.
|
|
5
|
+
|
|
6
|
+
events = []
|
|
7
|
+
reporter = Object.new
|
|
8
|
+
reporter.define_singleton_method(:begin_suite) { |suite| events << :begin_suite }
|
|
9
|
+
reporter.define_singleton_method(:begin_case) { |tcase| events << :begin_case }
|
|
10
|
+
reporter.define_singleton_method(:begin_test) { |test| events << :begin_test }
|
|
11
|
+
reporter.define_singleton_method(:record) { |result| events << result }
|
|
12
|
+
reporter.define_singleton_method(:end_test) { |test| events << :end_test }
|
|
13
|
+
reporter.define_singleton_method(:end_case) { |tcase| events << :end_case }
|
|
14
|
+
reporter.define_singleton_method(:finish) { |summary| events << summary }
|
|
15
|
+
|
|
16
|
+
runner_type = Class.new(Test::Runner) do
|
|
17
|
+
define_method(:reporter_load) { |_format| reporter }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
passing = -> { true }
|
|
21
|
+
failing = -> { raise Assertion, 'no' }
|
|
22
|
+
broken = -> { raise 'unexpected' }
|
|
23
|
+
pending = -> { raise NotImplementedError, 'later' }
|
|
24
|
+
skipped = -> { raise 'must not run' }
|
|
25
|
+
skipped.define_singleton_method(:skip?) { 'later' }
|
|
26
|
+
|
|
27
|
+
skipped_case = [-> { raise 'must not run' }]
|
|
28
|
+
skipped_case.define_singleton_method(:skip?) { 'case later' }
|
|
29
|
+
broken_case = Class.new(Array) do
|
|
30
|
+
def call
|
|
31
|
+
raise 'case setup failed'
|
|
32
|
+
end
|
|
33
|
+
end.new
|
|
34
|
+
|
|
35
|
+
runner = runner_type.new(suite: [[passing, failing], broken, pending, skipped,
|
|
36
|
+
skipped_case, broken_case], format: 'test')
|
|
37
|
+
runner.run.assert == false
|
|
38
|
+
|
|
39
|
+
statuses = runner.recorder.results.map(&:status)
|
|
40
|
+
statuses.assert == [:pass, :fail, :error, :todo, :skip, :skip, :error]
|
|
41
|
+
runner.recorder.results.map(&:kind).assert ==
|
|
42
|
+
[:test, :test, :test, :test, :test, :case, :case]
|
|
43
|
+
runner.recorder.results.find { |result| result.status == :fail }.test.assert.equal? failing
|
|
44
|
+
runner.recorder.summary.counts[:skip].assert == 2
|
|
45
|
+
events.grep(Test::Result).map(&:object_id).assert ==
|
|
46
|
+
runner.recorder.results.map(&:object_id)
|
|
47
|
+
events.count(:begin_case).assert == 2
|
|
48
|
+
events.count(:end_case).assert == 2
|
|
49
|
+
summary = events.last
|
|
50
|
+
summary.class.assert == Test::RunSummary
|
|
51
|
+
summary.counts[:error].assert == 2
|
|
52
|
+
summary.success?.assert == false
|
|
53
|
+
|
|
54
|
+
An after hook can turn an otherwise passing test into a failed result before
|
|
55
|
+
that result is recorded.
|
|
56
|
+
|
|
57
|
+
hook_reporter = Object.new
|
|
58
|
+
hook_reporter.define_singleton_method(:record) { |result| }
|
|
59
|
+
hook_reporter.define_singleton_method(:finish) { |summary| }
|
|
60
|
+
hook_runner_type = Class.new(Test::Runner) do
|
|
61
|
+
define_method(:reporter_load) { |_format| hook_reporter }
|
|
62
|
+
end
|
|
63
|
+
hook_runner = hook_runner_type.new(suite: [-> { true }], format: 'test')
|
|
64
|
+
hook_runner.after(:test) { raise Assertion, 'verification failed' }
|
|
65
|
+
hook_runner.run.assert == false
|
|
66
|
+
hook_runner.recorder.results.first.status.assert == :fail
|
|
67
|
+
hook_runner.recorder.results.first.exception.message.include?('verification failed').assert == true
|
|
68
|
+
|
|
69
|
+
If both a test and teardown fail, the error takes precedence while the result
|
|
70
|
+
keeps both exceptions. An outcome hook that raises also becomes a result.
|
|
71
|
+
|
|
72
|
+
double_runner = hook_runner_type.new(suite: [-> { raise Assertion, 'body failed' }],
|
|
73
|
+
format: 'test')
|
|
74
|
+
double_runner.after(:test) { raise 'cleanup failed' }
|
|
75
|
+
double_runner.run.assert == false
|
|
76
|
+
double_result = double_runner.recorder.results.first
|
|
77
|
+
double_result.status.assert == :error
|
|
78
|
+
double_result.exception.message.assert == 'cleanup failed'
|
|
79
|
+
double_result.exceptions.first.message.include?('body failed').assert == true
|
|
80
|
+
double_result.exceptions.last.message.assert == 'cleanup failed'
|
|
81
|
+
|
|
82
|
+
outcome_runner = hook_runner_type.new(suite: [-> { true }], format: 'test')
|
|
83
|
+
outcome_runner.upon(:pass) { raise 'outcome hook failed' }
|
|
84
|
+
outcome_runner.run.assert == false
|
|
85
|
+
outcome_runner.recorder.results.first.status.assert == :error
|
|
86
|
+
outcome_runner.recorder.results.first.exception.message.assert == 'outcome hook failed'
|
|
87
|
+
|
|
88
|
+
A failing before hook is recorded as an error; the test body is not called,
|
|
89
|
+
and the after hook still runs.
|
|
90
|
+
|
|
91
|
+
calls = 0
|
|
92
|
+
cleanup_ran = false
|
|
93
|
+
before_runner = hook_runner_type.new(suite: [-> { calls += 1 }], format: 'test')
|
|
94
|
+
before_runner.before(:test) { raise 'setup failed' }
|
|
95
|
+
before_runner.after(:test) { cleanup_ran = true }
|
|
96
|
+
before_runner.run.assert == false
|
|
97
|
+
before_runner.recorder.results.first.status.assert == :error
|
|
98
|
+
calls.assert == 0
|
|
99
|
+
cleanup_ran.assert == true
|
|
100
|
+
|
|
101
|
+
A case after hook error is recorded after its child test and case reporting
|
|
102
|
+
still ends.
|
|
103
|
+
|
|
104
|
+
case_end_events = []
|
|
105
|
+
case_end_reporter = Object.new
|
|
106
|
+
case_end_reporter.define_singleton_method(:record) { |result| case_end_events << result.status }
|
|
107
|
+
case_end_reporter.define_singleton_method(:end_case) { |tcase| case_end_events << :end_case }
|
|
108
|
+
case_end_reporter.define_singleton_method(:finish) { |summary| }
|
|
109
|
+
case_end_runner_type = Class.new(Test::Runner) do
|
|
110
|
+
define_method(:reporter_load) { |_format| case_end_reporter }
|
|
111
|
+
end
|
|
112
|
+
case_end_runner = case_end_runner_type.new(suite: [[-> { true }]], format: 'test')
|
|
113
|
+
case_end_runner.after(:case) { raise 'case teardown failed' }
|
|
114
|
+
case_end_runner.run.assert == false
|
|
115
|
+
case_end_events.assert == [:pass, :error, :end_case]
|
|
116
|
+
|
|
117
|
+
case_end_events.clear
|
|
118
|
+
case_body_calls = 0
|
|
119
|
+
case_cleanup_ran = false
|
|
120
|
+
case_start_runner = case_end_runner_type.new(suite: [[-> { case_body_calls += 1 }]], format: 'test')
|
|
121
|
+
case_start_runner.before(:case) { raise 'case setup failed' }
|
|
122
|
+
case_start_runner.after(:case) { case_cleanup_ran = true }
|
|
123
|
+
case_start_runner.run.assert == false
|
|
124
|
+
case_end_events.assert == [:error, :end_case]
|
|
125
|
+
case_body_calls.assert == 0
|
|
126
|
+
case_cleanup_ran.assert == true
|
|
127
|
+
|
|
128
|
+
The hash reporter receives captured output in its result, and the runner
|
|
129
|
+
restores the process streams after execution.
|
|
130
|
+
|
|
131
|
+
require 'rubytest/format/test'
|
|
132
|
+
hash_reporter_type = Class.new(Test::Reporters::Test) do
|
|
133
|
+
attr_reader :rows
|
|
134
|
+
|
|
135
|
+
def initialize(runner)
|
|
136
|
+
super
|
|
137
|
+
@rows = []
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def record(result)
|
|
141
|
+
@rows << super
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
hash_runner_type = Class.new(Test::Runner) do
|
|
145
|
+
define_method(:reporter_load) { |_format| hash_reporter_type.new(self) }
|
|
146
|
+
end
|
|
147
|
+
original_stdout, original_stderr = $stdout, $stderr
|
|
148
|
+
output_runner = hash_runner_type.new(suite: [-> { print 'hello'; warn 'oops' }], format: 'test')
|
|
149
|
+
output_runner.upon(:pass) { print ' from hook' }
|
|
150
|
+
output_runner.run.assert == true
|
|
151
|
+
row = output_runner.reporter.rows.first
|
|
152
|
+
row['stdout'].assert == 'hello from hook'
|
|
153
|
+
row['stderr'].assert == "oops\n"
|
|
154
|
+
($stdout.equal?(original_stdout)).assert == true
|
|
155
|
+
($stderr.equal?(original_stderr)).assert == true
|
|
156
|
+
|
|
157
|
+
case_output_runner = hash_runner_type.new(suite: [skipped_case], format: 'test')
|
|
158
|
+
case_output_runner.run.assert == true
|
|
159
|
+
case_output_runner.reporter.rows.first['type'].assert == 'case'
|
|
160
|
+
case_output_runner.reporter.rows.first['status'].assert == 'omit'
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
## Built-in reporters
|
|
2
|
+
|
|
3
|
+
The human-readable formats consume the same results, including pending tests
|
|
4
|
+
and skipped cases. They can be selected without installing another gem.
|
|
5
|
+
|
|
6
|
+
require 'stringio'
|
|
7
|
+
|
|
8
|
+
probe_type = Class.new do
|
|
9
|
+
def initialize(label, reason = nil, &action)
|
|
10
|
+
@label, @reason, @action = label, reason, action
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def call
|
|
14
|
+
@action.call
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def skip?
|
|
18
|
+
@reason
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def to_s
|
|
22
|
+
@label
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
passing = probe_type.new('passing') { true }
|
|
27
|
+
failing = probe_type.new('failing') { raise Assertion, 'wrong value' }
|
|
28
|
+
pending = probe_type.new('pending') { raise NotImplementedError, 'later' }
|
|
29
|
+
skipped = probe_type.new('skipped', 'not available') { raise 'must not run' }
|
|
30
|
+
skipped_case = Class.new(Array) do
|
|
31
|
+
def skip?; 'case unavailable'; end
|
|
32
|
+
def to_s; 'skipped case'; end
|
|
33
|
+
end.new([passing])
|
|
34
|
+
broken_case = Class.new(Array) do
|
|
35
|
+
def call; raise 'case setup failed'; end
|
|
36
|
+
def to_s; 'broken case'; end
|
|
37
|
+
end.new([passing])
|
|
38
|
+
suite = [[passing, failing], pending, skipped, skipped_case, broken_case]
|
|
39
|
+
|
|
40
|
+
render = lambda do |format, tests = suite|
|
|
41
|
+
require "rubytest/format/#{format}"
|
|
42
|
+
output = StringIO.new
|
|
43
|
+
reporter_type = Test::Reporters.const_get(format.capitalize)
|
|
44
|
+
runner_type = Class.new(Test::Runner) do
|
|
45
|
+
define_method(:reporter_load) { |_name| reporter_type.new(self, output: output) }
|
|
46
|
+
end
|
|
47
|
+
runner = runner_type.new(suite: tests, format: format)
|
|
48
|
+
[runner.run, output.string, runner.recorder.summary]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
%w[summary outline progress].each do |format|
|
|
52
|
+
success, output, report = render.call(format)
|
|
53
|
+
success.assert == false
|
|
54
|
+
report.total.assert == 6
|
|
55
|
+
output.include?('failing').assert == true
|
|
56
|
+
output.include?('skipped case').assert == true
|
|
57
|
+
output.include?('6').assert == true
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
TAP emits one test point for every result. Test output is captured and appears
|
|
61
|
+
as diagnostics, so it cannot corrupt the TAP stream. The final plan uses the
|
|
62
|
+
number actually recorded, including the broken and skipped cases.
|
|
63
|
+
|
|
64
|
+
output_test = probe_type.new('output') { print "ok 999 - injected\n"; true }
|
|
65
|
+
tap_suite = [[output_test, failing], pending, skipped, skipped_case, broken_case]
|
|
66
|
+
success, output, report = render.call('tap', tap_suite)
|
|
67
|
+
success.assert == false
|
|
68
|
+
lines = output.lines.map(&:chomp)
|
|
69
|
+
lines.first.assert == 'TAP version 13'
|
|
70
|
+
lines.last.assert == '1..6'
|
|
71
|
+
lines.grep(/\A(?:ok|not ok) \d+ - /).size.assert == report.total
|
|
72
|
+
lines.grep(/# SKIP /).size.assert == 2
|
|
73
|
+
lines.grep(/# TODO /).size.assert == 1
|
|
74
|
+
lines.grep(/\A# stdout: ok 999 - injected/).size.assert == 1
|
|
75
|
+
lines.grep(/\Anot ok \d+ - broken case/).size.assert == 1
|
|
76
|
+
|
|
77
|
+
TAP-Y and TAP-J carry the same results as structured records: one for the
|
|
78
|
+
suite, one for each case and test, and a final tally. TAP-Y is a stream of
|
|
79
|
+
YAML documents, TAP-J a line of JSON for each record.
|
|
80
|
+
|
|
81
|
+
require 'yaml'
|
|
82
|
+
require 'json'
|
|
83
|
+
|
|
84
|
+
readers = {
|
|
85
|
+
'tapy' => lambda { |text| YAML.load_stream(text) },
|
|
86
|
+
'tapj' => lambda { |text| text.lines.map { |line| JSON.parse(line) } }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
readers.each do |format, read|
|
|
90
|
+
success, output, report = render.call(format)
|
|
91
|
+
success.assert == false
|
|
92
|
+
records = read.call(output)
|
|
93
|
+
|
|
94
|
+
records.first['type'].assert == 'suite'
|
|
95
|
+
records.first['rev'].assert == 5
|
|
96
|
+
records.last['type'].assert == 'final'
|
|
97
|
+
records.last['counts']['total'].assert == report.total
|
|
98
|
+
records.last['counts']['omit'].assert == 2
|
|
99
|
+
|
|
100
|
+
results = records.select { |record| record['status'] }
|
|
101
|
+
results.size.assert == report.total
|
|
102
|
+
results.map { |record| record['status'] }.sort.assert == %w[error fail omit omit pass todo]
|
|
103
|
+
results.select { |record| record['status'] == 'omit' }.map { |record| record['type'] }.sort.assert == %w[case test]
|
|
104
|
+
records.count { |record| record['type'] == 'case' && !record['status'] }.assert == 2
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
A TAP-Y stream ends with the YAML end-of-document marker.
|
|
108
|
+
|
|
109
|
+
render.call('tapy')[1].lines.last.chomp.assert == '...'
|
|
110
|
+
|
|
111
|
+
The command line has a shortcut for each.
|
|
112
|
+
|
|
113
|
+
require 'rubytest/cli'
|
|
114
|
+
{ '-y' => 'tapy', '--tapj' => 'tapj' }.each do |flag, format|
|
|
115
|
+
cli = Test::CLI.new
|
|
116
|
+
cli.options.parse!([flag])
|
|
117
|
+
cli.format.assert == format
|
|
118
|
+
end
|
data/lib/rubytest/cli.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Test
|
|
|
9
9
|
# Test configuration file can be in `etc/test.rb` or `config/test.rb`, or
|
|
10
10
|
# `Testfile` or '.test` with optional `.rb` extension, in that order of
|
|
11
11
|
# precedence. To use a different file there is the -c/--config option.
|
|
12
|
-
GLOB_CONFIG = '{etc/test.rb,config/test.rb,testfile.rb,testfile,.test.rb,.test}'
|
|
12
|
+
GLOB_CONFIG = '{etc/test.rb,config/test.rb,Testfile.rb,Testfile,testfile.rb,testfile,.test.rb,.test}'
|
|
13
13
|
|
|
14
14
|
# Convenience method for invoking the CLI.
|
|
15
15
|
#
|
|
@@ -95,7 +95,7 @@ module Test
|
|
|
95
95
|
conf.requires.concat makelist(file)
|
|
96
96
|
end
|
|
97
97
|
opt.on '-c', '--config FILE', "use alternate config file" do |file|
|
|
98
|
-
conf.
|
|
98
|
+
conf.config_file = file
|
|
99
99
|
end
|
|
100
100
|
opt.on '-V' , '--verbose', 'provide extra detail in reports' do
|
|
101
101
|
conf.verbose = true
|
|
@@ -141,6 +141,10 @@ module Test
|
|
|
141
141
|
@config_file
|
|
142
142
|
end
|
|
143
143
|
|
|
144
|
+
def config_file=(file)
|
|
145
|
+
@config_file = file
|
|
146
|
+
end
|
|
147
|
+
|
|
144
148
|
def profile
|
|
145
149
|
@profile
|
|
146
150
|
end
|