rubytest 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/HISTORY.md +49 -1
- data/README.md +88 -67
- data/bin/rubytest +9 -0
- data/demo/03_runner_reliability.md +146 -0
- data/demo/04_result_pipeline.md +160 -0
- data/demo/05_builtin_reporters.md +118 -0
- data/lib/rubytest/cli.rb +227 -0
- data/lib/rubytest/config.rb +1 -1
- data/lib/rubytest/format/abstract.rb +0 -100
- data/lib/rubytest/format/abstract_hash.rb +70 -34
- data/lib/rubytest/format/dotprogress.rb +82 -56
- data/lib/rubytest/format/outline.rb +44 -0
- data/lib/rubytest/format/progress.rb +35 -0
- data/lib/rubytest/format/summary.rb +31 -0
- data/lib/rubytest/format/tap.rb +54 -0
- data/lib/rubytest/format/tapj.rb +76 -0
- data/lib/rubytest/format/tapy.rb +77 -0
- data/lib/rubytest/recorder.rb +36 -29
- data/lib/rubytest/result.rb +54 -0
- data/lib/rubytest/runner.rb +211 -77
- data/lib/rubytest.rb +10 -35
- data/lib/test.rb +1 -8
- metadata +65 -40
- data/.index +0 -54
- data/lib/rubytest.yml +0 -54
- data/test/basic_case.rb +0 -13
- data/test/helper.rb +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 87813efd760c9e9e813d32fe22a048da1c82e47bf580420892942931008039ba
|
|
4
|
+
data.tar.gz: 4f8c2918e8885f5a29f144d1315135ce61a44016711b4289e0419f7b8dfff02e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6d91841470661e6bfdc0131d970723154d6c6ff6eb18da980e4a4b205f43a1f5134192e73fef2430dbbe6f77b71d297d4b25db5c29ff5972a1179e0e3c155075
|
|
7
|
+
data.tar.gz: cc2829ce1797a428386e37cf2ab98a11f575ea0848e215ca3f65f19665d5e36057f0d8fd5ac488746bf43a0df1b7b9a275375791138a8693a0b18ce0021cd8e1
|
data/HISTORY.md
CHANGED
|
@@ -1,5 +1,54 @@
|
|
|
1
1
|
# RELEASE HISTORY
|
|
2
2
|
|
|
3
|
+
## 0.10.0 / 2026-11-17
|
|
4
|
+
|
|
5
|
+
Results now go through a recorder to the reporters, and the reporters that
|
|
6
|
+
were spun off into separate gems are back in Rubytest, so rubytest-cli,
|
|
7
|
+
rubytest-tap, rubytest-summary and rubytest-progress are no longer needed.
|
|
8
|
+
Supports Ruby 2.7 and later.
|
|
9
|
+
|
|
10
|
+
Changes:
|
|
11
|
+
|
|
12
|
+
* Support Ruby 2.7 and 3.0 again (0.9.0 required 3.1).
|
|
13
|
+
* Report an error raised outside of a test, such as while a test file loads,
|
|
14
|
+
with its class, where it was raised, and a pointer to `--debug`, instead of
|
|
15
|
+
the bare message.
|
|
16
|
+
* Bring summary, outline, and progress reporters into Rubytest and add a TAP 13
|
|
17
|
+
reporter that counts completed results, including skipped and broken cases.
|
|
18
|
+
* Bring the TAP-Y and TAP-J reporters in from rubytest-tap, with their `-y` and
|
|
19
|
+
`-j` shortcuts. The TAP-Y stream now opens with a YAML document (it was
|
|
20
|
+
JSON), the TAP-J stream no longer ends with a `...` line, and both report
|
|
21
|
+
skipped tests and cases.
|
|
22
|
+
* Remove the old Recorder status callbacks and status-indexed result table.
|
|
23
|
+
* Preserve test and teardown exceptions together in a result, and turn outcome
|
|
24
|
+
hook failures into recorded errors.
|
|
25
|
+
* Route structured test results through the recorder to reporters, while
|
|
26
|
+
retaining Advice hooks and older reporter callbacks.
|
|
27
|
+
* Fail empty runs and unmatched test file requests instead of reporting success.
|
|
28
|
+
* Apply test filters at every suite level and record skipped tests and cases.
|
|
29
|
+
* Record case errors, finish suite callbacks, and restore reporter output after
|
|
30
|
+
failures.
|
|
31
|
+
* Fix the CLI's alternate configuration option and `Testfile` discovery.
|
|
32
|
+
|
|
33
|
+
## 0.9.0 / 2026-03-31
|
|
34
|
+
|
|
35
|
+
Maintenance release. Modernized project tooling and merged CLI back in.
|
|
36
|
+
|
|
37
|
+
Changes:
|
|
38
|
+
|
|
39
|
+
* Merge rubytest-cli back into rubytest (bin/rubytest executable).
|
|
40
|
+
* Replace custom Indexer system with standard gemspec.
|
|
41
|
+
* Replace Travis CI with GitHub Actions.
|
|
42
|
+
* Replace Assembly with Rakefile.
|
|
43
|
+
* Add VERSION constant to Test module.
|
|
44
|
+
* Remove Ruby 1.9 compatibility code.
|
|
45
|
+
* Simplify version handling (remove metadata YAML).
|
|
46
|
+
* Fix typos and update URLs to HTTPS.
|
|
47
|
+
* Move site from gh-pages to docs/.
|
|
48
|
+
* Remove obsolete files.
|
|
49
|
+
* Clean up .gitignore.
|
|
50
|
+
|
|
51
|
+
|
|
3
52
|
## 0.8.0 / 2014-07-18
|
|
4
53
|
|
|
5
54
|
We have whittled Rubytest down to it core functionaltiy and spun the rest
|
|
@@ -203,4 +252,3 @@ First release of Ruby Test.
|
|
|
203
252
|
Changes:
|
|
204
253
|
|
|
205
254
|
* It's Your Birthday!
|
|
206
|
-
|
data/README.md
CHANGED
|
@@ -1,50 +1,65 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
[Homepage](http://rubyworks.github.com/rubytest) /
|
|
4
|
-
[User Guide](http://wiki.github.com/rubyworks/rubytest) /
|
|
5
|
-
[Support](http://github.com/rubyworks/rubytest/issues) /
|
|
6
|
-
[Development](http://github.com/rubyworks/rubytest)
|
|
1
|
+
[](https://rubyworks.github.io/rubytest)
|
|
7
2
|
|
|
8
|
-
|
|
9
|
-
[](http://badge.fury.io/rb/rubytest)
|
|
3
|
+
# Rubytest
|
|
10
4
|
|
|
5
|
+
[](https://rubygems.org/gems/rubytest)
|
|
6
|
+
[](https://github.com/rubyworks/rubytest/actions/workflows/test.yml)
|
|
7
|
+
[](https://github.com/rubyworks/rubytest/issues)
|
|
11
8
|
|
|
12
|
-
## Description
|
|
13
9
|
|
|
14
|
-
Rubytest is
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
can be used
|
|
19
|
-
|
|
20
|
-
can be used in a single test suite all of which can run through a single
|
|
21
|
-
uniform interface in a process.
|
|
10
|
+
Rubytest is Ruby's Universal Test Harness. Think of it as a *testing meta-framework*.
|
|
11
|
+
It defines a straight-forward specification that any application can use to create
|
|
12
|
+
their own testing DSLs. Rubytest can be used for testing end-user applications directly
|
|
13
|
+
or as the backend of a test framework. Since Rubytest controls the backend, multiple test
|
|
14
|
+
frameworks can be used in a single test suite, all of which can be run through one uniform
|
|
15
|
+
interface in a single process!
|
|
22
16
|
|
|
23
17
|
|
|
24
18
|
## Specification
|
|
25
19
|
|
|
26
20
|
The universal access point for testing is the `$TEST_SUITE` global array. A test
|
|
27
|
-
framework need only add compliant test objects to `$TEST_SUITE`.
|
|
21
|
+
framework need only add compliant test objects to `$TEST_SUITE`.
|
|
28
22
|
Rubytest will iterate through these objects. If a test object responds to
|
|
29
23
|
`#call`, it is run as a test procedure. If it responds to `#each` it is iterated
|
|
30
|
-
over as a test case with each entry handled in the same manner. All test
|
|
24
|
+
over as a test case with each entry handled in the same manner. All test
|
|
31
25
|
objects must respond to `#to_s` so their description can be used in test
|
|
32
26
|
reports.
|
|
33
27
|
|
|
34
|
-
Rubytest handles assertions with [BRASS](
|
|
28
|
+
Rubytest handles assertions with [BRASS](https://github.com/rubyworks/brass)
|
|
35
29
|
compliance. Any raised exception that responds to `#assertion?` in the
|
|
36
|
-
affirmative is taken to be a failed assertion rather than simply an error.
|
|
30
|
+
affirmative is taken to be a failed assertion rather than simply an error.
|
|
37
31
|
A test framework may raise a `NotImplementedError` to have a test recorded
|
|
38
32
|
as *todo* --a _pending_ exception to remind the developer of tests that still
|
|
39
33
|
need to be written. The `NotImplementedError` is a standard Ruby exception
|
|
40
34
|
and a subclass of `ScriptError`. The exception can also set a priority level
|
|
41
35
|
to indicate the urgency of the pending test. Priorities of -1 or lower
|
|
42
|
-
will generally not be brought to the attention of testers unless explicitly
|
|
36
|
+
will generally not be brought to the attention of testers unless explicitly
|
|
43
37
|
configured to do so.
|
|
44
38
|
|
|
45
39
|
That is the crux of Rubytest specification. Rubytest supports some
|
|
46
|
-
additional features that can
|
|
47
|
-
|
|
40
|
+
additional features that can make its usage even more convenient.
|
|
41
|
+
|
|
42
|
+
### Results and reporting
|
|
43
|
+
|
|
44
|
+
The runner turns each completed test into a `Test::Result` with its status,
|
|
45
|
+
exception or skip reason, elapsed time, and captured output when requested.
|
|
46
|
+
If both the test and a hook raise, the result retains both exceptions.
|
|
47
|
+
Skipped and broken cases also produce results. `Test::Recorder` keeps these
|
|
48
|
+
results, calculates the run summary, and forwards reporting events to the
|
|
49
|
+
selected reporter. Advice hooks remain with the runner, where they can affect
|
|
50
|
+
test execution.
|
|
51
|
+
|
|
52
|
+
Reporters implement `record(result)` for completed outcomes and
|
|
53
|
+
`finish(summary)` for the final totals. Suite and case boundary callbacks are
|
|
54
|
+
available for formats that show nesting.
|
|
55
|
+
|
|
56
|
+
The built-in formats are `dotprogress` (default), `summary` (one status line per
|
|
57
|
+
test), `outline` (nested cases), `progress` (running count and timing), `tap`
|
|
58
|
+
(TAP version 13), and `tapy` and `tapj` (TAP-Y and TAP-J, the structured
|
|
59
|
+
streams that [tapout](https://github.com/rubyworks/tapout) reads). Select one
|
|
60
|
+
with `rubytest -f FORMAT`; `-y` and `-j` are short for `-f tapy` and `-f tapj`.
|
|
61
|
+
Formats load only when selected. The TAP formats capture test output: TAP
|
|
62
|
+
emits it as diagnostics, and TAP-Y and TAP-J record it with each test.
|
|
48
63
|
|
|
49
64
|
|
|
50
65
|
## Installation
|
|
@@ -53,57 +68,71 @@ Rubytest is available as a Gem package.
|
|
|
53
68
|
|
|
54
69
|
$ gem install rubytest
|
|
55
70
|
|
|
56
|
-
Rubytest is compliant with Setup.rb layout standard, so it can
|
|
57
|
-
also be installed in an FHS compliant fashion if necessary.
|
|
58
|
-
|
|
59
71
|
|
|
60
72
|
## Running Tests
|
|
61
73
|
|
|
62
|
-
There are a few ways to run tests.
|
|
74
|
+
There are a few ways to run tests.
|
|
63
75
|
|
|
64
76
|
### Via Command-line Tool
|
|
65
77
|
|
|
66
|
-
The easiest way to run tests is via the command line tool
|
|
67
|
-
it on its [manpage](http://rubyworks.github.com/rubytest/man/rubytest.1.html),
|
|
68
|
-
but we will quickly go over it here.
|
|
69
|
-
|
|
70
|
-
The basic usage example is:
|
|
78
|
+
The easiest way to run tests is via the command line tool:
|
|
71
79
|
|
|
72
80
|
$ rubytest -Ilib test/test_*.rb
|
|
73
81
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
82
|
+
This would add `lib` to Ruby's `$LOAD_PATH` and then load all the
|
|
83
|
+
test files matching the glob. When running tests, be sure to load your
|
|
84
|
+
test framework or its adapter, e.g.
|
|
77
85
|
|
|
78
|
-
|
|
79
|
-
out to `rubytest` is a good way to go as it keeps your test environment as
|
|
80
|
-
pristine as possible, e.g.
|
|
86
|
+
$ rubytest -r lemon -r ae test/test_*.rb
|
|
81
87
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
sh "rubytest"
|
|
85
|
-
end
|
|
88
|
+
Rubytest exits unsuccessfully if a requested path matches no files or if the
|
|
89
|
+
selection runs no tests.
|
|
86
90
|
|
|
87
|
-
|
|
91
|
+
Use `-h/--help` to see all available options.
|
|
88
92
|
|
|
89
|
-
|
|
90
|
-
Surf over to its [webpage](http://rubyworks.github.com/rubytest-rake) for details.
|
|
91
|
-
A basic example in its case, add to ones Rakefile:
|
|
93
|
+
#### Configuration File
|
|
92
94
|
|
|
93
|
-
|
|
95
|
+
To avoid repeating options, add an `etc/test.rb` file to your project
|
|
96
|
+
with `Test.run` (or `Test.configure`) entries:
|
|
94
97
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
+
```ruby
|
|
99
|
+
Test.run do |r|
|
|
100
|
+
r.loadpath 'lib'
|
|
101
|
+
r.test_files << 'test/*_test.rb'
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
Test.run 'coverage' do |r|
|
|
105
|
+
r.loadpath 'lib'
|
|
106
|
+
r.test_files << 'test/*_test.rb'
|
|
107
|
+
r.before do
|
|
108
|
+
require 'simplecov'
|
|
109
|
+
SimpleCov.start do |s|
|
|
110
|
+
s.filter 'test/'
|
|
111
|
+
s.coverage_dir 'log/coverage'
|
|
98
112
|
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Now `rubytest` will use the first configuration by default. To use a
|
|
118
|
+
named profile, use the `-p/--profile` option:
|
|
119
|
+
|
|
120
|
+
$ rubytest -p coverage
|
|
99
121
|
|
|
100
|
-
|
|
122
|
+
The configuration file can also be at `config/test.rb`, `Testfile`,
|
|
123
|
+
or `.test` (with optional `.rb` extension).
|
|
124
|
+
|
|
125
|
+
If you are using Rake, shelling out to `rubytest` keeps your test
|
|
126
|
+
environment pristine:
|
|
127
|
+
|
|
128
|
+
desc "run tests"
|
|
129
|
+
task :test do
|
|
130
|
+
sh "rubytest"
|
|
131
|
+
end
|
|
101
132
|
|
|
102
133
|
### Via Runner Scripts
|
|
103
134
|
|
|
104
|
-
|
|
105
|
-
you simply write you own runner script using the Rubytest API.
|
|
106
|
-
Here is the basic example:
|
|
135
|
+
You can write your own runner script using the Rubytest API:
|
|
107
136
|
|
|
108
137
|
require 'rubytest'
|
|
109
138
|
|
|
@@ -120,32 +149,24 @@ runner.
|
|
|
120
149
|
|
|
121
150
|
## Requirements
|
|
122
151
|
|
|
123
|
-
Rubytest uses the [ANSI](
|
|
124
|
-
|
|
125
|
-
Because of the "foundational" nature of this library we will look at removing
|
|
126
|
-
this dependency for future versions, but for early development the
|
|
127
|
-
requirements does the job and does it well.
|
|
152
|
+
Rubytest uses the [ANSI](https://github.com/rubyworks/ansi) gem for color output.
|
|
128
153
|
|
|
129
154
|
|
|
130
155
|
## Development
|
|
131
156
|
|
|
132
|
-
Rubytest is
|
|
133
|
-
|
|
134
|
-
Rubytest is a [Rubyworks](http://rubyworks.github.com) project. If you can't
|
|
135
|
-
contribute code, you can still help out by contributing to our development fund.
|
|
157
|
+
Rubytest is a [Rubyworks](https://rubyworks.github.io) project.
|
|
136
158
|
|
|
137
159
|
|
|
138
160
|
## Reference Material
|
|
139
161
|
|
|
140
162
|
* [Standard Definition Of Unit Test](http://c2.com/cgi/wiki?StandardDefinitionOfUnitTest)
|
|
141
|
-
* [BRASS Assertions Standard](
|
|
163
|
+
* [BRASS Assertions Standard](https://github.com/rubyworks/brass)
|
|
142
164
|
|
|
143
165
|
|
|
144
166
|
## Copyrights
|
|
145
167
|
|
|
146
168
|
Copyright (c) 2011 Rubyworks
|
|
147
169
|
|
|
148
|
-
Made available according to the terms of the
|
|
170
|
+
Made available according to the terms of the **BSD-2-Clause** license.
|
|
149
171
|
|
|
150
172
|
See LICENSE.txt for details.
|
|
151
|
-
|
data/bin/rubytest
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
## Runner reliability
|
|
2
|
+
|
|
3
|
+
RubyTest reports an explicitly requested file that does not exist, rather than
|
|
4
|
+
passing an empty run.
|
|
5
|
+
|
|
6
|
+
runner = Test::Runner.new(files: ['__rubytest_missing_file__.rb'], format: 'test')
|
|
7
|
+
cleanup_ran = false
|
|
8
|
+
runner.config.after { cleanup_ran = true }
|
|
9
|
+
missing_file = begin
|
|
10
|
+
runner.run
|
|
11
|
+
nil
|
|
12
|
+
rescue ArgumentError => error
|
|
13
|
+
error
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
missing_file.class.assert == ArgumentError
|
|
17
|
+
missing_file.message.include?('__rubytest_missing_file__.rb').assert == true
|
|
18
|
+
cleanup_ran.assert == true
|
|
19
|
+
|
|
20
|
+
An empty suite also has an unsuccessful result.
|
|
21
|
+
|
|
22
|
+
empty_runner = Test::Runner.new(suite: [], format: 'test')
|
|
23
|
+
empty_runner.run.assert == false
|
|
24
|
+
|
|
25
|
+
### Selection and skips
|
|
26
|
+
|
|
27
|
+
Test descriptions can be matched at the top level and inside cases. A skipped
|
|
28
|
+
test that matches the selection is still reported.
|
|
29
|
+
|
|
30
|
+
class RunnerProbe
|
|
31
|
+
def initialize(description, skip_reason = nil, &block)
|
|
32
|
+
@description = description
|
|
33
|
+
@skip_reason = skip_reason
|
|
34
|
+
@block = block
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def call
|
|
38
|
+
@block.call
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def skip?
|
|
42
|
+
@skip_reason
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def to_s
|
|
46
|
+
@description
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
top = RunnerProbe.new('wanted top') { true }
|
|
51
|
+
other = RunnerProbe.new('other') { true }
|
|
52
|
+
nested = RunnerProbe.new('wanted nested') { true }
|
|
53
|
+
skipped = RunnerProbe.new('wanted skipped', 'later') { raise 'should not run' }
|
|
54
|
+
|
|
55
|
+
selected = Test::Runner.new(suite: [top, other, [nested, skipped]],
|
|
56
|
+
match: ['wanted'], format: 'test')
|
|
57
|
+
selected.run.assert == true
|
|
58
|
+
selected.recorder.summary.counts[:pass].assert == 2
|
|
59
|
+
selected.recorder.summary.counts[:skip].assert == 1
|
|
60
|
+
|
|
61
|
+
Skipping an entire case is recorded too.
|
|
62
|
+
|
|
63
|
+
skipped_case = Class.new(Array) do
|
|
64
|
+
def skip?
|
|
65
|
+
'later'
|
|
66
|
+
end
|
|
67
|
+
end.new
|
|
68
|
+
case_skip_runner = Test::Runner.new(suite: [skipped_case], format: 'test')
|
|
69
|
+
case_skip_runner.run.assert == true
|
|
70
|
+
case_skip_runner.recorder.summary.counts[:skip].assert == 1
|
|
71
|
+
|
|
72
|
+
### Case errors and cleanup
|
|
73
|
+
|
|
74
|
+
A case setup error is recorded, and the next test and suite cleanup still run.
|
|
75
|
+
|
|
76
|
+
broken_case = Class.new(Array) do
|
|
77
|
+
def call
|
|
78
|
+
raise 'setup failed'
|
|
79
|
+
end
|
|
80
|
+
end.new
|
|
81
|
+
following_test = RunnerProbe.new('following test') { true }
|
|
82
|
+
case_runner = Test::Runner.new(suite: [broken_case, following_test], format: 'test')
|
|
83
|
+
suite_ended = false
|
|
84
|
+
case_runner.after(:suite) { suite_ended = true }
|
|
85
|
+
|
|
86
|
+
case_runner.run.assert == false
|
|
87
|
+
case_runner.recorder.summary.counts[:error].assert == 1
|
|
88
|
+
case_runner.recorder.summary.counts[:pass].assert == 1
|
|
89
|
+
suite_ended.assert == true
|
|
90
|
+
|
|
91
|
+
An assertion failure in a hash reporter records the failure and restores
|
|
92
|
+
standard output.
|
|
93
|
+
|
|
94
|
+
original_stdout = $stdout
|
|
95
|
+
failing_test = RunnerProbe.new('failing test') { raise Assertion, 'expected failure' }
|
|
96
|
+
failure_runner = Test::Runner.new(suite: [failing_test], format: 'test')
|
|
97
|
+
|
|
98
|
+
failure_runner.run.assert == false
|
|
99
|
+
failure_runner.recorder.summary.counts[:fail].assert == 1
|
|
100
|
+
($stdout.equal?(original_stdout)).assert == true
|
|
101
|
+
|
|
102
|
+
Global assertionless mode treats a false return as a failure.
|
|
103
|
+
|
|
104
|
+
previous_assertionless = Test::Config.assertionless
|
|
105
|
+
begin
|
|
106
|
+
Test::Config.assertionless = true
|
|
107
|
+
false_test = RunnerProbe.new('false result') { false }
|
|
108
|
+
hard_runner = Test::Runner.new(suite: [false_test], format: 'test')
|
|
109
|
+
hard_runner.run.assert == false
|
|
110
|
+
hard_runner.recorder.summary.counts[:fail].assert == 1
|
|
111
|
+
ensure
|
|
112
|
+
Test::Config.assertionless = previous_assertionless
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
### CLI configuration
|
|
116
|
+
|
|
117
|
+
The `--config` option accepts an alternate configuration file.
|
|
118
|
+
|
|
119
|
+
require 'rubytest/cli'
|
|
120
|
+
cli = Test::CLI.new
|
|
121
|
+
cli.options.parse!(['--config', 'custom-test.rb'])
|
|
122
|
+
cli.config_file.assert == 'custom-test.rb'
|
|
123
|
+
|
|
124
|
+
### Errors outside of tests
|
|
125
|
+
|
|
126
|
+
An error raised outside of any test, such as one raised while a test file
|
|
127
|
+
loads, ends the run. The report names the error's class, says where it was
|
|
128
|
+
raised, and points to the full backtrace.
|
|
129
|
+
|
|
130
|
+
load_error = begin
|
|
131
|
+
''.freeze << 'x'
|
|
132
|
+
rescue FrozenError => error
|
|
133
|
+
error
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
report = Test.error_report(load_error)
|
|
137
|
+
lines = report.lines.map(&:chomp)
|
|
138
|
+
|
|
139
|
+
lines[0].assert.start_with?('ERROR: FrozenError: ')
|
|
140
|
+
lines[1].assert.start_with?(' at ')
|
|
141
|
+
lines[1].include?(load_error.backtrace.first).assert == true
|
|
142
|
+
lines[2].include?('--debug').assert == true
|
|
143
|
+
|
|
144
|
+
An error without a message or a backtrace still names its class.
|
|
145
|
+
|
|
146
|
+
Test.error_report(RuntimeError.new).lines.first.chomp.assert == 'ERROR: RuntimeError'
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
## Result pipeline
|
|
2
|
+
|
|
3
|
+
The runner sends completed outcomes to the recorder. The recorder keeps them
|
|
4
|
+
and forwards the same result objects, along with case boundaries, to a reporter.
|
|
5
|
+
|
|
6
|
+
events = []
|
|
7
|
+
reporter = Object.new
|
|
8
|
+
reporter.define_singleton_method(:begin_suite) { |suite| events << :begin_suite }
|
|
9
|
+
reporter.define_singleton_method(:begin_case) { |tcase| events << :begin_case }
|
|
10
|
+
reporter.define_singleton_method(:begin_test) { |test| events << :begin_test }
|
|
11
|
+
reporter.define_singleton_method(:record) { |result| events << result }
|
|
12
|
+
reporter.define_singleton_method(:end_test) { |test| events << :end_test }
|
|
13
|
+
reporter.define_singleton_method(:end_case) { |tcase| events << :end_case }
|
|
14
|
+
reporter.define_singleton_method(:finish) { |summary| events << summary }
|
|
15
|
+
|
|
16
|
+
runner_type = Class.new(Test::Runner) do
|
|
17
|
+
define_method(:reporter_load) { |_format| reporter }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
passing = -> { true }
|
|
21
|
+
failing = -> { raise Assertion, 'no' }
|
|
22
|
+
broken = -> { raise 'unexpected' }
|
|
23
|
+
pending = -> { raise NotImplementedError, 'later' }
|
|
24
|
+
skipped = -> { raise 'must not run' }
|
|
25
|
+
skipped.define_singleton_method(:skip?) { 'later' }
|
|
26
|
+
|
|
27
|
+
skipped_case = [-> { raise 'must not run' }]
|
|
28
|
+
skipped_case.define_singleton_method(:skip?) { 'case later' }
|
|
29
|
+
broken_case = Class.new(Array) do
|
|
30
|
+
def call
|
|
31
|
+
raise 'case setup failed'
|
|
32
|
+
end
|
|
33
|
+
end.new
|
|
34
|
+
|
|
35
|
+
runner = runner_type.new(suite: [[passing, failing], broken, pending, skipped,
|
|
36
|
+
skipped_case, broken_case], format: 'test')
|
|
37
|
+
runner.run.assert == false
|
|
38
|
+
|
|
39
|
+
statuses = runner.recorder.results.map(&:status)
|
|
40
|
+
statuses.assert == [:pass, :fail, :error, :todo, :skip, :skip, :error]
|
|
41
|
+
runner.recorder.results.map(&:kind).assert ==
|
|
42
|
+
[:test, :test, :test, :test, :test, :case, :case]
|
|
43
|
+
runner.recorder.results.find { |result| result.status == :fail }.test.assert.equal? failing
|
|
44
|
+
runner.recorder.summary.counts[:skip].assert == 2
|
|
45
|
+
events.grep(Test::Result).map(&:object_id).assert ==
|
|
46
|
+
runner.recorder.results.map(&:object_id)
|
|
47
|
+
events.count(:begin_case).assert == 2
|
|
48
|
+
events.count(:end_case).assert == 2
|
|
49
|
+
summary = events.last
|
|
50
|
+
summary.class.assert == Test::RunSummary
|
|
51
|
+
summary.counts[:error].assert == 2
|
|
52
|
+
summary.success?.assert == false
|
|
53
|
+
|
|
54
|
+
An after hook can turn an otherwise passing test into a failed result before
|
|
55
|
+
that result is recorded.
|
|
56
|
+
|
|
57
|
+
hook_reporter = Object.new
|
|
58
|
+
hook_reporter.define_singleton_method(:record) { |result| }
|
|
59
|
+
hook_reporter.define_singleton_method(:finish) { |summary| }
|
|
60
|
+
hook_runner_type = Class.new(Test::Runner) do
|
|
61
|
+
define_method(:reporter_load) { |_format| hook_reporter }
|
|
62
|
+
end
|
|
63
|
+
hook_runner = hook_runner_type.new(suite: [-> { true }], format: 'test')
|
|
64
|
+
hook_runner.after(:test) { raise Assertion, 'verification failed' }
|
|
65
|
+
hook_runner.run.assert == false
|
|
66
|
+
hook_runner.recorder.results.first.status.assert == :fail
|
|
67
|
+
hook_runner.recorder.results.first.exception.message.include?('verification failed').assert == true
|
|
68
|
+
|
|
69
|
+
If both a test and teardown fail, the error takes precedence while the result
|
|
70
|
+
keeps both exceptions. An outcome hook that raises also becomes a result.
|
|
71
|
+
|
|
72
|
+
double_runner = hook_runner_type.new(suite: [-> { raise Assertion, 'body failed' }],
|
|
73
|
+
format: 'test')
|
|
74
|
+
double_runner.after(:test) { raise 'cleanup failed' }
|
|
75
|
+
double_runner.run.assert == false
|
|
76
|
+
double_result = double_runner.recorder.results.first
|
|
77
|
+
double_result.status.assert == :error
|
|
78
|
+
double_result.exception.message.assert == 'cleanup failed'
|
|
79
|
+
double_result.exceptions.first.message.include?('body failed').assert == true
|
|
80
|
+
double_result.exceptions.last.message.assert == 'cleanup failed'
|
|
81
|
+
|
|
82
|
+
outcome_runner = hook_runner_type.new(suite: [-> { true }], format: 'test')
|
|
83
|
+
outcome_runner.upon(:pass) { raise 'outcome hook failed' }
|
|
84
|
+
outcome_runner.run.assert == false
|
|
85
|
+
outcome_runner.recorder.results.first.status.assert == :error
|
|
86
|
+
outcome_runner.recorder.results.first.exception.message.assert == 'outcome hook failed'
|
|
87
|
+
|
|
88
|
+
A failing before hook is recorded as an error; the test body is not called,
|
|
89
|
+
and the after hook still runs.
|
|
90
|
+
|
|
91
|
+
calls = 0
|
|
92
|
+
cleanup_ran = false
|
|
93
|
+
before_runner = hook_runner_type.new(suite: [-> { calls += 1 }], format: 'test')
|
|
94
|
+
before_runner.before(:test) { raise 'setup failed' }
|
|
95
|
+
before_runner.after(:test) { cleanup_ran = true }
|
|
96
|
+
before_runner.run.assert == false
|
|
97
|
+
before_runner.recorder.results.first.status.assert == :error
|
|
98
|
+
calls.assert == 0
|
|
99
|
+
cleanup_ran.assert == true
|
|
100
|
+
|
|
101
|
+
A case after hook error is recorded after its child test and case reporting
|
|
102
|
+
still ends.
|
|
103
|
+
|
|
104
|
+
case_end_events = []
|
|
105
|
+
case_end_reporter = Object.new
|
|
106
|
+
case_end_reporter.define_singleton_method(:record) { |result| case_end_events << result.status }
|
|
107
|
+
case_end_reporter.define_singleton_method(:end_case) { |tcase| case_end_events << :end_case }
|
|
108
|
+
case_end_reporter.define_singleton_method(:finish) { |summary| }
|
|
109
|
+
case_end_runner_type = Class.new(Test::Runner) do
|
|
110
|
+
define_method(:reporter_load) { |_format| case_end_reporter }
|
|
111
|
+
end
|
|
112
|
+
case_end_runner = case_end_runner_type.new(suite: [[-> { true }]], format: 'test')
|
|
113
|
+
case_end_runner.after(:case) { raise 'case teardown failed' }
|
|
114
|
+
case_end_runner.run.assert == false
|
|
115
|
+
case_end_events.assert == [:pass, :error, :end_case]
|
|
116
|
+
|
|
117
|
+
case_end_events.clear
|
|
118
|
+
case_body_calls = 0
|
|
119
|
+
case_cleanup_ran = false
|
|
120
|
+
case_start_runner = case_end_runner_type.new(suite: [[-> { case_body_calls += 1 }]], format: 'test')
|
|
121
|
+
case_start_runner.before(:case) { raise 'case setup failed' }
|
|
122
|
+
case_start_runner.after(:case) { case_cleanup_ran = true }
|
|
123
|
+
case_start_runner.run.assert == false
|
|
124
|
+
case_end_events.assert == [:error, :end_case]
|
|
125
|
+
case_body_calls.assert == 0
|
|
126
|
+
case_cleanup_ran.assert == true
|
|
127
|
+
|
|
128
|
+
The hash reporter receives captured output in its result, and the runner
|
|
129
|
+
restores the process streams after execution.
|
|
130
|
+
|
|
131
|
+
require 'rubytest/format/test'
|
|
132
|
+
hash_reporter_type = Class.new(Test::Reporters::Test) do
|
|
133
|
+
attr_reader :rows
|
|
134
|
+
|
|
135
|
+
def initialize(runner)
|
|
136
|
+
super
|
|
137
|
+
@rows = []
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def record(result)
|
|
141
|
+
@rows << super
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
hash_runner_type = Class.new(Test::Runner) do
|
|
145
|
+
define_method(:reporter_load) { |_format| hash_reporter_type.new(self) }
|
|
146
|
+
end
|
|
147
|
+
original_stdout, original_stderr = $stdout, $stderr
|
|
148
|
+
output_runner = hash_runner_type.new(suite: [-> { print 'hello'; warn 'oops' }], format: 'test')
|
|
149
|
+
output_runner.upon(:pass) { print ' from hook' }
|
|
150
|
+
output_runner.run.assert == true
|
|
151
|
+
row = output_runner.reporter.rows.first
|
|
152
|
+
row['stdout'].assert == 'hello from hook'
|
|
153
|
+
row['stderr'].assert == "oops\n"
|
|
154
|
+
($stdout.equal?(original_stdout)).assert == true
|
|
155
|
+
($stderr.equal?(original_stderr)).assert == true
|
|
156
|
+
|
|
157
|
+
case_output_runner = hash_runner_type.new(suite: [skipped_case], format: 'test')
|
|
158
|
+
case_output_runner.run.assert == true
|
|
159
|
+
case_output_runner.reporter.rows.first['type'].assert == 'case'
|
|
160
|
+
case_output_runner.reporter.rows.first['status'].assert == 'omit'
|