rubytest 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.md +30 -1
- data/README.md +67 -7
- data/demo/03_runner_reliability.md +146 -0
- data/demo/04_result_pipeline.md +160 -0
- data/demo/05_builtin_reporters.md +118 -0
- data/lib/rubytest/cli.rb +6 -2
- data/lib/rubytest/config.rb +1 -1
- data/lib/rubytest/format/abstract.rb +0 -100
- data/lib/rubytest/format/abstract_hash.rb +70 -34
- data/lib/rubytest/format/dotprogress.rb +82 -56
- data/lib/rubytest/format/outline.rb +44 -0
- data/lib/rubytest/format/progress.rb +35 -0
- data/lib/rubytest/format/summary.rb +31 -0
- data/lib/rubytest/format/tap.rb +54 -0
- data/lib/rubytest/format/tapj.rb +76 -0
- data/lib/rubytest/format/tapy.rb +77 -0
- data/lib/rubytest/recorder.rb +36 -29
- data/lib/rubytest/result.rb +54 -0
- data/lib/rubytest/runner.rb +211 -77
- data/lib/rubytest.rb +2 -1
- metadata +12 -2
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
|
|
3
|
+
module Test::Reporters
|
|
4
|
+
|
|
5
|
+
# TAP-J Reporter
|
|
6
|
+
#
|
|
7
|
+
# Writes the run as a TAP-J stream (revision 5): a line of JSON for
|
|
8
|
+
# the suite, each case, each test and the final tally.
|
|
9
|
+
#
|
|
10
|
+
class Tapj < AbstractHash
|
|
11
|
+
|
|
12
|
+
REVISION = 5
|
|
13
|
+
|
|
14
|
+
#
|
|
15
|
+
def initialize(runner, output: $stdout)
|
|
16
|
+
require 'json'
|
|
17
|
+
super(runner)
|
|
18
|
+
@output = output
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
#
|
|
22
|
+
def begin_suite(suite)
|
|
23
|
+
emit super(suite).merge('rev' => REVISION)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
#
|
|
27
|
+
def begin_case(test_case)
|
|
28
|
+
emit super(test_case)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
#
|
|
32
|
+
def pass(test)
|
|
33
|
+
emit super(test)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
#
|
|
37
|
+
def fail(test, exception)
|
|
38
|
+
emit super(test, exception)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
#
|
|
42
|
+
def error(test, exception)
|
|
43
|
+
emit super(test, exception)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
def todo(test, exception)
|
|
48
|
+
emit super(test, exception)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
#
|
|
52
|
+
def skip_test(test, reason=nil)
|
|
53
|
+
emit super(test, reason)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
#
|
|
57
|
+
def skip_case(test_case, reason=nil)
|
|
58
|
+
emit super(test_case, reason)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
#
|
|
62
|
+
def end_suite(suite)
|
|
63
|
+
emit super(suite)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
# Write one record to the stream.
|
|
69
|
+
def emit(hash)
|
|
70
|
+
@output.puts hash.to_json
|
|
71
|
+
hash
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
|
|
3
|
+
module Test::Reporters
|
|
4
|
+
|
|
5
|
+
# TAP-Y Reporter
|
|
6
|
+
#
|
|
7
|
+
# Writes the run as a TAP-Y stream (revision 5): a YAML document for
|
|
8
|
+
# the suite, each case, each test and the final tally.
|
|
9
|
+
#
|
|
10
|
+
class Tapy < AbstractHash
|
|
11
|
+
|
|
12
|
+
REVISION = 5
|
|
13
|
+
|
|
14
|
+
#
|
|
15
|
+
def initialize(runner, output: $stdout)
|
|
16
|
+
require 'yaml'
|
|
17
|
+
super(runner)
|
|
18
|
+
@output = output
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
#
|
|
22
|
+
def begin_suite(suite)
|
|
23
|
+
emit super(suite).merge('rev' => REVISION)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
#
|
|
27
|
+
def begin_case(test_case)
|
|
28
|
+
emit super(test_case)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
#
|
|
32
|
+
def pass(test)
|
|
33
|
+
emit super(test)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
#
|
|
37
|
+
def fail(test, exception)
|
|
38
|
+
emit super(test, exception)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
#
|
|
42
|
+
def error(test, exception)
|
|
43
|
+
emit super(test, exception)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
def todo(test, exception)
|
|
48
|
+
emit super(test, exception)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
#
|
|
52
|
+
def skip_test(test, reason=nil)
|
|
53
|
+
emit super(test, reason)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
#
|
|
57
|
+
def skip_case(test_case, reason=nil)
|
|
58
|
+
emit super(test_case, reason)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
#
|
|
62
|
+
def end_suite(suite)
|
|
63
|
+
emit super(suite)
|
|
64
|
+
@output.puts '...'
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
# Write one record to the stream.
|
|
70
|
+
def emit(hash)
|
|
71
|
+
@output.puts hash.to_yaml
|
|
72
|
+
hash
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
end
|
data/lib/rubytest/recorder.rb
CHANGED
|
@@ -1,53 +1,60 @@
|
|
|
1
1
|
module Test
|
|
2
|
-
|
|
3
|
-
# Recorder class is an observer that tracks all tests
|
|
4
|
-
# that are run and categorizes them according to their
|
|
5
|
-
# test status.
|
|
2
|
+
# Owns run results and controls when they reach the reporter.
|
|
6
3
|
class Recorder
|
|
4
|
+
attr_reader :results, :reporter, :summary
|
|
7
5
|
|
|
8
|
-
def initialize
|
|
9
|
-
@
|
|
6
|
+
def initialize(reporter = nil)
|
|
7
|
+
@reporter = reporter
|
|
8
|
+
@results = []
|
|
10
9
|
end
|
|
11
10
|
|
|
12
|
-
def
|
|
13
|
-
@
|
|
11
|
+
def begin_suite(suite)
|
|
12
|
+
@suite = suite
|
|
13
|
+
@started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
14
|
+
reporter.begin_suite(suite) if reporter&.respond_to?(:begin_suite)
|
|
14
15
|
end
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
self[:skip] << [test, reason]
|
|
17
|
+
def begin_case(test_case)
|
|
18
|
+
reporter.begin_case(test_case) if reporter&.respond_to?(:begin_case)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
self[:pass] << test
|
|
21
|
+
def begin_test(test)
|
|
22
|
+
reporter.begin_test(test) if reporter&.respond_to?(:begin_test)
|
|
24
23
|
end
|
|
25
24
|
|
|
26
|
-
def
|
|
27
|
-
|
|
25
|
+
def capture_output?
|
|
26
|
+
!!(reporter && reporter.respond_to?(:capture_output?) && reporter.capture_output?)
|
|
28
27
|
end
|
|
29
28
|
|
|
30
|
-
def
|
|
31
|
-
|
|
29
|
+
def record(result)
|
|
30
|
+
@results << result
|
|
31
|
+
reporter.record(result) if reporter
|
|
32
|
+
result
|
|
32
33
|
end
|
|
33
34
|
|
|
34
|
-
def
|
|
35
|
-
|
|
35
|
+
def end_test(test)
|
|
36
|
+
reporter.end_test(test) if reporter&.respond_to?(:end_test)
|
|
36
37
|
end
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
def end_case(test_case)
|
|
40
|
+
reporter.end_case(test_case) if reporter&.respond_to?(:end_case)
|
|
41
|
+
end
|
|
41
42
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
def end_suite(suite)
|
|
44
|
+
@summary = RunSummary.new(suite: suite, results: results, elapsed: elapsed)
|
|
45
|
+
reporter.finish(@summary) if reporter
|
|
46
|
+
@summary
|
|
45
47
|
end
|
|
46
48
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
+
def success?
|
|
50
|
+
(@summary || RunSummary.new(suite: @suite, results: results, elapsed: elapsed)).success?
|
|
49
51
|
end
|
|
50
52
|
|
|
51
|
-
|
|
53
|
+
private
|
|
52
54
|
|
|
55
|
+
def elapsed
|
|
56
|
+
return 0.0 unless @started_at
|
|
57
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at
|
|
58
|
+
end
|
|
59
|
+
end
|
|
53
60
|
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
module Test
|
|
2
|
+
# The outcome of one test, or of a case that was skipped or raised an error.
|
|
3
|
+
class Result
|
|
4
|
+
STATUSES = [:pass, :fail, :error, :todo, :skip].freeze
|
|
5
|
+
KINDS = [:test, :case].freeze
|
|
6
|
+
|
|
7
|
+
attr_reader :test, :kind, :status, :exception, :exceptions, :reason, :elapsed, :stdout, :stderr
|
|
8
|
+
|
|
9
|
+
def initialize(test:, status:, kind: :test, exception: nil, reason: nil,
|
|
10
|
+
elapsed: 0.0, stdout: '', stderr: '', exceptions: nil)
|
|
11
|
+
raise ArgumentError, "unknown result status: #{status.inspect}" unless STATUSES.include?(status)
|
|
12
|
+
raise ArgumentError, "unknown result kind: #{kind.inspect}" unless KINDS.include?(kind)
|
|
13
|
+
|
|
14
|
+
@test = test
|
|
15
|
+
@kind = kind
|
|
16
|
+
@status = status
|
|
17
|
+
@exception = exception
|
|
18
|
+
@exceptions = (exceptions || (exception ? [exception] : [])).dup.freeze
|
|
19
|
+
@reason = reason
|
|
20
|
+
@elapsed = elapsed
|
|
21
|
+
@stdout = stdout.to_s.dup.freeze
|
|
22
|
+
@stderr = stderr.to_s.dup.freeze
|
|
23
|
+
freeze
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# A snapshot of everything recorded during one run.
|
|
28
|
+
class RunSummary
|
|
29
|
+
attr_reader :suite, :results, :counts, :elapsed
|
|
30
|
+
|
|
31
|
+
def initialize(suite:, results:, elapsed:)
|
|
32
|
+
@suite = suite
|
|
33
|
+
@results = results.dup.freeze
|
|
34
|
+
@elapsed = elapsed
|
|
35
|
+
@counts = Result::STATUSES.each_with_object({}) do |status, counts|
|
|
36
|
+
counts[status] = @results.count { |result| result.status == status }
|
|
37
|
+
end.freeze
|
|
38
|
+
freeze
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def total
|
|
42
|
+
results.size
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def results_for(status)
|
|
46
|
+
results.select { |result| result.status == status }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def success?
|
|
50
|
+
counts[:fail].zero? && counts[:error].zero? &&
|
|
51
|
+
(counts[:pass] + counts[:todo] + counts[:skip]).positive?
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
data/lib/rubytest/runner.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'stringio'
|
|
2
|
+
|
|
1
3
|
module Test
|
|
2
4
|
|
|
3
5
|
# Alias for `Test.configure`.
|
|
@@ -19,11 +21,27 @@ module Test
|
|
|
19
21
|
exit -1 unless success
|
|
20
22
|
rescue => error
|
|
21
23
|
raise error if $DEBUG
|
|
22
|
-
$stderr.puts(
|
|
24
|
+
$stderr.puts(error_report(error))
|
|
23
25
|
exit -1
|
|
24
26
|
end
|
|
25
27
|
end
|
|
26
28
|
|
|
29
|
+
# What `run!` prints for an error raised outside of any test, such as one
|
|
30
|
+
# raised while a test file loads: the error's class and message, where it
|
|
31
|
+
# was raised, and how to see the rest of the backtrace.
|
|
32
|
+
#
|
|
33
|
+
# @return [String]
|
|
34
|
+
def self.error_report(error)
|
|
35
|
+
message = error.message.to_s
|
|
36
|
+
report = "ERROR: #{error.class}"
|
|
37
|
+
report << ": #{message}" unless message.empty? || message == error.class.name
|
|
38
|
+
if location = (error.backtrace || []).first
|
|
39
|
+
report << "\n at #{location}"
|
|
40
|
+
end
|
|
41
|
+
report << "\n(Use --debug, or ruby -d, for the full backtrace.)"
|
|
42
|
+
report
|
|
43
|
+
end
|
|
44
|
+
|
|
27
45
|
# The Test::Runner class handles the execution of tests.
|
|
28
46
|
#
|
|
29
47
|
class Runner
|
|
@@ -117,8 +135,7 @@ module Test
|
|
|
117
135
|
# Record pass, fail, error and pending tests.
|
|
118
136
|
attr :recorder
|
|
119
137
|
|
|
120
|
-
#
|
|
121
|
-
# reporter instances.
|
|
138
|
+
# Execution hooks and the reporting pipeline entry point.
|
|
122
139
|
attr :observers
|
|
123
140
|
|
|
124
141
|
# Run test suite.
|
|
@@ -139,20 +156,34 @@ module Test
|
|
|
139
156
|
# applied and before test files are required.
|
|
140
157
|
config.before.call if config.before
|
|
141
158
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
@reporter = reporter_load(format)
|
|
147
|
-
@recorder = Recorder.new
|
|
148
|
-
|
|
149
|
-
@observers = [advice, @recorder, @reporter]
|
|
150
|
-
|
|
151
|
-
observers.each{ |o| o.begin_suite(suite) }
|
|
152
|
-
run_thru(suite)
|
|
153
|
-
observers.each{ |o| o.end_suite(suite) }
|
|
159
|
+
begin
|
|
160
|
+
test_files.each do |test_file|
|
|
161
|
+
require test_file
|
|
162
|
+
end
|
|
154
163
|
|
|
155
|
-
|
|
164
|
+
@reporter = reporter_load(format)
|
|
165
|
+
@recorder = Recorder.new(@reporter)
|
|
166
|
+
|
|
167
|
+
@observers = [advice, @recorder]
|
|
168
|
+
|
|
169
|
+
started = false
|
|
170
|
+
begin
|
|
171
|
+
advice.begin_suite(suite)
|
|
172
|
+
recorder.begin_suite(suite)
|
|
173
|
+
started = true
|
|
174
|
+
run_thru(suite)
|
|
175
|
+
ensure
|
|
176
|
+
if started
|
|
177
|
+
begin
|
|
178
|
+
advice.end_suite(suite)
|
|
179
|
+
ensure
|
|
180
|
+
recorder.end_suite(suite)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
ensure
|
|
185
|
+
config.after.call if config.after
|
|
186
|
+
end
|
|
156
187
|
end
|
|
157
188
|
|
|
158
189
|
recorder.success?
|
|
@@ -174,13 +205,13 @@ module Test
|
|
|
174
205
|
|
|
175
206
|
#
|
|
176
207
|
def run_thru(list)
|
|
177
|
-
list.each do |t|
|
|
208
|
+
select(list).each do |t|
|
|
178
209
|
if t.respond_to?(:each)
|
|
179
210
|
run_case(t)
|
|
180
211
|
elsif t.respond_to?(:call)
|
|
181
212
|
run_test(t)
|
|
182
213
|
else
|
|
183
|
-
#
|
|
214
|
+
raise TypeError, "not a test or test case: #{t.inspect}"
|
|
184
215
|
end
|
|
185
216
|
end
|
|
186
217
|
end
|
|
@@ -189,20 +220,51 @@ module Test
|
|
|
189
220
|
#
|
|
190
221
|
def run_case(tcase)
|
|
191
222
|
if tcase.respond_to?(:skip?) && (reason = tcase.skip?)
|
|
192
|
-
|
|
223
|
+
advice.skip_case(tcase, reason)
|
|
224
|
+
return recorder.record(Result.new(test: tcase, kind: :case,
|
|
225
|
+
status: :skip, reason: reason))
|
|
193
226
|
end
|
|
194
227
|
|
|
195
|
-
|
|
228
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
229
|
+
before_error = nil
|
|
230
|
+
begin
|
|
231
|
+
advice.begin_case(tcase)
|
|
232
|
+
rescue *OPEN_ERRORS
|
|
233
|
+
raise
|
|
234
|
+
rescue Exception => hook_error
|
|
235
|
+
before_error = hook_error
|
|
236
|
+
end
|
|
237
|
+
recorder.begin_case(tcase)
|
|
196
238
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
239
|
+
begin
|
|
240
|
+
if before_error
|
|
241
|
+
raise before_error
|
|
242
|
+
elsif tcase.respond_to?(:call)
|
|
243
|
+
tcase.call do
|
|
244
|
+
run_thru(tcase)
|
|
245
|
+
end
|
|
246
|
+
else
|
|
247
|
+
run_thru(tcase)
|
|
248
|
+
end
|
|
249
|
+
rescue *OPEN_ERRORS
|
|
250
|
+
raise
|
|
251
|
+
rescue Exception => exception
|
|
252
|
+
advice.error(tcase, exception)
|
|
253
|
+
recorder.record(Result.new(test: tcase, kind: :case, status: :error,
|
|
254
|
+
exception: exception, elapsed: elapsed_since(started)))
|
|
255
|
+
ensure
|
|
256
|
+
begin
|
|
257
|
+
advice.end_case(tcase)
|
|
258
|
+
rescue *OPEN_ERRORS
|
|
259
|
+
raise
|
|
260
|
+
rescue Exception => exception
|
|
261
|
+
advice.error(tcase, exception)
|
|
262
|
+
recorder.record(Result.new(test: tcase, kind: :case, status: :error,
|
|
263
|
+
exception: exception, elapsed: elapsed_since(started)))
|
|
264
|
+
ensure
|
|
265
|
+
recorder.end_case(tcase)
|
|
200
266
|
end
|
|
201
|
-
else
|
|
202
|
-
run_thru( select(tcase) )
|
|
203
267
|
end
|
|
204
|
-
|
|
205
|
-
observers.each{ |o| o.end_case(tcase) }
|
|
206
268
|
end
|
|
207
269
|
|
|
208
270
|
# Run a test.
|
|
@@ -212,33 +274,88 @@ module Test
|
|
|
212
274
|
#
|
|
213
275
|
def run_test(test)
|
|
214
276
|
if test.respond_to?(:skip?) && (reason = test.skip?)
|
|
215
|
-
|
|
277
|
+
advice.skip_test(test, reason)
|
|
278
|
+
return recorder.record(Result.new(test: test, status: :skip, reason: reason))
|
|
216
279
|
end
|
|
217
280
|
|
|
218
|
-
|
|
281
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
282
|
+
exceptions = []
|
|
219
283
|
begin
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
284
|
+
advice.begin_test(test)
|
|
285
|
+
rescue *OPEN_ERRORS
|
|
286
|
+
raise
|
|
287
|
+
rescue Exception => hook_error
|
|
288
|
+
exceptions << hook_error
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
recorder.begin_test(test)
|
|
292
|
+
report_started = true
|
|
293
|
+
capture = recorder.capture_output?
|
|
294
|
+
original_stdout, original_stderr = $stdout, $stderr if capture
|
|
295
|
+
captured_stdout, captured_stderr = StringIO.new, StringIO.new if capture
|
|
296
|
+
begin
|
|
297
|
+
$stdout, $stderr = captured_stdout, captured_stderr if capture
|
|
298
|
+
begin
|
|
299
|
+
if exceptions.empty?
|
|
300
|
+
success = test.call
|
|
301
|
+
raise Assertion, "failure of #{test}" if config.hard? && !success
|
|
302
|
+
end
|
|
303
|
+
rescue *OPEN_ERRORS
|
|
304
|
+
raise
|
|
305
|
+
rescue Exception => error
|
|
306
|
+
exceptions << error
|
|
307
|
+
ensure
|
|
308
|
+
begin
|
|
309
|
+
advice.end_test(test)
|
|
310
|
+
rescue *OPEN_ERRORS
|
|
311
|
+
raise
|
|
312
|
+
rescue Exception => hook_error
|
|
313
|
+
exceptions << hook_error
|
|
314
|
+
end
|
|
225
315
|
end
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
if exception.assertion?
|
|
236
|
-
observers.each{ |o| o.fail(test, exception) }
|
|
237
|
-
else
|
|
238
|
-
observers.each{ |o| o.error(test, exception) }
|
|
316
|
+
|
|
317
|
+
status, exception = outcome_for(exceptions)
|
|
318
|
+
begin
|
|
319
|
+
exception ? advice.public_send(status, test, exception) : advice.pass(test)
|
|
320
|
+
rescue *OPEN_ERRORS
|
|
321
|
+
raise
|
|
322
|
+
rescue Exception => hook_error
|
|
323
|
+
exceptions << hook_error
|
|
324
|
+
status, exception = outcome_for(exceptions)
|
|
239
325
|
end
|
|
326
|
+
result = Result.new(test: test, status: status, exception: exception,
|
|
327
|
+
exceptions: exceptions,
|
|
328
|
+
elapsed: elapsed_since(started),
|
|
329
|
+
stdout: captured_stdout&.string,
|
|
330
|
+
stderr: captured_stderr&.string)
|
|
331
|
+
ensure
|
|
332
|
+
$stdout, $stderr = original_stdout, original_stderr if capture
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
recorder.record(result)
|
|
336
|
+
ensure
|
|
337
|
+
recorder.end_test(test) if report_started
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def elapsed_since(started)
|
|
341
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def outcome_for(exceptions)
|
|
345
|
+
return [:pass, nil] if exceptions.empty?
|
|
346
|
+
|
|
347
|
+
exception = exceptions.find do |error|
|
|
348
|
+
!(NotImplementedError === error) && !error.assertion?
|
|
240
349
|
end
|
|
241
|
-
|
|
350
|
+
exception ||= exceptions.find(&:assertion?)
|
|
351
|
+
exception ||= exceptions.first
|
|
352
|
+
|
|
353
|
+
status = if NotImplementedError === exception
|
|
354
|
+
:todo
|
|
355
|
+
else
|
|
356
|
+
exception.assertion? ? :fail : :error
|
|
357
|
+
end
|
|
358
|
+
[status, exception]
|
|
242
359
|
end
|
|
243
360
|
|
|
244
361
|
# TODO: Make sure this filtering code is correct for the complex
|
|
@@ -249,29 +366,36 @@ module Test
|
|
|
249
366
|
#
|
|
250
367
|
# @return [Array] selected test cases
|
|
251
368
|
def select(cases)
|
|
369
|
+
return cases if cases.respond_to?(:ordered?) && cases.ordered?
|
|
370
|
+
return cases if config.match.empty? && config.units.empty? && config.tags.empty?
|
|
371
|
+
|
|
252
372
|
selected = []
|
|
253
|
-
|
|
254
|
-
|
|
373
|
+
cases.each do |tc|
|
|
374
|
+
unless tc.respond_to?(:each) || tc.respond_to?(:call)
|
|
375
|
+
raise TypeError, "not a test or test case: #{tc.inspect}"
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# Keep cases so their descendants can be filtered. The case itself
|
|
379
|
+
# may have a different description, unit, or tags from its tests.
|
|
380
|
+
if tc.respond_to?(:each)
|
|
255
381
|
selected << tc
|
|
382
|
+
next
|
|
256
383
|
end
|
|
257
|
-
else
|
|
258
|
-
cases.each do |tc|
|
|
259
|
-
next if tc.respond_to?(:skip?) && tc.skip?
|
|
260
|
-
next if !config.match.empty? && !config.match.any?{ |m| m =~ tc.to_s }
|
|
261
384
|
|
|
262
|
-
|
|
263
|
-
next unless tc.respond_to?(:unit)
|
|
264
|
-
next unless config.units.find{ |u| tc.unit.start_with?(u) }
|
|
265
|
-
end
|
|
385
|
+
next if !config.match.empty? && !config.match.any?{ |m| tc.to_s.include?(m) }
|
|
266
386
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
end
|
|
387
|
+
if !config.units.empty?
|
|
388
|
+
next unless tc.respond_to?(:unit)
|
|
389
|
+
next unless config.units.any?{ |u| tc.unit.to_s.start_with?(u) }
|
|
390
|
+
end
|
|
272
391
|
|
|
273
|
-
|
|
392
|
+
if !config.tags.empty?
|
|
393
|
+
next unless tc.respond_to?(:tags)
|
|
394
|
+
tc_tags = [tc.tags].flatten.map{ |t| t.to_s }
|
|
395
|
+
next if (config.tags & tc_tags).empty?
|
|
274
396
|
end
|
|
397
|
+
|
|
398
|
+
selected << tc
|
|
275
399
|
end
|
|
276
400
|
selected
|
|
277
401
|
end
|
|
@@ -283,16 +407,25 @@ module Test
|
|
|
283
407
|
def reporter_load(format)
|
|
284
408
|
format = DEFAULT_REPORT_FORMAT unless format
|
|
285
409
|
format = format.to_s.downcase
|
|
286
|
-
name = reporter_list.find{ |r|
|
|
410
|
+
name = reporter_list.find{ |r| r.start_with?(format) } || format
|
|
287
411
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
412
|
+
if KNOWN_FORMATS.include?(name)
|
|
413
|
+
require_relative "format/#{name}"
|
|
414
|
+
else
|
|
415
|
+
begin
|
|
416
|
+
require "rubytest/format/#{name}"
|
|
417
|
+
rescue LoadError => error
|
|
418
|
+
raise ArgumentError, "unknown report format #{name.inspect}" if error.path == "rubytest/format/#{name}"
|
|
419
|
+
raise
|
|
420
|
+
end
|
|
292
421
|
end
|
|
293
422
|
|
|
294
423
|
reporter = Test::Reporters.const_get(name.capitalize)
|
|
295
|
-
reporter.new(self)
|
|
424
|
+
reporter = reporter.new(self)
|
|
425
|
+
unless reporter.respond_to?(:record) && reporter.respond_to?(:finish)
|
|
426
|
+
raise ArgumentError, "report format #{name} does not support result reporting"
|
|
427
|
+
end
|
|
428
|
+
reporter
|
|
296
429
|
end
|
|
297
430
|
|
|
298
431
|
# List of known report formats.
|
|
@@ -300,7 +433,7 @@ module Test
|
|
|
300
433
|
# TODO: Could use finder gem to look these up, but that's yet another dependency.
|
|
301
434
|
#
|
|
302
435
|
KNOWN_FORMATS = %w{
|
|
303
|
-
dotprogress
|
|
436
|
+
dotprogress progress outline summary tap tapy tapj test
|
|
304
437
|
}
|
|
305
438
|
|
|
306
439
|
# Returns a list of available report types.
|
|
@@ -320,12 +453,13 @@ module Test
|
|
|
320
453
|
#
|
|
321
454
|
# @return [Array<String>]
|
|
322
455
|
def resolve_test_files
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
456
|
+
config.files.flatten.flat_map do |pattern|
|
|
457
|
+
files = Dir[pattern].flat_map do |file|
|
|
458
|
+
File.directory?(file) ? Dir[File.join(file, '**/*.rb')] : [file]
|
|
459
|
+
end
|
|
460
|
+
raise ArgumentError, "no test files match #{pattern.inspect}" if files.empty?
|
|
461
|
+
files
|
|
462
|
+
end.uniq.map{ |file| File.expand_path(file) }
|
|
329
463
|
end
|
|
330
464
|
|
|
331
465
|
# Change to directory and run block.
|