rubytest 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,76 @@
1
+ # encoding: UTF-8
2
+
3
+ module Test::Reporters
4
+
5
+ # TAP-J Reporter
6
+ #
7
+ # Writes the run as a TAP-J stream (revision 5): a line of JSON for
8
+ # the suite, each case, each test and the final tally.
9
+ #
10
+ class Tapj < AbstractHash
11
+
12
+ REVISION = 5
13
+
14
+ #
15
+ def initialize(runner, output: $stdout)
16
+ require 'json'
17
+ super(runner)
18
+ @output = output
19
+ end
20
+
21
+ #
22
+ def begin_suite(suite)
23
+ emit super(suite).merge('rev' => REVISION)
24
+ end
25
+
26
+ #
27
+ def begin_case(test_case)
28
+ emit super(test_case)
29
+ end
30
+
31
+ #
32
+ def pass(test)
33
+ emit super(test)
34
+ end
35
+
36
+ #
37
+ def fail(test, exception)
38
+ emit super(test, exception)
39
+ end
40
+
41
+ #
42
+ def error(test, exception)
43
+ emit super(test, exception)
44
+ end
45
+
46
+ #
47
+ def todo(test, exception)
48
+ emit super(test, exception)
49
+ end
50
+
51
+ #
52
+ def skip_test(test, reason=nil)
53
+ emit super(test, reason)
54
+ end
55
+
56
+ #
57
+ def skip_case(test_case, reason=nil)
58
+ emit super(test_case, reason)
59
+ end
60
+
61
+ #
62
+ def end_suite(suite)
63
+ emit super(suite)
64
+ end
65
+
66
+ private
67
+
68
+ # Write one record to the stream.
69
+ def emit(hash)
70
+ @output.puts hash.to_json
71
+ hash
72
+ end
73
+
74
+ end
75
+
76
+ end
@@ -0,0 +1,77 @@
1
+ # encoding: UTF-8
2
+
3
+ module Test::Reporters
4
+
5
+ # TAP-Y Reporter
6
+ #
7
+ # Writes the run as a TAP-Y stream (revision 5): a YAML document for
8
+ # the suite, each case, each test and the final tally.
9
+ #
10
+ class Tapy < AbstractHash
11
+
12
+ REVISION = 5
13
+
14
+ #
15
+ def initialize(runner, output: $stdout)
16
+ require 'yaml'
17
+ super(runner)
18
+ @output = output
19
+ end
20
+
21
+ #
22
+ def begin_suite(suite)
23
+ emit super(suite).merge('rev' => REVISION)
24
+ end
25
+
26
+ #
27
+ def begin_case(test_case)
28
+ emit super(test_case)
29
+ end
30
+
31
+ #
32
+ def pass(test)
33
+ emit super(test)
34
+ end
35
+
36
+ #
37
+ def fail(test, exception)
38
+ emit super(test, exception)
39
+ end
40
+
41
+ #
42
+ def error(test, exception)
43
+ emit super(test, exception)
44
+ end
45
+
46
+ #
47
+ def todo(test, exception)
48
+ emit super(test, exception)
49
+ end
50
+
51
+ #
52
+ def skip_test(test, reason=nil)
53
+ emit super(test, reason)
54
+ end
55
+
56
+ #
57
+ def skip_case(test_case, reason=nil)
58
+ emit super(test_case, reason)
59
+ end
60
+
61
+ #
62
+ def end_suite(suite)
63
+ emit super(suite)
64
+ @output.puts '...'
65
+ end
66
+
67
+ private
68
+
69
+ # Write one record to the stream.
70
+ def emit(hash)
71
+ @output.puts hash.to_yaml
72
+ hash
73
+ end
74
+
75
+ end
76
+
77
+ end
@@ -1,53 +1,60 @@
1
1
  module Test
2
-
3
- # Recorder class is an observer that tracks all tests
4
- # that are run and categorizes them according to their
5
- # test status.
2
+ # Owns run results and controls when they reach the reporter.
6
3
  class Recorder
4
+ attr_reader :results, :reporter, :summary
7
5
 
8
- def initialize
9
- @table = Hash.new{ |h,k| h[k] = [] }
6
+ def initialize(reporter = nil)
7
+ @reporter = reporter
8
+ @results = []
10
9
  end
11
10
 
12
- def [](key)
13
- @table[key.to_sym]
11
+ def begin_suite(suite)
12
+ @suite = suite
13
+ @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
14
+ reporter.begin_suite(suite) if reporter&.respond_to?(:begin_suite)
14
15
  end
15
16
 
16
- #
17
- def skip_test(test, reason)
18
- self[:skip] << [test, reason]
17
+ def begin_case(test_case)
18
+ reporter.begin_case(test_case) if reporter&.respond_to?(:begin_case)
19
19
  end
20
20
 
21
- # Add `test` to pass set.
22
- def pass(test)
23
- self[:pass] << test
21
+ def begin_test(test)
22
+ reporter.begin_test(test) if reporter&.respond_to?(:begin_test)
24
23
  end
25
24
 
26
- def fail(test, exception)
27
- self[:fail] << [test, exception]
25
+ def capture_output?
26
+ !!(reporter && reporter.respond_to?(:capture_output?) && reporter.capture_output?)
28
27
  end
29
28
 
30
- def error(test, exception)
31
- self[:error] << [test, exception]
29
+ def record(result)
30
+ @results << result
31
+ reporter.record(result) if reporter
32
+ result
32
33
  end
33
34
 
34
- def todo(test, exception)
35
- self[:todo] << [test, exception]
35
+ def end_test(test)
36
+ reporter.end_test(test) if reporter&.respond_to?(:end_test)
36
37
  end
37
38
 
38
- #def omit(test, exception)
39
- # self[:omit] << [test, exception]
40
- #end
39
+ def end_case(test_case)
40
+ reporter.end_case(test_case) if reporter&.respond_to?(:end_case)
41
+ end
41
42
 
42
- # Returns true if their are no test errors or failures.
43
- def success?
44
- self[:error].size + self[:fail].size > 0 ? false : true
43
+ def end_suite(suite)
44
+ @summary = RunSummary.new(suite: suite, results: results, elapsed: elapsed)
45
+ reporter.finish(@summary) if reporter
46
+ @summary
45
47
  end
46
48
 
47
- # Ignore any other signals.
48
- def method_missing(*a)
49
+ def success?
50
+ (@summary || RunSummary.new(suite: @suite, results: results, elapsed: elapsed)).success?
49
51
  end
50
52
 
51
- end
53
+ private
52
54
 
55
+ def elapsed
56
+ return 0.0 unless @started_at
57
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at
58
+ end
59
+ end
53
60
  end
@@ -0,0 +1,54 @@
1
+ module Test
2
+ # The outcome of one test, or of a case that was skipped or raised an error.
3
+ class Result
4
+ STATUSES = [:pass, :fail, :error, :todo, :skip].freeze
5
+ KINDS = [:test, :case].freeze
6
+
7
+ attr_reader :test, :kind, :status, :exception, :exceptions, :reason, :elapsed, :stdout, :stderr
8
+
9
+ def initialize(test:, status:, kind: :test, exception: nil, reason: nil,
10
+ elapsed: 0.0, stdout: '', stderr: '', exceptions: nil)
11
+ raise ArgumentError, "unknown result status: #{status.inspect}" unless STATUSES.include?(status)
12
+ raise ArgumentError, "unknown result kind: #{kind.inspect}" unless KINDS.include?(kind)
13
+
14
+ @test = test
15
+ @kind = kind
16
+ @status = status
17
+ @exception = exception
18
+ @exceptions = (exceptions || (exception ? [exception] : [])).dup.freeze
19
+ @reason = reason
20
+ @elapsed = elapsed
21
+ @stdout = stdout.to_s.dup.freeze
22
+ @stderr = stderr.to_s.dup.freeze
23
+ freeze
24
+ end
25
+ end
26
+
27
+ # A snapshot of everything recorded during one run.
28
+ class RunSummary
29
+ attr_reader :suite, :results, :counts, :elapsed
30
+
31
+ def initialize(suite:, results:, elapsed:)
32
+ @suite = suite
33
+ @results = results.dup.freeze
34
+ @elapsed = elapsed
35
+ @counts = Result::STATUSES.each_with_object({}) do |status, counts|
36
+ counts[status] = @results.count { |result| result.status == status }
37
+ end.freeze
38
+ freeze
39
+ end
40
+
41
+ def total
42
+ results.size
43
+ end
44
+
45
+ def results_for(status)
46
+ results.select { |result| result.status == status }
47
+ end
48
+
49
+ def success?
50
+ counts[:fail].zero? && counts[:error].zero? &&
51
+ (counts[:pass] + counts[:todo] + counts[:skip]).positive?
52
+ end
53
+ end
54
+ end
@@ -1,3 +1,5 @@
1
+ require 'stringio'
2
+
1
3
  module Test
2
4
 
3
5
  # Alias for `Test.configure`.
@@ -19,11 +21,27 @@ module Test
19
21
  exit -1 unless success
20
22
  rescue => error
21
23
  raise error if $DEBUG
22
- $stderr.puts('ERROR: ' + error.to_s)
24
+ $stderr.puts(error_report(error))
23
25
  exit -1
24
26
  end
25
27
  end
26
28
 
29
+ # What `run!` prints for an error raised outside of any test, such as one
30
+ # raised while a test file loads: the error's class and message, where it
31
+ # was raised, and how to see the rest of the backtrace.
32
+ #
33
+ # @return [String]
34
+ def self.error_report(error)
35
+ message = error.message.to_s
36
+ report = "ERROR: #{error.class}"
37
+ report << ": #{message}" unless message.empty? || message == error.class.name
38
+ if location = (error.backtrace || []).first
39
+ report << "\n at #{location}"
40
+ end
41
+ report << "\n(Use --debug, or ruby -d, for the full backtrace.)"
42
+ report
43
+ end
44
+
27
45
  # The Test::Runner class handles the execution of tests.
28
46
  #
29
47
  class Runner
@@ -117,8 +135,7 @@ module Test
117
135
  # Record pass, fail, error and pending tests.
118
136
  attr :recorder
119
137
 
120
- # Array of observers, typically this just contains the recorder and
121
- # reporter instances.
138
+ # Execution hooks and the reporting pipeline entry point.
122
139
  attr :observers
123
140
 
124
141
  # Run test suite.
@@ -139,20 +156,34 @@ module Test
139
156
  # applied and before test files are required.
140
157
  config.before.call if config.before
141
158
 
142
- test_files.each do |test_file|
143
- require test_file
144
- end
145
-
146
- @reporter = reporter_load(format)
147
- @recorder = Recorder.new
148
-
149
- @observers = [advice, @recorder, @reporter]
150
-
151
- observers.each{ |o| o.begin_suite(suite) }
152
- run_thru(suite)
153
- observers.each{ |o| o.end_suite(suite) }
159
+ begin
160
+ test_files.each do |test_file|
161
+ require test_file
162
+ end
154
163
 
155
- config.after.call if config.after
164
+ @reporter = reporter_load(format)
165
+ @recorder = Recorder.new(@reporter)
166
+
167
+ @observers = [advice, @recorder]
168
+
169
+ started = false
170
+ begin
171
+ advice.begin_suite(suite)
172
+ recorder.begin_suite(suite)
173
+ started = true
174
+ run_thru(suite)
175
+ ensure
176
+ if started
177
+ begin
178
+ advice.end_suite(suite)
179
+ ensure
180
+ recorder.end_suite(suite)
181
+ end
182
+ end
183
+ end
184
+ ensure
185
+ config.after.call if config.after
186
+ end
156
187
  end
157
188
 
158
189
  recorder.success?
@@ -174,13 +205,13 @@ module Test
174
205
 
175
206
  #
176
207
  def run_thru(list)
177
- list.each do |t|
208
+ select(list).each do |t|
178
209
  if t.respond_to?(:each)
179
210
  run_case(t)
180
211
  elsif t.respond_to?(:call)
181
212
  run_test(t)
182
213
  else
183
- #run_note(t) ?
214
+ raise TypeError, "not a test or test case: #{t.inspect}"
184
215
  end
185
216
  end
186
217
  end
@@ -189,20 +220,51 @@ module Test
189
220
  #
190
221
  def run_case(tcase)
191
222
  if tcase.respond_to?(:skip?) && (reason = tcase.skip?)
192
- return observers.each{ |o| o.skip_case(tcase, reason) }
223
+ advice.skip_case(tcase, reason)
224
+ return recorder.record(Result.new(test: tcase, kind: :case,
225
+ status: :skip, reason: reason))
193
226
  end
194
227
 
195
- observers.each{ |o| o.begin_case(tcase) }
228
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
229
+ before_error = nil
230
+ begin
231
+ advice.begin_case(tcase)
232
+ rescue *OPEN_ERRORS
233
+ raise
234
+ rescue Exception => hook_error
235
+ before_error = hook_error
236
+ end
237
+ recorder.begin_case(tcase)
196
238
 
197
- if tcase.respond_to?(:call)
198
- tcase.call do
199
- run_thru( select(tcase) )
239
+ begin
240
+ if before_error
241
+ raise before_error
242
+ elsif tcase.respond_to?(:call)
243
+ tcase.call do
244
+ run_thru(tcase)
245
+ end
246
+ else
247
+ run_thru(tcase)
248
+ end
249
+ rescue *OPEN_ERRORS
250
+ raise
251
+ rescue Exception => exception
252
+ advice.error(tcase, exception)
253
+ recorder.record(Result.new(test: tcase, kind: :case, status: :error,
254
+ exception: exception, elapsed: elapsed_since(started)))
255
+ ensure
256
+ begin
257
+ advice.end_case(tcase)
258
+ rescue *OPEN_ERRORS
259
+ raise
260
+ rescue Exception => exception
261
+ advice.error(tcase, exception)
262
+ recorder.record(Result.new(test: tcase, kind: :case, status: :error,
263
+ exception: exception, elapsed: elapsed_since(started)))
264
+ ensure
265
+ recorder.end_case(tcase)
200
266
  end
201
- else
202
- run_thru( select(tcase) )
203
267
  end
204
-
205
- observers.each{ |o| o.end_case(tcase) }
206
268
  end
207
269
 
208
270
  # Run a test.
@@ -212,33 +274,88 @@ module Test
212
274
  #
213
275
  def run_test(test)
214
276
  if test.respond_to?(:skip?) && (reason = test.skip?)
215
- return observers.each{ |o| o.skip_test(test, reason) }
277
+ advice.skip_test(test, reason)
278
+ return recorder.record(Result.new(test: test, status: :skip, reason: reason))
216
279
  end
217
280
 
218
- observers.each{ |o| o.begin_test(test) }
281
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
282
+ exceptions = []
219
283
  begin
220
- success = test.call
221
- if config.hard? && !success # TODO: separate run_test method to speed things up?
222
- raise Assertion, "failure of #{test}"
223
- else
224
- observers.each{ |o| o.pass(test) }
284
+ advice.begin_test(test)
285
+ rescue *OPEN_ERRORS
286
+ raise
287
+ rescue Exception => hook_error
288
+ exceptions << hook_error
289
+ end
290
+
291
+ recorder.begin_test(test)
292
+ report_started = true
293
+ capture = recorder.capture_output?
294
+ original_stdout, original_stderr = $stdout, $stderr if capture
295
+ captured_stdout, captured_stderr = StringIO.new, StringIO.new if capture
296
+ begin
297
+ $stdout, $stderr = captured_stdout, captured_stderr if capture
298
+ begin
299
+ if exceptions.empty?
300
+ success = test.call
301
+ raise Assertion, "failure of #{test}" if config.hard? && !success
302
+ end
303
+ rescue *OPEN_ERRORS
304
+ raise
305
+ rescue Exception => error
306
+ exceptions << error
307
+ ensure
308
+ begin
309
+ advice.end_test(test)
310
+ rescue *OPEN_ERRORS
311
+ raise
312
+ rescue Exception => hook_error
313
+ exceptions << hook_error
314
+ end
225
315
  end
226
- rescue *OPEN_ERRORS => exception
227
- raise exception
228
- rescue NotImplementedError => exception
229
- #if exception.assertion? # TODO: May require assertion? for todo in future
230
- observers.each{ |o| o.todo(test, exception) }
231
- #else
232
- # observers.each{ |o| o.error(test, exception) }
233
- #end
234
- rescue Exception => exception
235
- if exception.assertion?
236
- observers.each{ |o| o.fail(test, exception) }
237
- else
238
- observers.each{ |o| o.error(test, exception) }
316
+
317
+ status, exception = outcome_for(exceptions)
318
+ begin
319
+ exception ? advice.public_send(status, test, exception) : advice.pass(test)
320
+ rescue *OPEN_ERRORS
321
+ raise
322
+ rescue Exception => hook_error
323
+ exceptions << hook_error
324
+ status, exception = outcome_for(exceptions)
239
325
  end
326
+ result = Result.new(test: test, status: status, exception: exception,
327
+ exceptions: exceptions,
328
+ elapsed: elapsed_since(started),
329
+ stdout: captured_stdout&.string,
330
+ stderr: captured_stderr&.string)
331
+ ensure
332
+ $stdout, $stderr = original_stdout, original_stderr if capture
333
+ end
334
+
335
+ recorder.record(result)
336
+ ensure
337
+ recorder.end_test(test) if report_started
338
+ end
339
+
340
+ def elapsed_since(started)
341
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
342
+ end
343
+
344
+ def outcome_for(exceptions)
345
+ return [:pass, nil] if exceptions.empty?
346
+
347
+ exception = exceptions.find do |error|
348
+ !(NotImplementedError === error) && !error.assertion?
240
349
  end
241
- observers.each{ |o| o.end_test(test) }
350
+ exception ||= exceptions.find(&:assertion?)
351
+ exception ||= exceptions.first
352
+
353
+ status = if NotImplementedError === exception
354
+ :todo
355
+ else
356
+ exception.assertion? ? :fail : :error
357
+ end
358
+ [status, exception]
242
359
  end
243
360
 
244
361
  # TODO: Make sure this filtering code is correct for the complex
@@ -249,29 +366,36 @@ module Test
249
366
  #
250
367
  # @return [Array] selected test cases
251
368
  def select(cases)
369
+ return cases if cases.respond_to?(:ordered?) && cases.ordered?
370
+ return cases if config.match.empty? && config.units.empty? && config.tags.empty?
371
+
252
372
  selected = []
253
- if cases.respond_to?(:ordered?) && cases.ordered?
254
- cases.each do |tc|
373
+ cases.each do |tc|
374
+ unless tc.respond_to?(:each) || tc.respond_to?(:call)
375
+ raise TypeError, "not a test or test case: #{tc.inspect}"
376
+ end
377
+
378
+ # Keep cases so their descendants can be filtered. The case itself
379
+ # may have a different description, unit, or tags from its tests.
380
+ if tc.respond_to?(:each)
255
381
  selected << tc
382
+ next
256
383
  end
257
- else
258
- cases.each do |tc|
259
- next if tc.respond_to?(:skip?) && tc.skip?
260
- next if !config.match.empty? && !config.match.any?{ |m| m =~ tc.to_s }
261
384
 
262
- if !config.units.empty?
263
- next unless tc.respond_to?(:unit)
264
- next unless config.units.find{ |u| tc.unit.start_with?(u) }
265
- end
385
+ next if !config.match.empty? && !config.match.any?{ |m| tc.to_s.include?(m) }
266
386
 
267
- if !config.tags.empty?
268
- next unless tc.respond_to?(:tags)
269
- tc_tags = [tc.tags].flatten.map{ |t| t.to_s }
270
- next if (config.tags & tc_tags).empty?
271
- end
387
+ if !config.units.empty?
388
+ next unless tc.respond_to?(:unit)
389
+ next unless config.units.any?{ |u| tc.unit.to_s.start_with?(u) }
390
+ end
272
391
 
273
- selected << tc
392
+ if !config.tags.empty?
393
+ next unless tc.respond_to?(:tags)
394
+ tc_tags = [tc.tags].flatten.map{ |t| t.to_s }
395
+ next if (config.tags & tc_tags).empty?
274
396
  end
397
+
398
+ selected << tc
275
399
  end
276
400
  selected
277
401
  end
@@ -283,16 +407,25 @@ module Test
283
407
  def reporter_load(format)
284
408
  format = DEFAULT_REPORT_FORMAT unless format
285
409
  format = format.to_s.downcase
286
- name = reporter_list.find{ |r| /^#{format}/ =~ r } || format
410
+ name = reporter_list.find{ |r| r.start_with?(format) } || format
287
411
 
288
- begin
289
- require "rubytest/format/#{name}"
290
- rescue LoadError
291
- raise "mistyped or uninstalled report format" unless format
412
+ if KNOWN_FORMATS.include?(name)
413
+ require_relative "format/#{name}"
414
+ else
415
+ begin
416
+ require "rubytest/format/#{name}"
417
+ rescue LoadError => error
418
+ raise ArgumentError, "unknown report format #{name.inspect}" if error.path == "rubytest/format/#{name}"
419
+ raise
420
+ end
292
421
  end
293
422
 
294
423
  reporter = Test::Reporters.const_get(name.capitalize)
295
- reporter.new(self)
424
+ reporter = reporter.new(self)
425
+ unless reporter.respond_to?(:record) && reporter.respond_to?(:finish)
426
+ raise ArgumentError, "report format #{name} does not support result reporting"
427
+ end
428
+ reporter
296
429
  end
297
430
 
298
431
  # List of known report formats.
@@ -300,7 +433,7 @@ module Test
300
433
  # TODO: Could use finder gem to look these up, but that's yet another dependency.
301
434
  #
302
435
  KNOWN_FORMATS = %w{
303
- dotprogress html progress outline summary tap tapy tapj test
436
+ dotprogress progress outline summary tap tapy tapj test
304
437
  }
305
438
 
306
439
  # Returns a list of available report types.
@@ -320,12 +453,13 @@ module Test
320
453
  #
321
454
  # @return [Array<String>]
322
455
  def resolve_test_files
323
- list = config.files.flatten
324
- list = list.map{ |f| Dir[f] }.flatten
325
- list = list.map{ |f| File.directory?(f) ? Dir[File.join(f, '**/*.rb')] : f }
326
- list = list.flatten.uniq
327
- list = list.map{ |f| File.expand_path(f) }
328
- list
456
+ config.files.flatten.flat_map do |pattern|
457
+ files = Dir[pattern].flat_map do |file|
458
+ File.directory?(file) ? Dir[File.join(file, '**/*.rb')] : [file]
459
+ end
460
+ raise ArgumentError, "no test files match #{pattern.inspect}" if files.empty?
461
+ files
462
+ end.uniq.map{ |file| File.expand_path(file) }
329
463
  end
330
464
 
331
465
  # Change to directory and run block.