rubytest 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,118 @@
1
+ ## Built-in reporters
2
+
3
+ The human-readable formats consume the same results, including pending tests
4
+ and skipped cases. They can be selected without installing another gem.
5
+
6
+ require 'stringio'
7
+
8
+ probe_type = Class.new do
9
+ def initialize(label, reason = nil, &action)
10
+ @label, @reason, @action = label, reason, action
11
+ end
12
+
13
+ def call
14
+ @action.call
15
+ end
16
+
17
+ def skip?
18
+ @reason
19
+ end
20
+
21
+ def to_s
22
+ @label
23
+ end
24
+ end
25
+
26
+ passing = probe_type.new('passing') { true }
27
+ failing = probe_type.new('failing') { raise Assertion, 'wrong value' }
28
+ pending = probe_type.new('pending') { raise NotImplementedError, 'later' }
29
+ skipped = probe_type.new('skipped', 'not available') { raise 'must not run' }
30
+ skipped_case = Class.new(Array) do
31
+ def skip?; 'case unavailable'; end
32
+ def to_s; 'skipped case'; end
33
+ end.new([passing])
34
+ broken_case = Class.new(Array) do
35
+ def call; raise 'case setup failed'; end
36
+ def to_s; 'broken case'; end
37
+ end.new([passing])
38
+ suite = [[passing, failing], pending, skipped, skipped_case, broken_case]
39
+
40
+ render = lambda do |format, tests = suite|
41
+ require "rubytest/format/#{format}"
42
+ output = StringIO.new
43
+ reporter_type = Test::Reporters.const_get(format.capitalize)
44
+ runner_type = Class.new(Test::Runner) do
45
+ define_method(:reporter_load) { |_name| reporter_type.new(self, output: output) }
46
+ end
47
+ runner = runner_type.new(suite: tests, format: format)
48
+ [runner.run, output.string, runner.recorder.summary]
49
+ end
50
+
51
+ %w[summary outline progress].each do |format|
52
+ success, output, report = render.call(format)
53
+ success.assert == false
54
+ report.total.assert == 6
55
+ output.include?('failing').assert == true
56
+ output.include?('skipped case').assert == true
57
+ output.include?('6').assert == true
58
+ end
59
+
60
+ TAP emits one test point for every result. Test output is captured and appears
61
+ as diagnostics, so it cannot corrupt the TAP stream. The final plan uses the
62
+ number actually recorded, including the broken and skipped cases.
63
+
64
+ output_test = probe_type.new('output') { print "ok 999 - injected\n"; true }
65
+ tap_suite = [[output_test, failing], pending, skipped, skipped_case, broken_case]
66
+ success, output, report = render.call('tap', tap_suite)
67
+ success.assert == false
68
+ lines = output.lines.map(&:chomp)
69
+ lines.first.assert == 'TAP version 13'
70
+ lines.last.assert == '1..6'
71
+ lines.grep(/\A(?:ok|not ok) \d+ - /).size.assert == report.total
72
+ lines.grep(/# SKIP /).size.assert == 2
73
+ lines.grep(/# TODO /).size.assert == 1
74
+ lines.grep(/\A# stdout: ok 999 - injected/).size.assert == 1
75
+ lines.grep(/\Anot ok \d+ - broken case/).size.assert == 1
76
+
77
+ TAP-Y and TAP-J carry the same results as structured records: one for the
78
+ suite, one for each case and test, and a final tally. TAP-Y is a stream of
79
+ YAML documents, TAP-J a line of JSON for each record.
80
+
81
+ require 'yaml'
82
+ require 'json'
83
+
84
+ readers = {
85
+ 'tapy' => lambda { |text| YAML.load_stream(text) },
86
+ 'tapj' => lambda { |text| text.lines.map { |line| JSON.parse(line) } }
87
+ }
88
+
89
+ readers.each do |format, read|
90
+ success, output, report = render.call(format)
91
+ success.assert == false
92
+ records = read.call(output)
93
+
94
+ records.first['type'].assert == 'suite'
95
+ records.first['rev'].assert == 5
96
+ records.last['type'].assert == 'final'
97
+ records.last['counts']['total'].assert == report.total
98
+ records.last['counts']['omit'].assert == 2
99
+
100
+ results = records.select { |record| record['status'] }
101
+ results.size.assert == report.total
102
+ results.map { |record| record['status'] }.sort.assert == %w[error fail omit omit pass todo]
103
+ results.select { |record| record['status'] == 'omit' }.map { |record| record['type'] }.sort.assert == %w[case test]
104
+ records.count { |record| record['type'] == 'case' && !record['status'] }.assert == 2
105
+ end
106
+
107
+ A TAP-Y stream ends with the YAML end-of-document marker.
108
+
109
+ render.call('tapy')[1].lines.last.chomp.assert == '...'
110
+
111
+ The command line has a shortcut for each.
112
+
113
+ require 'rubytest/cli'
114
+ { '-y' => 'tapy', '--tapj' => 'tapj' }.each do |flag, format|
115
+ cli = Test::CLI.new
116
+ cli.options.parse!([flag])
117
+ cli.format.assert == format
118
+ end
@@ -0,0 +1,227 @@
1
+ require 'rubytest'
2
+
3
+ module Test
4
+
5
+ # Command line interface to test runner.
6
+ #
7
+ class CLI
8
+
9
+ # Test configuration file can be in `etc/test.rb` or `config/test.rb`, or
10
+ # `Testfile` or '.test` with optional `.rb` extension, in that order of
11
+ # precedence. To use a different file there is the -c/--config option.
12
+ GLOB_CONFIG = '{etc/test.rb,config/test.rb,Testfile.rb,Testfile,testfile.rb,testfile,.test.rb,.test}'
13
+
14
+ # Convenience method for invoking the CLI.
15
+ #
16
+ # @return nothing
17
+ def self.run(*argv)
18
+ new.run(*argv)
19
+ end
20
+
21
+ # Initialize CLI instance.
22
+ #
23
+ # @return nothing
24
+ def initialize
25
+ require 'optparse'
26
+
27
+ @config = {}
28
+ @config_file = nil
29
+ end
30
+
31
+ # Run tests.
32
+ #
33
+ # @return nothing
34
+ def run(argv=nil)
35
+ argv = (argv || ARGV.dup)
36
+ options.parse!(argv)
37
+
38
+ @config[:files] = argv unless argv.empty?
39
+
40
+ load_config
41
+
42
+ test_config = Test.configuration(profile)
43
+ test_config.apply_environment_defaults
44
+ test_config.apply(@config)
45
+
46
+ Test.run!(test_config)
47
+ end
48
+
49
+ # Setup OptionsParser instance.
50
+ #
51
+ # @return [OptionParser]
52
+ def options
53
+ conf = self
54
+
55
+ OptionParser.new do |opt|
56
+ opt.banner = "Usage: #{File.basename($0)} [options] [files ...]"
57
+
58
+ opt.on '-p', '--profile NAME', 'use configuration profile' do |name|
59
+ conf.profile = name
60
+ end
61
+
62
+ opt.on '-f', '--format NAME', 'report format' do |name|
63
+ conf.format = name
64
+ end
65
+ opt.on '-y', '--tapy', 'shortcut for -f tapy' do
66
+ conf.format = 'tapy'
67
+ end
68
+ opt.on '-j', '--tapj', 'shortcut for -f tapj' do
69
+ conf.format = 'tapj'
70
+ end
71
+
72
+ opt.on '-t', '--tag TAG', 'select tests by tag' do |tag|
73
+ conf.tags.concat makelist(tag)
74
+ end
75
+ opt.on '-u', '--unit TAG', 'select tests by software unit' do |unit|
76
+ conf.units.concat makelist(unit)
77
+ end
78
+ opt.on '-m', '--match TEXT', 'select tests by description' do |text|
79
+ conf.match.concat makelist(text)
80
+ end
81
+
82
+ opt.on '-A', '--autopath', 'automatically add paths to $LOAD_PATH' do |paths|
83
+ conf.autopath = true
84
+ end
85
+ opt.on '-I', '--loadpath PATH', 'add given path to $LOAD_PATH' do |paths|
86
+ conf.loadpath.concat makelist(paths)
87
+ end
88
+ opt.on '-C', '--chdir DIR', 'change directory before running tests' do |dir|
89
+ conf.chdir = dir
90
+ end
91
+ opt.on '-R', '--chroot', 'change to project root directory before running tests' do
92
+ conf.chdir = Config.root
93
+ end
94
+ opt.on '-r', '--require FILE', 'require file' do |file|
95
+ conf.requires.concat makelist(file)
96
+ end
97
+ opt.on '-c', '--config FILE', "use alternate config file" do |file|
98
+ conf.config_file = file
99
+ end
100
+ opt.on '-V' , '--verbose', 'provide extra detail in reports' do
101
+ conf.verbose = true
102
+ end
103
+ opt.on("--[no-]ansi" , 'turn on/off ANSI colors'){ |v| $ansi = v }
104
+ opt.on("--debug" , 'turn on debugging mode'){ $DEBUG = true }
105
+
106
+ opt.on('--version' , 'display rubytest version') do
107
+ puts "rubytest v#{Test::VERSION}"
108
+ exit
109
+ end
110
+ opt.on('-h', '--help', 'display this help message'){
111
+ puts opt
112
+ exit
113
+ }
114
+ end
115
+ end
116
+
117
+ # Test run configuration.
118
+ #
119
+ # @return [Hash]
120
+ def config
121
+ @config
122
+ end
123
+
124
+ # Load configuration file.
125
+ def load_config
126
+ file = config_file
127
+ unless file
128
+ if chdir
129
+ file = Dir.glob(File.join(chdir, GLOB_CONFIG)).first
130
+ else
131
+ file = Dir.glob(GLOB_CONFIG).first
132
+ end
133
+ end
134
+ load file if file
135
+ end
136
+
137
+ # Find traditional configuration file.
138
+ #
139
+ # @return [String] Config file path.
140
+ def config_file
141
+ @config_file
142
+ end
143
+
144
+ def config_file=(file)
145
+ @config_file = file
146
+ end
147
+
148
+ def profile
149
+ @profile
150
+ end
151
+
152
+ def profile=(name)
153
+ @profile = name.to_s
154
+ end
155
+
156
+ def format
157
+ @config[:format]
158
+ end
159
+
160
+ def format=(format)
161
+ @config[:format] = format.to_s
162
+ end
163
+
164
+ def tags
165
+ @config[:tags] ||= []
166
+ end
167
+
168
+ def units
169
+ @config[:units] ||= []
170
+ end
171
+
172
+ def match
173
+ @config[:match] ||= []
174
+ end
175
+
176
+ def autopath?
177
+ @config[:autopath]
178
+ end
179
+
180
+ def autopath=(bool)
181
+ @config[:autopath] = !! bool
182
+ end
183
+
184
+ def chdir
185
+ @config[:chdir]
186
+ end
187
+
188
+ def chdir=(dir)
189
+ @config[:chdir] = dir.to_str
190
+ end
191
+
192
+ def loadpath
193
+ @config[:loadpath] ||= []
194
+ end
195
+
196
+ def requires
197
+ @config[:requires] ||= []
198
+ end
199
+
200
+ def verbose?
201
+ @config[:verbose]
202
+ end
203
+
204
+ def verbose=(bool)
205
+ @config[:verbose] = !! bool
206
+ end
207
+
208
+ private
209
+
210
+ # If given a String then split up at `:` and `;` markers.
211
+ # Otherwise ensure the list is an Array and the entries are
212
+ # all strings and not empty.
213
+ #
214
+ # @return [Array<String>]
215
+ def makelist(list)
216
+ case list
217
+ when String
218
+ list = list.split(/[:;]/)
219
+ else
220
+ list = Array(list).map{ |path| path.to_s }
221
+ end
222
+ list.reject{ |path| path.strip.empty? }
223
+ end
224
+
225
+ end
226
+
227
+ end
@@ -51,7 +51,7 @@ module Test
51
51
 
52
52
  #
53
53
  def self.assertionless=(boolean)
54
- @assertionaless = !!boolean
54
+ @assertionless = !!boolean
55
55
  end
56
56
 
57
57
  # Find and cache project root directory.
@@ -43,34 +43,6 @@ module Test
43
43
  def begin_test(test)
44
44
  end
45
45
 
46
- #
47
- def skip_case(test_case)
48
- end
49
-
50
- #
51
- def skip_test(test)
52
- end
53
-
54
- #
55
- #def test(test)
56
- #end
57
-
58
- #
59
- def pass(test)
60
- end
61
-
62
- #
63
- def fail(test, exception)
64
- end
65
-
66
- # Report a test error.
67
- def error(test, exception)
68
- end
69
-
70
- # Report a pending test.
71
- def todo(test, exception)
72
- end
73
-
74
46
  #
75
47
  def end_test(test)
76
48
  end
@@ -79,21 +51,8 @@ module Test
79
51
  def end_case(test_case)
80
52
  end
81
53
 
82
- #
83
- def end_suite(test_suite)
84
- end
85
-
86
54
  protected
87
55
 
88
- def record
89
- runner.recorder
90
- end
91
-
92
- # Is coverage information requested?
93
- #def cover?
94
- # runner.cover?
95
- #end
96
-
97
56
  # Count up the total number of tests.
98
57
  def total_count(suite)
99
58
  c = 0
@@ -107,73 +66,14 @@ module Test
107
66
  return c
108
67
  end
109
68
 
110
- # Common timestamp any reporter can use.
111
- def timestamp
112
- seconds = Time.now - @start_time
113
-
114
- "Finished in %.5fs, %.2f tests/s." % [seconds, total/seconds]
115
- end
116
-
117
- #
118
- def total
119
- @total ||= subtotal
120
- end
121
-
122
- #
123
- def subtotal
124
- [:todo, :pass, :fail, :error, :omit, :skip].inject(0) do |s,r|
125
- s += record[r.to_sym].size; s
126
- end
127
- end
128
-
129
- # TODO: lump skipped and omitted into one group ?
130
-
131
69
  TITLES = {
132
70
  :pass => 'passing',
133
71
  :fail => 'failures',
134
72
  :error => 'errors',
135
73
  :todo => 'pending',
136
- :omit => 'omissions',
137
74
  :skip => 'skipped'
138
75
  }
139
76
 
140
- # TODO: Add assertion counts (if reasonably possible).
141
-
142
- # Common tally stamp any reporter can use.
143
- #
144
- # @return [String] tally stamp
145
- def tally
146
- sizes = {}
147
- names = %w{pass error fail todo omit skip}.map{ |n| n.to_sym }
148
- names.each do |r|
149
- sizes[r] = record[r].size
150
- end
151
-
152
- #names.unshift(:tests)
153
- #sizes[:tests] = total
154
-
155
- s = []
156
- names.each do |n|
157
- next unless sizes[n] > 0
158
- s << tally_item(n, sizes)
159
- end
160
-
161
- 'Executed ' + "#{total}".ansi(:bold) + ' tests with ' + s.join(', ') + '.'
162
- end
163
-
164
- #
165
- def tally_item(name, sizes)
166
- x = []
167
- x << "%s" % sizes[name].to_s.ansi(:bold)
168
- x << " %s" % TITLES[name].downcase
169
- x << " (%.1f%%)" % ((sizes[name].to_f/total*100)) if runner.verbose?
170
- x.join('')
171
- end
172
-
173
- #--
174
- # TODO: Matching `bin/ruby-test` is not robust.
175
- #++
176
-
177
77
  # Remove reference to lemon library from backtrace.
178
78
  #
179
79
  # @param [Exception] exception
@@ -2,11 +2,34 @@
2
2
 
3
3
  module Test::Reporters
4
4
 
5
- # Hash Abstract is a base class for the TAP-Y
6
- # and TAP-J reporters.
5
+ # Structured records used by the internal test reporter.
7
6
  #
8
7
  class AbstractHash < Abstract
9
8
 
9
+ def capture_output?
10
+ true
11
+ end
12
+
13
+ # Return a structured record of the completed result.
14
+ def record(result)
15
+ @current_result = result
16
+ if result.status == :skip
17
+ method = result.kind == :case ? :skip_case : :skip_test
18
+ public_send(method, result.test, result.reason)
19
+ elsif result.status == :pass
20
+ pass(result.test)
21
+ else
22
+ public_send(result.status, result.test, result.exception)
23
+ end
24
+ ensure
25
+ @current_result = nil if result
26
+ end
27
+
28
+ def finish(summary)
29
+ @summary = summary
30
+ end_suite(summary.suite)
31
+ end
32
+
10
33
  #
11
34
  # @return [Hash]
12
35
  #
@@ -51,9 +74,6 @@ module Test::Reporters
51
74
  #
52
75
  def begin_test(test)
53
76
  @test_index += 1
54
-
55
- @stdout, @stderr = $stdout, $stderr
56
- $stdout, $stderr = StringIO.new, StringIO.new
57
77
  end
58
78
 
59
79
  # Ruby Test use the term "skip", where as TAP-Y/J uses "omit".
@@ -62,22 +82,16 @@ module Test::Reporters
62
82
  #
63
83
  # @return [Hash]
64
84
  #
65
- def skip_test(test)
66
- h = {}
67
- h['type' ] = 'test'
68
- h['status'] = 'omit'
69
-
70
- merge_subtype h, test
71
- merge_setup h, test
72
- merge_label h, test
73
- #merge_comparison h, test, exception
74
- #merge_coverage h, test
75
- merge_source h, test
76
- #merge_exception h, test, exception
77
- merge_output h
78
- merge_time h
85
+ def skip_test(test, reason=nil)
86
+ skip_record(test, reason)
87
+ end
79
88
 
80
- return h
89
+ # A skipped case is reported like a skipped test, as a case.
90
+ #
91
+ # @return [Hash]
92
+ #
93
+ def skip_case(test_case, reason=nil)
94
+ skip_record(test_case, reason).merge('type' => 'case')
81
95
  end
82
96
 
83
97
  #
@@ -169,7 +183,6 @@ module Test::Reporters
169
183
  #
170
184
  def end_test(test)
171
185
  super(test)
172
- $stdout, $stderr = @stdout, @stderr
173
186
  end
174
187
 
175
188
  #
@@ -183,14 +196,14 @@ module Test::Reporters
183
196
  def end_suite(suite)
184
197
  h = {
185
198
  'type' => 'final',
186
- 'time' => Time.now - @start_time,
199
+ 'time' => @summary.elapsed,
187
200
  'counts' => {
188
- 'total' => total,
189
- 'pass' => record[:pass].size,
190
- 'fail' => record[:fail].size,
191
- 'error' => record[:error].size,
192
- 'omit' => record[:omit].size,
193
- 'todo' => record[:todo].size
201
+ 'total' => @summary.total,
202
+ 'pass' => @summary.counts[:pass],
203
+ 'fail' => @summary.counts[:fail],
204
+ 'error' => @summary.counts[:error],
205
+ 'omit' => @summary.counts[:skip],
206
+ 'todo' => @summary.counts[:todo]
194
207
  }
195
208
  }
196
209
  return h
@@ -198,6 +211,23 @@ module Test::Reporters
198
211
 
199
212
  private
200
213
 
214
+ # The record of a skipped test or case.
215
+ def skip_record(test, reason)
216
+ h = {}
217
+ h['type' ] = 'test'
218
+ h['status'] = 'omit'
219
+ h['reason'] = reason if reason
220
+
221
+ merge_subtype h, test
222
+ merge_setup h, test
223
+ merge_label h, test
224
+ merge_source h, test
225
+ merge_output h
226
+ merge_time h
227
+
228
+ return h
229
+ end
230
+
201
231
  # For todo entries in particulr, i.e. NotImplementedError
202
232
  # exception, the return value represents the "todo" priority
203
233
  # level. The `Exception#priority` method returns an Integer
@@ -206,7 +236,7 @@ module Test::Reporters
206
236
  #
207
237
  def merge_priority(hash, test, exception)
208
238
  level = exception.priority
209
- h['priority'] = level.to_i
239
+ hash['priority'] = level.to_i
210
240
  end
211
241
 
212
242
  #
@@ -233,8 +263,8 @@ module Test::Reporters
233
263
 
234
264
  # Add source location information to hash.
235
265
  def merge_source(hash, test)
236
- if test.respond_to?('source_location')
237
- file, line = source_location
266
+ if test.respond_to?(:source_location)
267
+ file, line = test.source_location
238
268
  hash['file' ] = file
239
269
  hash['line' ] = line
240
270
  hash['source' ] = code(file, line).to_str
@@ -251,6 +281,12 @@ module Test::Reporters
251
281
  hash['exception']['snippet' ] = code(exception).to_omap
252
282
  hash['exception']['message' ] = exception.message
253
283
  hash['exception']['backtrace'] = clean_backtrace(exception) if bt
284
+ if @current_result && @current_result.exceptions.size > 1
285
+ hash['exceptions'] = @current_result.exceptions.map do |error|
286
+ {'class' => error.class.to_s, 'message' => error.message,
287
+ 'backtrace' => clean_backtrace(error)}
288
+ end
289
+ end
254
290
  end
255
291
 
256
292
  # TODO: This is still an "idea in progress" for both RybyTest and Tap-Y/J.
@@ -280,13 +316,13 @@ module Test::Reporters
280
316
 
281
317
  #
282
318
  def merge_output(hash)
283
- hash['stdout'] = $stdout.string
284
- hash['stderr'] = $stderr.string
319
+ hash['stdout'] = @current_result.stdout
320
+ hash['stderr'] = @current_result.stderr
285
321
  end
286
322
 
287
323
  #
288
324
  def merge_time(hash)
289
- hash['time'] = Time.now - @start_time
325
+ hash['time'] = @current_result.elapsed
290
326
  end
291
327
 
292
328
  end