padded_cell 0.1.0-x86_64-linux-gnu → 0.2.0-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'chart/delta'
4
+ require_relative 'chart/probe'
5
+ require_relative 'chart/raw'
6
+ require_relative 'chart/accumulator'
7
+ require_relative 'chart/filter'
8
+ require_relative 'chart/config'
9
+ require_relative 'chart/source_files'
10
+ require_relative 'chart/document'
11
+ require_relative 'chart/merge'
12
+ require_relative 'chart/report'
13
+
14
+ module PaddedCell
15
+ # Built-in coverage: Ruby's Coverage measured in the
16
+ # runner, every worker and every cell, summed exactly in the runner. The
17
+ # chart, as the asylum keeps one per patient: inside, a name that is not
18
+ # Ruby's own Coverage; outside, it is all --coverage.
19
+ #
20
+ # Measuring starts before the suite loads, so what is measured comes from
21
+ # the command line (--coverage=lines,branches,methods,eval); the report is
22
+ # configured in the test helper, with PaddedCell.coverage (see Config).
23
+ module Chart
24
+ CRITERIA = %i[lines branches methods eval].freeze
25
+ DEFAULT_CRITERIA = %i[lines branches].freeze
26
+
27
+ class << self
28
+ # Every PaddedCell.coverage block, in one Config.
29
+ def config = @config ||= Config.new
30
+
31
+ def configure(profile = nil, &block)
32
+ config.profile(profile) if profile
33
+ config.instance_eval(&block) if block
34
+ config
35
+ end
36
+ end
37
+ end
38
+ end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'optparse'
4
4
  require_relative 'runner'
5
+ require_relative 'chart'
5
6
 
6
7
  module PaddedCell
7
8
  # The padded_cell command.
@@ -18,13 +19,16 @@ module PaddedCell
18
19
 
19
20
  module_function
20
21
 
21
- # Exit status: 0 passed, 1 failed, 2 usage or load error, 130 interrupted.
22
+ # Exit status: 0 passed, 1 failed, 2 usage or load error, 3 coverage below
23
+ # a minimum, 130 interrupted.
22
24
  def start(argv, io: $stdout)
23
- options = { io: }
25
+ options = { io:, coverage: coverage_criteria(ENV.fetch('PADDED_CELL_COVERAGE', nil)) }
24
26
  parser(options).parse!(argv)
27
+ return merge(options) if options[:coverage_merge]
28
+
25
29
  options[:paths] = argv
26
30
  Runner.new(**options).run
27
- rescue OptionParser::ParseError, ArgumentError, LoadFailure, Unsupported, SystemCallError => e
31
+ rescue OptionParser::ParseError, ArgumentError, RegexpError, LoadFailure, Unsupported, SystemCallError => e
28
32
  warn "padded_cell: #{e.message}"
29
33
  2
30
34
  end
@@ -34,6 +38,7 @@ module PaddedCell
34
38
  opts.banner = BANNER
35
39
  opts.version = VERSION
36
40
  runner_options(opts, options)
41
+ coverage_options(opts, options)
37
42
  output_options(opts, options)
38
43
  opts.on('--version', 'Print the version') do
39
44
  options[:io].puts "padded_cell #{VERSION}"
@@ -50,7 +55,7 @@ module PaddedCell
50
55
  opts.on('-j', '--jobs N', Integer, "Workers (default: #{Etc.nprocessors})") do
51
56
  options[:jobs] = positive(it)
52
57
  end
53
- opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = it }
58
+ opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = seed(it) }
54
59
  opts.on('-n', '-i', '--name', '--include PATTERN', 'Only tests matching (/regexp/ or exact name)') do
55
60
  options[:include] = it
56
61
  end
@@ -66,9 +71,36 @@ module PaddedCell
66
71
  options[:timeout] = positive(it)
67
72
  end
68
73
  opts.on('--fail-fast', 'Stop dispatching after the first failure') { options[:fail_fast] = true }
74
+ opts.on('--asylum[=N]', Integer, 'Retry failed tests up to N times (default 2) after the run, alone, ' \
75
+ 'each in a fresh cell; report the flaky ones') do
76
+ options[:asylum] = it.nil? ? 2 : positive(it)
77
+ end
69
78
  opts.on('--list', 'List the tests that would run (static, loads nothing)') { options[:list] = true }
70
79
  end
71
80
 
81
+ def coverage_options(opts, options)
82
+ opts.on('--[no-]coverage[=CRITERIA]', 'Measure coverage: lines, branches, methods, eval ' \
83
+ "(default: #{Chart::DEFAULT_CRITERIA.join(',')}; " \
84
+ 'or $PADDED_CELL_COVERAGE)') do
85
+ options[:coverage] = coverage_criteria(it.to_s)
86
+ end
87
+ opts.on('--coverage-dir DIR', 'Write coverage reports to DIR (default: coverage, or the test helper\'s)') do
88
+ options[:coverage_dir] = it
89
+ end
90
+ opts.on('--coverage-merge DIR,DIR', Array, 'Merge the coverage of shards (their --coverage-dir) ' \
91
+ 'into one report; runs no test') do
92
+ options[:coverage_merge] = it
93
+ end
94
+ end
95
+
96
+ # --coverage-merge: 0, or 3 when a threshold is missed.
97
+ def merge(options)
98
+ document = Chart::Merge.call(options[:coverage_merge])
99
+ dir = options[:coverage_dir] || 'coverage'
100
+ document.write(dir)
101
+ Chart::Report.new(document, dir:, io: options[:io]).call(check: document.complete?) ? 0 : 3
102
+ end
103
+
72
104
  def output_options(opts, options)
73
105
  opts.on('-v', '--verbose', 'One line per test, with timings') { options[:verbose] = true }
74
106
  opts.on('--show-skips', 'Show skipped tests at the end') { options[:show_skips] = true }
@@ -78,6 +110,13 @@ module PaddedCell
78
110
  opts.on('--no-timings', 'Neither read nor write timings') { options[:timings] = nil }
79
111
  end
80
112
 
113
+ # The planner takes an unsigned 64-bit seed.
114
+ def seed(value)
115
+ raise OptionParser::InvalidArgument, "#{value} (must be 0 to 2**64 - 1)" unless value.between?(0, (2**64) - 1)
116
+
117
+ value
118
+ end
119
+
81
120
  # OptionParser names the switch in the message.
82
121
  def positive(value)
83
122
  raise OptionParser::InvalidArgument, "#{value} (must be positive)" unless value.positive?
@@ -85,6 +124,22 @@ module PaddedCell
85
124
  value
86
125
  end
87
126
 
127
+ # "branches,methods" => [:lines, :branches, :methods] (lines always);
128
+ # "1" or "true" => the defaults; nil, "", "0" or "false" => off.
129
+ def coverage_criteria(value)
130
+ value = value.to_s.strip.downcase
131
+ return if ['', '0', 'false'].include?(value)
132
+ return Chart::DEFAULT_CRITERIA if %w[1 true].include?(value)
133
+
134
+ names = value.split(',').map { it.strip.to_sym }
135
+ unknown = names - Chart::CRITERIA
136
+ unless unknown.empty?
137
+ raise OptionParser::InvalidArgument, "coverage #{unknown.join(',')} (known: #{Chart::CRITERIA.join(', ')})"
138
+ end
139
+
140
+ [:lines] | names
141
+ end
142
+
88
143
  def shard(value)
89
144
  index, count = value.split('/', 2).map { Integer(it, 10, exception: false) }
90
145
  unless index && count&.positive? && index.between?(1, count)
@@ -53,11 +53,15 @@ module PaddedCell
53
53
  end
54
54
 
55
55
  def call(klass, name)
56
+ matches_name?(klass, name) && (!@selection.line_filtered? || line_selected?(klass, name))
57
+ end
58
+
59
+ def matches_name?(klass, name)
56
60
  id = "#{klass}##{name}"
57
61
  return false if @include && !matches?(@include, name, id)
58
62
  return false if @exclude && matches?(@exclude, name, id)
59
63
 
60
- !@selection.line_filtered? || line_selected?(klass, name)
64
+ true
61
65
  end
62
66
 
63
67
  private
@@ -19,12 +19,14 @@ module PaddedCell
19
19
  Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch)
20
20
 
21
21
  # tests: [[klass, method_name], ...], dispatched by index.
22
- def initialize(tests, jobs:, isolation:, seed:, timeout: nil)
22
+ # coverage: a Chart::Accumulator when measuring coverage
23
+ def initialize(tests, jobs:, isolation:, seed:, timeout: nil, coverage: nil)
23
24
  @tests = tests
24
25
  @jobs = jobs
25
26
  @isolation = isolation
26
27
  @seed = seed
27
28
  @timeout = timeout
29
+ @coverage = coverage
28
30
  end
29
31
 
30
32
  # Runs batches of test indices in order (consuming them), yielding
@@ -47,6 +49,8 @@ module PaddedCell
47
49
  shutdown(force: !$!.nil?)
48
50
  end
49
51
 
52
+ def success? = @success
53
+
50
54
  # Dispatch nothing more; tests already running still report.
51
55
  def stop!
52
56
  @queue.clear
@@ -63,7 +67,8 @@ module PaddedCell
63
67
  # A worker holding another worker's socket would keep it from ever
64
68
  # seeing EOF: keep only its own end.
65
69
  [parent, *@sockets].each(&:close)
66
- work(slot.worker, child)
70
+ Worker.new(@tests, isolation: @isolation, seed: @seed, timeout: @timeout, probe: @coverage&.probe)
71
+ .run(slot.worker, child)
67
72
  end
68
73
  child.close
69
74
  @sockets << parent
@@ -71,45 +76,6 @@ module PaddedCell
71
76
  slot
72
77
  end
73
78
 
74
- # The worker process.
75
- def work(worker, socket)
76
- status = 1
77
- begin
78
- srand(@seed)
79
- setup_error = after_fork(worker)
80
- while (line = socket.gets)
81
- payload = Results.dump(perform(Integer(line), socket, setup_error))
82
- socket.write([payload.bytesize].pack('N'), payload)
83
- end
84
- Hooks.run_cleanup(worker) unless setup_error
85
- status = 0
86
- rescue Exception => e # anything: report it, then exit!
87
- warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
88
- ensure
89
- exit!(status)
90
- end
91
- end
92
-
93
- # A failing after_fork hook fails every test of the worker, the way Rails
94
- # parallel testing reports a broken parallelize_setup.
95
- def after_fork(worker)
96
- Hooks.run_after_fork(worker)
97
- nil
98
- rescue StandardError => e
99
- e
100
- end
101
-
102
- def perform(index, socket, setup_error)
103
- klass, name = @tests.fetch(index)
104
- if setup_error
105
- Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
106
- elsif @isolation == :test
107
- Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket])
108
- else
109
- Cell.execute(klass, name)
110
- end
111
- end
112
-
113
79
  def feed(slot)
114
80
  slot.batch = @queue.shift || steal if slot.batch.empty?
115
81
  slot.test = slot.batch.shift
@@ -121,6 +87,7 @@ module PaddedCell
121
87
  rescue Errno::EPIPE, Errno::ECONNRESET
122
88
  # The worker died between tests: its replacement gets this one.
123
89
  reap(slot)
90
+ @coverage&.lost_worker
124
91
  spawn(slot).socket.puts(slot.test)
125
92
  end
126
93
  end
@@ -175,6 +142,7 @@ module PaddedCell
175
142
 
176
143
  # Reports the running test of a dead worker and starts a new one.
177
144
  def replace(slot, message, time)
145
+ @coverage&.lost_worker
178
146
  klass, name = @tests.fetch(slot.test)
179
147
  report(slot, Results.crashed(klass, name, message, time:))
180
148
  return if @queue.empty? && @slots.all? { it.batch.empty? }
@@ -184,8 +152,16 @@ module PaddedCell
184
152
  end
185
153
 
186
154
  def reap(slot)
155
+ close_socket(slot)
156
+ wait_for(slot)
157
+ end
158
+
159
+ def close_socket(slot)
187
160
  @sockets.delete(slot.socket)
188
161
  slot.socket.close
162
+ end
163
+
164
+ def wait_for(slot)
189
165
  _, status = Process.wait2(slot.pid)
190
166
  slot.pid = nil
191
167
  status
@@ -196,7 +172,8 @@ module PaddedCell
196
172
  def shutdown(force:)
197
173
  live = Array(@slots).select(&:pid)
198
174
  live.each { Process.kill(:KILL, it.pid) } if force
199
- live.each { reap(it) }
175
+ live.each { close_socket(it) }
176
+ @success = live.map { wait_for(it) }.all?(&:success?)
200
177
  end
201
178
  end
202
179
  end
@@ -36,6 +36,12 @@ module PaddedCell
36
36
  end
37
37
 
38
38
  def relative(file) = file.delete_prefix("#{Dir.pwd}/")
39
+
40
+ # --asylum's report on a test that passed on retry, if it did.
41
+ def flaky(result) = result.metadata? ? result.metadata[:flaky] : nil
42
+
43
+ # --asylum's attempts at a test that never passed.
44
+ def asylum(result) = result.metadata? ? result.metadata[:asylum] : nil
39
45
  end
40
46
 
41
47
  # JUnit XML, as read by GitHub, GitLab, Jenkins and friends.
@@ -67,10 +73,20 @@ module PaddedCell
67
73
  file, line = file_and_line(result)
68
74
  attributes = %(classname="#{escape(result.klass)}" name="#{escape(result.name)}" file="#{escape(file)}" ) +
69
75
  %(line="#{line}" time="#{format('%.6f', result.time)}" assertions="#{result.assertions}")
70
- body = outcome(result)
76
+ body = outcome(result) + flaky_failure(result)
71
77
  body.empty? ? %( <testcase #{attributes}/>\n) : %( <testcase #{attributes}>\n#{body} </testcase>\n)
72
78
  end
73
79
 
80
+ # Surefire's element for a test that failed, then passed on retry.
81
+ def flaky_failure(result)
82
+ info = flaky(result)
83
+ return '' unless info
84
+
85
+ first = info[:first_failure]
86
+ %( <flakyFailure type="#{escape(first[:type])}" message="#{escape(first[:message])}">) +
87
+ %(#{escape(info[:cause])}</flakyFailure>\n)
88
+ end
89
+
74
90
  def outcome(result)
75
91
  failure = result.failure
76
92
  case status(result)
@@ -98,8 +114,9 @@ module PaddedCell
98
114
  {
99
115
  id: "#{result.klass}##{result.name}", class: result.klass, name: result.name, file:, line:,
100
116
  status: status(result), time: result.time, assertions: result.assertions,
101
- failures: result.failures.map { { type: it.result_label, message: it.message.scrub } }
102
- }
117
+ failures: result.failures.map { { type: it.result_label, message: it.message.scrub } },
118
+ flaky: flaky(result), asylum: asylum(result)
119
+ }.compact
103
120
  end
104
121
  end
105
122
  end
@@ -74,6 +74,19 @@ module PaddedCell
74
74
  placeholders << [owner, last]
75
75
  end
76
76
 
77
+ # What a cell sends its worker: its coverage delta (nil without
78
+ # coverage) beside its result. The outer layer holds only core types, so
79
+ # it loads even when the result does not.
80
+ def envelope(delta, result) = Marshal.dump([delta, dump(result)])
81
+
82
+ # [delta, result] from an envelope; never raises.
83
+ def unwrap(data, klass, name)
84
+ delta, payload = Marshal.load(data) # written by our own cell
85
+ [delta, load(payload, klass, name)]
86
+ rescue StandardError => e
87
+ [nil, crashed(klass, name, "its result could not be loaded: #{e.class}: #{e.message}")]
88
+ end
89
+
77
90
  # Marshal.dump, after replacing failures that cannot cross a pipe (an
78
91
  # exception holding a Proc, an IO...) with plain copies.
79
92
  def dump(result)
@@ -5,7 +5,9 @@ require 'minitest'
5
5
  require_relative '../padded_cell'
6
6
  require_relative 'results'
7
7
  require_relative 'cell'
8
+ require_relative 'worker'
8
9
  require_relative 'pool'
10
+ require_relative 'asylum'
9
11
  require_relative 'loader'
10
12
  require_relative 'timings'
11
13
  require_relative 'reporters'
@@ -18,8 +20,9 @@ module PaddedCell
18
20
  class Runner
19
21
  DEFAULTS = {
20
22
  paths: [], jobs: Etc.nprocessors, seed: nil, include: nil, exclude: nil, shard: [0, 1],
21
- isolation: nil, timeout: nil, fail_fast: false, verbose: false, show_skips: false,
22
- list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', io: $stdout
23
+ isolation: nil, timeout: nil, asylum: nil, fail_fast: false, verbose: false, show_skips: false,
24
+ list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', coverage: nil,
25
+ coverage_dir: nil, io: $stdout
23
26
  }.freeze
24
27
 
25
28
  attr_reader :options
@@ -29,33 +32,52 @@ module PaddedCell
29
32
  @options[:seed] ||= (ENV['SEED'] || Random.new_seed).to_i % 0xFFFF
30
33
  end
31
34
 
32
- # Exit status: 0 when everything passed, 1 otherwise, 130 on Ctrl-C.
35
+ # Exit status: 0 when everything passed, 1 otherwise, 3 when only
36
+ # coverage is below a minimum, 130 on Ctrl-C.
33
37
  def run
34
38
  selection = Selection.parse(options[:paths])
35
39
  refs = Discovery.call(selection.files)
36
- return list(selection, refs) if options[:list]
40
+ filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
41
+ return list(selection, refs, filter) if options[:list]
37
42
  raise Unsupported, 'padded_cell needs fork, which this Ruby does not have' unless Process.respond_to?(:fork)
38
43
 
44
+ @coverage = start_coverage if options[:coverage]
39
45
  Loader.call(selection.files, seed: options[:seed])
46
+ refuse_simplecov if @coverage
40
47
  if defined?(ActiveSupport::TestCase)
41
48
  require_relative 'active_support_hooks'
42
49
  ActiveSupportHooks.install
43
50
  end
44
51
  options[:isolation] ||= default_isolation
45
- filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
52
+ # Coverage thresholds hold for the whole suite only.
53
+ @whole = !selection.line_filtered? && !options[:include] && !options[:exclude]
46
54
  execute(Loader.tests.select { filter.call(*it) })
47
55
  end
48
56
 
49
57
  private
50
58
 
59
+ # Before the suite loads: nothing loaded earlier is measured.
60
+ def start_coverage
61
+ require_relative 'chart'
62
+ Chart::Accumulator.new(options[:coverage]).start
63
+ end
64
+
65
+ # Two tools cannot share Ruby's Coverage.
66
+ def refuse_simplecov
67
+ return unless defined?(::SimpleCov) && ::SimpleCov.respond_to?(:pid) && ::SimpleCov.pid
68
+
69
+ raise Unsupported, 'SimpleCov is running, and --coverage replaces it: remove SimpleCov.start (and any ' \
70
+ 'SimpleCov.result in parallelize_teardown) from the test helper'
71
+ end
72
+
51
73
  # A fork per test discards Active Record's connection pool, and with it
52
74
  # the fixtures loaded through it: every test would reload them. With
53
75
  # fixtures around, tests share their worker instead.
54
76
  def default_isolation = defined?(ActiveRecord::FixtureSet) ? :worker : :test
55
77
 
56
78
  # Tests as they would run (static discovery, no loading), in file order.
57
- def list(selection, refs)
58
- refs = refs.select { selection.covers?(it.file, it.lines) }
79
+ def list(selection, refs, filter)
80
+ refs = refs.select { selection.covers?(it.file, it.lines) && filter.matches_name?(it.class_name, it.name) }
59
81
  kept = Scheduler.batches(refs.map(&:class_name), seed: options[:seed], shard: options[:shard]).flatten.sort
60
82
  kept.each do |index|
61
83
  ref = refs[index]
@@ -69,37 +91,112 @@ module PaddedCell
69
91
  timings = Timings.load(options[:timings])
70
92
  batches = Scheduler.batches(tests.map { |klass, _| klass.to_s }, durations: ids.map { timings[it] },
71
93
  seed: options[:seed], shard: options[:shard])
94
+ nothing_ran = batches.empty?
72
95
  reporter = build_reporter
73
96
  reporter.start
74
- interrupted = dispatch(tests, batches, reporter) { |index, result| timings[ids[index]] = result.time }
97
+ retried = []
98
+ @coverage&.snapshot
99
+ status = dispatch(tests, batches, reporter, retried) { |index, result| timings[ids[index]] = result.time }
75
100
  reporter.report
101
+ report_asylum(retried)
102
+ covered = @coverage ? report_coverage(status, reporter) : true
76
103
  Timings.save(options[:timings], timings)
77
104
  after_run
78
- finish(reporter, interrupted, batches.empty?)
105
+ result = finish(reporter, status, nothing_ran)
106
+ result.zero? && !covered ? 3 : result
79
107
  end
80
108
 
81
- # Yields each result after reporting it. True when interrupted.
82
- def dispatch(tests, batches, reporter)
83
- pool = nil
84
- on_result = lambda do |index, result|
109
+ # Writes the coverage reports, even for a failing run; false when the
110
+ # coverage of a whole, passing, complete run is below a minimum.
111
+ def report_coverage(status, reporter)
112
+ raw = @coverage.finish(interrupted: status == 130)
113
+ return true unless raw
114
+
115
+ dir = options[:coverage_dir] || Chart.config.directory
116
+ document = Chart::Document.build(raw, config: Chart.config, shard: options[:shard], whole: @whole)
117
+ document.write(dir)
118
+ check = status.zero? && reporter.passed? && document.complete?
119
+ Chart::Report.new(document, dir:, io: options[:io]).call(check:)
120
+ end
121
+
122
+ # Yields each result as it arrives. With --asylum a failure is held back:
123
+ # only its final result, after the retries, is reported. Returns 1 when a
124
+ # worker failed (its cleanup hook), 130 when interrupted, 0 otherwise.
125
+ def dispatch(tests, batches, reporter, retried)
126
+ held = {}
127
+ record = lambda do |index, result|
85
128
  reporter.prerecord(*tests[index])
86
129
  reporter.record(result)
87
- yield index, result
88
- pool&.stop! if options[:fail_fast] && !result.passed? && !result.skipped?
130
+ retried << result if result.metadata? && (result.metadata[:flaky] || result.metadata[:asylum])
89
131
  end
90
132
  pool = Pool.new(tests, jobs: options[:jobs], isolation: options[:isolation], seed: options[:seed],
91
- timeout: options[:timeout])
92
- pool.run(batches, &on_result)
93
- false
133
+ timeout: options[:timeout], coverage: @coverage)
134
+ pool.run(batches) do |index, result|
135
+ yield index, result
136
+ failed = failed?(result)
137
+ options[:asylum] && failed ? held[index] = result : record.call(index, result)
138
+ pool.stop! if options[:fail_fast] && failed
139
+ end
140
+ retries_ok = retry_held(tests, held, record)
141
+ pool.success? && retries_ok ? 0 : 1
94
142
  rescue Interrupt
95
143
  warn 'Interrupted. Exiting...'
96
- true
144
+ # Failures still waiting for a retry are failures.
145
+ held.each { |index, result| record.call(index, result) }
146
+ 130
147
+ end
148
+
149
+ # False when a retry's worker failed.
150
+ def retry_held(tests, held, record)
151
+ return true if held.empty?
152
+
153
+ asylum = Asylum.new(tests, attempts: options[:asylum], seed: options[:seed], timeout: options[:timeout])
154
+ asylum.call(held) do |index, result|
155
+ held.delete(index)
156
+ record.call(index, result)
157
+ end
158
+ asylum.success?
159
+ end
160
+
161
+ def failed?(result) = !result.passed? && !result.skipped?
162
+
163
+ # After Minitest's summary: what --asylum found, attempt by attempt.
164
+ def report_asylum(retried)
165
+ flaky, failing = retried.partition { it.metadata[:flaky] }
166
+ report_flaky(flaky) if flaky.any?
167
+ report_failing(failing) if failing.any?
168
+ end
169
+
170
+ def report_flaky(flaky)
171
+ io = options[:io]
172
+ tests = flaky.size == 1 ? '1 test' : "#{flaky.size} tests"
173
+ io.puts "\nYou're not crazy, the world is. #{tests} failed, then passed:"
174
+ width = flaky.flat_map { it.metadata[:flaky][:history] }.map { it[:seed].to_s.size }.max
175
+ flaky.each do |result|
176
+ info = result.metadata[:flaky]
177
+ io.puts "\n #{result.klass}##{result.name}"
178
+ info[:history].each do |attempt|
179
+ io.puts " seed #{attempt[:seed].to_s.ljust(width)} #{attempt_line(attempt, info[:cause])}"
180
+ end
181
+ end
182
+ end
183
+
184
+ def attempt_line(attempt, cause)
185
+ return "passed, alone → #{cause}" if attempt[:outcome] == 'passed'
186
+
187
+ attempt[:alone] ? "#{attempt[:outcome]} (alone)" : attempt[:outcome]
188
+ end
189
+
190
+ def report_failing(failing)
191
+ io = options[:io]
192
+ io.puts "\nIt's not the world, it's the test:"
193
+ failing.each { io.puts " #{it.klass}##{it.name} failed all #{it.metadata[:asylum][:attempts]} attempts" }
97
194
  end
98
195
 
99
196
  def build_reporter
100
197
  reporter = Minitest::CompositeReporter.new
101
198
  minitest_options = {
102
- io: options[:io], seed: options[:seed], args: "--seed #{options[:seed]} --isolation #{options[:isolation]}",
199
+ io: options[:io], seed: options[:seed], args: run_options,
103
200
  verbose: options[:verbose], show_skips: options[:show_skips]
104
201
  }
105
202
  reporter << Minitest::SummaryReporter.new(options[:io], minitest_options)
@@ -113,13 +210,19 @@ module PaddedCell
113
210
  reporter
114
211
  end
115
212
 
213
+ # What a rerun needs to reproduce this run's order and settings.
214
+ def run_options
215
+ args = "--seed #{options[:seed]} --isolation #{options[:isolation]}"
216
+ options[:asylum] ? "#{args} --asylum=#{options[:asylum]}" : args
217
+ end
218
+
116
219
  # Minitest.after_run blocks, which autorun would have called.
117
220
  def after_run
118
221
  Minitest.class_variable_get(:@@after_run).reverse_each(&:call)
119
222
  end
120
223
 
121
- def finish(reporter, interrupted, nothing_ran)
122
- return 130 if interrupted
224
+ def finish(reporter, status, nothing_ran)
225
+ return status unless status.zero?
123
226
  if nothing_ran && options[:include]
124
227
  warn "Nothing ran for filter: #{options[:include]}"
125
228
  return 1
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PaddedCell
4
- VERSION = '0.1.0'
4
+ VERSION = '0.2.0'
5
5
  end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PaddedCell
4
+ # A Pool's worker process. Reads test indices from its socket, runs each
5
+ # (in a Cell, or itself with isolation :worker) and writes back its
6
+ # Minitest::Result as a length-prefixed Marshal frame. Exits once the
7
+ # socket closes.
8
+ class Worker
9
+ # probe: a Chart::Probe when measuring coverage
10
+ def initialize(tests, isolation:, seed:, timeout:, probe:)
11
+ @tests = tests
12
+ @isolation = isolation
13
+ @seed = seed
14
+ @timeout = timeout
15
+ @probe = probe
16
+ end
17
+
18
+ # Never returns.
19
+ def run(worker, socket)
20
+ status = 1
21
+ begin
22
+ srand(@seed)
23
+ @probe&.start
24
+ setup_error = after_fork(worker)
25
+ while (line = socket.gets)
26
+ payload = Results.dump(perform(Integer(line), socket, setup_error))
27
+ socket.write([payload.bytesize].pack('N'), payload)
28
+ @probe&.tick
29
+ end
30
+ # Before cleanup, which may stop Coverage.
31
+ @probe&.flush
32
+ Hooks.run_cleanup(worker) unless setup_error
33
+ status = 0
34
+ rescue Exception => e # anything: report it, then exit!
35
+ warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
36
+ ensure
37
+ exit!(status)
38
+ end
39
+ end
40
+
41
+ private
42
+
43
+ # A failing after_fork hook fails every test of the worker, the way Rails
44
+ # parallel testing reports a broken parallelize_setup.
45
+ def after_fork(worker)
46
+ Hooks.run_after_fork(worker)
47
+ nil
48
+ rescue StandardError => e
49
+ e
50
+ end
51
+
52
+ def perform(index, socket, setup_error)
53
+ klass, name = @tests.fetch(index)
54
+ if setup_error
55
+ Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
56
+ elsif @isolation == :test
57
+ Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket], probe: @probe)
58
+ else
59
+ Cell.execute(klass, name)
60
+ end
61
+ end
62
+ end
63
+ end
data/lib/padded_cell.rb CHANGED
@@ -23,6 +23,14 @@ module PaddedCell
23
23
  def after_fork(&) = Hooks.after_fork(&)
24
24
 
25
25
  def cleanup(&) = Hooks.cleanup(&)
26
+
27
+ # Configures the coverage report (see Chart::Config). Inert unless
28
+ # padded_cell runs with --coverage. A helper also run without
29
+ # padded_cell guards it: `if defined?(PaddedCell.coverage)`.
30
+ def coverage(profile = nil, &)
31
+ require_relative 'padded_cell/chart'
32
+ Chart.configure(profile, &)
33
+ end
26
34
  end
27
35
  end
28
36