padded_cell 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +38 -0
  3. data/Cargo.lock +16 -2
  4. data/Cargo.toml +2 -0
  5. data/README.md +170 -8
  6. data/ext/padded_cell_native/core/Cargo.toml +2 -1
  7. data/ext/padded_cell_native/core/src/chart/directives/tests.rs +56 -0
  8. data/ext/padded_cell_native/core/src/chart/directives.rs +144 -0
  9. data/ext/padded_cell_native/core/src/chart/json/tests.rs +46 -0
  10. data/ext/padded_cell_native/core/src/chart/json.rs +175 -0
  11. data/ext/padded_cell_native/core/src/chart/lcov.rs +50 -0
  12. data/ext/padded_cell_native/core/src/chart/markdown/tests.rs +93 -0
  13. data/ext/padded_cell_native/core/src/chart/markdown.rs +201 -0
  14. data/ext/padded_cell_native/core/src/chart/methods/tests.rs +38 -0
  15. data/ext/padded_cell_native/core/src/chart/methods.rs +62 -0
  16. data/ext/padded_cell_native/core/src/chart/output/tests.rs +45 -0
  17. data/ext/padded_cell_native/core/src/chart/output.rs +103 -0
  18. data/ext/padded_cell_native/core/src/chart/results/tests.rs +96 -0
  19. data/ext/padded_cell_native/core/src/chart/results.rs +379 -0
  20. data/ext/padded_cell_native/core/src/chart/source/tests.rs +64 -0
  21. data/ext/padded_cell_native/core/src/chart/source.rs +153 -0
  22. data/ext/padded_cell_native/core/src/chart/stats/tests.rs +52 -0
  23. data/ext/padded_cell_native/core/src/chart/stats.rs +218 -0
  24. data/ext/padded_cell_native/core/src/chart/test_support.rs +93 -0
  25. data/ext/padded_cell_native/core/src/chart/tests.rs +97 -0
  26. data/ext/padded_cell_native/core/src/chart/thresholds/tests.rs +36 -0
  27. data/ext/padded_cell_native/core/src/chart/thresholds.rs +116 -0
  28. data/ext/padded_cell_native/core/src/chart.rs +302 -0
  29. data/ext/padded_cell_native/core/src/discover/tests.rs +2 -0
  30. data/ext/padded_cell_native/core/src/discover.rs +7 -53
  31. data/ext/padded_cell_native/core/src/error.rs +24 -0
  32. data/ext/padded_cell_native/core/src/lib.rs +10 -3
  33. data/ext/padded_cell_native/core/src/parallel.rs +43 -0
  34. data/ext/padded_cell_native/core/src/restarts/tests.rs +29 -0
  35. data/ext/padded_cell_native/core/src/restarts.rs +64 -0
  36. data/ext/padded_cell_native/ffi/Cargo.toml +2 -1
  37. data/ext/padded_cell_native/ffi/src/chart.rs +205 -0
  38. data/ext/padded_cell_native/ffi/src/lib.rs +5 -0
  39. data/ext/padded_cell_native/ffi/src/limits.rs +83 -0
  40. data/lib/padded_cell/asylum.rb +93 -0
  41. data/lib/padded_cell/cell.rb +23 -14
  42. data/lib/padded_cell/chart/accumulator.rb +103 -0
  43. data/lib/padded_cell/chart/config.rb +129 -0
  44. data/lib/padded_cell/chart/delta.rb +148 -0
  45. data/lib/padded_cell/chart/document.rb +64 -0
  46. data/lib/padded_cell/chart/filter.rb +55 -0
  47. data/lib/padded_cell/chart/merge.rb +63 -0
  48. data/lib/padded_cell/chart/probe.rb +70 -0
  49. data/lib/padded_cell/chart/raw.rb +48 -0
  50. data/lib/padded_cell/chart/report.rb +128 -0
  51. data/lib/padded_cell/chart/source_files.rb +90 -0
  52. data/lib/padded_cell/chart.rb +38 -0
  53. data/lib/padded_cell/cli.rb +70 -4
  54. data/lib/padded_cell/limits.rb +45 -0
  55. data/lib/padded_cell/loader.rb +5 -1
  56. data/lib/padded_cell/pool.rb +82 -58
  57. data/lib/padded_cell/reporters.rb +20 -3
  58. data/lib/padded_cell/results.rb +13 -0
  59. data/lib/padded_cell/runner.rb +150 -22
  60. data/lib/padded_cell/version.rb +1 -1
  61. data/lib/padded_cell/worker.rb +63 -0
  62. data/lib/padded_cell.rb +8 -0
  63. metadata +43 -1
@@ -13,18 +13,34 @@ module PaddedCell
13
13
  #
14
14
  # With isolation :test a worker forks a Cell per test; with :worker it runs
15
15
  # tests itself, faster but sharing process state. A worker that dies
16
- # mid-test, or runs past the timeout, gets its test reported as an error and
17
- # is replaced; its replacement continues the class.
16
+ # mid-test, or runs past its deadline (--timeout, --budget), gets its test
17
+ # reported as an error and is replaced; its replacement continues the class.
18
+ #
19
+ # Replaced, up to a point: workers that keep dying (a boot that segfaults,
20
+ # an after_fork hook that exits) would turn every remaining test into a
21
+ # crash report. Past PaddedCellNative::Restarts' limit the pool gives up.
22
+ # Once the run's budget is spent, it dispatches nothing more.
18
23
  class Pool
19
- Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch)
24
+ Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch, :deadline)
25
+
26
+ # Tests left undispatched by stop! (--fail-fast, the budget, giving up).
27
+ attr_reader :undispatched
28
+
29
+ # Why the pool gave up on its workers, or nil.
30
+ attr_reader :gave_up
20
31
 
21
32
  # tests: [[klass, method_name], ...], dispatched by index.
22
- def initialize(tests, jobs:, isolation:, seed:, timeout: nil)
33
+ # limits: the run's Limits
34
+ # coverage: a Chart::Accumulator when measuring coverage
35
+ def initialize(tests, jobs:, isolation:, seed:, limits: Limits.new, coverage: nil)
23
36
  @tests = tests
24
37
  @jobs = jobs
25
38
  @isolation = isolation
26
39
  @seed = seed
27
- @timeout = timeout
40
+ @limits = limits
41
+ @coverage = coverage
42
+ @restarts = PaddedCellNative::Restarts.new(jobs)
43
+ @undispatched = 0
28
44
  end
29
45
 
30
46
  # Runs batches of test indices in order (consuming them), yielding
@@ -34,7 +50,7 @@ module PaddedCell
34
50
  @on_result = on_result
35
51
  @sockets = []
36
52
  Hooks.run_before_fork
37
- @slots = Array.new([@jobs, batches.sum(&:size)].min) { spawn(Slot.new(it, nil, nil, nil, nil, [])) }
53
+ @slots = Array.new([@jobs, batches.sum(&:size)].min) { spawn(Slot.new(it, nil, nil, nil, nil, [], nil)) }
38
54
  @slots.each { feed(it) }
39
55
  loop do
40
56
  busy = @slots.select(&:test)
@@ -47,8 +63,11 @@ module PaddedCell
47
63
  shutdown(force: !$!.nil?)
48
64
  end
49
65
 
66
+ def success? = @success
67
+
50
68
  # Dispatch nothing more; tests already running still report.
51
69
  def stop!
70
+ @undispatched += @queue.sum(&:size) + @slots.sum { it.batch.size }
52
71
  @queue.clear
53
72
  @slots.each { it.batch.clear }
54
73
  end
@@ -63,7 +82,8 @@ module PaddedCell
63
82
  # A worker holding another worker's socket would keep it from ever
64
83
  # seeing EOF: keep only its own end.
65
84
  [parent, *@sockets].each(&:close)
66
- work(slot.worker, child)
85
+ Worker.new(@tests, isolation: @isolation, seed: @seed, limits: @limits, probe: @coverage&.probe)
86
+ .run(slot.worker, child)
67
87
  end
68
88
  child.close
69
89
  @sockets << parent
@@ -71,57 +91,26 @@ module PaddedCell
71
91
  slot
72
92
  end
73
93
 
74
- # The worker process.
75
- def work(worker, socket)
76
- status = 1
77
- begin
78
- srand(@seed)
79
- setup_error = after_fork(worker)
80
- while (line = socket.gets)
81
- payload = Results.dump(perform(Integer(line), socket, setup_error))
82
- socket.write([payload.bytesize].pack('N'), payload)
83
- end
84
- Hooks.run_cleanup(worker) unless setup_error
85
- status = 0
86
- rescue Exception => e # anything: report it, then exit!
87
- warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
88
- ensure
89
- exit!(status)
90
- end
91
- end
92
-
93
- # A failing after_fork hook fails every test of the worker, the way Rails
94
- # parallel testing reports a broken parallelize_setup.
95
- def after_fork(worker)
96
- Hooks.run_after_fork(worker)
97
- nil
98
- rescue StandardError => e
99
- e
100
- end
101
-
102
- def perform(index, socket, setup_error)
103
- klass, name = @tests.fetch(index)
104
- if setup_error
105
- Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
106
- elsif @isolation == :test
107
- Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket])
108
- else
109
- Cell.execute(klass, name)
110
- end
111
- end
112
-
113
94
  def feed(slot)
95
+ return stop! if @limits.spent?
96
+
114
97
  slot.batch = @queue.shift || steal if slot.batch.empty?
115
98
  slot.test = slot.batch.shift
116
99
  return unless slot.test
117
100
 
118
101
  slot.started = Minitest.clock_time
102
+ slot.deadline = @limits.for_test
119
103
  begin
120
104
  slot.socket.puts(slot.test)
121
105
  rescue Errno::EPIPE, Errno::ECONNRESET
122
- # The worker died between tests: its replacement gets this one.
106
+ # The worker died between tests: its replacement gets this one, unless
107
+ # it dies too. Restarts bounds the retries.
123
108
  reap(slot)
124
- spawn(slot).socket.puts(slot.test)
109
+ @coverage&.lost_worker
110
+ return abandon(slot) unless @restarts.died(Limits.now)
111
+
112
+ spawn(slot)
113
+ retry
125
114
  end
126
115
  end
127
116
 
@@ -132,17 +121,24 @@ module PaddedCell
132
121
  end
133
122
 
134
123
  def receive(slot)
135
- header = slot.socket.read(4)
124
+ header = read_frame(slot, 4)
136
125
  return died(slot) unless header&.bytesize == 4
137
126
 
138
127
  size = header.unpack1('N')
139
- payload = slot.socket.read(size)
128
+ payload = read_frame(slot, size)
140
129
  return died(slot) unless payload&.bytesize == size
141
130
 
142
131
  report(slot, Results.load(payload, *@tests.fetch(slot.test)))
143
132
  feed(slot)
144
133
  end
145
134
 
135
+ # nil when the worker died: EOF, or a reset when it left data unread.
136
+ def read_frame(slot, size)
137
+ slot.socket.read(size)
138
+ rescue Errno::ECONNRESET
139
+ nil
140
+ end
141
+
146
142
  def report(slot, result)
147
143
  index = slot.test
148
144
  slot.test = nil
@@ -151,32 +147,51 @@ module PaddedCell
151
147
 
152
148
  def died(slot)
153
149
  status = reap(slot)
150
+ give_up unless @restarts.died(Limits.now)
154
151
  replace(slot, Results.describe(status, 'worker'), Minitest.clock_time - slot.started)
155
152
  end
156
153
 
157
- # Cells time out inside their worker; with :worker isolation the runner
158
- # kills the worker itself.
154
+ # Stops replacing workers: whatever was not dispatched is not run.
155
+ def give_up
156
+ window = PaddedCellNative::Restarts.window_ns / Limits::NS
157
+ @gave_up = "workers died more than #{@restarts.limit} times in #{window}s"
158
+ stop!
159
+ end
160
+
161
+ # A test taken for a worker that died before receiving it, and is not
162
+ # replaced.
163
+ def abandon(slot)
164
+ slot.test = nil
165
+ @undispatched += 1
166
+ give_up
167
+ end
168
+
169
+ # Cells meet their deadline inside their worker; with :worker isolation
170
+ # the runner kills the worker itself.
159
171
  def wait_time(busy)
160
- return unless @timeout && @isolation == :worker
172
+ return unless @isolation == :worker
161
173
 
162
- deadline = busy.map { it.started + @timeout }.min
163
- [deadline - Minitest.clock_time, 0].max
174
+ now = Limits.now
175
+ busy.filter_map { Limits.remaining(it.deadline, now) }.min
164
176
  end
165
177
 
166
178
  def expire(busy)
179
+ now = Limits.now
167
180
  busy.each do |slot|
168
- next if Minitest.clock_time - slot.started < @timeout
181
+ next unless slot.deadline.expired?(now)
169
182
 
170
183
  Process.kill(:KILL, slot.pid)
171
184
  reap(slot)
172
- replace(slot, "timed out after #{@timeout}s", @timeout)
185
+ replace(slot, @limits.message(slot.deadline), Minitest.clock_time - slot.started)
173
186
  end
174
187
  end
175
188
 
176
189
  # Reports the running test of a dead worker and starts a new one.
177
190
  def replace(slot, message, time)
191
+ @coverage&.lost_worker
178
192
  klass, name = @tests.fetch(slot.test)
179
193
  report(slot, Results.crashed(klass, name, message, time:))
194
+ stop! if @limits.spent?
180
195
  return if @queue.empty? && @slots.all? { it.batch.empty? }
181
196
 
182
197
  spawn(slot)
@@ -184,8 +199,16 @@ module PaddedCell
184
199
  end
185
200
 
186
201
  def reap(slot)
202
+ close_socket(slot)
203
+ wait_for(slot)
204
+ end
205
+
206
+ def close_socket(slot)
187
207
  @sockets.delete(slot.socket)
188
208
  slot.socket.close
209
+ end
210
+
211
+ def wait_for(slot)
189
212
  _, status = Process.wait2(slot.pid)
190
213
  slot.pid = nil
191
214
  status
@@ -196,7 +219,8 @@ module PaddedCell
196
219
  def shutdown(force:)
197
220
  live = Array(@slots).select(&:pid)
198
221
  live.each { Process.kill(:KILL, it.pid) } if force
199
- live.each { reap(it) }
222
+ live.each { close_socket(it) }
223
+ @success = live.map { wait_for(it) }.all?(&:success?)
200
224
  end
201
225
  end
202
226
  end
@@ -36,6 +36,12 @@ module PaddedCell
36
36
  end
37
37
 
38
38
  def relative(file) = file.delete_prefix("#{Dir.pwd}/")
39
+
40
+ # --asylum's report on a test that passed on retry, if it did.
41
+ def flaky(result) = result.metadata? ? result.metadata[:flaky] : nil
42
+
43
+ # --asylum's attempts at a test that never passed.
44
+ def asylum(result) = result.metadata? ? result.metadata[:asylum] : nil
39
45
  end
40
46
 
41
47
  # JUnit XML, as read by GitHub, GitLab, Jenkins and friends.
@@ -67,10 +73,20 @@ module PaddedCell
67
73
  file, line = file_and_line(result)
68
74
  attributes = %(classname="#{escape(result.klass)}" name="#{escape(result.name)}" file="#{escape(file)}" ) +
69
75
  %(line="#{line}" time="#{format('%.6f', result.time)}" assertions="#{result.assertions}")
70
- body = outcome(result)
76
+ body = outcome(result) + flaky_failure(result)
71
77
  body.empty? ? %( <testcase #{attributes}/>\n) : %( <testcase #{attributes}>\n#{body} </testcase>\n)
72
78
  end
73
79
 
80
+ # Surefire's element for a test that failed, then passed on retry.
81
+ def flaky_failure(result)
82
+ info = flaky(result)
83
+ return '' unless info
84
+
85
+ first = info[:first_failure]
86
+ %( <flakyFailure type="#{escape(first[:type])}" message="#{escape(first[:message])}">) +
87
+ %(#{escape(info[:cause])}</flakyFailure>\n)
88
+ end
89
+
74
90
  def outcome(result)
75
91
  failure = result.failure
76
92
  case status(result)
@@ -98,8 +114,9 @@ module PaddedCell
98
114
  {
99
115
  id: "#{result.klass}##{result.name}", class: result.klass, name: result.name, file:, line:,
100
116
  status: status(result), time: result.time, assertions: result.assertions,
101
- failures: result.failures.map { { type: it.result_label, message: it.message.scrub } }
102
- }
117
+ failures: result.failures.map { { type: it.result_label, message: it.message.scrub } },
118
+ flaky: flaky(result), asylum: asylum(result)
119
+ }.compact
103
120
  end
104
121
  end
105
122
  end
@@ -74,6 +74,19 @@ module PaddedCell
74
74
  placeholders << [owner, last]
75
75
  end
76
76
 
77
+ # What a cell sends its worker: its coverage delta (nil without
78
+ # coverage) beside its result. The outer layer holds only core types, so
79
+ # it loads even when the result does not.
80
+ def envelope(delta, result) = Marshal.dump([delta, dump(result)])
81
+
82
+ # [delta, result] from an envelope; never raises.
83
+ def unwrap(data, klass, name)
84
+ delta, payload = Marshal.load(data) # written by our own cell
85
+ [delta, load(payload, klass, name)]
86
+ rescue StandardError => e
87
+ [nil, crashed(klass, name, "its result could not be loaded: #{e.class}: #{e.message}")]
88
+ end
89
+
77
90
  # Marshal.dump, after replacing failures that cannot cross a pipe (an
78
91
  # exception holding a Proc, an IO...) with plain copies.
79
92
  def dump(result)
@@ -4,8 +4,11 @@ require 'etc'
4
4
  require 'minitest'
5
5
  require_relative '../padded_cell'
6
6
  require_relative 'results'
7
+ require_relative 'limits'
7
8
  require_relative 'cell'
9
+ require_relative 'worker'
8
10
  require_relative 'pool'
11
+ require_relative 'asylum'
9
12
  require_relative 'loader'
10
13
  require_relative 'timings'
11
14
  require_relative 'reporters'
@@ -18,8 +21,9 @@ module PaddedCell
18
21
  class Runner
19
22
  DEFAULTS = {
20
23
  paths: [], jobs: Etc.nprocessors, seed: nil, include: nil, exclude: nil, shard: [0, 1],
21
- isolation: nil, timeout: nil, fail_fast: false, verbose: false, show_skips: false,
22
- list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', io: $stdout
24
+ isolation: nil, timeout: nil, budget: nil, asylum: nil, fail_fast: false, verbose: false, show_skips: false,
25
+ list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', coverage: nil,
26
+ coverage_dir: nil, io: $stdout
23
27
  }.freeze
24
28
 
25
29
  attr_reader :options
@@ -29,33 +33,54 @@ module PaddedCell
29
33
  @options[:seed] ||= (ENV['SEED'] || Random.new_seed).to_i % 0xFFFF
30
34
  end
31
35
 
32
- # Exit status: 0 when everything passed, 1 otherwise, 130 on Ctrl-C.
36
+ # Exit status: 0 when everything passed, 1 otherwise, 3 when only
37
+ # coverage is below a minimum, 130 on Ctrl-C.
33
38
  def run
39
+ # The budget covers the whole run, loading included.
40
+ @limits = Limits.new(timeout: options[:timeout], budget: options[:budget])
34
41
  selection = Selection.parse(options[:paths])
35
42
  refs = Discovery.call(selection.files)
36
- return list(selection, refs) if options[:list]
43
+ filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
44
+ return list(selection, refs, filter) if options[:list]
37
45
  raise Unsupported, 'padded_cell needs fork, which this Ruby does not have' unless Process.respond_to?(:fork)
38
46
 
47
+ @coverage = start_coverage if options[:coverage]
39
48
  Loader.call(selection.files, seed: options[:seed])
49
+ refuse_simplecov if @coverage
40
50
  if defined?(ActiveSupport::TestCase)
41
51
  require_relative 'active_support_hooks'
42
52
  ActiveSupportHooks.install
43
53
  end
44
54
  options[:isolation] ||= default_isolation
45
- filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
55
+ # Coverage thresholds hold for the whole suite only.
56
+ @whole = !selection.line_filtered? && !options[:include] && !options[:exclude]
46
57
  execute(Loader.tests.select { filter.call(*it) })
47
58
  end
48
59
 
49
60
  private
50
61
 
62
+ # Before the suite loads: nothing loaded earlier is measured.
63
+ def start_coverage
64
+ require_relative 'chart'
65
+ Chart::Accumulator.new(options[:coverage]).start
66
+ end
67
+
68
+ # Two tools cannot share Ruby's Coverage.
69
+ def refuse_simplecov
70
+ return unless defined?(::SimpleCov) && ::SimpleCov.respond_to?(:pid) && ::SimpleCov.pid
71
+
72
+ raise Unsupported, 'SimpleCov is running, and --coverage replaces it: remove SimpleCov.start (and any ' \
73
+ 'SimpleCov.result in parallelize_teardown) from the test helper'
74
+ end
75
+
51
76
  # A fork per test discards Active Record's connection pool, and with it
52
77
  # the fixtures loaded through it: every test would reload them. With
53
78
  # fixtures around, tests share their worker instead.
54
79
  def default_isolation = defined?(ActiveRecord::FixtureSet) ? :worker : :test
55
80
 
56
81
  # Tests as they would run (static discovery, no loading), in file order.
57
- def list(selection, refs)
58
- refs = refs.select { selection.covers?(it.file, it.lines) }
82
+ def list(selection, refs, filter)
83
+ refs = refs.select { selection.covers?(it.file, it.lines) && filter.matches_name?(it.class_name, it.name) }
59
84
  kept = Scheduler.batches(refs.map(&:class_name), seed: options[:seed], shard: options[:shard]).flatten.sort
60
85
  kept.each do |index|
61
86
  ref = refs[index]
@@ -69,37 +94,134 @@ module PaddedCell
69
94
  timings = Timings.load(options[:timings])
70
95
  batches = Scheduler.batches(tests.map { |klass, _| klass.to_s }, durations: ids.map { timings[it] },
71
96
  seed: options[:seed], shard: options[:shard])
97
+ nothing_ran = batches.empty?
72
98
  reporter = build_reporter
73
99
  reporter.start
74
- interrupted = dispatch(tests, batches, reporter) { |index, result| timings[ids[index]] = result.time }
100
+ retried = []
101
+ @coverage&.snapshot
102
+ status = dispatch(tests, batches, reporter, retried) { |index, result| timings[ids[index]] = result.time }
75
103
  reporter.report
104
+ report_asylum(retried)
105
+ covered = @coverage ? report_coverage(status, reporter) : true
76
106
  Timings.save(options[:timings], timings)
77
107
  after_run
78
- finish(reporter, interrupted, batches.empty?)
108
+ result = finish(reporter, status, nothing_ran)
109
+ result.zero? && !covered ? 3 : result
110
+ end
111
+
112
+ # Writes the coverage reports, even for a failing run; false when the
113
+ # coverage of a whole, passing, complete run is below a minimum.
114
+ def report_coverage(status, reporter)
115
+ raw = @coverage.finish(interrupted: status == 130)
116
+ return true unless raw
117
+
118
+ dir = options[:coverage_dir] || Chart.config.directory
119
+ document = Chart::Document.build(raw, config: Chart.config, shard: options[:shard], whole: @whole)
120
+ document.write(dir)
121
+ check = status.zero? && reporter.passed? && document.complete?
122
+ Chart::Report.new(document, dir:, io: options[:io]).call(check:)
79
123
  end
80
124
 
81
- # Yields each result after reporting it. True when interrupted.
82
- def dispatch(tests, batches, reporter)
83
- pool = nil
84
- on_result = lambda do |index, result|
125
+ # Yields each result as it arrives. With --asylum a failure is held back:
126
+ # only its final result, after the retries, is reported. Returns 1 when a
127
+ # worker failed (its cleanup hook) or tests were left out (the budget ran
128
+ # out, workers kept dying), 130 when interrupted, 0 otherwise.
129
+ def dispatch(tests, batches, reporter, retried)
130
+ held = {}
131
+ record = lambda do |index, result|
85
132
  reporter.prerecord(*tests[index])
86
133
  reporter.record(result)
87
- yield index, result
88
- pool&.stop! if options[:fail_fast] && !result.passed? && !result.skipped?
134
+ retried << result if result.metadata? && (result.metadata[:flaky] || result.metadata[:asylum])
89
135
  end
90
136
  pool = Pool.new(tests, jobs: options[:jobs], isolation: options[:isolation], seed: options[:seed],
91
- timeout: options[:timeout])
92
- pool.run(batches, &on_result)
93
- false
137
+ limits: @limits, coverage: @coverage)
138
+ pool.run(batches) do |index, result|
139
+ yield index, result
140
+ failed = failed?(result)
141
+ options[:asylum] && failed ? held[index] = result : record.call(index, result)
142
+ pool.stop! if options[:fail_fast] && failed
143
+ end
144
+ complete = report_left_out(pool)
145
+ retries_ok = retry_held(tests, held, record)
146
+ pool.success? && retries_ok && complete ? 0 : 1
94
147
  rescue Interrupt
95
148
  warn 'Interrupted. Exiting...'
96
- true
149
+ # Failures still waiting for a retry are failures.
150
+ held.each { |index, result| record.call(index, result) }
151
+ 130
152
+ end
153
+
154
+ # False when tests never ran because the budget ran out or workers kept
155
+ # dying. --fail-fast leaving them out is what it asked for.
156
+ def report_left_out(pool)
157
+ left_out = pool.undispatched
158
+ if pool.gave_up
159
+ warn "padded_cell: #{pool.gave_up}; gave up, #{left_out} tests not run"
160
+ elsif @limits.spent? && left_out.positive?
161
+ warn "padded_cell: run budget of #{@limits.budget}s spent, #{left_out} tests not run"
162
+ else
163
+ return true
164
+ end
165
+ false
166
+ end
167
+
168
+ # False when a retry's worker failed.
169
+ def retry_held(tests, held, record)
170
+ return true if held.empty?
171
+
172
+ # No budget left to retry in: failures stay failures.
173
+ if @limits.spent?
174
+ held.each { |index, result| record.call(index, result) }
175
+ return true
176
+ end
177
+
178
+ asylum = Asylum.new(tests, attempts: options[:asylum], seed: options[:seed], limits: @limits)
179
+ asylum.call(held) do |index, result|
180
+ held.delete(index)
181
+ record.call(index, result)
182
+ end
183
+ asylum.success?
184
+ end
185
+
186
+ def failed?(result) = !result.passed? && !result.skipped?
187
+
188
+ # After Minitest's summary: what --asylum found, attempt by attempt.
189
+ def report_asylum(retried)
190
+ flaky, failing = retried.partition { it.metadata[:flaky] }
191
+ report_flaky(flaky) if flaky.any?
192
+ report_failing(failing) if failing.any?
193
+ end
194
+
195
+ def report_flaky(flaky)
196
+ io = options[:io]
197
+ tests = flaky.size == 1 ? '1 test' : "#{flaky.size} tests"
198
+ io.puts "\nYou're not crazy, the world is. #{tests} failed, then passed:"
199
+ width = flaky.flat_map { it.metadata[:flaky][:history] }.map { it[:seed].to_s.size }.max
200
+ flaky.each do |result|
201
+ info = result.metadata[:flaky]
202
+ io.puts "\n #{result.klass}##{result.name}"
203
+ info[:history].each do |attempt|
204
+ io.puts " seed #{attempt[:seed].to_s.ljust(width)} #{attempt_line(attempt, info[:cause])}"
205
+ end
206
+ end
207
+ end
208
+
209
+ def attempt_line(attempt, cause)
210
+ return "passed, alone → #{cause}" if attempt[:outcome] == 'passed'
211
+
212
+ attempt[:alone] ? "#{attempt[:outcome]} (alone)" : attempt[:outcome]
213
+ end
214
+
215
+ def report_failing(failing)
216
+ io = options[:io]
217
+ io.puts "\nIt's not the world, it's the test:"
218
+ failing.each { io.puts " #{it.klass}##{it.name} failed all #{it.metadata[:asylum][:attempts]} attempts" }
97
219
  end
98
220
 
99
221
  def build_reporter
100
222
  reporter = Minitest::CompositeReporter.new
101
223
  minitest_options = {
102
- io: options[:io], seed: options[:seed], args: "--seed #{options[:seed]} --isolation #{options[:isolation]}",
224
+ io: options[:io], seed: options[:seed], args: run_options,
103
225
  verbose: options[:verbose], show_skips: options[:show_skips]
104
226
  }
105
227
  reporter << Minitest::SummaryReporter.new(options[:io], minitest_options)
@@ -113,13 +235,19 @@ module PaddedCell
113
235
  reporter
114
236
  end
115
237
 
238
+ # What a rerun needs to reproduce this run's order and settings.
239
+ def run_options
240
+ args = "--seed #{options[:seed]} --isolation #{options[:isolation]}"
241
+ options[:asylum] ? "#{args} --asylum=#{options[:asylum]}" : args
242
+ end
243
+
116
244
  # Minitest.after_run blocks, which autorun would have called.
117
245
  def after_run
118
246
  Minitest.class_variable_get(:@@after_run).reverse_each(&:call)
119
247
  end
120
248
 
121
- def finish(reporter, interrupted, nothing_ran)
122
- return 130 if interrupted
249
+ def finish(reporter, status, nothing_ran)
250
+ return status unless status.zero?
123
251
  if nothing_ran && options[:include]
124
252
  warn "Nothing ran for filter: #{options[:include]}"
125
253
  return 1
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PaddedCell
4
- VERSION = '0.1.0'
4
+ VERSION = '0.3.0'
5
5
  end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PaddedCell
4
+ # A Pool's worker process. Reads test indices from its socket, runs each
5
+ # (in a Cell, or itself with isolation :worker) and writes back its
6
+ # Minitest::Result as a length-prefixed Marshal frame. Exits once the
7
+ # socket closes.
8
+ class Worker
9
+ # limits: the run's Limits; probe: a Chart::Probe when measuring coverage
10
+ def initialize(tests, isolation:, seed:, limits:, probe:)
11
+ @tests = tests
12
+ @isolation = isolation
13
+ @seed = seed
14
+ @limits = limits
15
+ @probe = probe
16
+ end
17
+
18
+ # Never returns.
19
+ def run(worker, socket)
20
+ status = 1
21
+ begin
22
+ srand(@seed)
23
+ @probe&.start
24
+ setup_error = after_fork(worker)
25
+ while (line = socket.gets)
26
+ payload = Results.dump(perform(Integer(line), socket, setup_error))
27
+ socket.write([payload.bytesize].pack('N'), payload)
28
+ @probe&.tick
29
+ end
30
+ # Before cleanup, which may stop Coverage.
31
+ @probe&.flush
32
+ Hooks.run_cleanup(worker) unless setup_error
33
+ status = 0
34
+ rescue Exception => e # anything: report it, then exit!
35
+ warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
36
+ ensure
37
+ exit!(status)
38
+ end
39
+ end
40
+
41
+ private
42
+
43
+ # A failing after_fork hook fails every test of the worker, the way Rails
44
+ # parallel testing reports a broken parallelize_setup.
45
+ def after_fork(worker)
46
+ Hooks.run_after_fork(worker)
47
+ nil
48
+ rescue StandardError => e
49
+ e
50
+ end
51
+
52
+ def perform(index, socket, setup_error)
53
+ klass, name = @tests.fetch(index)
54
+ if setup_error
55
+ Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
56
+ elsif @isolation == :test
57
+ Cell.run(klass, name, seed: @seed, limits: @limits, close: [socket], probe: @probe)
58
+ else
59
+ Cell.execute(klass, name)
60
+ end
61
+ end
62
+ end
63
+ end
data/lib/padded_cell.rb CHANGED
@@ -23,6 +23,14 @@ module PaddedCell
23
23
  def after_fork(&) = Hooks.after_fork(&)
24
24
 
25
25
  def cleanup(&) = Hooks.cleanup(&)
26
+
27
+ # Configures the coverage report (see Chart::Config). Inert unless
28
+ # padded_cell runs with --coverage. A helper also run without
29
+ # padded_cell guards it: `if defined?(PaddedCell.coverage)`.
30
+ def coverage(profile = nil, &)
31
+ require_relative 'padded_cell/chart'
32
+ Chart.configure(profile, &)
33
+ end
26
34
  end
27
35
  end
28
36