padded_cell 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/Cargo.lock +2 -2
- data/README.md +148 -7
- data/ext/padded_cell_native/core/Cargo.toml +1 -1
- data/ext/padded_cell_native/core/src/chart/directives/tests.rs +56 -0
- data/ext/padded_cell_native/core/src/chart/directives.rs +144 -0
- data/ext/padded_cell_native/core/src/chart/json/tests.rs +46 -0
- data/ext/padded_cell_native/core/src/chart/json.rs +175 -0
- data/ext/padded_cell_native/core/src/chart/lcov.rs +50 -0
- data/ext/padded_cell_native/core/src/chart/markdown/tests.rs +93 -0
- data/ext/padded_cell_native/core/src/chart/markdown.rs +201 -0
- data/ext/padded_cell_native/core/src/chart/methods/tests.rs +38 -0
- data/ext/padded_cell_native/core/src/chart/methods.rs +62 -0
- data/ext/padded_cell_native/core/src/chart/output/tests.rs +45 -0
- data/ext/padded_cell_native/core/src/chart/output.rs +103 -0
- data/ext/padded_cell_native/core/src/chart/results/tests.rs +96 -0
- data/ext/padded_cell_native/core/src/chart/results.rs +379 -0
- data/ext/padded_cell_native/core/src/chart/source/tests.rs +64 -0
- data/ext/padded_cell_native/core/src/chart/source.rs +153 -0
- data/ext/padded_cell_native/core/src/chart/stats/tests.rs +52 -0
- data/ext/padded_cell_native/core/src/chart/stats.rs +218 -0
- data/ext/padded_cell_native/core/src/chart/test_support.rs +93 -0
- data/ext/padded_cell_native/core/src/chart/tests.rs +97 -0
- data/ext/padded_cell_native/core/src/chart/thresholds/tests.rs +36 -0
- data/ext/padded_cell_native/core/src/chart/thresholds.rs +116 -0
- data/ext/padded_cell_native/core/src/chart.rs +302 -0
- data/ext/padded_cell_native/core/src/discover/tests.rs +2 -0
- data/ext/padded_cell_native/core/src/discover.rs +7 -53
- data/ext/padded_cell_native/core/src/error.rs +24 -0
- data/ext/padded_cell_native/core/src/lib.rs +5 -1
- data/ext/padded_cell_native/core/src/parallel.rs +43 -0
- data/ext/padded_cell_native/ffi/Cargo.toml +1 -1
- data/ext/padded_cell_native/ffi/src/chart.rs +205 -0
- data/ext/padded_cell_native/ffi/src/lib.rs +3 -0
- data/lib/padded_cell/asylum.rb +91 -0
- data/lib/padded_cell/cell.rb +14 -8
- data/lib/padded_cell/chart/accumulator.rb +103 -0
- data/lib/padded_cell/chart/config.rb +129 -0
- data/lib/padded_cell/chart/delta.rb +148 -0
- data/lib/padded_cell/chart/document.rb +64 -0
- data/lib/padded_cell/chart/filter.rb +55 -0
- data/lib/padded_cell/chart/merge.rb +63 -0
- data/lib/padded_cell/chart/probe.rb +70 -0
- data/lib/padded_cell/chart/raw.rb +48 -0
- data/lib/padded_cell/chart/report.rb +128 -0
- data/lib/padded_cell/chart/source_files.rb +90 -0
- data/lib/padded_cell/chart.rb +38 -0
- data/lib/padded_cell/cli.rb +59 -4
- data/lib/padded_cell/loader.rb +5 -1
- data/lib/padded_cell/pool.rb +19 -42
- data/lib/padded_cell/reporters.rb +20 -3
- data/lib/padded_cell/results.rb +13 -0
- data/lib/padded_cell/runner.rb +125 -22
- data/lib/padded_cell/version.rb +1 -1
- data/lib/padded_cell/worker.rb +63 -0
- data/lib/padded_cell.rb +8 -0
- metadata +39 -1
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'chart/delta'
|
|
4
|
+
require_relative 'chart/probe'
|
|
5
|
+
require_relative 'chart/raw'
|
|
6
|
+
require_relative 'chart/accumulator'
|
|
7
|
+
require_relative 'chart/filter'
|
|
8
|
+
require_relative 'chart/config'
|
|
9
|
+
require_relative 'chart/source_files'
|
|
10
|
+
require_relative 'chart/document'
|
|
11
|
+
require_relative 'chart/merge'
|
|
12
|
+
require_relative 'chart/report'
|
|
13
|
+
|
|
14
|
+
module PaddedCell
|
|
15
|
+
# Built-in coverage: Ruby's Coverage measured in the
|
|
16
|
+
# runner, every worker and every cell, summed exactly in the runner. The
|
|
17
|
+
# chart, as the asylum keeps one per patient: inside, a name that is not
|
|
18
|
+
# Ruby's own Coverage; outside, it is all --coverage.
|
|
19
|
+
#
|
|
20
|
+
# Measuring starts before the suite loads, so what is measured comes from
|
|
21
|
+
# the command line (--coverage=lines,branches,methods,eval); the report is
|
|
22
|
+
# configured in the test helper, with PaddedCell.coverage (see Config).
|
|
23
|
+
module Chart
|
|
24
|
+
CRITERIA = %i[lines branches methods eval].freeze
|
|
25
|
+
DEFAULT_CRITERIA = %i[lines branches].freeze
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
# Every PaddedCell.coverage block, in one Config.
|
|
29
|
+
def config = @config ||= Config.new
|
|
30
|
+
|
|
31
|
+
def configure(profile = nil, &block)
|
|
32
|
+
config.profile(profile) if profile
|
|
33
|
+
config.instance_eval(&block) if block
|
|
34
|
+
config
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
data/lib/padded_cell/cli.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'optparse'
|
|
4
4
|
require_relative 'runner'
|
|
5
|
+
require_relative 'chart'
|
|
5
6
|
|
|
6
7
|
module PaddedCell
|
|
7
8
|
# The padded_cell command.
|
|
@@ -18,13 +19,16 @@ module PaddedCell
|
|
|
18
19
|
|
|
19
20
|
module_function
|
|
20
21
|
|
|
21
|
-
# Exit status: 0 passed, 1 failed, 2 usage or load error,
|
|
22
|
+
# Exit status: 0 passed, 1 failed, 2 usage or load error, 3 coverage below
|
|
23
|
+
# a minimum, 130 interrupted.
|
|
22
24
|
def start(argv, io: $stdout)
|
|
23
|
-
options = { io: }
|
|
25
|
+
options = { io:, coverage: coverage_criteria(ENV.fetch('PADDED_CELL_COVERAGE', nil)) }
|
|
24
26
|
parser(options).parse!(argv)
|
|
27
|
+
return merge(options) if options[:coverage_merge]
|
|
28
|
+
|
|
25
29
|
options[:paths] = argv
|
|
26
30
|
Runner.new(**options).run
|
|
27
|
-
rescue OptionParser::ParseError, ArgumentError, LoadFailure, Unsupported, SystemCallError => e
|
|
31
|
+
rescue OptionParser::ParseError, ArgumentError, RegexpError, LoadFailure, Unsupported, SystemCallError => e
|
|
28
32
|
warn "padded_cell: #{e.message}"
|
|
29
33
|
2
|
|
30
34
|
end
|
|
@@ -34,6 +38,7 @@ module PaddedCell
|
|
|
34
38
|
opts.banner = BANNER
|
|
35
39
|
opts.version = VERSION
|
|
36
40
|
runner_options(opts, options)
|
|
41
|
+
coverage_options(opts, options)
|
|
37
42
|
output_options(opts, options)
|
|
38
43
|
opts.on('--version', 'Print the version') do
|
|
39
44
|
options[:io].puts "padded_cell #{VERSION}"
|
|
@@ -50,7 +55,7 @@ module PaddedCell
|
|
|
50
55
|
opts.on('-j', '--jobs N', Integer, "Workers (default: #{Etc.nprocessors})") do
|
|
51
56
|
options[:jobs] = positive(it)
|
|
52
57
|
end
|
|
53
|
-
opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = it }
|
|
58
|
+
opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = seed(it) }
|
|
54
59
|
opts.on('-n', '-i', '--name', '--include PATTERN', 'Only tests matching (/regexp/ or exact name)') do
|
|
55
60
|
options[:include] = it
|
|
56
61
|
end
|
|
@@ -66,9 +71,36 @@ module PaddedCell
|
|
|
66
71
|
options[:timeout] = positive(it)
|
|
67
72
|
end
|
|
68
73
|
opts.on('--fail-fast', 'Stop dispatching after the first failure') { options[:fail_fast] = true }
|
|
74
|
+
opts.on('--asylum[=N]', Integer, 'Retry failed tests up to N times (default 2) after the run, alone, ' \
|
|
75
|
+
'each in a fresh cell; report the flaky ones') do
|
|
76
|
+
options[:asylum] = it.nil? ? 2 : positive(it)
|
|
77
|
+
end
|
|
69
78
|
opts.on('--list', 'List the tests that would run (static, loads nothing)') { options[:list] = true }
|
|
70
79
|
end
|
|
71
80
|
|
|
81
|
+
def coverage_options(opts, options)
|
|
82
|
+
opts.on('--[no-]coverage[=CRITERIA]', 'Measure coverage: lines, branches, methods, eval ' \
|
|
83
|
+
"(default: #{Chart::DEFAULT_CRITERIA.join(',')}; " \
|
|
84
|
+
'or $PADDED_CELL_COVERAGE)') do
|
|
85
|
+
options[:coverage] = coverage_criteria(it.to_s)
|
|
86
|
+
end
|
|
87
|
+
opts.on('--coverage-dir DIR', 'Write coverage reports to DIR (default: coverage, or the test helper\'s)') do
|
|
88
|
+
options[:coverage_dir] = it
|
|
89
|
+
end
|
|
90
|
+
opts.on('--coverage-merge DIR,DIR', Array, 'Merge the coverage of shards (their --coverage-dir) ' \
|
|
91
|
+
'into one report; runs no test') do
|
|
92
|
+
options[:coverage_merge] = it
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# --coverage-merge: 0, or 3 when a threshold is missed.
|
|
97
|
+
def merge(options)
|
|
98
|
+
document = Chart::Merge.call(options[:coverage_merge])
|
|
99
|
+
dir = options[:coverage_dir] || 'coverage'
|
|
100
|
+
document.write(dir)
|
|
101
|
+
Chart::Report.new(document, dir:, io: options[:io]).call(check: document.complete?) ? 0 : 3
|
|
102
|
+
end
|
|
103
|
+
|
|
72
104
|
def output_options(opts, options)
|
|
73
105
|
opts.on('-v', '--verbose', 'One line per test, with timings') { options[:verbose] = true }
|
|
74
106
|
opts.on('--show-skips', 'Show skipped tests at the end') { options[:show_skips] = true }
|
|
@@ -78,6 +110,13 @@ module PaddedCell
|
|
|
78
110
|
opts.on('--no-timings', 'Neither read nor write timings') { options[:timings] = nil }
|
|
79
111
|
end
|
|
80
112
|
|
|
113
|
+
# The planner takes an unsigned 64-bit seed.
|
|
114
|
+
def seed(value)
|
|
115
|
+
raise OptionParser::InvalidArgument, "#{value} (must be 0 to 2**64 - 1)" unless value.between?(0, (2**64) - 1)
|
|
116
|
+
|
|
117
|
+
value
|
|
118
|
+
end
|
|
119
|
+
|
|
81
120
|
# OptionParser names the switch in the message.
|
|
82
121
|
def positive(value)
|
|
83
122
|
raise OptionParser::InvalidArgument, "#{value} (must be positive)" unless value.positive?
|
|
@@ -85,6 +124,22 @@ module PaddedCell
|
|
|
85
124
|
value
|
|
86
125
|
end
|
|
87
126
|
|
|
127
|
+
# "branches,methods" => [:lines, :branches, :methods] (lines always);
|
|
128
|
+
# "1" or "true" => the defaults; nil, "", "0" or "false" => off.
|
|
129
|
+
def coverage_criteria(value)
|
|
130
|
+
value = value.to_s.strip.downcase
|
|
131
|
+
return if ['', '0', 'false'].include?(value)
|
|
132
|
+
return Chart::DEFAULT_CRITERIA if %w[1 true].include?(value)
|
|
133
|
+
|
|
134
|
+
names = value.split(',').map { it.strip.to_sym }
|
|
135
|
+
unknown = names - Chart::CRITERIA
|
|
136
|
+
unless unknown.empty?
|
|
137
|
+
raise OptionParser::InvalidArgument, "coverage #{unknown.join(',')} (known: #{Chart::CRITERIA.join(', ')})"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
[:lines] | names
|
|
141
|
+
end
|
|
142
|
+
|
|
88
143
|
def shard(value)
|
|
89
144
|
index, count = value.split('/', 2).map { Integer(it, 10, exception: false) }
|
|
90
145
|
unless index && count&.positive? && index.between?(1, count)
|
data/lib/padded_cell/loader.rb
CHANGED
|
@@ -53,11 +53,15 @@ module PaddedCell
|
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
def call(klass, name)
|
|
56
|
+
matches_name?(klass, name) && (!@selection.line_filtered? || line_selected?(klass, name))
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def matches_name?(klass, name)
|
|
56
60
|
id = "#{klass}##{name}"
|
|
57
61
|
return false if @include && !matches?(@include, name, id)
|
|
58
62
|
return false if @exclude && matches?(@exclude, name, id)
|
|
59
63
|
|
|
60
|
-
|
|
64
|
+
true
|
|
61
65
|
end
|
|
62
66
|
|
|
63
67
|
private
|
data/lib/padded_cell/pool.rb
CHANGED
|
@@ -19,12 +19,14 @@ module PaddedCell
|
|
|
19
19
|
Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch)
|
|
20
20
|
|
|
21
21
|
# tests: [[klass, method_name], ...], dispatched by index.
|
|
22
|
-
|
|
22
|
+
# coverage: a Chart::Accumulator when measuring coverage
|
|
23
|
+
def initialize(tests, jobs:, isolation:, seed:, timeout: nil, coverage: nil)
|
|
23
24
|
@tests = tests
|
|
24
25
|
@jobs = jobs
|
|
25
26
|
@isolation = isolation
|
|
26
27
|
@seed = seed
|
|
27
28
|
@timeout = timeout
|
|
29
|
+
@coverage = coverage
|
|
28
30
|
end
|
|
29
31
|
|
|
30
32
|
# Runs batches of test indices in order (consuming them), yielding
|
|
@@ -47,6 +49,8 @@ module PaddedCell
|
|
|
47
49
|
shutdown(force: !$!.nil?)
|
|
48
50
|
end
|
|
49
51
|
|
|
52
|
+
def success? = @success
|
|
53
|
+
|
|
50
54
|
# Dispatch nothing more; tests already running still report.
|
|
51
55
|
def stop!
|
|
52
56
|
@queue.clear
|
|
@@ -63,7 +67,8 @@ module PaddedCell
|
|
|
63
67
|
# A worker holding another worker's socket would keep it from ever
|
|
64
68
|
# seeing EOF: keep only its own end.
|
|
65
69
|
[parent, *@sockets].each(&:close)
|
|
66
|
-
|
|
70
|
+
Worker.new(@tests, isolation: @isolation, seed: @seed, timeout: @timeout, probe: @coverage&.probe)
|
|
71
|
+
.run(slot.worker, child)
|
|
67
72
|
end
|
|
68
73
|
child.close
|
|
69
74
|
@sockets << parent
|
|
@@ -71,45 +76,6 @@ module PaddedCell
|
|
|
71
76
|
slot
|
|
72
77
|
end
|
|
73
78
|
|
|
74
|
-
# The worker process.
|
|
75
|
-
def work(worker, socket)
|
|
76
|
-
status = 1
|
|
77
|
-
begin
|
|
78
|
-
srand(@seed)
|
|
79
|
-
setup_error = after_fork(worker)
|
|
80
|
-
while (line = socket.gets)
|
|
81
|
-
payload = Results.dump(perform(Integer(line), socket, setup_error))
|
|
82
|
-
socket.write([payload.bytesize].pack('N'), payload)
|
|
83
|
-
end
|
|
84
|
-
Hooks.run_cleanup(worker) unless setup_error
|
|
85
|
-
status = 0
|
|
86
|
-
rescue Exception => e # anything: report it, then exit!
|
|
87
|
-
warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
|
|
88
|
-
ensure
|
|
89
|
-
exit!(status)
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# A failing after_fork hook fails every test of the worker, the way Rails
|
|
94
|
-
# parallel testing reports a broken parallelize_setup.
|
|
95
|
-
def after_fork(worker)
|
|
96
|
-
Hooks.run_after_fork(worker)
|
|
97
|
-
nil
|
|
98
|
-
rescue StandardError => e
|
|
99
|
-
e
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def perform(index, socket, setup_error)
|
|
103
|
-
klass, name = @tests.fetch(index)
|
|
104
|
-
if setup_error
|
|
105
|
-
Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
|
|
106
|
-
elsif @isolation == :test
|
|
107
|
-
Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket])
|
|
108
|
-
else
|
|
109
|
-
Cell.execute(klass, name)
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
|
|
113
79
|
def feed(slot)
|
|
114
80
|
slot.batch = @queue.shift || steal if slot.batch.empty?
|
|
115
81
|
slot.test = slot.batch.shift
|
|
@@ -121,6 +87,7 @@ module PaddedCell
|
|
|
121
87
|
rescue Errno::EPIPE, Errno::ECONNRESET
|
|
122
88
|
# The worker died between tests: its replacement gets this one.
|
|
123
89
|
reap(slot)
|
|
90
|
+
@coverage&.lost_worker
|
|
124
91
|
spawn(slot).socket.puts(slot.test)
|
|
125
92
|
end
|
|
126
93
|
end
|
|
@@ -175,6 +142,7 @@ module PaddedCell
|
|
|
175
142
|
|
|
176
143
|
# Reports the running test of a dead worker and starts a new one.
|
|
177
144
|
def replace(slot, message, time)
|
|
145
|
+
@coverage&.lost_worker
|
|
178
146
|
klass, name = @tests.fetch(slot.test)
|
|
179
147
|
report(slot, Results.crashed(klass, name, message, time:))
|
|
180
148
|
return if @queue.empty? && @slots.all? { it.batch.empty? }
|
|
@@ -184,8 +152,16 @@ module PaddedCell
|
|
|
184
152
|
end
|
|
185
153
|
|
|
186
154
|
def reap(slot)
|
|
155
|
+
close_socket(slot)
|
|
156
|
+
wait_for(slot)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def close_socket(slot)
|
|
187
160
|
@sockets.delete(slot.socket)
|
|
188
161
|
slot.socket.close
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def wait_for(slot)
|
|
189
165
|
_, status = Process.wait2(slot.pid)
|
|
190
166
|
slot.pid = nil
|
|
191
167
|
status
|
|
@@ -196,7 +172,8 @@ module PaddedCell
|
|
|
196
172
|
def shutdown(force:)
|
|
197
173
|
live = Array(@slots).select(&:pid)
|
|
198
174
|
live.each { Process.kill(:KILL, it.pid) } if force
|
|
199
|
-
live.each {
|
|
175
|
+
live.each { close_socket(it) }
|
|
176
|
+
@success = live.map { wait_for(it) }.all?(&:success?)
|
|
200
177
|
end
|
|
201
178
|
end
|
|
202
179
|
end
|
|
@@ -36,6 +36,12 @@ module PaddedCell
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def relative(file) = file.delete_prefix("#{Dir.pwd}/")
|
|
39
|
+
|
|
40
|
+
# --asylum's report on a test that passed on retry, if it did.
|
|
41
|
+
def flaky(result) = result.metadata? ? result.metadata[:flaky] : nil
|
|
42
|
+
|
|
43
|
+
# --asylum's attempts at a test that never passed.
|
|
44
|
+
def asylum(result) = result.metadata? ? result.metadata[:asylum] : nil
|
|
39
45
|
end
|
|
40
46
|
|
|
41
47
|
# JUnit XML, as read by GitHub, GitLab, Jenkins and friends.
|
|
@@ -67,10 +73,20 @@ module PaddedCell
|
|
|
67
73
|
file, line = file_and_line(result)
|
|
68
74
|
attributes = %(classname="#{escape(result.klass)}" name="#{escape(result.name)}" file="#{escape(file)}" ) +
|
|
69
75
|
%(line="#{line}" time="#{format('%.6f', result.time)}" assertions="#{result.assertions}")
|
|
70
|
-
body = outcome(result)
|
|
76
|
+
body = outcome(result) + flaky_failure(result)
|
|
71
77
|
body.empty? ? %( <testcase #{attributes}/>\n) : %( <testcase #{attributes}>\n#{body} </testcase>\n)
|
|
72
78
|
end
|
|
73
79
|
|
|
80
|
+
# Surefire's element for a test that failed, then passed on retry.
|
|
81
|
+
def flaky_failure(result)
|
|
82
|
+
info = flaky(result)
|
|
83
|
+
return '' unless info
|
|
84
|
+
|
|
85
|
+
first = info[:first_failure]
|
|
86
|
+
%( <flakyFailure type="#{escape(first[:type])}" message="#{escape(first[:message])}">) +
|
|
87
|
+
%(#{escape(info[:cause])}</flakyFailure>\n)
|
|
88
|
+
end
|
|
89
|
+
|
|
74
90
|
def outcome(result)
|
|
75
91
|
failure = result.failure
|
|
76
92
|
case status(result)
|
|
@@ -98,8 +114,9 @@ module PaddedCell
|
|
|
98
114
|
{
|
|
99
115
|
id: "#{result.klass}##{result.name}", class: result.klass, name: result.name, file:, line:,
|
|
100
116
|
status: status(result), time: result.time, assertions: result.assertions,
|
|
101
|
-
failures: result.failures.map { { type: it.result_label, message: it.message.scrub } }
|
|
102
|
-
|
|
117
|
+
failures: result.failures.map { { type: it.result_label, message: it.message.scrub } },
|
|
118
|
+
flaky: flaky(result), asylum: asylum(result)
|
|
119
|
+
}.compact
|
|
103
120
|
end
|
|
104
121
|
end
|
|
105
122
|
end
|
data/lib/padded_cell/results.rb
CHANGED
|
@@ -74,6 +74,19 @@ module PaddedCell
|
|
|
74
74
|
placeholders << [owner, last]
|
|
75
75
|
end
|
|
76
76
|
|
|
77
|
+
# What a cell sends its worker: its coverage delta (nil without
|
|
78
|
+
# coverage) beside its result. The outer layer holds only core types, so
|
|
79
|
+
# it loads even when the result does not.
|
|
80
|
+
def envelope(delta, result) = Marshal.dump([delta, dump(result)])
|
|
81
|
+
|
|
82
|
+
# [delta, result] from an envelope; never raises.
|
|
83
|
+
def unwrap(data, klass, name)
|
|
84
|
+
delta, payload = Marshal.load(data) # written by our own cell
|
|
85
|
+
[delta, load(payload, klass, name)]
|
|
86
|
+
rescue StandardError => e
|
|
87
|
+
[nil, crashed(klass, name, "its result could not be loaded: #{e.class}: #{e.message}")]
|
|
88
|
+
end
|
|
89
|
+
|
|
77
90
|
# Marshal.dump, after replacing failures that cannot cross a pipe (an
|
|
78
91
|
# exception holding a Proc, an IO...) with plain copies.
|
|
79
92
|
def dump(result)
|
data/lib/padded_cell/runner.rb
CHANGED
|
@@ -5,7 +5,9 @@ require 'minitest'
|
|
|
5
5
|
require_relative '../padded_cell'
|
|
6
6
|
require_relative 'results'
|
|
7
7
|
require_relative 'cell'
|
|
8
|
+
require_relative 'worker'
|
|
8
9
|
require_relative 'pool'
|
|
10
|
+
require_relative 'asylum'
|
|
9
11
|
require_relative 'loader'
|
|
10
12
|
require_relative 'timings'
|
|
11
13
|
require_relative 'reporters'
|
|
@@ -18,8 +20,9 @@ module PaddedCell
|
|
|
18
20
|
class Runner
|
|
19
21
|
DEFAULTS = {
|
|
20
22
|
paths: [], jobs: Etc.nprocessors, seed: nil, include: nil, exclude: nil, shard: [0, 1],
|
|
21
|
-
isolation: nil, timeout: nil, fail_fast: false, verbose: false, show_skips: false,
|
|
22
|
-
list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv',
|
|
23
|
+
isolation: nil, timeout: nil, asylum: nil, fail_fast: false, verbose: false, show_skips: false,
|
|
24
|
+
list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', coverage: nil,
|
|
25
|
+
coverage_dir: nil, io: $stdout
|
|
23
26
|
}.freeze
|
|
24
27
|
|
|
25
28
|
attr_reader :options
|
|
@@ -29,33 +32,52 @@ module PaddedCell
|
|
|
29
32
|
@options[:seed] ||= (ENV['SEED'] || Random.new_seed).to_i % 0xFFFF
|
|
30
33
|
end
|
|
31
34
|
|
|
32
|
-
# Exit status: 0 when everything passed, 1 otherwise,
|
|
35
|
+
# Exit status: 0 when everything passed, 1 otherwise, 3 when only
|
|
36
|
+
# coverage is below a minimum, 130 on Ctrl-C.
|
|
33
37
|
def run
|
|
34
38
|
selection = Selection.parse(options[:paths])
|
|
35
39
|
refs = Discovery.call(selection.files)
|
|
36
|
-
|
|
40
|
+
filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
|
|
41
|
+
return list(selection, refs, filter) if options[:list]
|
|
37
42
|
raise Unsupported, 'padded_cell needs fork, which this Ruby does not have' unless Process.respond_to?(:fork)
|
|
38
43
|
|
|
44
|
+
@coverage = start_coverage if options[:coverage]
|
|
39
45
|
Loader.call(selection.files, seed: options[:seed])
|
|
46
|
+
refuse_simplecov if @coverage
|
|
40
47
|
if defined?(ActiveSupport::TestCase)
|
|
41
48
|
require_relative 'active_support_hooks'
|
|
42
49
|
ActiveSupportHooks.install
|
|
43
50
|
end
|
|
44
51
|
options[:isolation] ||= default_isolation
|
|
45
|
-
|
|
52
|
+
# Coverage thresholds hold for the whole suite only.
|
|
53
|
+
@whole = !selection.line_filtered? && !options[:include] && !options[:exclude]
|
|
46
54
|
execute(Loader.tests.select { filter.call(*it) })
|
|
47
55
|
end
|
|
48
56
|
|
|
49
57
|
private
|
|
50
58
|
|
|
59
|
+
# Before the suite loads: nothing loaded earlier is measured.
|
|
60
|
+
def start_coverage
|
|
61
|
+
require_relative 'chart'
|
|
62
|
+
Chart::Accumulator.new(options[:coverage]).start
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Two tools cannot share Ruby's Coverage.
|
|
66
|
+
def refuse_simplecov
|
|
67
|
+
return unless defined?(::SimpleCov) && ::SimpleCov.respond_to?(:pid) && ::SimpleCov.pid
|
|
68
|
+
|
|
69
|
+
raise Unsupported, 'SimpleCov is running, and --coverage replaces it: remove SimpleCov.start (and any ' \
|
|
70
|
+
'SimpleCov.result in parallelize_teardown) from the test helper'
|
|
71
|
+
end
|
|
72
|
+
|
|
51
73
|
# A fork per test discards Active Record's connection pool, and with it
|
|
52
74
|
# the fixtures loaded through it: every test would reload them. With
|
|
53
75
|
# fixtures around, tests share their worker instead.
|
|
54
76
|
def default_isolation = defined?(ActiveRecord::FixtureSet) ? :worker : :test
|
|
55
77
|
|
|
56
78
|
# Tests as they would run (static discovery, no loading), in file order.
|
|
57
|
-
def list(selection, refs)
|
|
58
|
-
refs = refs.select { selection.covers?(it.file, it.lines) }
|
|
79
|
+
def list(selection, refs, filter)
|
|
80
|
+
refs = refs.select { selection.covers?(it.file, it.lines) && filter.matches_name?(it.class_name, it.name) }
|
|
59
81
|
kept = Scheduler.batches(refs.map(&:class_name), seed: options[:seed], shard: options[:shard]).flatten.sort
|
|
60
82
|
kept.each do |index|
|
|
61
83
|
ref = refs[index]
|
|
@@ -69,37 +91,112 @@ module PaddedCell
|
|
|
69
91
|
timings = Timings.load(options[:timings])
|
|
70
92
|
batches = Scheduler.batches(tests.map { |klass, _| klass.to_s }, durations: ids.map { timings[it] },
|
|
71
93
|
seed: options[:seed], shard: options[:shard])
|
|
94
|
+
nothing_ran = batches.empty?
|
|
72
95
|
reporter = build_reporter
|
|
73
96
|
reporter.start
|
|
74
|
-
|
|
97
|
+
retried = []
|
|
98
|
+
@coverage&.snapshot
|
|
99
|
+
status = dispatch(tests, batches, reporter, retried) { |index, result| timings[ids[index]] = result.time }
|
|
75
100
|
reporter.report
|
|
101
|
+
report_asylum(retried)
|
|
102
|
+
covered = @coverage ? report_coverage(status, reporter) : true
|
|
76
103
|
Timings.save(options[:timings], timings)
|
|
77
104
|
after_run
|
|
78
|
-
finish(reporter,
|
|
105
|
+
result = finish(reporter, status, nothing_ran)
|
|
106
|
+
result.zero? && !covered ? 3 : result
|
|
79
107
|
end
|
|
80
108
|
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
109
|
+
# Writes the coverage reports, even for a failing run; false when the
|
|
110
|
+
# coverage of a whole, passing, complete run is below a minimum.
|
|
111
|
+
def report_coverage(status, reporter)
|
|
112
|
+
raw = @coverage.finish(interrupted: status == 130)
|
|
113
|
+
return true unless raw
|
|
114
|
+
|
|
115
|
+
dir = options[:coverage_dir] || Chart.config.directory
|
|
116
|
+
document = Chart::Document.build(raw, config: Chart.config, shard: options[:shard], whole: @whole)
|
|
117
|
+
document.write(dir)
|
|
118
|
+
check = status.zero? && reporter.passed? && document.complete?
|
|
119
|
+
Chart::Report.new(document, dir:, io: options[:io]).call(check:)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Yields each result as it arrives. With --asylum a failure is held back:
|
|
123
|
+
# only its final result, after the retries, is reported. Returns 1 when a
|
|
124
|
+
# worker failed (its cleanup hook), 130 when interrupted, 0 otherwise.
|
|
125
|
+
def dispatch(tests, batches, reporter, retried)
|
|
126
|
+
held = {}
|
|
127
|
+
record = lambda do |index, result|
|
|
85
128
|
reporter.prerecord(*tests[index])
|
|
86
129
|
reporter.record(result)
|
|
87
|
-
|
|
88
|
-
pool&.stop! if options[:fail_fast] && !result.passed? && !result.skipped?
|
|
130
|
+
retried << result if result.metadata? && (result.metadata[:flaky] || result.metadata[:asylum])
|
|
89
131
|
end
|
|
90
132
|
pool = Pool.new(tests, jobs: options[:jobs], isolation: options[:isolation], seed: options[:seed],
|
|
91
|
-
timeout: options[:timeout])
|
|
92
|
-
pool.run(batches,
|
|
93
|
-
|
|
133
|
+
timeout: options[:timeout], coverage: @coverage)
|
|
134
|
+
pool.run(batches) do |index, result|
|
|
135
|
+
yield index, result
|
|
136
|
+
failed = failed?(result)
|
|
137
|
+
options[:asylum] && failed ? held[index] = result : record.call(index, result)
|
|
138
|
+
pool.stop! if options[:fail_fast] && failed
|
|
139
|
+
end
|
|
140
|
+
retries_ok = retry_held(tests, held, record)
|
|
141
|
+
pool.success? && retries_ok ? 0 : 1
|
|
94
142
|
rescue Interrupt
|
|
95
143
|
warn 'Interrupted. Exiting...'
|
|
96
|
-
|
|
144
|
+
# Failures still waiting for a retry are failures.
|
|
145
|
+
held.each { |index, result| record.call(index, result) }
|
|
146
|
+
130
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# False when a retry's worker failed.
|
|
150
|
+
def retry_held(tests, held, record)
|
|
151
|
+
return true if held.empty?
|
|
152
|
+
|
|
153
|
+
asylum = Asylum.new(tests, attempts: options[:asylum], seed: options[:seed], timeout: options[:timeout])
|
|
154
|
+
asylum.call(held) do |index, result|
|
|
155
|
+
held.delete(index)
|
|
156
|
+
record.call(index, result)
|
|
157
|
+
end
|
|
158
|
+
asylum.success?
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def failed?(result) = !result.passed? && !result.skipped?
|
|
162
|
+
|
|
163
|
+
# After Minitest's summary: what --asylum found, attempt by attempt.
|
|
164
|
+
def report_asylum(retried)
|
|
165
|
+
flaky, failing = retried.partition { it.metadata[:flaky] }
|
|
166
|
+
report_flaky(flaky) if flaky.any?
|
|
167
|
+
report_failing(failing) if failing.any?
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def report_flaky(flaky)
|
|
171
|
+
io = options[:io]
|
|
172
|
+
tests = flaky.size == 1 ? '1 test' : "#{flaky.size} tests"
|
|
173
|
+
io.puts "\nYou're not crazy, the world is. #{tests} failed, then passed:"
|
|
174
|
+
width = flaky.flat_map { it.metadata[:flaky][:history] }.map { it[:seed].to_s.size }.max
|
|
175
|
+
flaky.each do |result|
|
|
176
|
+
info = result.metadata[:flaky]
|
|
177
|
+
io.puts "\n #{result.klass}##{result.name}"
|
|
178
|
+
info[:history].each do |attempt|
|
|
179
|
+
io.puts " seed #{attempt[:seed].to_s.ljust(width)} #{attempt_line(attempt, info[:cause])}"
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def attempt_line(attempt, cause)
|
|
185
|
+
return "passed, alone → #{cause}" if attempt[:outcome] == 'passed'
|
|
186
|
+
|
|
187
|
+
attempt[:alone] ? "#{attempt[:outcome]} (alone)" : attempt[:outcome]
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def report_failing(failing)
|
|
191
|
+
io = options[:io]
|
|
192
|
+
io.puts "\nIt's not the world, it's the test:"
|
|
193
|
+
failing.each { io.puts " #{it.klass}##{it.name} failed all #{it.metadata[:asylum][:attempts]} attempts" }
|
|
97
194
|
end
|
|
98
195
|
|
|
99
196
|
def build_reporter
|
|
100
197
|
reporter = Minitest::CompositeReporter.new
|
|
101
198
|
minitest_options = {
|
|
102
|
-
io: options[:io], seed: options[:seed], args:
|
|
199
|
+
io: options[:io], seed: options[:seed], args: run_options,
|
|
103
200
|
verbose: options[:verbose], show_skips: options[:show_skips]
|
|
104
201
|
}
|
|
105
202
|
reporter << Minitest::SummaryReporter.new(options[:io], minitest_options)
|
|
@@ -113,13 +210,19 @@ module PaddedCell
|
|
|
113
210
|
reporter
|
|
114
211
|
end
|
|
115
212
|
|
|
213
|
+
# What a rerun needs to reproduce this run's order and settings.
|
|
214
|
+
def run_options
|
|
215
|
+
args = "--seed #{options[:seed]} --isolation #{options[:isolation]}"
|
|
216
|
+
options[:asylum] ? "#{args} --asylum=#{options[:asylum]}" : args
|
|
217
|
+
end
|
|
218
|
+
|
|
116
219
|
# Minitest.after_run blocks, which autorun would have called.
|
|
117
220
|
def after_run
|
|
118
221
|
Minitest.class_variable_get(:@@after_run).reverse_each(&:call)
|
|
119
222
|
end
|
|
120
223
|
|
|
121
|
-
def finish(reporter,
|
|
122
|
-
return
|
|
224
|
+
def finish(reporter, status, nothing_ran)
|
|
225
|
+
return status unless status.zero?
|
|
123
226
|
if nothing_ran && options[:include]
|
|
124
227
|
warn "Nothing ran for filter: #{options[:include]}"
|
|
125
228
|
return 1
|
data/lib/padded_cell/version.rb
CHANGED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PaddedCell
|
|
4
|
+
# A Pool's worker process. Reads test indices from its socket, runs each
|
|
5
|
+
# (in a Cell, or itself with isolation :worker) and writes back its
|
|
6
|
+
# Minitest::Result as a length-prefixed Marshal frame. Exits once the
|
|
7
|
+
# socket closes.
|
|
8
|
+
class Worker
|
|
9
|
+
# probe: a Chart::Probe when measuring coverage
|
|
10
|
+
def initialize(tests, isolation:, seed:, timeout:, probe:)
|
|
11
|
+
@tests = tests
|
|
12
|
+
@isolation = isolation
|
|
13
|
+
@seed = seed
|
|
14
|
+
@timeout = timeout
|
|
15
|
+
@probe = probe
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Never returns.
|
|
19
|
+
def run(worker, socket)
|
|
20
|
+
status = 1
|
|
21
|
+
begin
|
|
22
|
+
srand(@seed)
|
|
23
|
+
@probe&.start
|
|
24
|
+
setup_error = after_fork(worker)
|
|
25
|
+
while (line = socket.gets)
|
|
26
|
+
payload = Results.dump(perform(Integer(line), socket, setup_error))
|
|
27
|
+
socket.write([payload.bytesize].pack('N'), payload)
|
|
28
|
+
@probe&.tick
|
|
29
|
+
end
|
|
30
|
+
# Before cleanup, which may stop Coverage.
|
|
31
|
+
@probe&.flush
|
|
32
|
+
Hooks.run_cleanup(worker) unless setup_error
|
|
33
|
+
status = 0
|
|
34
|
+
rescue Exception => e # anything: report it, then exit!
|
|
35
|
+
warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
|
|
36
|
+
ensure
|
|
37
|
+
exit!(status)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# A failing after_fork hook fails every test of the worker, the way Rails
|
|
44
|
+
# parallel testing reports a broken parallelize_setup.
|
|
45
|
+
def after_fork(worker)
|
|
46
|
+
Hooks.run_after_fork(worker)
|
|
47
|
+
nil
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
e
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def perform(index, socket, setup_error)
|
|
53
|
+
klass, name = @tests.fetch(index)
|
|
54
|
+
if setup_error
|
|
55
|
+
Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
|
|
56
|
+
elsif @isolation == :test
|
|
57
|
+
Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket], probe: @probe)
|
|
58
|
+
else
|
|
59
|
+
Cell.execute(klass, name)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
data/lib/padded_cell.rb
CHANGED
|
@@ -23,6 +23,14 @@ module PaddedCell
|
|
|
23
23
|
def after_fork(&) = Hooks.after_fork(&)
|
|
24
24
|
|
|
25
25
|
def cleanup(&) = Hooks.cleanup(&)
|
|
26
|
+
|
|
27
|
+
# Configures the coverage report (see Chart::Config). Inert unless
|
|
28
|
+
# padded_cell runs with --coverage. A helper also run without
|
|
29
|
+
# padded_cell guards it: `if defined?(PaddedCell.coverage)`.
|
|
30
|
+
def coverage(profile = nil, &)
|
|
31
|
+
require_relative 'padded_cell/chart'
|
|
32
|
+
Chart.configure(profile, &)
|
|
33
|
+
end
|
|
26
34
|
end
|
|
27
35
|
end
|
|
28
36
|
|